From jim at digicool.com Sun Apr 1 14:38:36 2001 From: jim at digicool.com (Jim Fulton) Date: Sun Aug 10 16:31:14 2008 Subject: [ZEO-Checkins] CVS: Packages/ZEO - fap.py:1.2 start.py:1.19 __init__.py:1.6 Message-ID: <20010401183836.D4D68510E2@korak.digicool.com> Update of /cvs-repository/Packages/ZEO In directory korak:/tmp/cvs-serv18812 Modified Files: fap.py start.py __init__.py Log Message: Changed the way we try to find ZServer to take advantage of the known location of the start.py script (when it is used). --- Updated File fap.py in package Packages/ZEO -- --- fap.py 2001/03/27 23:36:23 1.1 +++ fap.py 2001/04/01 18:38:34 1.2 @@ -90,47 +90,54 @@ """ import sys -# if we are using an old version of Python, our asyncore is likely to -# be out of date. If ZServer is sitting around, we can get a current -# version of ayncore from it. In any case, if we are going to be used -# with Zope, it's important to use the version from Zope. -try: - from ZServer.medusa import asyncore -except: - # Try a little harder to import ZServer +def whiff(where): + if not where: return 0 import os, imp - try: m=imp.find_module('ZServer', ['.']) - except: - try: m=imp.find_module('ZServer', [os.path.join('..','..')]) - except: - import asyncore - else: - sys.path.append(os.path.join('..','..')) - from ZServer.medusa import asyncore - else: - sys.path.append('.') + try: m=imp.find_module('ZServer', [where]) + except: return 0 + else: return 1 + + +def fap(where=''): + # if we are using an old version of Python, our asyncore is likely to + # be out of date. If ZServer is sitting around, we can get a current + # version of ayncore from it. In any case, if we are going to be used + # with Zope, it's important to use the version from Zope. + try: from ZServer.medusa import asyncore + except: + # Try a little harder to import ZServer + import os, imp -if sys.version[:1] < '2' and asyncore.loop.func_code.co_argcount < 3: - raise ImportError, 'Cannot import an up-to-date asyncore' + for location in where, '.', os.path.join('..','..'): + if whiff(location): + sys.path.append(location) + try: + from ZServer.medusa import asyncore + except: + import asyncore + break -sys.modules['ZEO.asyncore']=asyncore + if sys.version[:1] < '2' and asyncore.loop.func_code.co_argcount < 3: + raise ImportError, 'Cannot import an up-to-date asyncore' -# We need a recent version of cPickle too. -if sys.version[:3] < '1.6': - try: - from ZODB import cPickle - sys.modules['ZEO.cPickle']=cPickle - except: - # Try a little harder + sys.modules['ZEO.asyncore']=asyncore + + # We need a recent version of cPickle too. + if sys.version[:3] < '1.6': + try: + from ZODB import cPickle + sys.modules['ZEO.cPickle']=cPickle + except: + # Try a little harder + import cPickle + else: import cPickle -else: - import cPickle -import cStringIO -p=cPickle.Pickler(cStringIO.StringIO(),1) -try: - p.fast=1 -except: - raise ImportError, 'Cannot import an up-to-date cPickle' + import cStringIO + p=cPickle.Pickler(cStringIO.StringIO(),1) + try: + p.fast=1 + except: + raise ImportError, 'Cannot import an up-to-date cPickle' --- Updated File start.py in package Packages/ZEO -- --- start.py 2001/03/28 21:55:06 1.18 +++ start.py 2001/04/01 18:38:34 1.19 @@ -207,7 +207,8 @@ elif o=='-d': detailed=1 elif o=='-s': Z=0 - import fap # fixup asyncore/cPickle dependencies + import fap + fap.fap(directory(me, 4)) # fixup asyncore/cPickle dependencies if port is None and unix is None: print usage --- Updated File __init__.py in package Packages/ZEO -- --- __init__.py 2001/03/27 23:39:02 1.5 +++ __init__.py 2001/04/01 18:38:34 1.6 @@ -83,4 +83,5 @@ # ############################################################################## -import fap # fixup asyncore/cPickle dependencies +import fap +fap.fap() # fixup asyncore/cPickle dependencies --- Updated File fap.py in package Packages/ZEO -- --- fap.py 2001/03/27 23:36:23 1.1 +++ fap.py 2001/04/01 18:38:34 1.2 @@ -90,47 +90,54 @@ """ import sys -# if we are using an old version of Python, our asyncore is likely to -# be out of date. If ZServer is sitting around, we can get a current -# version of ayncore from it. In any case, if we are going to be used -# with Zope, it's important to use the version from Zope. -try: - from ZServer.medusa import asyncore -except: - # Try a little harder to import ZServer +def whiff(where): + if not where: return 0 import os, imp - try: m=imp.find_module('ZServer', ['.']) - except: - try: m=imp.find_module('ZServer', [os.path.join('..','..')]) - except: - import asyncore - else: - sys.path.append(os.path.join('..','..')) - from ZServer.medusa import asyncore - else: - sys.path.append('.') + try: m=imp.find_module('ZServer', [where]) + except: return 0 + else: return 1 + + +def fap(where=''): + # if we are using an old version of Python, our asyncore is likely to + # be out of date. If ZServer is sitting around, we can get a current + # version of ayncore from it. In any case, if we are going to be used + # with Zope, it's important to use the version from Zope. + try: from ZServer.medusa import asyncore + except: + # Try a little harder to import ZServer + import os, imp -if sys.version[:1] < '2' and asyncore.loop.func_code.co_argcount < 3: - raise ImportError, 'Cannot import an up-to-date asyncore' + for location in where, '.', os.path.join('..','..'): + if whiff(location): + sys.path.append(location) + try: + from ZServer.medusa import asyncore + except: + import asyncore + break -sys.modules['ZEO.asyncore']=asyncore + if sys.version[:1] < '2' and asyncore.loop.func_code.co_argcount < 3: + raise ImportError, 'Cannot import an up-to-date asyncore' -# We need a recent version of cPickle too. -if sys.version[:3] < '1.6': - try: - from ZODB import cPickle - sys.modules['ZEO.cPickle']=cPickle - except: - # Try a little harder + sys.modules['ZEO.asyncore']=asyncore + + # We need a recent version of cPickle too. + if sys.version[:3] < '1.6': + try: + from ZODB import cPickle + sys.modules['ZEO.cPickle']=cPickle + except: + # Try a little harder + import cPickle + else: import cPickle -else: - import cPickle -import cStringIO -p=cPickle.Pickler(cStringIO.StringIO(),1) -try: - p.fast=1 -except: - raise ImportError, 'Cannot import an up-to-date cPickle' + import cStringIO + p=cPickle.Pickler(cStringIO.StringIO(),1) + try: + p.fast=1 + except: + raise ImportError, 'Cannot import an up-to-date cPickle' --- Updated File start.py in package Packages/ZEO -- --- start.py 2001/03/28 21:55:06 1.18 +++ start.py 2001/04/01 18:38:34 1.19 @@ -207,7 +207,8 @@ elif o=='-d': detailed=1 elif o=='-s': Z=0 - import fap # fixup asyncore/cPickle dependencies + import fap + fap.fap(directory(me, 4)) # fixup asyncore/cPickle dependencies if port is None and unix is None: print usage --- Updated File __init__.py in package Packages/ZEO -- --- __init__.py 2001/03/27 23:39:02 1.5 +++ __init__.py 2001/04/01 18:38:34 1.6 @@ -83,4 +83,5 @@ # ############################################################################## -import fap # fixup asyncore/cPickle dependencies +import fap +fap.fap() # fixup asyncore/cPickle dependencies From jim at digicool.com Sun Apr 1 14:41:15 2001 From: jim at digicool.com (Jim Fulton) Date: Sun Aug 10 16:31:14 2008 Subject: [ZEO-Checkins] CVS: Packages/ZEO - zrpc.py:1.14 Message-ID: <20010401184115.A3109510E2@korak.digicool.com> Update of /cvs-repository/Packages/ZEO In directory korak:/tmp/cvs-serv19452 Modified Files: zrpc.py Log Message: - Renamed and commented the flag formerly known as __map. - Commented the Wakeup calls. - Fixed a bug in the reconnection logic that caused the dispatcher to get reinitialized incorrectly when a main loop was active. This could cause a client to seem to hang on occasions. --- Updated File zrpc.py in package Packages/ZEO -- --- zrpc.py 2001/03/27 23:43:32 1.13 +++ zrpc.py 2001/04/01 18:41:13 1.14 @@ -107,7 +107,9 @@ class asyncRPC(SizedMessageAsyncConnection): - __map=0 + # Flag indicating whether a main loop is running. If one isn't running, + # then we'll have to provide our own main loop at times. + __haveMainLoop=0 def __Wakeup(*args): pass def __init__(self, connection, outOfBand=None, tmin=5, tmax=300, debug=0): @@ -159,7 +161,9 @@ return 1 def finishConnect(self, s): - SizedMessageAsyncConnection.__init__(self, s, '', {}) + if self.__haveMainLoop: map=None # use the main loop map + else: map = {} # provide a dummy map + SizedMessageAsyncConnection.__init__(self, s, '', map) # we are our own socket map! def keys(self): return (self._fileno,) @@ -171,7 +175,7 @@ raise KeyError, key def sync(self): - if self.__map: return # in async mode + if self.__haveMainLoop: return # in async mode # Ick, I have to do my own select loop, which sucks while 1: @@ -188,10 +192,10 @@ self.__lr() def setLoop(self, map=None, Wakeup=lambda : None): - if map is None: self.__map=0 + if map is None: self.__haveMainLoop=0 else: self.add_channel(map) # asyncore registration - self.__map=1 + self.__haveMainLoop=1 self.__Wakeup=Wakeup @@ -202,7 +206,8 @@ self._last_args=args=dump(args,1) self.message_output(args) - if self.__map: self.__Wakeup() # You dumb bastard + if self.__haveMainLoop: + self.__Wakeup() # Wakeup the main loop else: self.readLoop() while 1: @@ -231,7 +236,8 @@ def sendMessage(self, *args): self.message_output(dump(args,1)) - if self.__map: self.__Wakeup() # You dumb bastard + if self.__haveMainLoop: + self.__Wakeup() # Wake up the main loop else: asyncore.poll(0.0, self) def setOutOfBand(self, f): @@ -275,8 +281,10 @@ return self.__r def closeIntensionally(self): - if self.__map: - self.__Wakeup(lambda self=self: self.close()) # You dumb bastard + if self.__haveMainLoop: + # We aren't willing to close until told to by the main loop. + # So we'll tell the main loop to tell us. :) + self.__Wakeup(lambda self=self: self.close()) else: self.close() def close(self): --- Updated File zrpc.py in package Packages/ZEO -- --- zrpc.py 2001/03/27 23:43:32 1.13 +++ zrpc.py 2001/04/01 18:41:13 1.14 @@ -107,7 +107,9 @@ class asyncRPC(SizedMessageAsyncConnection): - __map=0 + # Flag indicating whether a main loop is running. If one isn't running, + # then we'll have to provide our own main loop at times. + __haveMainLoop=0 def __Wakeup(*args): pass def __init__(self, connection, outOfBand=None, tmin=5, tmax=300, debug=0): @@ -159,7 +161,9 @@ return 1 def finishConnect(self, s): - SizedMessageAsyncConnection.__init__(self, s, '', {}) + if self.__haveMainLoop: map=None # use the main loop map + else: map = {} # provide a dummy map + SizedMessageAsyncConnection.__init__(self, s, '', map) # we are our own socket map! def keys(self): return (self._fileno,) @@ -171,7 +175,7 @@ raise KeyError, key def sync(self): - if self.__map: return # in async mode + if self.__haveMainLoop: return # in async mode # Ick, I have to do my own select loop, which sucks while 1: @@ -188,10 +192,10 @@ self.__lr() def setLoop(self, map=None, Wakeup=lambda : None): - if map is None: self.__map=0 + if map is None: self.__haveMainLoop=0 else: self.add_channel(map) # asyncore registration - self.__map=1 + self.__haveMainLoop=1 self.__Wakeup=Wakeup @@ -202,7 +206,8 @@ self._last_args=args=dump(args,1) self.message_output(args) - if self.__map: self.__Wakeup() # You dumb bastard + if self.__haveMainLoop: + self.__Wakeup() # Wakeup the main loop else: self.readLoop() while 1: @@ -231,7 +236,8 @@ def sendMessage(self, *args): self.message_output(dump(args,1)) - if self.__map: self.__Wakeup() # You dumb bastard + if self.__haveMainLoop: + self.__Wakeup() # Wake up the main loop else: asyncore.poll(0.0, self) def setOutOfBand(self, f): @@ -275,8 +281,10 @@ return self.__r def closeIntensionally(self): - if self.__map: - self.__Wakeup(lambda self=self: self.close()) # You dumb bastard + if self.__haveMainLoop: + # We aren't willing to close until told to by the main loop. + # So we'll tell the main loop to tell us. :) + self.__Wakeup(lambda self=self: self.close()) else: self.close() def close(self): From jim at digicool.com Sun Apr 1 15:11:55 2001 From: jim at digicool.com (Jim Fulton) Date: Sun Aug 10 16:31:14 2008 Subject: [ZEO-Checkins] CVS: Packages/ZEO - start.py:1.20 Message-ID: <20010401191155.CB762510E2@korak.digicool.com> Update of /cvs-repository/Packages/ZEO In directory korak:/tmp/cvs-serv24265 Modified Files: start.py Log Message: Added some extra exception handlers to, hopefully, ease running on Windows. --- Updated File start.py in package Packages/ZEO -- --- start.py 2001/04/01 18:38:34 1.19 +++ start.py 2001/04/01 19:11:53 1.20 @@ -296,7 +296,8 @@ signal.signal(signal.SIGINT, lambda sig, frame, s=storages: shutdown(s, 0) ) - signal.signal(signal.SIGHUP, rotate_logs_handler) + try: signal.signal(signal.SIGHUP, rotate_logs_handler) + finally: pass finally: pass @@ -309,8 +310,9 @@ ZEO.StorageServer.StorageServer(unix, storages) - - open(zeo_pid,'w').write("%s %s" % (os.getppid(), os.getpid())) + try: ppid, pid = os.getppid(), os.getpid() + except: pass # getpid not supported + else: open(zeo_pid,'w').write("%s %s" % (ppid, pid)) asyncore.loop() except: --- Updated File start.py in package Packages/ZEO -- --- start.py 2001/04/01 18:38:34 1.19 +++ start.py 2001/04/01 19:11:53 1.20 @@ -296,7 +296,8 @@ signal.signal(signal.SIGINT, lambda sig, frame, s=storages: shutdown(s, 0) ) - signal.signal(signal.SIGHUP, rotate_logs_handler) + try: signal.signal(signal.SIGHUP, rotate_logs_handler) + finally: pass finally: pass @@ -309,8 +310,9 @@ ZEO.StorageServer.StorageServer(unix, storages) - - open(zeo_pid,'w').write("%s %s" % (os.getppid(), os.getpid())) + try: ppid, pid = os.getppid(), os.getpid() + except: pass # getpid not supported + else: open(zeo_pid,'w').write("%s %s" % (ppid, pid)) asyncore.loop() except: From jim at digicool.com Mon Apr 2 17:34:02 2001 From: jim at digicool.com (Jim Fulton) Date: Sun Aug 10 16:31:14 2008 Subject: [ZEO-Checkins] CVS: Packages/ZEO - ClientStorage.py:1.27 Message-ID: <20010402213402.ED424510E2@korak.digicool.com> Update of /cvs-repository/Packages/ZEO In directory korak:/tmp/cvs-serv13271 Modified Files: ClientStorage.py Log Message: Added logic to support conflict resolution. --- Updated File ClientStorage.py in package Packages/ZEO -- --- ClientStorage.py 2001/01/11 21:52:02 1.26 +++ ClientStorage.py 2001/04/02 21:34:01 1.27 @@ -96,6 +96,9 @@ from ZODB.TimeStamp import TimeStamp from zLOG import LOG, PROBLEM, INFO +try: from ZODB.ConflictResolution import ResolvedSerial +except: ResolvedSerial='rs' + TupleType=type(()) class ClientStorageError(POSException.StorageError): @@ -517,7 +520,10 @@ "Unexpected end of file in client storage " "temporary file." ) - update(oid, s, v, p) + if s==ResolvedSerial: + cache.invalidate(oid, v) + else: + update(oid, s, v, p) i=i+14+vlen+dlen seek(0) --- Updated File ClientStorage.py in package Packages/ZEO -- --- ClientStorage.py 2001/01/11 21:52:02 1.26 +++ ClientStorage.py 2001/04/02 21:34:01 1.27 @@ -96,6 +96,9 @@ from ZODB.TimeStamp import TimeStamp from zLOG import LOG, PROBLEM, INFO +try: from ZODB.ConflictResolution import ResolvedSerial +except: ResolvedSerial='rs' + TupleType=type(()) class ClientStorageError(POSException.StorageError): @@ -517,7 +520,10 @@ "Unexpected end of file in client storage " "temporary file." ) - update(oid, s, v, p) + if s==ResolvedSerial: + cache.invalidate(oid, v) + else: + update(oid, s, v, p) i=i+14+vlen+dlen seek(0) From jim at digicool.com Mon Apr 2 17:39:29 2001 From: jim at digicool.com (Jim Fulton) Date: Sun Aug 10 16:31:14 2008 Subject: [ZEO-Checkins] CVS: Packages/ZEO - CHANGES.txt:1.20 Message-ID: <20010402213929.757B9510E2@korak.digicool.com> Update of /cvs-repository/Packages/ZEO In directory korak:/tmp/cvs-serv14131 Modified Files: CHANGES.txt Log Message: *** empty log message *** --- Updated File CHANGES.txt in package Packages/ZEO -- --- CHANGES.txt 2001/03/27 23:35:34 1.19 +++ CHANGES.txt 2001/04/02 21:39:27 1.20 @@ -37,6 +37,10 @@ Bugs Fixed + - Application-level conflict resolution, introduced in Zope + 2.3.1, was not supported. This caused the ZEO cache to be + written incorrectly. + - A possible (but unobserved) race condition that could lead to ZEO cache corruption was corrected. --- Updated File CHANGES.txt in package Packages/ZEO -- --- CHANGES.txt 2001/03/27 23:35:34 1.19 +++ CHANGES.txt 2001/04/02 21:39:27 1.20 @@ -37,6 +37,10 @@ Bugs Fixed + - Application-level conflict resolution, introduced in Zope + 2.3.1, was not supported. This caused the ZEO cache to be + written incorrectly. + - A possible (but unobserved) race condition that could lead to ZEO cache corruption was corrected. From jim at digicool.com Mon Apr 2 17:44:41 2001 From: jim at digicool.com (Jim Fulton) Date: Sun Aug 10 16:31:14 2008 Subject: [ZEO-Checkins] CVS: Packages/ZEO - ZopeREADME.txt:1.3 Message-ID: <20010402214441.EB17E510E2@korak.digicool.com> Update of /cvs-repository/Packages/ZEO/doc In directory korak:/tmp/cvs-serv14960 Modified Files: ZopeREADME.txt Log Message: Added mention of FORCE_PRODUCT_LOAD. --- Updated File ZopeREADME.txt in package Packages/ZEO -- --- ZopeREADME.txt 2001/03/27 23:54:42 1.2 +++ ZopeREADME.txt 2001/04/02 21:44:40 1.3 @@ -74,7 +74,7 @@ If you do install new Zope products, then you need to take a special step to cause the new products to be properly registered in the database. The easiest way to do this is to start Zope - once without ZEO_CLIENT set. + once with the environment variable FORCE_PRODUCT_LOAD set. The interaction between ZEO and Zope product installation is unfortunate. In the future, this interaction will be removed by --- Updated File ZopeREADME.txt in package Packages/ZEO -- --- ZopeREADME.txt 2001/03/27 23:54:42 1.2 +++ ZopeREADME.txt 2001/04/02 21:44:40 1.3 @@ -74,7 +74,7 @@ If you do install new Zope products, then you need to take a special step to cause the new products to be properly registered in the database. The easiest way to do this is to start Zope - once without ZEO_CLIENT set. + once with the environment variable FORCE_PRODUCT_LOAD set. The interaction between ZEO and Zope product installation is unfortunate. In the future, this interaction will be removed by From jeremy at digicool.com Mon Apr 2 19:16:14 2001 From: jeremy at digicool.com (jeremy@digicool.com) Date: Sun Aug 10 16:31:14 2008 Subject: [ZEO-Checkins] CVS: Packages/ZEO - TransactionBuffer.py:1.1.2.1 ClientStorage.py:1.26.4.6 Message-ID: <20010402231614.9B9C5510E2@korak.digicool.com> Update of /cvs-repository/Packages/ZEO In directory korak:/tmp/cvs-serv29215 Modified Files: Tag: ZEO-ZRPC-Dev ClientStorage.py Added Files: Tag: ZEO-ZRPC-Dev TransactionBuffer.py Log Message: Replace explicit management of TemporaryFile with TransactionBuffer --- Added File TransactionBuffer.py in package Packages/ZEO --- """A TransactionBuffer store transaction updates until commit or abort. A transaction may generate enough data that it is not practical to always hold pending updates in memory. Instead, a TransactionBuffer is used to store the data until a commit or abort. """ # XXX Figure out what a sensible storage format is # XXX A faster implementation might store trans data in memory until # it reaches a certain size. import tempfile import marshal class TransactionBuffer: def __init__(self): self.file = tempfile.TemporaryFile() self.count = 0 self.size = 0 def store(self, oid, version, data): """Store oid, version, data for later retrieval""" marshal.dump((oid, version, data), self.file) self.count = self.count + 1 # Estimate per-record cache size self.size = self.size + len(data) + (27 + 12) if version: self.size = self.size + len(version) + 4 def clear(self): """Mark the buffer as empty""" self.file.seek(0) self.count = 0 self.size = 0 # XXX unchecked constraints: # 1. can't call store() after begin_iterate() # 2. must call clear() after iteration finishes def begin_iterate(self): """Move the file pointer in advance of iteration""" self.file.flush() self.file.seek(0) def next(self): """Return next tuple of data or None if EOF""" if not self.count: return None try: oid_ver_data = marshal.load(self.file) except EOFError: return None self.count = self.count - 1 return oid_ver_data def get_size(self): """Return size of data stored in buffer (just a hint).""" return self.size --- Updated File ClientStorage.py in package Packages/ZEO -- --- ClientStorage.py 2001/03/30 21:16:02 1.26.4.5 +++ ClientStorage.py 2001/04/02 23:16:12 1.26.4.6 @@ -90,13 +90,14 @@ import tempfile, ExtensionClass, thread import zrpc2 import ServerStub +from TransactionBuffer import TransactionBuffer import cPickle from struct import pack, unpack from ZODB import POSException, BaseStorage from ZODB.TimeStamp import TimeStamp -from zLOG import LOG, PROBLEM, INFO +from zLOG import LOG, PROBLEM, INFO, BLATHER import sys from types import TupleType @@ -110,7 +111,7 @@ class ClientDisconnected(ClientStorageError): """The database storage is disconnected from the storage.""" -class ClientStorage(ExtensionClass.Base, BaseStorage.BaseStorage): +class ClientStorage(BaseStorage.BaseStorage): __super_init = BaseStorage.BaseStorage.__init__ _server = None @@ -133,7 +134,7 @@ name = name or str(addr) - self._tfile = tempfile.TemporaryFile() + self._tbuf = TransactionBuffer() self._oids = [] # XXX It's confusing to have _serial, _serials, and _seriald. self._serials = [] @@ -270,13 +271,12 @@ def load(self, oid, version, _stuff=None): self._lock_acquire() try: - cache = self._cache - p = cache.load(oid, version) + p = self._cache.load(oid, version) if p: return p p, s, v, pv, sv = self._server.zeoLoad(oid) - cache.checkSize(0) - cache.store(oid, p, s, v, pv, sv) + self._cache.checkSize(0) + self._cache.store(oid, p, s, v, pv, sv) if not v or not version or version != v: if s: return p, s @@ -338,11 +338,7 @@ # sendMessage and then return it if _serials was None. # But sendMessage always returned None. self._server.storea(oid, serial, data, version, self._serial) - - write = self._tfile.write - write(oid+pack(">HI", len(version), len(data))+version) - write(data) - + self._tbuf.store(oid, version, data) return self._check_serials() finally: self._lock_release() @@ -370,7 +366,7 @@ return self._server.tpc_abort(self._serial) self._transaction = None - self._tfile.seek(0) + self._tbuf.clear() self._seriald.clear() del self._serials[:] self._commit_lock_release() @@ -412,7 +408,8 @@ # We have *BOTH* the local and distributed commit # lock, now we can actually get ready to get started. self._serial = id - self._tfile.seek(0) +## # _tbuf should always be in the clear state +## self._tfile.seek(0) self._seriald.clear() del self._serials[:] @@ -436,37 +433,25 @@ seriald=self._seriald r = self._check_serials() + + self._cache.checkSize(self._tbuf.get_size()) - tfile=self._tfile - seek=tfile.seek - read=tfile.read - cache=self._cache - update=cache.update - size=tfile.tell() - cache.checkSize(size) - seek(0) - i=0 - while i < size: - oid=read(8) + self._tbuf.begin_iterate() + while 1: try: - s=seriald[oid] - except KeyError: - print "failed to find oid: %s" % repr(oid) - raise - h=read(6) - vlen, dlen = unpack(">HI", h) - if vlen: v=read(vlen) - else: v='' - p=read(dlen) - if len(p) != dlen: + t = self._tbuf.next() + except ValueError, msg: raise ClientStorageError, ( - "Unexpected end of file in client storage " - "temporary file." - ) - update(oid, s, v, p) - i=i+14+vlen+dlen - - seek(0) + "Unexpected error reading temporary file in " + "client storage: %s" % msg) + if t is None: + break + oid, v, p = t + LOG("tbuf", BLATHER, "oid=%s v=%s len(p)=%d" % ( + repr(oid), repr(v), len(p))) + s = seriald[oid] + self._cache.update(oid, s, v, p) + self._tbuf.clear() self._transaction=None self._commit_lock_release() @@ -539,13 +524,13 @@ if self._pickler is None: return self._pickler.dump((0,0)) - self._pickler.dump = None +## self._pickler.dump = None self._tfile.seek(0) - load = cPickle.Unpickler(self._tfile).load + unpick = cPickle.Unpickler(self._tfile) self._tfile = None while 1: - oid, version = load() + oid, version = unpick.load() if not oid: break self._cache.invalidate(oid, version=version) --- Added File TransactionBuffer.py in package Packages/ZEO --- """A TransactionBuffer store transaction updates until commit or abort. A transaction may generate enough data that it is not practical to always hold pending updates in memory. Instead, a TransactionBuffer is used to store the data until a commit or abort. """ # XXX Figure out what a sensible storage format is # XXX A faster implementation might store trans data in memory until # it reaches a certain size. import tempfile import marshal class TransactionBuffer: def __init__(self): self.file = tempfile.TemporaryFile() self.count = 0 self.size = 0 def store(self, oid, version, data): """Store oid, version, data for later retrieval""" marshal.dump((oid, version, data), self.file) self.count = self.count + 1 # Estimate per-record cache size self.size = self.size + len(data) + (27 + 12) if version: self.size = self.size + len(version) + 4 def clear(self): """Mark the buffer as empty""" self.file.seek(0) self.count = 0 self.size = 0 # XXX unchecked constraints: # 1. can't call store() after begin_iterate() # 2. must call clear() after iteration finishes def begin_iterate(self): """Move the file pointer in advance of iteration""" self.file.flush() self.file.seek(0) def next(self): """Return next tuple of data or None if EOF""" if not self.count: return None try: oid_ver_data = marshal.load(self.file) except EOFError: return None self.count = self.count - 1 return oid_ver_data def get_size(self): """Return size of data stored in buffer (just a hint).""" return self.size --- Updated File ClientStorage.py in package Packages/ZEO -- --- ClientStorage.py 2001/03/30 21:16:02 1.26.4.5 +++ ClientStorage.py 2001/04/02 23:16:12 1.26.4.6 @@ -90,13 +90,14 @@ import tempfile, ExtensionClass, thread import zrpc2 import ServerStub +from TransactionBuffer import TransactionBuffer import cPickle from struct import pack, unpack from ZODB import POSException, BaseStorage from ZODB.TimeStamp import TimeStamp -from zLOG import LOG, PROBLEM, INFO +from zLOG import LOG, PROBLEM, INFO, BLATHER import sys from types import TupleType @@ -110,7 +111,7 @@ class ClientDisconnected(ClientStorageError): """The database storage is disconnected from the storage.""" -class ClientStorage(ExtensionClass.Base, BaseStorage.BaseStorage): +class ClientStorage(BaseStorage.BaseStorage): __super_init = BaseStorage.BaseStorage.__init__ _server = None @@ -133,7 +134,7 @@ name = name or str(addr) - self._tfile = tempfile.TemporaryFile() + self._tbuf = TransactionBuffer() self._oids = [] # XXX It's confusing to have _serial, _serials, and _seriald. self._serials = [] @@ -270,13 +271,12 @@ def load(self, oid, version, _stuff=None): self._lock_acquire() try: - cache = self._cache - p = cache.load(oid, version) + p = self._cache.load(oid, version) if p: return p p, s, v, pv, sv = self._server.zeoLoad(oid) - cache.checkSize(0) - cache.store(oid, p, s, v, pv, sv) + self._cache.checkSize(0) + self._cache.store(oid, p, s, v, pv, sv) if not v or not version or version != v: if s: return p, s @@ -338,11 +338,7 @@ # sendMessage and then return it if _serials was None. # But sendMessage always returned None. self._server.storea(oid, serial, data, version, self._serial) - - write = self._tfile.write - write(oid+pack(">HI", len(version), len(data))+version) - write(data) - + self._tbuf.store(oid, version, data) return self._check_serials() finally: self._lock_release() @@ -370,7 +366,7 @@ return self._server.tpc_abort(self._serial) self._transaction = None - self._tfile.seek(0) + self._tbuf.clear() self._seriald.clear() del self._serials[:] self._commit_lock_release() @@ -412,7 +408,8 @@ # We have *BOTH* the local and distributed commit # lock, now we can actually get ready to get started. self._serial = id - self._tfile.seek(0) +## # _tbuf should always be in the clear state +## self._tfile.seek(0) self._seriald.clear() del self._serials[:] @@ -436,37 +433,25 @@ seriald=self._seriald r = self._check_serials() + + self._cache.checkSize(self._tbuf.get_size()) - tfile=self._tfile - seek=tfile.seek - read=tfile.read - cache=self._cache - update=cache.update - size=tfile.tell() - cache.checkSize(size) - seek(0) - i=0 - while i < size: - oid=read(8) + self._tbuf.begin_iterate() + while 1: try: - s=seriald[oid] - except KeyError: - print "failed to find oid: %s" % repr(oid) - raise - h=read(6) - vlen, dlen = unpack(">HI", h) - if vlen: v=read(vlen) - else: v='' - p=read(dlen) - if len(p) != dlen: + t = self._tbuf.next() + except ValueError, msg: raise ClientStorageError, ( - "Unexpected end of file in client storage " - "temporary file." - ) - update(oid, s, v, p) - i=i+14+vlen+dlen - - seek(0) + "Unexpected error reading temporary file in " + "client storage: %s" % msg) + if t is None: + break + oid, v, p = t + LOG("tbuf", BLATHER, "oid=%s v=%s len(p)=%d" % ( + repr(oid), repr(v), len(p))) + s = seriald[oid] + self._cache.update(oid, s, v, p) + self._tbuf.clear() self._transaction=None self._commit_lock_release() @@ -539,13 +524,13 @@ if self._pickler is None: return self._pickler.dump((0,0)) - self._pickler.dump = None +## self._pickler.dump = None self._tfile.seek(0) - load = cPickle.Unpickler(self._tfile).load + unpick = cPickle.Unpickler(self._tfile) self._tfile = None while 1: - oid, version = load() + oid, version = unpick.load() if not oid: break self._cache.invalidate(oid, version=version) From jeremy at digicool.com Wed Apr 4 13:54:06 2001 From: jeremy at digicool.com (jeremy@digicool.com) Date: Sun Aug 10 16:31:14 2008 Subject: [ZEO-Checkins] CVS: Packages/ZEO - fap.py:1.3 Message-ID: <20010404175406.9C8B3510E2@korak.digicool.com> Update of /cvs-repository/Packages/ZEO In directory korak:/tmp/cvs-serv9545 Modified Files: fap.py Log Message: Make sure asyncore is defined one way or another --- Updated File fap.py in package Packages/ZEO -- --- fap.py 2001/04/01 18:38:34 1.2 +++ fap.py 2001/04/04 17:54:04 1.3 @@ -109,6 +109,8 @@ # Try a little harder to import ZServer import os, imp + asyncore = None + for location in where, '.', os.path.join('..','..'): if whiff(location): sys.path.append(location) @@ -117,6 +119,9 @@ except: import asyncore break + + if asyncore is None: + import asyncore if sys.version[:1] < '2' and asyncore.loop.func_code.co_argcount < 3: raise ImportError, 'Cannot import an up-to-date asyncore' --- Updated File fap.py in package Packages/ZEO -- --- fap.py 2001/04/01 18:38:34 1.2 +++ fap.py 2001/04/04 17:54:04 1.3 @@ -109,6 +109,8 @@ # Try a little harder to import ZServer import os, imp + asyncore = None + for location in where, '.', os.path.join('..','..'): if whiff(location): sys.path.append(location) @@ -117,6 +119,9 @@ except: import asyncore break + + if asyncore is None: + import asyncore if sys.version[:1] < '2' and asyncore.loop.func_code.co_argcount < 3: raise ImportError, 'Cannot import an up-to-date asyncore' From jim at digicool.com Thu Apr 5 19:33:42 2001 From: jim at digicool.com (Jim Fulton) Date: Sun Aug 10 16:31:14 2008 Subject: [ZEO-Checkins] CVS: Packages/ZEO - fap.py:1.4 Message-ID: <20010405233342.7F8AF510E2@korak.digicool.com> Update of /cvs-repository/Packages/ZEO In directory korak:/tmp/cvs-serv23788 Modified Files: fap.py Log Message: Redid ZServer search to use package path info. --- Updated File fap.py in package Packages/ZEO -- --- fap.py 2001/04/04 17:54:04 1.3 +++ fap.py 2001/04/05 23:33:39 1.4 @@ -88,41 +88,41 @@ Try to fix up the imports of these to make these dependencies work, localizing the hacks^H^H^H^H^Hchanges here. """ -import sys +import sys, os def whiff(where): if not where: return 0 - import os, imp + import imp try: m=imp.find_module('ZServer', [where]) except: return 0 else: return 1 -def fap(where=''): +def fap(): # if we are using an old version of Python, our asyncore is likely to # be out of date. If ZServer is sitting around, we can get a current # version of ayncore from it. In any case, if we are going to be used # with Zope, it's important to use the version from Zope. try: - from ZServer.medusa import asyncore + import ZServer except: # Try a little harder to import ZServer import os, imp + + location = package_home() + location = os.path.split(location)[0] + location = os.path.split(location)[0] + location = os.path.split(location)[0] + + if whiff(location): + sys.path.append(location) + try: + import ZServer + except: + pass - asyncore = None + import asyncore - for location in where, '.', os.path.join('..','..'): - if whiff(location): - sys.path.append(location) - try: - from ZServer.medusa import asyncore - except: - import asyncore - break - - if asyncore is None: - import asyncore - if sys.version[:1] < '2' and asyncore.loop.func_code.co_argcount < 3: raise ImportError, 'Cannot import an up-to-date asyncore' @@ -146,3 +146,16 @@ except: raise ImportError, 'Cannot import an up-to-date cPickle' + +def package_home(): + m=sys.modules[__name__] + if hasattr(m,'__path__'): + r=m.__path__[0] + elif "." in __name__: + from string import rfind + r=sys.modules[__name__[:rfind(__name__,'.')]].__path__[0] + else: + r=__name__ + return os.path.join(os.getcwd(), r) + +fap() --- Updated File fap.py in package Packages/ZEO -- --- fap.py 2001/04/04 17:54:04 1.3 +++ fap.py 2001/04/05 23:33:39 1.4 @@ -88,41 +88,41 @@ Try to fix up the imports of these to make these dependencies work, localizing the hacks^H^H^H^H^Hchanges here. """ -import sys +import sys, os def whiff(where): if not where: return 0 - import os, imp + import imp try: m=imp.find_module('ZServer', [where]) except: return 0 else: return 1 -def fap(where=''): +def fap(): # if we are using an old version of Python, our asyncore is likely to # be out of date. If ZServer is sitting around, we can get a current # version of ayncore from it. In any case, if we are going to be used # with Zope, it's important to use the version from Zope. try: - from ZServer.medusa import asyncore + import ZServer except: # Try a little harder to import ZServer import os, imp + + location = package_home() + location = os.path.split(location)[0] + location = os.path.split(location)[0] + location = os.path.split(location)[0] + + if whiff(location): + sys.path.append(location) + try: + import ZServer + except: + pass - asyncore = None + import asyncore - for location in where, '.', os.path.join('..','..'): - if whiff(location): - sys.path.append(location) - try: - from ZServer.medusa import asyncore - except: - import asyncore - break - - if asyncore is None: - import asyncore - if sys.version[:1] < '2' and asyncore.loop.func_code.co_argcount < 3: raise ImportError, 'Cannot import an up-to-date asyncore' @@ -146,3 +146,16 @@ except: raise ImportError, 'Cannot import an up-to-date cPickle' + +def package_home(): + m=sys.modules[__name__] + if hasattr(m,'__path__'): + r=m.__path__[0] + elif "." in __name__: + from string import rfind + r=sys.modules[__name__[:rfind(__name__,'.')]].__path__[0] + else: + r=__name__ + return os.path.join(os.getcwd(), r) + +fap() From jim at digicool.com Thu Apr 5 19:34:33 2001 From: jim at digicool.com (Jim Fulton) Date: Sun Aug 10 16:31:14 2008 Subject: [ZEO-Checkins] CVS: Packages/ZEO - __init__.py:1.7 Message-ID: <20010405233433.AC3CD510E2@korak.digicool.com> Update of /cvs-repository/Packages/ZEO In directory korak:/tmp/cvs-serv24075 Modified Files: __init__.py Log Message: Went back to just importing fap, since we no longer pass anything in. --- Updated File __init__.py in package Packages/ZEO -- --- __init__.py 2001/04/01 18:38:34 1.6 +++ __init__.py 2001/04/05 23:34:32 1.7 @@ -84,4 +84,3 @@ ############################################################################## import fap -fap.fap() # fixup asyncore/cPickle dependencies --- Updated File __init__.py in package Packages/ZEO -- --- __init__.py 2001/04/01 18:38:34 1.6 +++ __init__.py 2001/04/05 23:34:32 1.7 @@ -84,4 +84,3 @@ ############################################################################## import fap -fap.fap() # fixup asyncore/cPickle dependencies From jim at digicool.com Thu Apr 5 19:36:00 2001 From: jim at digicool.com (Jim Fulton) Date: Sun Aug 10 16:31:14 2008 Subject: [ZEO-Checkins] CVS: Packages/ZEO - start.py:1.21 Message-ID: <20010405233600.0EF9D510E2@korak.digicool.com> Update of /cvs-repository/Packages/ZEO In directory korak:/tmp/cvs-serv24278 Modified Files: start.py Log Message: We now rely on importing ZEO before importing asyncore. This lests us rely on the ZEO __init__, which relies on fap, to sort things out. --- Updated File start.py in package Packages/ZEO -- --- start.py 2001/04/01 19:11:53 1.20 +++ start.py 2001/04/05 23:35:58 1.21 @@ -207,9 +207,6 @@ elif o=='-d': detailed=1 elif o=='-s': Z=0 - import fap - fap.fap(directory(me, 4)) # fixup asyncore/cPickle dependencies - if port is None and unix is None: print usage print 'No port specified.' --- Updated File start.py in package Packages/ZEO -- --- start.py 2001/04/01 19:11:53 1.20 +++ start.py 2001/04/05 23:35:58 1.21 @@ -207,9 +207,6 @@ elif o=='-d': detailed=1 elif o=='-s': Z=0 - import fap - fap.fap(directory(me, 4)) # fixup asyncore/cPickle dependencies - if port is None and unix is None: print usage print 'No port specified.' From jeremy at digicool.com Fri Apr 6 20:05:52 2001 From: jeremy at digicool.com (jeremy@digicool.com) Date: Sun Aug 10 16:31:14 2008 Subject: [ZEO-Checkins] CVS: Packages/ZEO - start.py:1.22 Message-ID: <20010407000552.56791510E2@korak.digicool.com> Update of /cvs-repository/Packages/ZEO In directory korak:/tmp/cvs-serv23135 Modified Files: start.py Log Message: Replace to useful "finally: pass" clauses with "except: pass" clauses that allow start.py to recover. Thanks for Chris Withers for reminding about this. --- Updated File start.py in package Packages/ZEO -- --- start.py 2001/04/05 23:35:58 1.21 +++ start.py 2001/04/07 00:05:49 1.22 @@ -294,9 +294,9 @@ lambda sig, frame, s=storages: shutdown(s, 0) ) try: signal.signal(signal.SIGHUP, rotate_logs_handler) - finally: pass + except: pass - finally: pass + except: pass items=storages.items() items.sort() --- Updated File start.py in package Packages/ZEO -- --- start.py 2001/04/05 23:35:58 1.21 +++ start.py 2001/04/07 00:05:49 1.22 @@ -294,9 +294,9 @@ lambda sig, frame, s=storages: shutdown(s, 0) ) try: signal.signal(signal.SIGHUP, rotate_logs_handler) - finally: pass + except: pass - finally: pass + except: pass items=storages.items() items.sort() From jeremy at digicool.com Fri Apr 6 20:06:42 2001 From: jeremy at digicool.com (jeremy@digicool.com) Date: Sun Aug 10 16:31:14 2008 Subject: [ZEO-Checkins] CVS: Packages/ZEO - fap.py:1.5 Message-ID: <20010407000642.A24FC51014@korak.digicool.com> Update of /cvs-repository/Packages/ZEO In directory korak:/tmp/cvs-serv23455 Modified Files: fap.py Log Message: Make sure that Unpickler has a find_global attribute. --- Updated File fap.py in package Packages/ZEO -- --- fap.py 2001/04/05 23:33:39 1.4 +++ fap.py 2001/04/07 00:06:40 1.5 @@ -145,6 +145,11 @@ p.fast=1 except: raise ImportError, 'Cannot import an up-to-date cPickle' + p=cPickle.Unpickler(cStringIO.StringIO()) + try: + p.find_global=1 + except: + raise ImportError, 'Cannot import an up-to-date cPickle' def package_home(): --- Updated File fap.py in package Packages/ZEO -- --- fap.py 2001/04/05 23:33:39 1.4 +++ fap.py 2001/04/07 00:06:40 1.5 @@ -145,6 +145,11 @@ p.fast=1 except: raise ImportError, 'Cannot import an up-to-date cPickle' + p=cPickle.Unpickler(cStringIO.StringIO()) + try: + p.find_global=1 + except: + raise ImportError, 'Cannot import an up-to-date cPickle' def package_home(): From jeremy at digicool.com Fri Apr 6 20:07:39 2001 From: jeremy at digicool.com (jeremy@digicool.com) Date: Sun Aug 10 16:31:14 2008 Subject: [ZEO-Checkins] CVS: Packages/ZEO - README:NONE Message-ID: <20010407000739.CD72951014@korak.digicool.com> Update of /cvs-repository/Packages/ZEO In directory korak:/tmp/cvs-serv23587 Removed Files: README Log Message: For the future, this README file will live in Releases/ZEO --- Removed file README from package Packages/ZEO -- --- Removed file README from package Packages/ZEO -- From jeremy at digicool.com Fri Apr 6 20:26:05 2001 From: jeremy at digicool.com (jeremy@digicool.com) Date: Sun Aug 10 16:31:14 2008 Subject: [ZEO-Checkins] CVS: Packages/ZEO - NonZopeREADME.txt:1.2 Message-ID: <20010407002605.A4049510E2@korak.digicool.com> Update of /cvs-repository/Packages/ZEO/doc In directory korak:/tmp/cvs-serv26377 Modified Files: NonZopeREADME.txt Log Message: Small changes, primarily to mention that start.py works if you copy it to a directory other than the ZEO package directory. --- Updated File NonZopeREADME.txt in package Packages/ZEO -- --- NonZopeREADME.txt 2001/03/28 00:12:58 1.1 +++ NonZopeREADME.txt 2001/04/07 00:26:03 1.2 @@ -1,27 +1,29 @@ Zope Enterprize Objects - Put this package (the ZEO directory, without any wrapping directory - included in a distribution) in your python path. + Put the ZEO package in a directory on your Python path. On a Unix + system, you can use the site-packages directory of your Python lib + directory. The ZEO package is the directory named ZEO that contains + an __init__.py file. Starting (and configuring) the ZEO Server - To start the storage server, go to your Zope install directory and:: + To start the storage server, run the start.py script contained in + the ZEO package. You can run the script from the package + directory or copy it to a directory on your path. - python ZEO/start.py -p port_number + Specify the port number when you run the script:: - (Run start without arguments to see options.) + python ZEO/start.py -p port_number - Of course, the server and the client don't have to be on the same - machine. + Or run start.py without arguments to see options. The options are + documented in start.txt. - If the server and client *are* on the same machine, then you can use - a Unix domain socket:: + The server and the client don't have to be on the same machine. + If the server and client *are* on the same machine, then you can + use a Unix domain socket:: python ZEO/start.py -U filename - The start script provides a number of options not documented here. - See doc/start.txt for more information. - Running a ZEO client In your application, create a ClientStorage, rather than, say, a @@ -42,7 +44,7 @@ db=ZODB.DB(Storage) There are a number of configuration options available for the - ClientStorage. See doc/ClientStorage.txt for details. + ClientStorage. See ClientStorage.txt for details. If you want a persistent client cache which retains cache contents across ClientStorage restarts, you need to define the environment @@ -53,7 +55,7 @@ Dependencies on other modules - - The module, ThreadedAsync must be in the python path. + - The module ThreadedAsync must be on the Python path. - The zdaemon module is necessary if you want to run your storage server as a daemon that automatically restarts itself --- Updated File NonZopeREADME.txt in package Packages/ZEO -- --- NonZopeREADME.txt 2001/03/28 00:12:58 1.1 +++ NonZopeREADME.txt 2001/04/07 00:26:03 1.2 @@ -1,27 +1,29 @@ Zope Enterprize Objects - Put this package (the ZEO directory, without any wrapping directory - included in a distribution) in your python path. + Put the ZEO package in a directory on your Python path. On a Unix + system, you can use the site-packages directory of your Python lib + directory. The ZEO package is the directory named ZEO that contains + an __init__.py file. Starting (and configuring) the ZEO Server - To start the storage server, go to your Zope install directory and:: + To start the storage server, run the start.py script contained in + the ZEO package. You can run the script from the package + directory or copy it to a directory on your path. - python ZEO/start.py -p port_number + Specify the port number when you run the script:: - (Run start without arguments to see options.) + python ZEO/start.py -p port_number - Of course, the server and the client don't have to be on the same - machine. + Or run start.py without arguments to see options. The options are + documented in start.txt. - If the server and client *are* on the same machine, then you can use - a Unix domain socket:: + The server and the client don't have to be on the same machine. + If the server and client *are* on the same machine, then you can + use a Unix domain socket:: python ZEO/start.py -U filename - The start script provides a number of options not documented here. - See doc/start.txt for more information. - Running a ZEO client In your application, create a ClientStorage, rather than, say, a @@ -42,7 +44,7 @@ db=ZODB.DB(Storage) There are a number of configuration options available for the - ClientStorage. See doc/ClientStorage.txt for details. + ClientStorage. See ClientStorage.txt for details. If you want a persistent client cache which retains cache contents across ClientStorage restarts, you need to define the environment @@ -53,7 +55,7 @@ Dependencies on other modules - - The module, ThreadedAsync must be in the python path. + - The module ThreadedAsync must be on the Python path. - The zdaemon module is necessary if you want to run your storage server as a daemon that automatically restarts itself From jim at digicool.com Sat Apr 7 13:46:47 2001 From: jim at digicool.com (Jim Fulton) Date: Sun Aug 10 16:31:14 2008 Subject: [ZEO-Checkins] CVS: Packages/ZEO - start.py:1.23 Message-ID: <20010407174647.F3EC2510E2@korak.digicool.com> Update of /cvs-repository/Packages/ZEO In directory korak:/tmp/cvs-serv23977 Modified Files: start.py Log Message: Needed to move loop call outside of startup exception handler. --- Updated File start.py in package Packages/ZEO -- --- start.py 2001/04/07 00:05:49 1.22 +++ start.py 2001/04/07 17:46:45 1.23 @@ -311,7 +311,6 @@ except: pass # getpid not supported else: open(zeo_pid,'w').write("%s %s" % (ppid, pid)) - asyncore.loop() except: # Log startup exception and tell zdaemon not to restart us. info=sys.exc_info() @@ -326,6 +325,9 @@ apply(traceback.print_exception, info2) sys.exit(0) + + asyncore.loop() + def rotate_logs(): import zLOG --- Updated File start.py in package Packages/ZEO -- --- start.py 2001/04/07 00:05:49 1.22 +++ start.py 2001/04/07 17:46:45 1.23 @@ -311,7 +311,6 @@ except: pass # getpid not supported else: open(zeo_pid,'w').write("%s %s" % (ppid, pid)) - asyncore.loop() except: # Log startup exception and tell zdaemon not to restart us. info=sys.exc_info() @@ -326,6 +325,9 @@ apply(traceback.print_exception, info2) sys.exit(0) + + asyncore.loop() + def rotate_logs(): import zLOG From jim at digicool.com Sat Apr 7 13:49:38 2001 From: jim at digicool.com (Jim Fulton) Date: Sun Aug 10 16:31:14 2008 Subject: [ZEO-Checkins] CVS: Packages/ZEO - CHANGES.txt:1.21 Message-ID: <20010407174938.B586351014@korak.digicool.com> Update of /cvs-repository/Packages/ZEO In directory korak:/tmp/cvs-serv24422 Modified Files: CHANGES.txt Log Message: *** empty log message *** --- Updated File CHANGES.txt in package Packages/ZEO -- --- CHANGES.txt 2001/04/02 21:39:27 1.20 +++ CHANGES.txt 2001/04/07 17:49:36 1.21 @@ -1,5 +1,21 @@ Zope Enterprise Objects (ZEO) Revision History + ZEO 1.0 beta 1 + + New Features + + - Improved release organization. + + Bugs fixed + + - Normal shutdown was reported as a panic. + + - The signal exception handler was disabled. + + - Errors arising from incompatable versions of cPickle were + uclear. + + ZEO 0.5.0 New Features --- Updated File CHANGES.txt in package Packages/ZEO -- --- CHANGES.txt 2001/04/02 21:39:27 1.20 +++ CHANGES.txt 2001/04/07 17:49:36 1.21 @@ -1,5 +1,21 @@ Zope Enterprise Objects (ZEO) Revision History + ZEO 1.0 beta 1 + + New Features + + - Improved release organization. + + Bugs fixed + + - Normal shutdown was reported as a panic. + + - The signal exception handler was disabled. + + - Errors arising from incompatable versions of cPickle were + uclear. + + ZEO 0.5.0 New Features From jeremy at digicool.com Mon Apr 9 10:12:16 2001 From: jeremy at digicool.com (jeremy@digicool.com) Date: Sun Aug 10 16:31:14 2008 Subject: [ZEO-Checkins] CVS: Packages/ZEO - CHANGES.txt:NONE Message-ID: <20010409141216.EE017510E2@korak.digicool.com> Update of /cvs-repository/Packages/ZEO In directory korak:/tmp/cvs-serv12563 Removed Files: CHANGES.txt Log Message: moved to Releases/ZEO --- Removed file CHANGES.txt from package Packages/ZEO -- --- Removed file CHANGES.txt from package Packages/ZEO -- From jeremy at digicool.com Mon Apr 9 10:14:03 2001 From: jeremy at digicool.com (jeremy@digicool.com) Date: Sun Aug 10 16:31:14 2008 Subject: [ZEO-Checkins] CVS: Packages/ZEO - ClientCache.txt:NONE ClientStorage.txt:NONE NonZopeREADME.txt:NONE ZopeREADME.txt:NONE start.txt:NONE Message-ID: <20010409141403.8F7AD510E2@korak.digicool.com> Update of /cvs-repository/Packages/ZEO/doc In directory korak:/tmp/cvs-serv13728 Removed Files: ClientCache.txt ClientStorage.txt NonZopeREADME.txt ZopeREADME.txt start.txt Log Message: moved to Releases/ZEO/docs --- Removed file ClientCache.txt from package Packages/ZEO -- --- Removed file ClientStorage.txt from package Packages/ZEO -- --- Removed file NonZopeREADME.txt from package Packages/ZEO -- --- Removed file ZopeREADME.txt from package Packages/ZEO -- --- Removed file start.txt from package Packages/ZEO -- --- Removed file ClientCache.txt from package Packages/ZEO -- --- Removed file ClientStorage.txt from package Packages/ZEO -- --- Removed file NonZopeREADME.txt from package Packages/ZEO -- --- Removed file ZopeREADME.txt from package Packages/ZEO -- --- Removed file start.txt from package Packages/ZEO -- From jim at digicool.com Tue Apr 10 15:02:46 2001 From: jim at digicool.com (Jim Fulton) Date: Sun Aug 10 16:31:14 2008 Subject: [ZEO-Checkins] CVS: Packages/ZEO - README:NONE custom_zodb.py:NONE stop_zeo:NONE Message-ID: <20010410190246.D7156510E2@korak.digicool.com> Update of /cvs-repository/Packages/ZEO/misc In directory korak:/tmp/cvs-serv23757 Removed Files: README custom_zodb.py stop_zeo Log Message: Moved misc files to release directory --- Removed file README from package Packages/ZEO -- --- Removed file custom_zodb.py from package Packages/ZEO -- --- Removed file stop_zeo from package Packages/ZEO -- --- Removed file README from package Packages/ZEO -- --- Removed file custom_zodb.py from package Packages/ZEO -- --- Removed file stop_zeo from package Packages/ZEO -- From jim at digicool.com Tue Apr 10 15:04:48 2001 From: jim at digicool.com (Jim Fulton) Date: Sun Aug 10 16:31:14 2008 Subject: [ZEO-Checkins] CVS: Packages/ZEO - start_zeo:NONE Message-ID: <20010410190448.51F53510E2@korak.digicool.com> Update of /cvs-repository/Packages/ZEO/misc In directory korak:/tmp/cvs-serv25043 Removed Files: start_zeo Log Message: Moved misc files to release dir --- Removed file start_zeo from package Packages/ZEO -- --- Removed file start_zeo from package Packages/ZEO -- From jeremy at digicool.com Wed Apr 18 11:49:18 2001 From: jeremy at digicool.com (jeremy@digicool.com) Date: Sun Aug 10 16:31:14 2008 Subject: [ZEO-Checkins] CVS: Packages/ZEO Packages/ZEO/tests - New directory Message-ID: <20010418154918.A61E6510E3@korak.digicool.com> Update of /cvs-repository/Packages/ZEO/tests In directory korak:/tmp/cvs-serv22581/tests Log Message: Directory /cvs-repository/Packages/ZEO/tests added to the repository --> Using per-directory sticky tag `ZEO-ZRPC-Dev' --- Added directory Packages/ZEO/tests in package Packages/ZEO --- --- Added directory Packages/ZEO/tests in package Packages/ZEO --- From jeremy at digicool.com Wed Apr 18 11:49:56 2001 From: jeremy at digicool.com (jeremy@digicool.com) Date: Sun Aug 10 16:31:14 2008 Subject: [ZEO-Checkins] CVS: Packages/ZEO - testZEO.py:1.1.2.1 Message-ID: <20010418154956.D08C6510E3@korak.digicool.com> Update of /cvs-repository/Packages/ZEO/tests In directory korak:/tmp/cvs-serv22621 Added Files: Tag: ZEO-ZRPC-Dev testZEO.py Log Message: An set of tests for ZEO. Only some of the tests succeed. --- Added File testZEO.py in package Packages/ZEO --- """Test suite for ZEO based on ZODB.tests""" import asyncore import os import tempfile import threading import unittest import ZEO.ClientStorage, ZEO.StorageServer import ThreadedAsync, ZEO.trigger import zLOG # XXX The ZODB.tests package contains a grab bad things, including, # apparently, a collection of modules that define mixin classes # containing tests cases. from ZODB.tests import StorageTestBase, BasicStorage, VersionStorage class MainloopThread(threading.Thread): __super_init = threading.Thread.__init__ def __init__(self, *args, **kw): self.ready_lock = threading.Lock() self.ready_lock.acquire() self.__super_init(*args, **kw) def run(self): ThreadedAsync.register_loop_callback(self.callback) asyncore.loop(timeout=1.0) def callback(self, map): self.ready_lock.release() class GenericTests(StorageTestBase.StorageTestBase, BasicStorage.BasicStorage, ## VersionStorage.VersionStorage ): """An abstract base class for ZEO tests A specific ZEO test run depends on having a real storage that the StorageServer provides access to. The GenericTests must be subclassed to provide an implementation of getStorage() that returns a specific storage, e.g. FileStorage. The storage server mainloop is run in a separate thread so that it does not interfere with the thread running the tests. The tearDown() method clears asyncore's socket_map dict, which causes asyncore.loop() to return and the thread to exit. """ __super_setUp = StorageTestBase.StorageTestBase.setUp __super_tearDown = StorageTestBase.StorageTestBase.tearDown def setUp(self): """Start a ZEO server using a Unix domain socket The ZEO server uses the storage object returned by the getStorage() method. """ self.running = 1 self.__trigger = ZEO.trigger.trigger() self.__map = asyncore.socket_map ThreadedAsync.register_loop_callback(self.start_loop) self.__storage = self.getStorage() self.__sock = tempfile.mktemp() self.__server = ZEO.StorageServer.StorageServer(self.__sock, {'1': self.__storage}) t = MainloopThread() t.start() # wait for mainloop to start t.ready_lock.acquire() self._storage = ZEO.ClientStorage.ClientStorage(self.__sock) self.__super_setUp() def tearDown(self): """Try to cause the tests to halt""" zLOG.LOG("test", zLOG.INFO, "tearDown()") self.running = 0 self.stop_loop() self.__storage.close() os.unlink(self.__sock) self.__super_tearDown() def start_loop(self, map): ## zLOG.LOG("test", zLOG.INFO, "start_loop(%s)" % repr(map)) self.__map = map if not self.running: self.stop_loop() def stop_loop(self): ## zLOG.LOG("test", zLOG.INFO, "map = %s" % id(self.__map)) self.__map.clear() self.__trigger.pull_trigger() def checkFirst(self): self._storage.tpc_begin(self._transaction) self._storage.tpc_abort(self._transaction) class ZEOFileStorageTests(GenericTests): __super_setUp = GenericTests.setUp __super_tearDown = GenericTests.tearDown from ZODB.FileStorage import FileStorage def setUp(self): self.__fs_base = tempfile.mktemp() self.__super_setUp() def tearDown(self): self.__super_tearDown() # file storage appears to create three files for ext in '', '.index', '.lock', '.tmp': path = self.__fs_base + ext os.unlink(path) def getStorage(self): return self.FileStorage(self.__fs_base, create=1) def main(): tests = unittest.makeSuite(ZEOFileStorageTests, 'check') runner = unittest.TextTestRunner() runner.run(tests) if __name__ == "__main__": main() --- Added File testZEO.py in package Packages/ZEO --- """Test suite for ZEO based on ZODB.tests""" import asyncore import os import tempfile import threading import unittest import ZEO.ClientStorage, ZEO.StorageServer import ThreadedAsync, ZEO.trigger import zLOG # XXX The ZODB.tests package contains a grab bad things, including, # apparently, a collection of modules that define mixin classes # containing tests cases. from ZODB.tests import StorageTestBase, BasicStorage, VersionStorage class MainloopThread(threading.Thread): __super_init = threading.Thread.__init__ def __init__(self, *args, **kw): self.ready_lock = threading.Lock() self.ready_lock.acquire() self.__super_init(*args, **kw) def run(self): ThreadedAsync.register_loop_callback(self.callback) asyncore.loop(timeout=1.0) def callback(self, map): self.ready_lock.release() class GenericTests(StorageTestBase.StorageTestBase, BasicStorage.BasicStorage, ## VersionStorage.VersionStorage ): """An abstract base class for ZEO tests A specific ZEO test run depends on having a real storage that the StorageServer provides access to. The GenericTests must be subclassed to provide an implementation of getStorage() that returns a specific storage, e.g. FileStorage. The storage server mainloop is run in a separate thread so that it does not interfere with the thread running the tests. The tearDown() method clears asyncore's socket_map dict, which causes asyncore.loop() to return and the thread to exit. """ __super_setUp = StorageTestBase.StorageTestBase.setUp __super_tearDown = StorageTestBase.StorageTestBase.tearDown def setUp(self): """Start a ZEO server using a Unix domain socket The ZEO server uses the storage object returned by the getStorage() method. """ self.running = 1 self.__trigger = ZEO.trigger.trigger() self.__map = asyncore.socket_map ThreadedAsync.register_loop_callback(self.start_loop) self.__storage = self.getStorage() self.__sock = tempfile.mktemp() self.__server = ZEO.StorageServer.StorageServer(self.__sock, {'1': self.__storage}) t = MainloopThread() t.start() # wait for mainloop to start t.ready_lock.acquire() self._storage = ZEO.ClientStorage.ClientStorage(self.__sock) self.__super_setUp() def tearDown(self): """Try to cause the tests to halt""" zLOG.LOG("test", zLOG.INFO, "tearDown()") self.running = 0 self.stop_loop() self.__storage.close() os.unlink(self.__sock) self.__super_tearDown() def start_loop(self, map): ## zLOG.LOG("test", zLOG.INFO, "start_loop(%s)" % repr(map)) self.__map = map if not self.running: self.stop_loop() def stop_loop(self): ## zLOG.LOG("test", zLOG.INFO, "map = %s" % id(self.__map)) self.__map.clear() self.__trigger.pull_trigger() def checkFirst(self): self._storage.tpc_begin(self._transaction) self._storage.tpc_abort(self._transaction) class ZEOFileStorageTests(GenericTests): __super_setUp = GenericTests.setUp __super_tearDown = GenericTests.tearDown from ZODB.FileStorage import FileStorage def setUp(self): self.__fs_base = tempfile.mktemp() self.__super_setUp() def tearDown(self): self.__super_tearDown() # file storage appears to create three files for ext in '', '.index', '.lock', '.tmp': path = self.__fs_base + ext os.unlink(path) def getStorage(self): return self.FileStorage(self.__fs_base, create=1) def main(): tests = unittest.makeSuite(ZEOFileStorageTests, 'check') runner = unittest.TextTestRunner() runner.run(tests) if __name__ == "__main__": main() From jeremy at digicool.com Wed Apr 18 16:46:44 2001 From: jeremy at digicool.com (jeremy@digicool.com) Date: Sun Aug 10 16:31:14 2008 Subject: [ZEO-Checkins] CVS: Packages/ZEO - __init__.py:1.1.2.1 Message-ID: <20010418204644.551BE510E3@korak.digicool.com> Update of /cvs-repository/Packages/ZEO/tests In directory korak:/tmp/cvs-serv7917 Added Files: Tag: ZEO-ZRPC-Dev __init__.py Log Message: distutils is happier if this is a package --- Added File __init__.py in package Packages/ZEO --- --- Added File __init__.py in package Packages/ZEO --- From jeremy at digicool.com Wed Apr 18 16:51:08 2001 From: jeremy at digicool.com (jeremy@digicool.com) Date: Sun Aug 10 16:31:14 2008 Subject: [ZEO-Checkins] CVS: Packages/ZEO - ClientStorage.py:1.26.4.7 Message-ID: <20010418205108.35C2F510E3@korak.digicool.com> Update of /cvs-repository/Packages/ZEO In directory korak:/tmp/cvs-serv8237 Modified Files: Tag: ZEO-ZRPC-Dev ClientStorage.py Log Message: Sundry changes to ClientStorage Add is_connected() method that returns true if the storage is currently connected to the server. Simply cache protocol by allowing cache itself to call verify(). Fix some stupid typos: self._serverabortVersion -> self._server.abortVersion Modify close() method to close the rpc mgr (so it doesn't open a new connection) and then the current connection. Get rid of unused variable in _check_serials(). Augment logging for now. --- Updated File ClientStorage.py in package Packages/ZEO -- --- ClientStorage.py 2001/04/02 23:16:12 1.26.4.6 +++ ClientStorage.py 2001/04/18 20:51:08 1.26.4.7 @@ -83,6 +83,8 @@ # ############################################################################## """Network ZODB storage client + +XXX support multiple outstanding requests up until the vote """ __version__='$Revision$'[11:-2] @@ -100,7 +102,7 @@ from zLOG import LOG, PROBLEM, INFO, BLATHER import sys -from types import TupleType +from types import TupleType, StringType class ClientStorageError(POSException.StorageError): """An error occured in the ZEO Client Storage""" @@ -144,7 +146,7 @@ self._db = None self._cache = ClientCache.ClientCache(storage, cache_size, - client=client, var=var) + client=client, var=var) self._cache.open() # XXX self._rpc_mgr = zrpc2.ConnectionManager(addr, self, @@ -160,6 +162,16 @@ """Register that the storage is controlled by the given DB.""" self._db = db + def is_connected(self): + self._lock_acquire() + try: + if self._server: + return 1 + else: + return 0 + finally: + self._lock_release() + def notifyConnected(self, c): LOG("ClientStorage", INFO, "Connected to storage") self._lock_acquire() @@ -175,13 +187,9 @@ self._lock_release() def verify_cache(self): - cached = self._cache.open() # XXX - ### This is a little expensive for large caches - if cached: - self._server.beginZeoVerify() - for oid, (s, vs) in cached: - self._server.zeoVerify(oid, s, vs) - self._server.endZeoVerify() + self._server.beginZeoVerify() + self._cache.verify(self._server.zeoVerify) + self._server.endZeoVerify() ### Is there a race condition between notifyConnected and ### notifyDisconnected? In Particular, what if we get @@ -211,7 +219,7 @@ raise POSException.StorageTransactionError(self, transaction) self._lock_acquire() try: - oids = self._serverabortVersion(src, self._serial) + oids = self._server.abortVersion(src, self._serial) invalidate = self._cache.invalidate for oid in oids: invalidate(oid, src) @@ -222,8 +230,10 @@ def close(self): self._lock_acquire() try: - self_.call.closeIntensionally() # XXX this can't work - self._rpc_mgr.close_nicely() # XXX or self._server + # Close the manager first, so that it doesn't attempt to + # re-open the connection. + self._rpc_mgr.close() + self._server.rpc.close() finally: self._lock_release() @@ -248,8 +258,6 @@ def getName(self): return "%s (%s)" % (self.__name__, "XXX") -## # XXX self._connected is gone -## self._connected and 'connected' or 'disconnected') def getSize(self): return self._info['size'] @@ -257,7 +265,7 @@ def history(self, oid, version, length=1): self._lock_acquire() try: - return self._serverhistory(oid, version, length) + return self._server.history(oid, version, length) finally: self._lock_release() @@ -324,8 +332,9 @@ l = len(self._serials) r = self._serials[:l] del self._serials[:l] - d = self._seriald for oid, s in r: + LOG("ClientStorage", BLATHER, + "serial: %s %s" % (repr(oid), repr(s))) self._seriald[oid] = s return r @@ -408,8 +417,6 @@ # We have *BOTH* the local and distributed commit # lock, now we can actually get ready to get started. self._serial = id -## # _tbuf should always be in the clear state -## self._tfile.seek(0) self._seriald.clear() del self._serials[:] @@ -431,11 +438,14 @@ transaction.description, transaction._extension) - seriald=self._seriald r = self._check_serials() + assert r is None or len(r) == 0, "unhandled serialnos: %s" % r self._cache.checkSize(self._tbuf.get_size()) + LOG("ClientStorage", BLATHER, "tpc_finish()") + LOG("ClientStorage", BLATHER, "serials: %s" % repr(self._seriald)) + self._tbuf.begin_iterate() while 1: try: @@ -449,7 +459,11 @@ oid, v, p = t LOG("tbuf", BLATHER, "oid=%s v=%s len(p)=%d" % ( repr(oid), repr(v), len(p))) - s = seriald[oid] + s = self._seriald[oid] + if type(s) != StringType: + LOG("ClientStorage", INFO, "bad serialno: %s for %s" % \ + (repr(s), repr(oid))) + assert type(s) == StringType, "bad serialno: %s" % s self._cache.update(oid, s, v, p) self._tbuf.clear() @@ -505,6 +519,7 @@ self.commit_lock_release() def serialno(self, arg): + LOG("ClientStorage", BLATHER, "serialno(%s)" % repr(arg)) self._serials.append(arg) def info(self, dict): --- Updated File ClientStorage.py in package Packages/ZEO -- --- ClientStorage.py 2001/04/02 23:16:12 1.26.4.6 +++ ClientStorage.py 2001/04/18 20:51:08 1.26.4.7 @@ -83,6 +83,8 @@ # ############################################################################## """Network ZODB storage client + +XXX support multiple outstanding requests up until the vote """ __version__='$Revision$'[11:-2] @@ -100,7 +102,7 @@ from zLOG import LOG, PROBLEM, INFO, BLATHER import sys -from types import TupleType +from types import TupleType, StringType class ClientStorageError(POSException.StorageError): """An error occured in the ZEO Client Storage""" @@ -144,7 +146,7 @@ self._db = None self._cache = ClientCache.ClientCache(storage, cache_size, - client=client, var=var) + client=client, var=var) self._cache.open() # XXX self._rpc_mgr = zrpc2.ConnectionManager(addr, self, @@ -160,6 +162,16 @@ """Register that the storage is controlled by the given DB.""" self._db = db + def is_connected(self): + self._lock_acquire() + try: + if self._server: + return 1 + else: + return 0 + finally: + self._lock_release() + def notifyConnected(self, c): LOG("ClientStorage", INFO, "Connected to storage") self._lock_acquire() @@ -175,13 +187,9 @@ self._lock_release() def verify_cache(self): - cached = self._cache.open() # XXX - ### This is a little expensive for large caches - if cached: - self._server.beginZeoVerify() - for oid, (s, vs) in cached: - self._server.zeoVerify(oid, s, vs) - self._server.endZeoVerify() + self._server.beginZeoVerify() + self._cache.verify(self._server.zeoVerify) + self._server.endZeoVerify() ### Is there a race condition between notifyConnected and ### notifyDisconnected? In Particular, what if we get @@ -211,7 +219,7 @@ raise POSException.StorageTransactionError(self, transaction) self._lock_acquire() try: - oids = self._serverabortVersion(src, self._serial) + oids = self._server.abortVersion(src, self._serial) invalidate = self._cache.invalidate for oid in oids: invalidate(oid, src) @@ -222,8 +230,10 @@ def close(self): self._lock_acquire() try: - self_.call.closeIntensionally() # XXX this can't work - self._rpc_mgr.close_nicely() # XXX or self._server + # Close the manager first, so that it doesn't attempt to + # re-open the connection. + self._rpc_mgr.close() + self._server.rpc.close() finally: self._lock_release() @@ -248,8 +258,6 @@ def getName(self): return "%s (%s)" % (self.__name__, "XXX") -## # XXX self._connected is gone -## self._connected and 'connected' or 'disconnected') def getSize(self): return self._info['size'] @@ -257,7 +265,7 @@ def history(self, oid, version, length=1): self._lock_acquire() try: - return self._serverhistory(oid, version, length) + return self._server.history(oid, version, length) finally: self._lock_release() @@ -324,8 +332,9 @@ l = len(self._serials) r = self._serials[:l] del self._serials[:l] - d = self._seriald for oid, s in r: + LOG("ClientStorage", BLATHER, + "serial: %s %s" % (repr(oid), repr(s))) self._seriald[oid] = s return r @@ -408,8 +417,6 @@ # We have *BOTH* the local and distributed commit # lock, now we can actually get ready to get started. self._serial = id -## # _tbuf should always be in the clear state -## self._tfile.seek(0) self._seriald.clear() del self._serials[:] @@ -431,11 +438,14 @@ transaction.description, transaction._extension) - seriald=self._seriald r = self._check_serials() + assert r is None or len(r) == 0, "unhandled serialnos: %s" % r self._cache.checkSize(self._tbuf.get_size()) + LOG("ClientStorage", BLATHER, "tpc_finish()") + LOG("ClientStorage", BLATHER, "serials: %s" % repr(self._seriald)) + self._tbuf.begin_iterate() while 1: try: @@ -449,7 +459,11 @@ oid, v, p = t LOG("tbuf", BLATHER, "oid=%s v=%s len(p)=%d" % ( repr(oid), repr(v), len(p))) - s = seriald[oid] + s = self._seriald[oid] + if type(s) != StringType: + LOG("ClientStorage", INFO, "bad serialno: %s for %s" % \ + (repr(s), repr(oid))) + assert type(s) == StringType, "bad serialno: %s" % s self._cache.update(oid, s, v, p) self._tbuf.clear() @@ -505,6 +519,7 @@ self.commit_lock_release() def serialno(self, arg): + LOG("ClientStorage", BLATHER, "serialno(%s)" % repr(arg)) self._serials.append(arg) def info(self, dict): From jeremy at digicool.com Wed Apr 18 16:52:34 2001 From: jeremy at digicool.com (jeremy@digicool.com) Date: Sun Aug 10 16:31:14 2008 Subject: [ZEO-Checkins] CVS: Packages/ZEO - ClientCache.py:1.14.4.2 Message-ID: <20010418205234.1A6DF510E3@korak.digicool.com> Update of /cvs-repository/Packages/ZEO In directory korak:/tmp/cvs-serv8305 Modified Files: Tag: ZEO-ZRPC-Dev ClientCache.py Log Message: Revise cache to use new validation protocol. Add local log() helper function. --- Updated File ClientCache.py in package Packages/ZEO -- --- ClientCache.py 2001/03/29 21:31:57 1.14.4.1 +++ ClientCache.py 2001/04/18 20:52:33 1.14.4.2 @@ -150,6 +150,12 @@ from struct import pack, unpack from thread import allocate_lock +import sys +import zLOG + +def log(msg, level=zLOG.INFO): + zLOG.LOG("ZEC", level, msg) + magic='ZEC0' class ClientCache: @@ -207,6 +213,8 @@ f[0].write(magic) current=0 + log("cache opened. current = %s" % current) + self._limit=size/2 self._current=current @@ -232,6 +240,8 @@ verifyFunc(oid, serialno, version) """ + for oid, (s, vs) in self.open(): + verifyFunc(oid, s, vs) def invalidate(self, oid, version): self._acquire() --- Updated File ClientCache.py in package Packages/ZEO -- --- ClientCache.py 2001/03/29 21:31:57 1.14.4.1 +++ ClientCache.py 2001/04/18 20:52:33 1.14.4.2 @@ -150,6 +150,12 @@ from struct import pack, unpack from thread import allocate_lock +import sys +import zLOG + +def log(msg, level=zLOG.INFO): + zLOG.LOG("ZEC", level, msg) + magic='ZEC0' class ClientCache: @@ -207,6 +213,8 @@ f[0].write(magic) current=0 + log("cache opened. current = %s" % current) + self._limit=size/2 self._current=current @@ -232,6 +240,8 @@ verifyFunc(oid, serialno, version) """ + for oid, (s, vs) in self.open(): + verifyFunc(oid, s, vs) def invalidate(self, oid, version): self._acquire() From jeremy at digicool.com Wed Apr 18 16:53:38 2001 From: jeremy at digicool.com (jeremy@digicool.com) Date: Sun Aug 10 16:31:14 2008 Subject: [ZEO-Checkins] CVS: Packages/ZEO - StorageServer.py:1.21.4.4 Message-ID: <20010418205338.333AA510E3@korak.digicool.com> Update of /cvs-repository/Packages/ZEO In directory korak:/tmp/cvs-serv8514 Modified Files: Tag: ZEO-ZRPC-Dev StorageServer.py Log Message: Sundry changes to StorageServer Rename closed() to close(). Fix typos. Add some logging. --- Updated File StorageServer.py in package Packages/ZEO -- --- StorageServer.py 2001/03/30 22:55:22 1.21.4.3 +++ StorageServer.py 2001/04/18 20:53:37 1.21.4.4 @@ -58,7 +58,7 @@ else: p.client.info(info) - def closed(self, conn): + def close(self, conn): # called when conn is closed # way too inefficient removed = 0 @@ -198,15 +198,17 @@ newserial = self.__storage.store(oid, serial, data, version, t) except TransactionError, v: # This is a normal transaction error such as a conflict error - # or a version lock or conflict error. It doen't need to be + # or a version lock or conflict error. It doesn't need to be # logged. + log("transaction error: %s" % repr(v)) newserial = v except: # all errors need to be serialized to prevent unexpected # returns, which would screw up the return handling. # IOW, Anything that ends up here is evil enough to be logged. error = sys.exc_info() - log('store error: %s: %s' % (error[0], error[1])) + log('store error: %s: %s' % (error[0], error[1]). + zLOG.ERROR) newserial = sys.exc_info()[1] else: if serial != '\0\0\0\0\0\0\0\0': @@ -215,6 +217,8 @@ try: nil = dump(newserial, 1) except: + log("couldn't pickle newserial: %s" % repr(newserial), + zLOG.ERROR) dump('', 1) # clear pickler r = StorageServerError("Couldn't pickle exception %s" % \ `newserial`) --- Updated File StorageServer.py in package Packages/ZEO -- --- StorageServer.py 2001/03/30 22:55:22 1.21.4.3 +++ StorageServer.py 2001/04/18 20:53:37 1.21.4.4 @@ -58,7 +58,7 @@ else: p.client.info(info) - def closed(self, conn): + def close(self, conn): # called when conn is closed # way too inefficient removed = 0 @@ -198,15 +198,17 @@ newserial = self.__storage.store(oid, serial, data, version, t) except TransactionError, v: # This is a normal transaction error such as a conflict error - # or a version lock or conflict error. It doen't need to be + # or a version lock or conflict error. It doesn't need to be # logged. + log("transaction error: %s" % repr(v)) newserial = v except: # all errors need to be serialized to prevent unexpected # returns, which would screw up the return handling. # IOW, Anything that ends up here is evil enough to be logged. error = sys.exc_info() - log('store error: %s: %s' % (error[0], error[1])) + log('store error: %s: %s' % (error[0], error[1]). + zLOG.ERROR) newserial = sys.exc_info()[1] else: if serial != '\0\0\0\0\0\0\0\0': @@ -215,6 +217,8 @@ try: nil = dump(newserial, 1) except: + log("couldn't pickle newserial: %s" % repr(newserial), + zLOG.ERROR) dump('', 1) # clear pickler r = StorageServerError("Couldn't pickle exception %s" % \ `newserial`) From jeremy at digicool.com Wed Apr 18 16:55:10 2001 From: jeremy at digicool.com (jeremy@digicool.com) Date: Sun Aug 10 16:31:14 2008 Subject: [ZEO-Checkins] CVS: Packages/ZEO - smac.py:1.9.6.3 Message-ID: <20010418205510.B9AC4510E3@korak.digicool.com> Update of /cvs-repository/Packages/ZEO In directory korak:/tmp/cvs-serv8591 Modified Files: Tag: ZEO-ZRPC-Dev smac.py Log Message: Fiddle with imports and remove commented-out lines --- Updated File smac.py in package Packages/ZEO -- --- smac.py 2001/03/29 13:30:33 1.9.6.2 +++ smac.py 2001/04/18 20:55:09 1.9.6.3 @@ -87,8 +87,8 @@ __version__ = "$Revision$"[11:-2] -import asyncore, string, struct, zLOG, sys, Acquisition -from zLOG import LOG, TRACE, ERROR, INFO +import asyncore, string, struct, sys, Acquisition +from zLOG import LOG, TRACE, ERROR, INFO, BLATHER from types import StringType class SizedMessageAsyncConnection(asyncore.dispatcher): @@ -119,8 +119,6 @@ def __nonzero__(self): return 1 -## __read_inprogress = None - def handle_read(self): # Use a single __inp buffer and integer indexes to make this # fast. @@ -128,7 +126,6 @@ if not d: return -## self.__read_inprogress = 1 input_len = self.__input_len + len(d) msg_size = self.__msg_size state = self.__state @@ -155,8 +152,6 @@ offset = 0 while (offset + msg_size) <= input_len: -## if state is None and msg_size != 4: -## print "OUT OF LUCK" msg = inp[offset:offset + msg_size] offset = offset + msg_size if state is None: @@ -239,7 +234,6 @@ break # we can't write any more else: del output[0] -# LOG(self._debug, INFO, "output written; remain=%d" % len(self.__output)) def handle_close(self): self.close() @@ -260,13 +254,12 @@ # do two separate appends to avoid copying the message string self.__output.append(struct.pack(">i", len(message))) self.__output.append(message) -# LOG(self._debug, INFO, "output remain=%d" % len(self.__output)) - def log_info(self, message, type='info'): + def log_info(self, message, type='trace'): if type == 'error': type = ERROR else: - type = INFO + type = TRACE LOG('ZEO', type, message) log = log_info --- Updated File smac.py in package Packages/ZEO -- --- smac.py 2001/03/29 13:30:33 1.9.6.2 +++ smac.py 2001/04/18 20:55:09 1.9.6.3 @@ -87,8 +87,8 @@ __version__ = "$Revision$"[11:-2] -import asyncore, string, struct, zLOG, sys, Acquisition -from zLOG import LOG, TRACE, ERROR, INFO +import asyncore, string, struct, sys, Acquisition +from zLOG import LOG, TRACE, ERROR, INFO, BLATHER from types import StringType class SizedMessageAsyncConnection(asyncore.dispatcher): @@ -119,8 +119,6 @@ def __nonzero__(self): return 1 -## __read_inprogress = None - def handle_read(self): # Use a single __inp buffer and integer indexes to make this # fast. @@ -128,7 +126,6 @@ if not d: return -## self.__read_inprogress = 1 input_len = self.__input_len + len(d) msg_size = self.__msg_size state = self.__state @@ -155,8 +152,6 @@ offset = 0 while (offset + msg_size) <= input_len: -## if state is None and msg_size != 4: -## print "OUT OF LUCK" msg = inp[offset:offset + msg_size] offset = offset + msg_size if state is None: @@ -239,7 +234,6 @@ break # we can't write any more else: del output[0] -# LOG(self._debug, INFO, "output written; remain=%d" % len(self.__output)) def handle_close(self): self.close() @@ -260,13 +254,12 @@ # do two separate appends to avoid copying the message string self.__output.append(struct.pack(">i", len(message))) self.__output.append(message) -# LOG(self._debug, INFO, "output remain=%d" % len(self.__output)) - def log_info(self, message, type='info'): + def log_info(self, message, type='trace'): if type == 'error': type = ERROR else: - type = INFO + type = TRACE LOG('ZEO', type, message) log = log_info From jeremy at digicool.com Wed Apr 18 16:57:29 2001 From: jeremy at digicool.com (jeremy@digicool.com) Date: Sun Aug 10 16:31:14 2008 Subject: [ZEO-Checkins] CVS: Packages/ZEO - zrpc2.py:1.1.2.6 Message-ID: <20010418205729.29265510E3@korak.digicool.com> Update of /cvs-repository/Packages/ZEO In directory korak:/tmp/cvs-serv8684 Modified Files: Tag: ZEO-ZRPC-Dev zrpc2.py Log Message: Sundry fixes and updates Make sure all places were sockets are created check to see if it's AF_INET or AF_UNIX. Rename closed() method to notify_closed() and add closed attribute. Update managed connection to reflect new names args here and in ClientStorage. --- Updated File zrpc2.py in package Packages/ZEO -- --- zrpc2.py 2001/03/30 21:19:36 1.1.2.5 +++ zrpc2.py 2001/04/18 20:57:28 1.1.2.6 @@ -10,6 +10,14 @@ method is a string specifying the method to invoke. For a reply, the method is ".reply". args is a tuple of the argument to pass to method. + +XXX need to specify a version number that describes the protocol. +allow for future revision. + +XXX support multiple outstanding calls + +XXX factor out common pattern of deciding what protocol to use based +on whether address is tuple or string """ import asyncore @@ -50,7 +58,10 @@ loop = asyncore.loop def connect(addr, client=None): - s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + if type(addr) == types.TupleType: + s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + else: + s = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) s.connect(addr) c = Connection(s, addr, client) return c @@ -171,8 +182,8 @@ # waiting for a response self.__super_init(sock, addr) self._prepare_async() - self.__reply_lock = DebugLock() -## self.__reply_lock = thread.allocate_lock() +## self.__reply_lock = DebugLock() + self.__reply_lock = thread.allocate_lock() self.__reply_lock.acquire() if isinstance(obj, Handler): self.set_caller = 1 @@ -191,6 +202,9 @@ def message_input(self, message): """Decoding an incoming message and dispatch it""" + # XXX Not sure what to do with errors that reach this level. + # Need to catch ZRPCErrors in handle_reply() and + # handle_request() so that they get back to the client. try: msgid, flags, name, args = self.marshal.decode(message) except DecodingError, msg: @@ -227,7 +241,7 @@ except (POSException.UndoError, POSException.VersionCommitError), msg: return self.return_error(msgid, flags, sys.exc_info()[0], - sys.exc_info()[1]) + sys.exc_info()[1]) except Exception, msg: return self.return_error(msgid, flags, sys.exc_info()[0], sys.exc_info()[1]) @@ -244,7 +258,10 @@ def handle_error(self): t, v, tb = sys.exc_info() - print t, v + if type(v) == types.StringType: + print t, repr(v) + else: + print t, v traceback.print_tb(tb) def check_method(self, name): @@ -405,8 +422,13 @@ self._connect_lock = threading.Lock() self.trigger = None self.async = 0 + self.closed = 0 ThreadedAsync.register_loop_callback(self.set_async) + def close(self): + """Prevent ConnectionManager from opening new connections""" + self.closed = 1 + def register_object(self, obj): self.obj = obj @@ -467,10 +489,11 @@ t = self.tmax return t - def closed(self, conn): + def notify_closed(self, conn): self.connected = 0 self.obj.notifyDisconnected(None) - self.connect() + if not self.closed: + self.connect() class ManagedServerConnection(ServerConnection): """A connection that notifies its ConnectionManager of closing""" @@ -483,7 +506,8 @@ def close(self): self.__super_close() - self.__mgr.closed(self) + log("self.__mgr = %s" % repr(self.__mgr)) + self.__mgr.close(self) class ManagedConnection(Connection): """A connection that notifies its ConnectionManager of closing. @@ -518,7 +542,7 @@ def close(self): self.__super_close() - self.__mgr.closed(self) + self.__mgr.notify_closed(self) class Dispatcher(asyncore.dispatcher): """A server that accepts incoming RPC connections""" @@ -537,7 +561,10 @@ self._open_socket() def _open_socket(self): - self.create_socket(socket.AF_INET, socket.SOCK_STREAM) + if type(self.addr) == types.TupleType: + self.create_socket(socket.AF_INET, socket.SOCK_STREAM) + else: + self.create_socket(socket.AF_UNIX, socket.SOCK_STREAM) self.set_reuse_addr() self.bind(self.addr) self.listen(5) --- Updated File zrpc2.py in package Packages/ZEO -- --- zrpc2.py 2001/03/30 21:19:36 1.1.2.5 +++ zrpc2.py 2001/04/18 20:57:28 1.1.2.6 @@ -10,6 +10,14 @@ method is a string specifying the method to invoke. For a reply, the method is ".reply". args is a tuple of the argument to pass to method. + +XXX need to specify a version number that describes the protocol. +allow for future revision. + +XXX support multiple outstanding calls + +XXX factor out common pattern of deciding what protocol to use based +on whether address is tuple or string """ import asyncore @@ -50,7 +58,10 @@ loop = asyncore.loop def connect(addr, client=None): - s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + if type(addr) == types.TupleType: + s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + else: + s = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) s.connect(addr) c = Connection(s, addr, client) return c @@ -171,8 +182,8 @@ # waiting for a response self.__super_init(sock, addr) self._prepare_async() - self.__reply_lock = DebugLock() -## self.__reply_lock = thread.allocate_lock() +## self.__reply_lock = DebugLock() + self.__reply_lock = thread.allocate_lock() self.__reply_lock.acquire() if isinstance(obj, Handler): self.set_caller = 1 @@ -191,6 +202,9 @@ def message_input(self, message): """Decoding an incoming message and dispatch it""" + # XXX Not sure what to do with errors that reach this level. + # Need to catch ZRPCErrors in handle_reply() and + # handle_request() so that they get back to the client. try: msgid, flags, name, args = self.marshal.decode(message) except DecodingError, msg: @@ -227,7 +241,7 @@ except (POSException.UndoError, POSException.VersionCommitError), msg: return self.return_error(msgid, flags, sys.exc_info()[0], - sys.exc_info()[1]) + sys.exc_info()[1]) except Exception, msg: return self.return_error(msgid, flags, sys.exc_info()[0], sys.exc_info()[1]) @@ -244,7 +258,10 @@ def handle_error(self): t, v, tb = sys.exc_info() - print t, v + if type(v) == types.StringType: + print t, repr(v) + else: + print t, v traceback.print_tb(tb) def check_method(self, name): @@ -405,8 +422,13 @@ self._connect_lock = threading.Lock() self.trigger = None self.async = 0 + self.closed = 0 ThreadedAsync.register_loop_callback(self.set_async) + def close(self): + """Prevent ConnectionManager from opening new connections""" + self.closed = 1 + def register_object(self, obj): self.obj = obj @@ -467,10 +489,11 @@ t = self.tmax return t - def closed(self, conn): + def notify_closed(self, conn): self.connected = 0 self.obj.notifyDisconnected(None) - self.connect() + if not self.closed: + self.connect() class ManagedServerConnection(ServerConnection): """A connection that notifies its ConnectionManager of closing""" @@ -483,7 +506,8 @@ def close(self): self.__super_close() - self.__mgr.closed(self) + log("self.__mgr = %s" % repr(self.__mgr)) + self.__mgr.close(self) class ManagedConnection(Connection): """A connection that notifies its ConnectionManager of closing. @@ -518,7 +542,7 @@ def close(self): self.__super_close() - self.__mgr.closed(self) + self.__mgr.notify_closed(self) class Dispatcher(asyncore.dispatcher): """A server that accepts incoming RPC connections""" @@ -537,7 +561,10 @@ self._open_socket() def _open_socket(self): - self.create_socket(socket.AF_INET, socket.SOCK_STREAM) + if type(self.addr) == types.TupleType: + self.create_socket(socket.AF_INET, socket.SOCK_STREAM) + else: + self.create_socket(socket.AF_UNIX, socket.SOCK_STREAM) self.set_reuse_addr() self.bind(self.addr) self.listen(5) From jeremy at digicool.com Wed Apr 18 17:02:39 2001 From: jeremy at digicool.com (jeremy@digicool.com) Date: Sun Aug 10 16:31:14 2008 Subject: [ZEO-Checkins] CVS: Packages/ZEO - testZEO.py:1.1.2.2 Message-ID: <20010418210239.E8CD9510E3@korak.digicool.com> Update of /cvs-repository/Packages/ZEO/tests In directory korak:/tmp/cvs-serv9000 Modified Files: Tag: ZEO-ZRPC-Dev testZEO.py Log Message: Replace multithreaded test approach with forked child process approach Most of the tests run successfully provided that you include some local hacks to the ZODB.tests package that work around limitations in the handling of ConflictErrors by ZEO ClientStorage. --- Updated File testZEO.py in package Packages/ZEO -- --- testZEO.py 2001/04/18 15:49:56 1.1.2.1 +++ testZEO.py 2001/04/18 21:02:39 1.1.2.2 @@ -2,11 +2,13 @@ import asyncore import os +import sys import tempfile -import threading +import time import unittest import ZEO.ClientStorage, ZEO.StorageServer +from ZEO.zrpc2 import ManagedServerConnection import ThreadedAsync, ZEO.trigger import zLOG @@ -17,24 +19,20 @@ from ZODB.tests import StorageTestBase, BasicStorage, VersionStorage -class MainloopThread(threading.Thread): - __super_init = threading.Thread.__init__ - - def __init__(self, *args, **kw): - self.ready_lock = threading.Lock() - self.ready_lock.acquire() - self.__super_init(*args, **kw) - - def run(self): - ThreadedAsync.register_loop_callback(self.callback) - asyncore.loop(timeout=1.0) - - def callback(self, map): - self.ready_lock.release() - +class TestStorageServer(ZEO.StorageServer.StorageServer): + def newConnection(self, sock, addr, nil): + c = ManagedServerConnection(sock, addr, None, self) + c.register_object(TestStorageProxy(self, c)) + zLOG.LOG("TSS", zLOG.INFO, "connected: %s" % c) + +class TestStorageProxy(ZEO.StorageServer.StorageProxy): + def shutdown(self): + self._StorageProxy__storage.close() + os._exit(0) + class GenericTests(StorageTestBase.StorageTestBase, BasicStorage.BasicStorage, -## VersionStorage.VersionStorage + VersionStorage.VersionStorage ): """An abstract base class for ZEO tests @@ -42,11 +40,6 @@ StorageServer provides access to. The GenericTests must be subclassed to provide an implementation of getStorage() that returns a specific storage, e.g. FileStorage. - - The storage server mainloop is run in a separate thread so that - it does not interfere with the thread running the tests. The - tearDown() method clears asyncore's socket_map dict, which causes - asyncore.loop() to return and the thread to exit. """ __super_setUp = StorageTestBase.StorageTestBase.setUp @@ -59,40 +52,33 @@ getStorage() method. """ self.running = 1 - self.__trigger = ZEO.trigger.trigger() - self.__map = asyncore.socket_map - ThreadedAsync.register_loop_callback(self.start_loop) - self.__storage = self.getStorage() self.__sock = tempfile.mktemp() - self.__server = ZEO.StorageServer.StorageServer(self.__sock, - {'1': self.__storage}) - t = MainloopThread() - t.start() - # wait for mainloop to start - t.ready_lock.acquire() + self.startServer() self._storage = ZEO.ClientStorage.ClientStorage(self.__sock) + while not self._storage.is_connected(): + time.sleep(0.1) self.__super_setUp() def tearDown(self): """Try to cause the tests to halt""" zLOG.LOG("test", zLOG.INFO, "tearDown()") self.running = 0 - self.stop_loop() - self.__storage.close() + self._storage._server.rpc.callAsync('shutdown') + self._storage._rpc_mgr.close() + os.waitpid(self.__pid, 0) os.unlink(self.__sock) self.__super_tearDown() - - def start_loop(self, map): -## zLOG.LOG("test", zLOG.INFO, "start_loop(%s)" % repr(map)) - self.__map = map - if not self.running: - self.stop_loop() - - def stop_loop(self): -## zLOG.LOG("test", zLOG.INFO, "map = %s" % id(self.__map)) - self.__map.clear() - self.__trigger.pull_trigger() + def startServer(self): + self.__pid = os.fork() + if self.__pid == 0: + self.__storage = self.getStorage() + d = {'1': self.__storage} + self.__server = TestStorageServer(self.__sock, d) + asyncore.loop() + else: + zLOG.LOG("test", zLOG.INFO, "forked %s" % self.__pid) + def checkFirst(self): self._storage.tpc_begin(self._transaction) self._storage.tpc_abort(self._transaction) --- Updated File testZEO.py in package Packages/ZEO -- --- testZEO.py 2001/04/18 15:49:56 1.1.2.1 +++ testZEO.py 2001/04/18 21:02:39 1.1.2.2 @@ -2,11 +2,13 @@ import asyncore import os +import sys import tempfile -import threading +import time import unittest import ZEO.ClientStorage, ZEO.StorageServer +from ZEO.zrpc2 import ManagedServerConnection import ThreadedAsync, ZEO.trigger import zLOG @@ -17,24 +19,20 @@ from ZODB.tests import StorageTestBase, BasicStorage, VersionStorage -class MainloopThread(threading.Thread): - __super_init = threading.Thread.__init__ - - def __init__(self, *args, **kw): - self.ready_lock = threading.Lock() - self.ready_lock.acquire() - self.__super_init(*args, **kw) - - def run(self): - ThreadedAsync.register_loop_callback(self.callback) - asyncore.loop(timeout=1.0) - - def callback(self, map): - self.ready_lock.release() - +class TestStorageServer(ZEO.StorageServer.StorageServer): + def newConnection(self, sock, addr, nil): + c = ManagedServerConnection(sock, addr, None, self) + c.register_object(TestStorageProxy(self, c)) + zLOG.LOG("TSS", zLOG.INFO, "connected: %s" % c) + +class TestStorageProxy(ZEO.StorageServer.StorageProxy): + def shutdown(self): + self._StorageProxy__storage.close() + os._exit(0) + class GenericTests(StorageTestBase.StorageTestBase, BasicStorage.BasicStorage, -## VersionStorage.VersionStorage + VersionStorage.VersionStorage ): """An abstract base class for ZEO tests @@ -42,11 +40,6 @@ StorageServer provides access to. The GenericTests must be subclassed to provide an implementation of getStorage() that returns a specific storage, e.g. FileStorage. - - The storage server mainloop is run in a separate thread so that - it does not interfere with the thread running the tests. The - tearDown() method clears asyncore's socket_map dict, which causes - asyncore.loop() to return and the thread to exit. """ __super_setUp = StorageTestBase.StorageTestBase.setUp @@ -59,40 +52,33 @@ getStorage() method. """ self.running = 1 - self.__trigger = ZEO.trigger.trigger() - self.__map = asyncore.socket_map - ThreadedAsync.register_loop_callback(self.start_loop) - self.__storage = self.getStorage() self.__sock = tempfile.mktemp() - self.__server = ZEO.StorageServer.StorageServer(self.__sock, - {'1': self.__storage}) - t = MainloopThread() - t.start() - # wait for mainloop to start - t.ready_lock.acquire() + self.startServer() self._storage = ZEO.ClientStorage.ClientStorage(self.__sock) + while not self._storage.is_connected(): + time.sleep(0.1) self.__super_setUp() def tearDown(self): """Try to cause the tests to halt""" zLOG.LOG("test", zLOG.INFO, "tearDown()") self.running = 0 - self.stop_loop() - self.__storage.close() + self._storage._server.rpc.callAsync('shutdown') + self._storage._rpc_mgr.close() + os.waitpid(self.__pid, 0) os.unlink(self.__sock) self.__super_tearDown() - - def start_loop(self, map): -## zLOG.LOG("test", zLOG.INFO, "start_loop(%s)" % repr(map)) - self.__map = map - if not self.running: - self.stop_loop() - - def stop_loop(self): -## zLOG.LOG("test", zLOG.INFO, "map = %s" % id(self.__map)) - self.__map.clear() - self.__trigger.pull_trigger() + def startServer(self): + self.__pid = os.fork() + if self.__pid == 0: + self.__storage = self.getStorage() + d = {'1': self.__storage} + self.__server = TestStorageServer(self.__sock, d) + asyncore.loop() + else: + zLOG.LOG("test", zLOG.INFO, "forked %s" % self.__pid) + def checkFirst(self): self._storage.tpc_begin(self._transaction) self._storage.tpc_abort(self._transaction) From jeremy at digicool.com Thu Apr 19 11:46:42 2001 From: jeremy at digicool.com (jeremy@digicool.com) Date: Sun Aug 10 16:31:14 2008 Subject: [ZEO-Checkins] CVS: Packages/ZEO - zrpc2.py:1.1.2.7 Message-ID: <20010419154642.933BA510E3@korak.digicool.com> Update of /cvs-repository/Packages/ZEO In directory korak:/tmp/cvs-serv11428 Modified Files: Tag: ZEO-ZRPC-Dev zrpc2.py Log Message: undo augmented assignments just because they confused object domain --- Updated File zrpc2.py in package Packages/ZEO -- --- zrpc2.py 2001/04/18 20:57:28 1.1.2.6 +++ zrpc2.py 2001/04/19 15:46:40 1.1.2.7 @@ -126,7 +126,7 @@ self.lock = thread.allocate_lock() # XXX this actually needs to be locked too self.__lock_id = self.__locks - self.__locks += 1 + self.__locks = self.__locks + 1 def _debug(self): method = sys._getframe().f_back @@ -327,7 +327,7 @@ if self.closed: raise DisconnectedError("This action is temporarily unavailable") msgid = self.msgid - self.msgid += 1 + self.msgid = self.msgid + 1 log("call %s %s" % (msgid, method)) self.message_output(self.marshal.encode(msgid, 0, method, args)) @@ -352,7 +352,7 @@ if self.closed: raise DisconnectedError("This action is temporarily unavailable") msgid = self.msgid - self.msgid += 1 + self.msgid = self.msgid + 1 log("async %s %s" % (msgid, method)) self.message_output(self.marshal.encode(msgid, ASYNC, method, args)) self._do_io() --- Updated File zrpc2.py in package Packages/ZEO -- --- zrpc2.py 2001/04/18 20:57:28 1.1.2.6 +++ zrpc2.py 2001/04/19 15:46:40 1.1.2.7 @@ -126,7 +126,7 @@ self.lock = thread.allocate_lock() # XXX this actually needs to be locked too self.__lock_id = self.__locks - self.__locks += 1 + self.__locks = self.__locks + 1 def _debug(self): method = sys._getframe().f_back @@ -327,7 +327,7 @@ if self.closed: raise DisconnectedError("This action is temporarily unavailable") msgid = self.msgid - self.msgid += 1 + self.msgid = self.msgid + 1 log("call %s %s" % (msgid, method)) self.message_output(self.marshal.encode(msgid, 0, method, args)) @@ -352,7 +352,7 @@ if self.closed: raise DisconnectedError("This action is temporarily unavailable") msgid = self.msgid - self.msgid += 1 + self.msgid = self.msgid + 1 log("async %s %s" % (msgid, method)) self.message_output(self.marshal.encode(msgid, ASYNC, method, args)) self._do_io() From jeremy at digicool.com Thu Apr 19 11:47:49 2001 From: jeremy at digicool.com (jeremy@digicool.com) Date: Sun Aug 10 16:31:14 2008 Subject: [ZEO-Checkins] CVS: Packages/ZEO - StorageServer.py:1.21.4.5 Message-ID: <20010419154749.82D50510E3@korak.digicool.com> Update of /cvs-repository/Packages/ZEO In directory korak:/tmp/cvs-serv11649 Modified Files: Tag: ZEO-ZRPC-Dev StorageServer.py Log Message: Remove reference to no-longer-defined name _noreturn (replace with None) --- Updated File StorageServer.py in package Packages/ZEO -- --- StorageServer.py 2001/04/18 20:53:37 1.21.4.4 +++ StorageServer.py 2001/04/19 15:47:48 1.21.4.5 @@ -139,7 +139,7 @@ try: p, os, v, pv, osv = self.zeoLoad(oid) except: # except what? - return _noreturn + return None p = pv = None # free the pickles if os != s: self.client.invalidate((oid, '')) --- Updated File StorageServer.py in package Packages/ZEO -- --- StorageServer.py 2001/04/18 20:53:37 1.21.4.4 +++ StorageServer.py 2001/04/19 15:47:48 1.21.4.5 @@ -139,7 +139,7 @@ try: p, os, v, pv, osv = self.zeoLoad(oid) except: # except what? - return _noreturn + return None p = pv = None # free the pickles if os != s: self.client.invalidate((oid, '')) From jeremy at digicool.com Thu Apr 19 12:59:54 2001 From: jeremy at digicool.com (jeremy@digicool.com) Date: Sun Aug 10 16:31:14 2008 Subject: [ZEO-Checkins] CVS: Packages/ZEO - testZEO.py:1.1.2.3 Message-ID: <20010419165954.4782C510E3@korak.digicool.com> Update of /cvs-repository/Packages/ZEO/tests In directory korak:/tmp/cvs-serv16022/tests Modified Files: Tag: ZEO-ZRPC-Dev testZEO.py Log Message: Add ZEOTestBase that enables many tests to succeed with current ZEO ZEOTestBase modifies _dostore() to account for the different way that ZEO currently handles serialno/ConflictError returns. --- Updated File testZEO.py in package Packages/ZEO -- --- testZEO.py 2001/04/18 21:02:39 1.1.2.2 +++ testZEO.py 2001/04/19 16:59:53 1.1.2.3 @@ -5,6 +5,7 @@ import sys import tempfile import time +import types import unittest import ZEO.ClientStorage, ZEO.StorageServer @@ -29,8 +30,66 @@ def shutdown(self): self._StorageProxy__storage.close() os._exit(0) + +ZERO = '\0'*8 +import pickle + +class ZEOTestBase(StorageTestBase.StorageTestBase): + """Version of the storage test class that supports ZEO. + + For ZEO, we don't always get the serialno/exception for a + particular store as the return value from the store. But we + will get no later than the return value from vote. + """ + + def _dostore(self, oid=None, revid=None, data=None, version=None): + """Do a complete storage transaction. + + The defaults are: + - oid=None, ask the storage for a new oid + - revid=None, use a revid of ZERO + - data=None, pickle up some arbitrary data (the integer 7) + - version=None, use the empty string version + + Returns the object's new revision id. + """ + if oid is None: + oid = self._storage.new_oid() + if revid is None: + revid = ZERO + if data is None: + data = pickle.dumps(7) + else: + data = pickle.dumps(data) + if version is None: + version = '' + # Begin the transaction + self._storage.tpc_begin(self._transaction) + # Store an object + r1 = self._storage.store(oid, revid, data, version, + self._transaction) + # Finish the transaction + r2 = self._storage.tpc_vote(self._transaction) + self._storage.tpc_finish(self._transaction) + return self._get_serial([r1, r2])[oid] + + def _get_serial(self, resps): + """Return oid -> serialno dict from sequence of ZEO replies.""" + d = {} + for r in resps: + if not r: + continue + if type(r) == types.StringType: + raise RuntimeError, "unexpected ZEO response: no oid" + else: + for oid, serial in r: + if type(serial) != types.StringType: + raise s + else: + d[oid] = serial + return d -class GenericTests(StorageTestBase.StorageTestBase, +class GenericTests(ZEOTestBase, BasicStorage.BasicStorage, VersionStorage.VersionStorage ): --- Updated File testZEO.py in package Packages/ZEO -- --- testZEO.py 2001/04/18 21:02:39 1.1.2.2 +++ testZEO.py 2001/04/19 16:59:53 1.1.2.3 @@ -5,6 +5,7 @@ import sys import tempfile import time +import types import unittest import ZEO.ClientStorage, ZEO.StorageServer @@ -29,8 +30,66 @@ def shutdown(self): self._StorageProxy__storage.close() os._exit(0) + +ZERO = '\0'*8 +import pickle + +class ZEOTestBase(StorageTestBase.StorageTestBase): + """Version of the storage test class that supports ZEO. + + For ZEO, we don't always get the serialno/exception for a + particular store as the return value from the store. But we + will get no later than the return value from vote. + """ + + def _dostore(self, oid=None, revid=None, data=None, version=None): + """Do a complete storage transaction. + + The defaults are: + - oid=None, ask the storage for a new oid + - revid=None, use a revid of ZERO + - data=None, pickle up some arbitrary data (the integer 7) + - version=None, use the empty string version + + Returns the object's new revision id. + """ + if oid is None: + oid = self._storage.new_oid() + if revid is None: + revid = ZERO + if data is None: + data = pickle.dumps(7) + else: + data = pickle.dumps(data) + if version is None: + version = '' + # Begin the transaction + self._storage.tpc_begin(self._transaction) + # Store an object + r1 = self._storage.store(oid, revid, data, version, + self._transaction) + # Finish the transaction + r2 = self._storage.tpc_vote(self._transaction) + self._storage.tpc_finish(self._transaction) + return self._get_serial([r1, r2])[oid] + + def _get_serial(self, resps): + """Return oid -> serialno dict from sequence of ZEO replies.""" + d = {} + for r in resps: + if not r: + continue + if type(r) == types.StringType: + raise RuntimeError, "unexpected ZEO response: no oid" + else: + for oid, serial in r: + if type(serial) != types.StringType: + raise s + else: + d[oid] = serial + return d -class GenericTests(StorageTestBase.StorageTestBase, +class GenericTests(ZEOTestBase, BasicStorage.BasicStorage, VersionStorage.VersionStorage ): From jeremy at digicool.com Thu Apr 19 14:24:03 2001 From: jeremy at digicool.com (jeremy@digicool.com) Date: Sun Aug 10 16:31:14 2008 Subject: [ZEO-Checkins] CVS: Packages/ZEO - ClientStorage.py:1.26.4.8 Message-ID: <20010419182403.6A8F9510E3@korak.digicool.com> Update of /cvs-repository/Packages/ZEO In directory korak:/tmp/cvs-serv21182 Modified Files: Tag: ZEO-ZRPC-Dev ClientStorage.py Log Message: fiddle with logging --- Updated File ClientStorage.py in package Packages/ZEO -- --- ClientStorage.py 2001/04/18 20:51:08 1.26.4.7 +++ ClientStorage.py 2001/04/19 18:24:02 1.26.4.8 @@ -443,9 +443,6 @@ self._cache.checkSize(self._tbuf.get_size()) - LOG("ClientStorage", BLATHER, "tpc_finish()") - LOG("ClientStorage", BLATHER, "serials: %s" % repr(self._seriald)) - self._tbuf.begin_iterate() while 1: try: @@ -457,13 +454,11 @@ if t is None: break oid, v, p = t - LOG("tbuf", BLATHER, "oid=%s v=%s len(p)=%d" % ( - repr(oid), repr(v), len(p))) s = self._seriald[oid] if type(s) != StringType: LOG("ClientStorage", INFO, "bad serialno: %s for %s" % \ (repr(s), repr(oid))) - assert type(s) == StringType, "bad serialno: %s" % s + assert type(s) == StringType, "bad serialno: %s" % repr(s) self._cache.update(oid, s, v, p) self._tbuf.clear() --- Updated File ClientStorage.py in package Packages/ZEO -- --- ClientStorage.py 2001/04/18 20:51:08 1.26.4.7 +++ ClientStorage.py 2001/04/19 18:24:02 1.26.4.8 @@ -443,9 +443,6 @@ self._cache.checkSize(self._tbuf.get_size()) - LOG("ClientStorage", BLATHER, "tpc_finish()") - LOG("ClientStorage", BLATHER, "serials: %s" % repr(self._seriald)) - self._tbuf.begin_iterate() while 1: try: @@ -457,13 +454,11 @@ if t is None: break oid, v, p = t - LOG("tbuf", BLATHER, "oid=%s v=%s len(p)=%d" % ( - repr(oid), repr(v), len(p))) s = self._seriald[oid] if type(s) != StringType: LOG("ClientStorage", INFO, "bad serialno: %s for %s" % \ (repr(s), repr(oid))) - assert type(s) == StringType, "bad serialno: %s" % s + assert type(s) == StringType, "bad serialno: %s" % repr(s) self._cache.update(oid, s, v, p) self._tbuf.clear() From jeremy at digicool.com Thu Apr 19 14:25:04 2001 From: jeremy at digicool.com (jeremy@digicool.com) Date: Sun Aug 10 16:31:14 2008 Subject: [ZEO-Checkins] CVS: Packages/ZEO - zrpc2.py:1.1.2.8 Message-ID: <20010419182504.158E1510E3@korak.digicool.com> Update of /cvs-repository/Packages/ZEO In directory korak:/tmp/cvs-serv21243 Modified Files: Tag: ZEO-ZRPC-Dev zrpc2.py Log Message: Remove prints from return_error() for cases where exception is sent to client --- Updated File zrpc2.py in package Packages/ZEO -- --- zrpc2.py 2001/04/19 15:46:40 1.1.2.7 +++ zrpc2.py 2001/04/19 18:25:03 1.1.2.8 @@ -284,8 +284,6 @@ if type(err_value) is not types.InstanceType: err_value = err_type, err_value - print self.handle_error() - try: msg = self.marshal.encode(msgid, 0, REPLY, (err_type, err_value)) except self.marshal.errors: @@ -293,8 +291,6 @@ msg = self.marshal.encode(msgid, 0, REPLY, (ZRPCError, err)) self.message_output(msg) self._do_io() - print "Sent error message for:" - self.handle_error() # The next five methods implement an asyncore socket map --- Updated File zrpc2.py in package Packages/ZEO -- --- zrpc2.py 2001/04/19 15:46:40 1.1.2.7 +++ zrpc2.py 2001/04/19 18:25:03 1.1.2.8 @@ -284,8 +284,6 @@ if type(err_value) is not types.InstanceType: err_value = err_type, err_value - print self.handle_error() - try: msg = self.marshal.encode(msgid, 0, REPLY, (err_type, err_value)) except self.marshal.errors: @@ -293,8 +291,6 @@ msg = self.marshal.encode(msgid, 0, REPLY, (ZRPCError, err)) self.message_output(msg) self._do_io() - print "Sent error message for:" - self.handle_error() # The next five methods implement an asyncore socket map From jeremy at digicool.com Thu Apr 19 14:26:05 2001 From: jeremy at digicool.com (jeremy@digicool.com) Date: Sun Aug 10 16:31:14 2008 Subject: [ZEO-Checkins] CVS: Packages/ZEO - testZEO.py:1.1.2.4 Message-ID: <20010419182605.C808D510E3@korak.digicool.com> Update of /cvs-repository/Packages/ZEO/tests In directory korak:/tmp/cvs-serv21302/tests Modified Files: Tag: ZEO-ZRPC-Dev testZEO.py Log Message: Fix uses of _get_serial() so that exceptions occur at the right time and values really get returned. 17 or 19 tests now succeed --- Updated File testZEO.py in package Packages/ZEO -- --- testZEO.py 2001/04/19 16:59:53 1.1.2.3 +++ testZEO.py 2001/04/19 18:26:05 1.1.2.4 @@ -68,25 +68,28 @@ # Store an object r1 = self._storage.store(oid, revid, data, version, self._transaction) + s1 = self._get_serial(r1) # Finish the transaction r2 = self._storage.tpc_vote(self._transaction) + s2 = self._get_serial(r2) self._storage.tpc_finish(self._transaction) - return self._get_serial([r1, r2])[oid] + # s1, s2 can be None or dict + return s1 and s1[oid] or s2 and s2[oid] - def _get_serial(self, resps): + def _get_serial(self, r): """Return oid -> serialno dict from sequence of ZEO replies.""" d = {} - for r in resps: - if not r: - continue - if type(r) == types.StringType: - raise RuntimeError, "unexpected ZEO response: no oid" - else: - for oid, serial in r: - if type(serial) != types.StringType: - raise s - else: - d[oid] = serial + zLOG.LOG('ZTB', zLOG.BLATHER, 'reply: %s' % repr(r)) + if r is None: + return None + if type(r) == types.StringType: + raise RuntimeError, "unexpected ZEO response: no oid" + else: + for oid, serial in r: + if type(serial) != types.StringType: + raise serial + else: + d[oid] = serial return d class GenericTests(ZEOTestBase, --- Updated File testZEO.py in package Packages/ZEO -- --- testZEO.py 2001/04/19 16:59:53 1.1.2.3 +++ testZEO.py 2001/04/19 18:26:05 1.1.2.4 @@ -68,25 +68,28 @@ # Store an object r1 = self._storage.store(oid, revid, data, version, self._transaction) + s1 = self._get_serial(r1) # Finish the transaction r2 = self._storage.tpc_vote(self._transaction) + s2 = self._get_serial(r2) self._storage.tpc_finish(self._transaction) - return self._get_serial([r1, r2])[oid] + # s1, s2 can be None or dict + return s1 and s1[oid] or s2 and s2[oid] - def _get_serial(self, resps): + def _get_serial(self, r): """Return oid -> serialno dict from sequence of ZEO replies.""" d = {} - for r in resps: - if not r: - continue - if type(r) == types.StringType: - raise RuntimeError, "unexpected ZEO response: no oid" - else: - for oid, serial in r: - if type(serial) != types.StringType: - raise s - else: - d[oid] = serial + zLOG.LOG('ZTB', zLOG.BLATHER, 'reply: %s' % repr(r)) + if r is None: + return None + if type(r) == types.StringType: + raise RuntimeError, "unexpected ZEO response: no oid" + else: + for oid, serial in r: + if type(serial) != types.StringType: + raise serial + else: + d[oid] = serial return d class GenericTests(ZEOTestBase, From jeremy at digicool.com Thu Apr 19 15:18:23 2001 From: jeremy at digicool.com (jeremy@digicool.com) Date: Sun Aug 10 16:31:14 2008 Subject: [ZEO-Checkins] CVS: Packages/ZEO - ClientCache.py:1.14.4.3 Message-ID: <20010419191823.01516510E4@korak.digicool.com> Update of /cvs-repository/Packages/ZEO In directory korak:/tmp/cvs-serv24680 Modified Files: Tag: ZEO-ZRPC-Dev ClientCache.py Log Message: Gotcha. The bare 'None' should have been a 'return None', otherwise the cache will return data from version X whenever versioned data is requested, regardless of whether the version in question is X. --- Updated File ClientCache.py in package Packages/ZEO -- --- ClientCache.py 2001/04/18 20:52:33 1.14.4.2 +++ ClientCache.py 2001/04/19 19:18:20 1.14.4.3 @@ -295,7 +295,8 @@ if dlen: seek(-dlen-vlen, 1) return read(dlen), h[19:] - else: None + else: + return None dlen=unpack(">i", read(4))[0] return read(dlen), read(8) --- Updated File ClientCache.py in package Packages/ZEO -- --- ClientCache.py 2001/04/18 20:52:33 1.14.4.2 +++ ClientCache.py 2001/04/19 19:18:20 1.14.4.3 @@ -295,7 +295,8 @@ if dlen: seek(-dlen-vlen, 1) return read(dlen), h[19:] - else: None + else: + return None dlen=unpack(">i", read(4))[0] return read(dlen), read(8) From jeremy at digicool.com Thu Apr 19 15:20:20 2001 From: jeremy at digicool.com (jeremy@digicool.com) Date: Sun Aug 10 16:31:14 2008 Subject: [ZEO-Checkins] CVS: Packages/ZEO - ClientCache.py:1.15 Message-ID: <20010419192020.5BC3E510E3@korak.digicool.com> Update of /cvs-repository/Packages/ZEO In directory korak:/tmp/cvs-serv24770 Modified Files: ClientCache.py Log Message: Gotcha. The bare 'None' should have been a 'return None', otherwise the cache will return data from version X whenever versioned data is requested, regardless of whether the version in question is X. --- Updated File ClientCache.py in package Packages/ZEO -- --- ClientCache.py 2000/12/11 18:02:13 1.14 +++ ClientCache.py 2001/04/19 19:20:19 1.15 @@ -277,7 +277,8 @@ if dlen: seek(-dlen-vlen, 1) return read(dlen), h[19:] - else: None + else: + return None dlen=unpack(">i", read(4))[0] return read(dlen), read(8) --- Updated File ClientCache.py in package Packages/ZEO -- --- ClientCache.py 2000/12/11 18:02:13 1.14 +++ ClientCache.py 2001/04/19 19:20:19 1.15 @@ -277,7 +277,8 @@ if dlen: seek(-dlen-vlen, 1) return read(dlen), h[19:] - else: None + else: + return None dlen=unpack(">i", read(4))[0] return read(dlen), read(8) From jeremy at digicool.com Fri Apr 20 09:51:25 2001 From: jeremy at digicool.com (jeremy@digicool.com) Date: Sun Aug 10 16:31:14 2008 Subject: [ZEO-Checkins] CVS: Packages/ZEO - ClientStorage.py:1.26.4.9 Message-ID: <20010420135125.8B643510E3@korak.digicool.com> Update of /cvs-repository/Packages/ZEO In directory korak:/tmp/cvs-serv13565 Modified Files: Tag: ZEO-ZRPC-Dev ClientStorage.py Log Message: More cleanups Add untested support for confliction resolution. Move timestamp generation process to function. XXX Might be better as a method. Simplify logic of new_oid(), depending on list[:] to have a side effect on an aliased attribute was too subtle. Eliminate some used-once local variables in tpc_begin(). --- Updated File ClientStorage.py in package Packages/ZEO -- --- ClientStorage.py 2001/04/19 18:24:02 1.26.4.8 +++ ClientStorage.py 2001/04/20 13:51:24 1.26.4.9 @@ -101,6 +101,11 @@ from ZODB.TimeStamp import TimeStamp from zLOG import LOG, PROBLEM, INFO, BLATHER +try: + from ZODB.ConflictResolution import ResolvedSerial +except ImportError: + ResolvedSerial = 'rs' + import sys from types import TupleType, StringType @@ -113,6 +118,12 @@ class ClientDisconnected(ClientStorageError): """The database storage is disconnected from the storage.""" +def get_timestamp(prev_ts): + t = time.time() + t = apply(TimeStamp, (time.gmtime(t)[:5] + (t % 60,))) + t = t.laterThan(prev_ts) + return t + class ClientStorage(BaseStorage.BaseStorage): __super_init = BaseStorage.BaseStorage.__init__ @@ -306,11 +317,10 @@ def new_oid(self, last=None): self._lock_acquire() try: - oids = self._oids - if not oids: - oids[:] = self._server.new_oids() - oids.reverse() - return oids.pop() + if not self._oids: + self._oids = self._server.new_oids() + self._oids.reverse() + return self._oids.pop() finally: self._lock_release() @@ -327,7 +337,6 @@ self._lock_release() def _check_serials(self): - # XXX I don't really understand what this does if self._serials: l = len(self._serials) r = self._serials[:l] @@ -343,9 +352,6 @@ raise POSException.StorageTransactionError(self, transaction) self._lock_acquire() try: - # XXX this code used to store the value returned by - # sendMessage and then return it if _serials was None. - # But sendMessage always returned None. self._server.storea(oid, serial, data, version, self._serial) self._tbuf.store(oid, version, data) return self._check_serials() @@ -387,26 +393,22 @@ # vote stage. self._lock_acquire() try: - if self._transaction is transaction: return + if self._transaction is transaction: + return # can start the same transaction many times - user=transaction.user - desc=transaction.description - ext=transaction._extension - while 1: self._lock_release() self._commit_lock_acquire() self._lock_acquire() - # We've got the local commit lock. Now get - # a (tentative) transaction time stamp. - t=time.time() - t=apply(TimeStamp,(time.gmtime(t)[:5]+(t%60,))) - self._ts=t=t.laterThan(self._ts) - id=`t` + self._ts = get_timestamp(self._ts) + id = `self._ts` try: - r = self._server.tpc_begin(id, user, desc, ext) + r = self._server.tpc_begin(id, + transaction.user, + transaction.description, + transaction._extension) except: self._commit_lock_release() raise @@ -421,7 +423,6 @@ del self._serials[:] self._transaction = transaction - finally: self._lock_release() @@ -459,7 +460,10 @@ LOG("ClientStorage", INFO, "bad serialno: %s for %s" % \ (repr(s), repr(oid))) assert type(s) == StringType, "bad serialno: %s" % repr(s) - self._cache.update(oid, s, v, p) + if s == ResolvedSerial: + self._cache.invalidate(oid, v) + else: + self._cache.update(oid, s, v, p) self._tbuf.clear() self._transaction=None --- Updated File ClientStorage.py in package Packages/ZEO -- --- ClientStorage.py 2001/04/19 18:24:02 1.26.4.8 +++ ClientStorage.py 2001/04/20 13:51:24 1.26.4.9 @@ -101,6 +101,11 @@ from ZODB.TimeStamp import TimeStamp from zLOG import LOG, PROBLEM, INFO, BLATHER +try: + from ZODB.ConflictResolution import ResolvedSerial +except ImportError: + ResolvedSerial = 'rs' + import sys from types import TupleType, StringType @@ -113,6 +118,12 @@ class ClientDisconnected(ClientStorageError): """The database storage is disconnected from the storage.""" +def get_timestamp(prev_ts): + t = time.time() + t = apply(TimeStamp, (time.gmtime(t)[:5] + (t % 60,))) + t = t.laterThan(prev_ts) + return t + class ClientStorage(BaseStorage.BaseStorage): __super_init = BaseStorage.BaseStorage.__init__ @@ -306,11 +317,10 @@ def new_oid(self, last=None): self._lock_acquire() try: - oids = self._oids - if not oids: - oids[:] = self._server.new_oids() - oids.reverse() - return oids.pop() + if not self._oids: + self._oids = self._server.new_oids() + self._oids.reverse() + return self._oids.pop() finally: self._lock_release() @@ -327,7 +337,6 @@ self._lock_release() def _check_serials(self): - # XXX I don't really understand what this does if self._serials: l = len(self._serials) r = self._serials[:l] @@ -343,9 +352,6 @@ raise POSException.StorageTransactionError(self, transaction) self._lock_acquire() try: - # XXX this code used to store the value returned by - # sendMessage and then return it if _serials was None. - # But sendMessage always returned None. self._server.storea(oid, serial, data, version, self._serial) self._tbuf.store(oid, version, data) return self._check_serials() @@ -387,26 +393,22 @@ # vote stage. self._lock_acquire() try: - if self._transaction is transaction: return + if self._transaction is transaction: + return # can start the same transaction many times - user=transaction.user - desc=transaction.description - ext=transaction._extension - while 1: self._lock_release() self._commit_lock_acquire() self._lock_acquire() - # We've got the local commit lock. Now get - # a (tentative) transaction time stamp. - t=time.time() - t=apply(TimeStamp,(time.gmtime(t)[:5]+(t%60,))) - self._ts=t=t.laterThan(self._ts) - id=`t` + self._ts = get_timestamp(self._ts) + id = `self._ts` try: - r = self._server.tpc_begin(id, user, desc, ext) + r = self._server.tpc_begin(id, + transaction.user, + transaction.description, + transaction._extension) except: self._commit_lock_release() raise @@ -421,7 +423,6 @@ del self._serials[:] self._transaction = transaction - finally: self._lock_release() @@ -459,7 +460,10 @@ LOG("ClientStorage", INFO, "bad serialno: %s for %s" % \ (repr(s), repr(oid))) assert type(s) == StringType, "bad serialno: %s" % repr(s) - self._cache.update(oid, s, v, p) + if s == ResolvedSerial: + self._cache.invalidate(oid, v) + else: + self._cache.update(oid, s, v, p) self._tbuf.clear() self._transaction=None From jeremy at digicool.com Fri Apr 20 09:56:11 2001 From: jeremy at digicool.com (jeremy@digicool.com) Date: Sun Aug 10 16:31:14 2008 Subject: [ZEO-Checkins] CVS: Packages/ZEO - ClientStorage.py:1.26.4.10 Message-ID: <20010420135611.BF076510E3@korak.digicool.com> Update of /cvs-repository/Packages/ZEO In directory korak:/tmp/cvs-serv13734 Modified Files: Tag: ZEO-ZRPC-Dev ClientStorage.py Log Message: Reorganize imports and remove some logging calls --- Updated File ClientStorage.py in package Packages/ZEO -- --- ClientStorage.py 2001/04/20 13:51:24 1.26.4.9 +++ ClientStorage.py 2001/04/20 13:56:10 1.26.4.10 @@ -88,15 +88,24 @@ """ __version__='$Revision$'[11:-2] -import struct, time, os, socket, string, Sync, ClientCache -import tempfile, ExtensionClass, thread +import cPickle +import os +import socket +import string +import struct +import sys +import tempfile +import thread +import time +from types import TupleType, StringType +from struct import pack, unpack + +import ExtensionClass, Sync +import ClientCache import zrpc2 import ServerStub from TransactionBuffer import TransactionBuffer -import cPickle - -from struct import pack, unpack from ZODB import POSException, BaseStorage from ZODB.TimeStamp import TimeStamp from zLOG import LOG, PROBLEM, INFO, BLATHER @@ -106,8 +115,6 @@ except ImportError: ResolvedSerial = 'rs' -import sys -from types import TupleType, StringType class ClientStorageError(POSException.StorageError): """An error occured in the ZEO Client Storage""" @@ -342,8 +349,6 @@ r = self._serials[:l] del self._serials[:l] for oid, s in r: - LOG("ClientStorage", BLATHER, - "serial: %s %s" % (repr(oid), repr(s))) self._seriald[oid] = s return r @@ -518,7 +523,6 @@ self.commit_lock_release() def serialno(self, arg): - LOG("ClientStorage", BLATHER, "serialno(%s)" % repr(arg)) self._serials.append(arg) def info(self, dict): --- Updated File ClientStorage.py in package Packages/ZEO -- --- ClientStorage.py 2001/04/20 13:51:24 1.26.4.9 +++ ClientStorage.py 2001/04/20 13:56:10 1.26.4.10 @@ -88,15 +88,24 @@ """ __version__='$Revision$'[11:-2] -import struct, time, os, socket, string, Sync, ClientCache -import tempfile, ExtensionClass, thread +import cPickle +import os +import socket +import string +import struct +import sys +import tempfile +import thread +import time +from types import TupleType, StringType +from struct import pack, unpack + +import ExtensionClass, Sync +import ClientCache import zrpc2 import ServerStub from TransactionBuffer import TransactionBuffer -import cPickle - -from struct import pack, unpack from ZODB import POSException, BaseStorage from ZODB.TimeStamp import TimeStamp from zLOG import LOG, PROBLEM, INFO, BLATHER @@ -106,8 +115,6 @@ except ImportError: ResolvedSerial = 'rs' -import sys -from types import TupleType, StringType class ClientStorageError(POSException.StorageError): """An error occured in the ZEO Client Storage""" @@ -342,8 +349,6 @@ r = self._serials[:l] del self._serials[:l] for oid, s in r: - LOG("ClientStorage", BLATHER, - "serial: %s %s" % (repr(oid), repr(s))) self._seriald[oid] = s return r @@ -518,7 +523,6 @@ self.commit_lock_release() def serialno(self, arg): - LOG("ClientStorage", BLATHER, "serialno(%s)" % repr(arg)) self._serials.append(arg) def info(self, dict): From jeremy at digicool.com Fri Apr 20 15:14:10 2001 From: jeremy at digicool.com (jeremy@digicool.com) Date: Sun Aug 10 16:31:15 2008 Subject: [ZEO-Checkins] CVS: Packages/ZEO - zeolog.py:1.1.2.1 ClientCache.py:1.14.4.4 ClientStorage.py:1.26.4.11 StorageServer.py:1.21.4.6 smac.py:1.9.6.4 zrpc2.py:1.1.2.9 Message-ID: <20010420191410.03840510E3@korak.digicool.com> Update of /cvs-repository/Packages/ZEO In directory korak:/tmp/cvs-serv24482 Modified Files: Tag: ZEO-ZRPC-Dev ClientCache.py ClientStorage.py StorageServer.py smac.py zrpc2.py Added Files: Tag: ZEO-ZRPC-Dev zeolog.py Log Message: Big changes to logging to improve performance. Replace use of zLOG with zeolog (a slimmed-down replacement). Protect all calls for TRACE level inside if __debug__. The zLOG code has a lot of extra indirections that slow down an application. zeolog appears to be about 15% faster. The use of __debug__ has a much more dramatic effect, because there is a lot of logging in the lower layers of zrpc2. By putting TRACE-level logging inside if __debug__ and running python -O, the net savings is another 20% (measured with speed.py). Also, add ZPL and doc string to zrpc2.py --- Added File zeolog.py in package Packages/ZEO --- """ZEO-specific replacement for the zLOG module""" import os import sys import time # the logging severity constants defined by zLOG from zLOG import TRACE, DEBUG, BLATHER, INFO, PROBLEM, WARNING, ERROR, PANIC severity_strings = {TRACE: 'TRACE', DEBUG: 'DEBUG', BLATHER: 'BLATHER', INFO: 'INFO', PROBLEM: 'PROBLEM', WARNING: 'WARNING', ERROR: 'ERROR', PANIC: 'PANIC', } def now(): t = time.time() hms = time.gmtime(t)[3:6] msec = int(1000 * (t - int(t))) return "%2.2d:%2.2d:%2.2d." % hms + "%03d" % msec def LOG(subsys, severity, summary, detail='', error=None, reraise=None): if log_file is None or severity < log_level: return s = "%s %7s %s %s" % (now(), severity_strings[severity], subsys, summary) if detail: s = "%s\n%s" % (s, detail) print >> log_file, s log_file.flush() if error: pass # XXX haven't implemented traceback writing if reraise and error: raise error[0], error[1], error[2] def initialize(): """Initialize the module from environment variables""" global log_file, log_level path = os.environ.get('STUPID_LOG_FILE', None) if path is None: log_file = None else: if path: log_file = open(path, 'a') else: log_file = sys.stderr severity = os.environ.get('STUPID_LOG_SEVERITY', None) if severity: log_level = int(severity) else: log_level = INFO __all__ = ["LOG", "TRACE", "DEBUG", "BLATHER", "INFO", "PROBLEM", "WARNING", "ERROR", "PANIC"] initialize() --- Updated File ClientCache.py in package Packages/ZEO -- --- ClientCache.py 2001/04/19 19:18:20 1.14.4.3 +++ ClientCache.py 2001/04/20 19:14:08 1.14.4.4 @@ -151,10 +151,10 @@ from thread import allocate_lock import sys -import zLOG +import zeolog -def log(msg, level=zLOG.INFO): - zLOG.LOG("ZEC", level, msg) +def log(msg, level=zeolog.INFO): + zeolog.LOG("ZEC", level, msg) magic='ZEC0' --- Updated File ClientStorage.py in package Packages/ZEO -- --- ClientStorage.py 2001/04/20 13:56:10 1.26.4.10 +++ ClientStorage.py 2001/04/20 19:14:08 1.26.4.11 @@ -108,7 +108,7 @@ from ZODB import POSException, BaseStorage from ZODB.TimeStamp import TimeStamp -from zLOG import LOG, PROBLEM, INFO, BLATHER +from zeolog import LOG, PROBLEM, INFO, BLATHER try: from ZODB.ConflictResolution import ResolvedSerial --- Updated File StorageServer.py in package Packages/ZEO -- --- StorageServer.py 2001/04/19 15:47:48 1.21.4.5 +++ StorageServer.py 2001/04/20 19:14:08 1.21.4.6 @@ -1,3 +1,93 @@ +############################################################################## +# +# Zope Public License (ZPL) Version 1.0 +# ------------------------------------- +# +# Copyright (c) Digital Creations. All rights reserved. +# +# This license has been certified as Open Source(tm). +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# 1. Redistributions in source code must retain the above copyright +# notice, this list of conditions, and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions, and the following disclaimer in +# the documentation and/or other materials provided with the +# distribution. +# +# 3. Digital Creations requests that attribution be given to Zope +# in any manner possible. Zope includes a "Powered by Zope" +# button that is installed by default. While it is not a license +# violation to remove this button, it is requested that the +# attribution remain. A significant investment has been put +# into Zope, and this effort will continue if the Zope community +# continues to grow. This is one way to assure that growth. +# +# 4. All advertising materials and documentation mentioning +# features derived from or use of this software must display +# the following acknowledgement: +# +# "This product includes software developed by Digital Creations +# for use in the Z Object Publishing Environment +# (http://www.zope.org/)." +# +# In the event that the product being advertised includes an +# intact Zope distribution (with copyright and license included) +# then this clause is waived. +# +# 5. Names associated with Zope or Digital Creations must not be used to +# endorse or promote products derived from this software without +# prior written permission from Digital Creations. +# +# 6. Modified redistributions of any form whatsoever must retain +# the following acknowledgment: +# +# "This product includes software developed by Digital Creations +# for use in the Z Object Publishing Environment +# (http://www.zope.org/)." +# +# Intact (re-)distributions of any official Zope release do not +# require an external acknowledgement. +# +# 7. Modifications are encouraged but must be packaged separately as +# patches to official Zope releases. Distributions that do not +# clearly separate the patches from the original work must be clearly +# labeled as unofficial distributions. Modifications which do not +# carry the name Zope may be packaged in any form, as long as they +# conform to all of the clauses above. +# +# +# Disclaimer +# +# THIS SOFTWARE IS PROVIDED BY DIGITAL CREATIONS ``AS IS'' AND ANY +# EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL DIGITAL CREATIONS OR ITS +# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF +# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT +# OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF +# SUCH DAMAGE. +# +# +# This software consists of contributions made by Digital Creations and +# many individuals on behalf of Digital Creations. Specific +# attributions are listed in the accompanying credits file. +# +############################################################################## +"""Network ZODB storage server + +This server acts as a front-end for one or more real storages, like +file storage or Berkeley storage. +""" + import cPickle import os import sys @@ -5,7 +95,7 @@ import ClientStub import zrpc2 -import zLOG +import zeolog from zrpc2 import Dispatcher, Handler, ManagedServerConnection, Delay from ZODB.POSException import StorageError, StorageTransactionError, \ @@ -20,8 +110,8 @@ pickler.fast = 1 # Don't use the memo dump = pickler.dump -def log(message, level=zLOG.INFO, label="zeoserver:%s" % os.getpid()): - zLOG.LOG(label, level, message) +def log(message, level=zeolog.INFO, label="zeoserver:%s" % os.getpid()): + zeolog.LOG(label, level, message) class StorageServerError(StorageError): pass @@ -157,7 +247,7 @@ try: self.__storage.pack(t, referencesf) except: - log('ZEO Server', zLOG.ERROR, + log('ZEO Server', zeolog.ERROR, 'Pack failed for %s' % self.__storage_id, error=sys.exc_info()) if wait: @@ -208,7 +298,7 @@ # IOW, Anything that ends up here is evil enough to be logged. error = sys.exc_info() log('store error: %s: %s' % (error[0], error[1]). - zLOG.ERROR) + zeolog.ERROR) newserial = sys.exc_info()[1] else: if serial != '\0\0\0\0\0\0\0\0': @@ -218,7 +308,7 @@ nil = dump(newserial, 1) except: log("couldn't pickle newserial: %s" % repr(newserial), - zLOG.ERROR) + zeolog.ERROR) dump('', 1) # clear pickler r = StorageServerError("Couldn't pickle exception %s" % \ `newserial`) --- Updated File smac.py in package Packages/ZEO -- --- smac.py 2001/04/18 20:55:09 1.9.6.3 +++ smac.py 2001/04/20 19:14:08 1.9.6.4 @@ -88,7 +88,7 @@ __version__ = "$Revision$"[11:-2] import asyncore, string, struct, sys, Acquisition -from zLOG import LOG, TRACE, ERROR, INFO, BLATHER +from zeolog import LOG, TRACE, ERROR, INFO, BLATHER from types import StringType class SizedMessageAsyncConnection(asyncore.dispatcher): --- Updated File zrpc2.py in package Packages/ZEO -- --- zrpc2.py 2001/04/19 18:25:03 1.1.2.8 +++ zrpc2.py 2001/04/20 19:14:09 1.1.2.9 @@ -36,14 +36,14 @@ from ZODB import POSException import smac import trigger -import zLOG +import zeolog import ThreadedAsync REPLY = ".reply" # message name used for replies ASYNC = 1 -def log(message, level=zLOG.BLATHER, label="zrpc:%s" % os.getpid()): - zLOG.LOG(label, level, message) +def log(message, level=zeolog.BLATHER, label="zrpc:%s" % os.getpid()): + zeolog.LOG(label, level, message) class ZRPCError(POSException.StorageError): pass @@ -97,11 +97,11 @@ unpickler.find_global = find_global try: + return unpickler.load() # msgid, flags, name, args msgid, flags, name, args = unpickler.load() except (cPickle.UnpicklingError, IndexError), err_msg: - log("can't decode %s" % repr(msg), level=zLOG.ERROR) + log("can't decode %s" % repr(msg), level=zeolog.ERROR) raise DecodingError(msg) - return msgid, flags, name, args class Delay: """Used to delay response to client for synchronous calls @@ -118,43 +118,6 @@ def reply(self, obj): self.send_reply(self.msgid, obj) -class DebugLock: - - __locks = 0 - - def __init__(self): - self.lock = thread.allocate_lock() - # XXX this actually needs to be locked too - self.__lock_id = self.__locks - self.__locks = self.__locks + 1 - - def _debug(self): - method = sys._getframe().f_back - caller = method.f_back - filename = os.path.split(caller.f_code.co_filename)[1] - log("LOCK %s (tid %s): " \ - "%s called by %s, %s, line %s" % (self.__lock_id, - thread.get_ident(), - method.f_code.co_name, - caller.f_code.co_name, - filename, - caller.f_lineno)) - - def acquire(self, wait=None): - self._debug() - if wait is not None: - return self.lock.acquire(wait) - else: - return self.lock.acquire() - - def release(self): - self._debug() - return self.lock.release() - - def locked(self): - self._debug() - return self.lock.locked() - class Connection(smac.SizedMessageAsyncConnection): """Dispatcher for RPC on object @@ -182,7 +145,6 @@ # waiting for a response self.__super_init(sock, addr) self._prepare_async() -## self.__reply_lock = DebugLock() self.__reply_lock = thread.allocate_lock() self.__reply_lock.acquire() if isinstance(obj, Handler): @@ -211,20 +173,25 @@ return self.return_error(None, None, sys.exc_info()[0], sys.exc_info()[1]) - log("message: %s, %s, %s, %s" % (msgid, flags, name, repr(args)[:40]), - level=zLOG.TRACE) + if __debug__: + log("message: %s, %s, %s, %s" % (msgid, flags, name, + repr(args)[:40]), + level=zeolog.TRACE) if name == REPLY: self.handle_reply(msgid, flags, args) else: self.handle_request(msgid, flags, name, args) def handle_reply(self, msgid, flags, args): - log("reply: %s, %s, %s" % (msgid, flags, str(args)[:40])) + if __debug__: + log("reply: %s, %s, %s" % (msgid, flags, str(args)[:40]), + level=zeolog.TRACE) self.__reply = msgid, flags, args self.__reply_lock.release() # will fail if lock is unlocked def handle_request(self, msgid, flags, name, args): - log("%s%s" % (name, repr(args)[:40]), zLOG.TRACE) + if __debug__: + log("%s%s" % (name, repr(args)[:40]), zeolog.TRACE) if not self.check_method(name): raise ZRPCError("Invalid method name: %s" % name) @@ -250,7 +217,8 @@ if ret is not None: raise ZRPCError("async method returned value") else: - log("%s reply %s" % (name, repr(ret)[:40]), zLOG.TRACE) + if __debug__: + log("%s reply %s" % (name, repr(ret)[:40]), zeolog.TRACE) if isinstance(ret, Delay): ret.set_sender(msgid, self.send_reply) else: @@ -324,7 +292,6 @@ raise DisconnectedError("This action is temporarily unavailable") msgid = self.msgid self.msgid = self.msgid + 1 - log("call %s %s" % (msgid, method)) self.message_output(self.marshal.encode(msgid, 0, method, args)) self.__reply = None @@ -333,15 +300,12 @@ # lock is held again... r_msgid, r_flags, r_args = self.__reply self.__reply_lock.acquire() - log("call acquired lock") assert r_msgid == msgid, "%s != %s: %s" % (r_msgid, msgid, r_args) if type(r_args) == types.TupleType \ and type(r_args[0]) == types.ClassType \ and issubclass(r_args[0], Exception): - log("call %s %s raised error" % (msgid, method)) raise r_args[1] - log("call %s %s returned" % (msgid, method)) return r_args def callAsync(self, method, *args): @@ -349,12 +313,14 @@ raise DisconnectedError("This action is temporarily unavailable") msgid = self.msgid self.msgid = self.msgid + 1 - log("async %s %s" % (msgid, method)) self.message_output(self.marshal.encode(msgid, ASYNC, method, args)) self._do_io() # handle IO, possibly in async mode + def sync(self): + pass # XXX what is this supposed to do? + def _prepare_async(self): self._async = 0 ThreadedAsync.register_loop_callback(self.set_async) @@ -373,9 +339,10 @@ def _do_io(self, wait=0): # XXX need better name # XXX invariant? lock must be held when calling with wait==1 # otherwise, in non-async mode, there will be no poll - - log("_do_io(wait=%d), async=%d" % (wait, self.is_async()), - level=zLOG.TRACE) + + if __debug__: + log("_do_io(wait=%d), async=%d" % (wait, self.is_async()), + level=zeolog.TRACE) if self.is_async(): self.trigger.pull_trigger() if wait: @@ -465,12 +432,12 @@ s.connect(self.addr) except socket.error: if self.debug: - log("Failed to connect to server", level=zLOG.DEBUG) + log("Failed to connect to server", level=zeolog.DEBUG) if repeat: t = self._wait(t) else: if self.debug: - log("Connected to server", level=zLOG.DEBUG) + log("Connected to server", level=zeolog.DEBUG) self.connected = 1 if self.connected: c = ManagedConnection(s, self.addr, self.obj, self) @@ -502,7 +469,6 @@ def close(self): self.__super_close() - log("self.__mgr = %s" % repr(self.__mgr)) self.__mgr.close(self) class ManagedConnection(Connection): @@ -578,7 +544,7 @@ log("accepted failed: %s" % msg) return c = self.factory(sock, addr, self.obj) - log("connect from %s: %s" % (addr, c)) + log("connect from %s: %s" % (repr(addr), c)) self.clients.append(c) class Handler: --- Added File zeolog.py in package Packages/ZEO --- """ZEO-specific replacement for the zLOG module""" import os import sys import time # the logging severity constants defined by zLOG from zLOG import TRACE, DEBUG, BLATHER, INFO, PROBLEM, WARNING, ERROR, PANIC severity_strings = {TRACE: 'TRACE', DEBUG: 'DEBUG', BLATHER: 'BLATHER', INFO: 'INFO', PROBLEM: 'PROBLEM', WARNING: 'WARNING', ERROR: 'ERROR', PANIC: 'PANIC', } def now(): t = time.time() hms = time.gmtime(t)[3:6] msec = int(1000 * (t - int(t))) return "%2.2d:%2.2d:%2.2d." % hms + "%03d" % msec def LOG(subsys, severity, summary, detail='', error=None, reraise=None): if log_file is None or severity < log_level: return s = "%s %7s %s %s" % (now(), severity_strings[severity], subsys, summary) if detail: s = "%s\n%s" % (s, detail) print >> log_file, s log_file.flush() if error: pass # XXX haven't implemented traceback writing if reraise and error: raise error[0], error[1], error[2] def initialize(): """Initialize the module from environment variables""" global log_file, log_level path = os.environ.get('STUPID_LOG_FILE', None) if path is None: log_file = None else: if path: log_file = open(path, 'a') else: log_file = sys.stderr severity = os.environ.get('STUPID_LOG_SEVERITY', None) if severity: log_level = int(severity) else: log_level = INFO __all__ = ["LOG", "TRACE", "DEBUG", "BLATHER", "INFO", "PROBLEM", "WARNING", "ERROR", "PANIC"] initialize() --- Updated File ClientCache.py in package Packages/ZEO -- --- ClientCache.py 2001/04/19 19:18:20 1.14.4.3 +++ ClientCache.py 2001/04/20 19:14:08 1.14.4.4 @@ -151,10 +151,10 @@ from thread import allocate_lock import sys -import zLOG +import zeolog -def log(msg, level=zLOG.INFO): - zLOG.LOG("ZEC", level, msg) +def log(msg, level=zeolog.INFO): + zeolog.LOG("ZEC", level, msg) magic='ZEC0' --- Updated File ClientStorage.py in package Packages/ZEO -- --- ClientStorage.py 2001/04/20 13:56:10 1.26.4.10 +++ ClientStorage.py 2001/04/20 19:14:08 1.26.4.11 @@ -108,7 +108,7 @@ from ZODB import POSException, BaseStorage from ZODB.TimeStamp import TimeStamp -from zLOG import LOG, PROBLEM, INFO, BLATHER +from zeolog import LOG, PROBLEM, INFO, BLATHER try: from ZODB.ConflictResolution import ResolvedSerial --- Updated File StorageServer.py in package Packages/ZEO -- --- StorageServer.py 2001/04/19 15:47:48 1.21.4.5 +++ StorageServer.py 2001/04/20 19:14:08 1.21.4.6 @@ -1,3 +1,93 @@ +############################################################################## +# +# Zope Public License (ZPL) Version 1.0 +# ------------------------------------- +# +# Copyright (c) Digital Creations. All rights reserved. +# +# This license has been certified as Open Source(tm). +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# 1. Redistributions in source code must retain the above copyright +# notice, this list of conditions, and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions, and the following disclaimer in +# the documentation and/or other materials provided with the +# distribution. +# +# 3. Digital Creations requests that attribution be given to Zope +# in any manner possible. Zope includes a "Powered by Zope" +# button that is installed by default. While it is not a license +# violation to remove this button, it is requested that the +# attribution remain. A significant investment has been put +# into Zope, and this effort will continue if the Zope community +# continues to grow. This is one way to assure that growth. +# +# 4. All advertising materials and documentation mentioning +# features derived from or use of this software must display +# the following acknowledgement: +# +# "This product includes software developed by Digital Creations +# for use in the Z Object Publishing Environment +# (http://www.zope.org/)." +# +# In the event that the product being advertised includes an +# intact Zope distribution (with copyright and license included) +# then this clause is waived. +# +# 5. Names associated with Zope or Digital Creations must not be used to +# endorse or promote products derived from this software without +# prior written permission from Digital Creations. +# +# 6. Modified redistributions of any form whatsoever must retain +# the following acknowledgment: +# +# "This product includes software developed by Digital Creations +# for use in the Z Object Publishing Environment +# (http://www.zope.org/)." +# +# Intact (re-)distributions of any official Zope release do not +# require an external acknowledgement. +# +# 7. Modifications are encouraged but must be packaged separately as +# patches to official Zope releases. Distributions that do not +# clearly separate the patches from the original work must be clearly +# labeled as unofficial distributions. Modifications which do not +# carry the name Zope may be packaged in any form, as long as they +# conform to all of the clauses above. +# +# +# Disclaimer +# +# THIS SOFTWARE IS PROVIDED BY DIGITAL CREATIONS ``AS IS'' AND ANY +# EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL DIGITAL CREATIONS OR ITS +# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF +# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT +# OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF +# SUCH DAMAGE. +# +# +# This software consists of contributions made by Digital Creations and +# many individuals on behalf of Digital Creations. Specific +# attributions are listed in the accompanying credits file. +# +############################################################################## +"""Network ZODB storage server + +This server acts as a front-end for one or more real storages, like +file storage or Berkeley storage. +""" + import cPickle import os import sys @@ -5,7 +95,7 @@ import ClientStub import zrpc2 -import zLOG +import zeolog from zrpc2 import Dispatcher, Handler, ManagedServerConnection, Delay from ZODB.POSException import StorageError, StorageTransactionError, \ @@ -20,8 +110,8 @@ pickler.fast = 1 # Don't use the memo dump = pickler.dump -def log(message, level=zLOG.INFO, label="zeoserver:%s" % os.getpid()): - zLOG.LOG(label, level, message) +def log(message, level=zeolog.INFO, label="zeoserver:%s" % os.getpid()): + zeolog.LOG(label, level, message) class StorageServerError(StorageError): pass @@ -157,7 +247,7 @@ try: self.__storage.pack(t, referencesf) except: - log('ZEO Server', zLOG.ERROR, + log('ZEO Server', zeolog.ERROR, 'Pack failed for %s' % self.__storage_id, error=sys.exc_info()) if wait: @@ -208,7 +298,7 @@ # IOW, Anything that ends up here is evil enough to be logged. error = sys.exc_info() log('store error: %s: %s' % (error[0], error[1]). - zLOG.ERROR) + zeolog.ERROR) newserial = sys.exc_info()[1] else: if serial != '\0\0\0\0\0\0\0\0': @@ -218,7 +308,7 @@ nil = dump(newserial, 1) except: log("couldn't pickle newserial: %s" % repr(newserial), - zLOG.ERROR) + zeolog.ERROR) dump('', 1) # clear pickler r = StorageServerError("Couldn't pickle exception %s" % \ `newserial`) --- Updated File smac.py in package Packages/ZEO -- --- smac.py 2001/04/18 20:55:09 1.9.6.3 +++ smac.py 2001/04/20 19:14:08 1.9.6.4 @@ -88,7 +88,7 @@ __version__ = "$Revision$"[11:-2] import asyncore, string, struct, sys, Acquisition -from zLOG import LOG, TRACE, ERROR, INFO, BLATHER +from zeolog import LOG, TRACE, ERROR, INFO, BLATHER from types import StringType class SizedMessageAsyncConnection(asyncore.dispatcher): --- Updated File zrpc2.py in package Packages/ZEO -- --- zrpc2.py 2001/04/19 18:25:03 1.1.2.8 +++ zrpc2.py 2001/04/20 19:14:09 1.1.2.9 @@ -36,14 +36,14 @@ from ZODB import POSException import smac import trigger -import zLOG +import zeolog import ThreadedAsync REPLY = ".reply" # message name used for replies ASYNC = 1 -def log(message, level=zLOG.BLATHER, label="zrpc:%s" % os.getpid()): - zLOG.LOG(label, level, message) +def log(message, level=zeolog.BLATHER, label="zrpc:%s" % os.getpid()): + zeolog.LOG(label, level, message) class ZRPCError(POSException.StorageError): pass @@ -97,11 +97,11 @@ unpickler.find_global = find_global try: + return unpickler.load() # msgid, flags, name, args msgid, flags, name, args = unpickler.load() except (cPickle.UnpicklingError, IndexError), err_msg: - log("can't decode %s" % repr(msg), level=zLOG.ERROR) + log("can't decode %s" % repr(msg), level=zeolog.ERROR) raise DecodingError(msg) - return msgid, flags, name, args class Delay: """Used to delay response to client for synchronous calls @@ -118,43 +118,6 @@ def reply(self, obj): self.send_reply(self.msgid, obj) -class DebugLock: - - __locks = 0 - - def __init__(self): - self.lock = thread.allocate_lock() - # XXX this actually needs to be locked too - self.__lock_id = self.__locks - self.__locks = self.__locks + 1 - - def _debug(self): - method = sys._getframe().f_back - caller = method.f_back - filename = os.path.split(caller.f_code.co_filename)[1] - log("LOCK %s (tid %s): " \ - "%s called by %s, %s, line %s" % (self.__lock_id, - thread.get_ident(), - method.f_code.co_name, - caller.f_code.co_name, - filename, - caller.f_lineno)) - - def acquire(self, wait=None): - self._debug() - if wait is not None: - return self.lock.acquire(wait) - else: - return self.lock.acquire() - - def release(self): - self._debug() - return self.lock.release() - - def locked(self): - self._debug() - return self.lock.locked() - class Connection(smac.SizedMessageAsyncConnection): """Dispatcher for RPC on object @@ -182,7 +145,6 @@ # waiting for a response self.__super_init(sock, addr) self._prepare_async() -## self.__reply_lock = DebugLock() self.__reply_lock = thread.allocate_lock() self.__reply_lock.acquire() if isinstance(obj, Handler): @@ -211,20 +173,25 @@ return self.return_error(None, None, sys.exc_info()[0], sys.exc_info()[1]) - log("message: %s, %s, %s, %s" % (msgid, flags, name, repr(args)[:40]), - level=zLOG.TRACE) + if __debug__: + log("message: %s, %s, %s, %s" % (msgid, flags, name, + repr(args)[:40]), + level=zeolog.TRACE) if name == REPLY: self.handle_reply(msgid, flags, args) else: self.handle_request(msgid, flags, name, args) def handle_reply(self, msgid, flags, args): - log("reply: %s, %s, %s" % (msgid, flags, str(args)[:40])) + if __debug__: + log("reply: %s, %s, %s" % (msgid, flags, str(args)[:40]), + level=zeolog.TRACE) self.__reply = msgid, flags, args self.__reply_lock.release() # will fail if lock is unlocked def handle_request(self, msgid, flags, name, args): - log("%s%s" % (name, repr(args)[:40]), zLOG.TRACE) + if __debug__: + log("%s%s" % (name, repr(args)[:40]), zeolog.TRACE) if not self.check_method(name): raise ZRPCError("Invalid method name: %s" % name) @@ -250,7 +217,8 @@ if ret is not None: raise ZRPCError("async method returned value") else: - log("%s reply %s" % (name, repr(ret)[:40]), zLOG.TRACE) + if __debug__: + log("%s reply %s" % (name, repr(ret)[:40]), zeolog.TRACE) if isinstance(ret, Delay): ret.set_sender(msgid, self.send_reply) else: @@ -324,7 +292,6 @@ raise DisconnectedError("This action is temporarily unavailable") msgid = self.msgid self.msgid = self.msgid + 1 - log("call %s %s" % (msgid, method)) self.message_output(self.marshal.encode(msgid, 0, method, args)) self.__reply = None @@ -333,15 +300,12 @@ # lock is held again... r_msgid, r_flags, r_args = self.__reply self.__reply_lock.acquire() - log("call acquired lock") assert r_msgid == msgid, "%s != %s: %s" % (r_msgid, msgid, r_args) if type(r_args) == types.TupleType \ and type(r_args[0]) == types.ClassType \ and issubclass(r_args[0], Exception): - log("call %s %s raised error" % (msgid, method)) raise r_args[1] - log("call %s %s returned" % (msgid, method)) return r_args def callAsync(self, method, *args): @@ -349,12 +313,14 @@ raise DisconnectedError("This action is temporarily unavailable") msgid = self.msgid self.msgid = self.msgid + 1 - log("async %s %s" % (msgid, method)) self.message_output(self.marshal.encode(msgid, ASYNC, method, args)) self._do_io() # handle IO, possibly in async mode + def sync(self): + pass # XXX what is this supposed to do? + def _prepare_async(self): self._async = 0 ThreadedAsync.register_loop_callback(self.set_async) @@ -373,9 +339,10 @@ def _do_io(self, wait=0): # XXX need better name # XXX invariant? lock must be held when calling with wait==1 # otherwise, in non-async mode, there will be no poll - - log("_do_io(wait=%d), async=%d" % (wait, self.is_async()), - level=zLOG.TRACE) + + if __debug__: + log("_do_io(wait=%d), async=%d" % (wait, self.is_async()), + level=zeolog.TRACE) if self.is_async(): self.trigger.pull_trigger() if wait: @@ -465,12 +432,12 @@ s.connect(self.addr) except socket.error: if self.debug: - log("Failed to connect to server", level=zLOG.DEBUG) + log("Failed to connect to server", level=zeolog.DEBUG) if repeat: t = self._wait(t) else: if self.debug: - log("Connected to server", level=zLOG.DEBUG) + log("Connected to server", level=zeolog.DEBUG) self.connected = 1 if self.connected: c = ManagedConnection(s, self.addr, self.obj, self) @@ -502,7 +469,6 @@ def close(self): self.__super_close() - log("self.__mgr = %s" % repr(self.__mgr)) self.__mgr.close(self) class ManagedConnection(Connection): @@ -578,7 +544,7 @@ log("accepted failed: %s" % msg) return c = self.factory(sock, addr, self.obj) - log("connect from %s: %s" % (addr, c)) + log("connect from %s: %s" % (repr(addr), c)) self.clients.append(c) class Handler: From jeremy at digicool.com Fri Apr 20 15:17:24 2001 From: jeremy at digicool.com (jeremy@digicool.com) Date: Sun Aug 10 16:31:15 2008 Subject: [ZEO-Checkins] CVS: Packages/ZEO - speed.py:1.1.2.1 Message-ID: <20010420191724.8FBC1510E3@korak.digicool.com> Update of /cvs-repository/Packages/ZEO/tests In directory korak:/tmp/cvs-serv24625 Added Files: Tag: ZEO-ZRPC-Dev speed.py Log Message: Add a version of speed.py that runs a ZEO server with a FileStorage as a subprocess. --- Added File speed.py in package Packages/ZEO --- ############################################################################## # # Zope Public License (ZPL) Version 1.0 # ------------------------------------- # # Copyright (c) Digital Creations. All rights reserved. # # This license has been certified as Open Source(tm). # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions in source code must retain the above copyright # notice, this list of conditions, and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions, and the following disclaimer in # the documentation and/or other materials provided with the # distribution. # # 3. Digital Creations requests that attribution be given to Zope # in any manner possible. Zope includes a "Powered by Zope" # button that is installed by default. While it is not a license # violation to remove this button, it is requested that the # attribution remain. A significant investment has been put # into Zope, and this effort will continue if the Zope community # continues to grow. This is one way to assure that growth. # # 4. All advertising materials and documentation mentioning # features derived from or use of this software must display # the following acknowledgement: # # "This product includes software developed by Digital Creations # for use in the Z Object Publishing Environment # (http://www.zope.org/)." # # In the event that the product being advertised includes an # intact Zope distribution (with copyright and license included) # then this clause is waived. # # 5. Names associated with Zope or Digital Creations must not be used to # endorse or promote products derived from this software without # prior written permission from Digital Creations. # # 6. Modified redistributions of any form whatsoever must retain # the following acknowledgment: # # "This product includes software developed by Digital Creations # for use in the Z Object Publishing Environment # (http://www.zope.org/)." # # Intact (re-)distributions of any official Zope release do not # require an external acknowledgement. # # 7. Modifications are encouraged but must be packaged separately as # patches to official Zope releases. Distributions that do not # clearly separate the patches from the original work must be clearly # labeled as unofficial distributions. Modifications which do not # carry the name Zope may be packaged in any form, as long as they # conform to all of the clauses above. # # # Disclaimer # # THIS SOFTWARE IS PROVIDED BY DIGITAL CREATIONS ``AS IS'' AND ANY # EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL DIGITAL CREATIONS OR ITS # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF # USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND # ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT # OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF # SUCH DAMAGE. # # # This software consists of contributions made by Digital Creations and # many individuals on behalf of Digital Creations. Specific # attributions are listed in the accompanying credits file. # ############################################################################## usage="""Test speed of a ZODB storage Options: -d file The data file to use as input. The default is this script. -n n The number of repititions -s module A module that defines a 'Storage' attribute, which is an open storage. If not specified, a FileStorage will ne used. -z Test compressing data -D Run in debug mode -L Test loads as well as stores by minimizing the cache after eachrun -M Output means only -C Run with a persistent client cache """ import asyncore import sys, os, getopt, string, time ##sys.path.insert(0, os.getcwd()) import ZODB, ZODB.FileStorage import Persistence import ZEO.ClientStorage, ZEO.StorageServer class P(Persistence.Persistent): pass fs_name = "zeo-speed.fs" class ZEOExit(asyncore.file_dispatcher): """Used to exit ZEO.StorageServer when run is done""" def writable(self): return 0 def readable(self): return 1 def handle_read(self): buf = self.recv(4) assert buf == "done" self.delete_fs() os._exit(0) def handle_close(self): print "Parent process exited unexpectedly" self.delete_fs() os._exit(0) def delete_fs(self): os.unlink(fs_name) os.unlink(fs_name + ".lock") os.unlink(fs_name + ".tmp") def main(args): opts, args = getopt.getopt(args, 'zd:n:Ds:LM') s = None compress = None data=sys.argv[0] nrep=5 minimize=0 detailed=1 cache = None for o, v in opts: if o=='-n': nrep = int(v) elif o=='-d': data = v elif o=='-s': s = v elif o=='-z': import zlib compress = zlib.compress elif o=='-L': minimize=1 elif o=='-M': detailed=0 elif o=='-D': global debug os.environ['STUPID_LOG_FILE']='' os.environ['STUPID_LOG_SEVERITY']='-999' debug = 1 elif o == '-C': cache = 'speed' zeo_pipe = None if s: s = __import__(s, globals(), globals(), ('__doc__',)) s = s.Storage else: rd, wr = os.pipe() pid = os.fork() if pid: # in the child, run the storage server os.close(wr) import asyncore ZEOExit(rd) fs = ZODB.FileStorage.FileStorage(fs_name, create=1) serv = ZEO.StorageServer.StorageServer(('', 1975), {'1':fs}) asyncore.loop() else: os.close(rd) zeo_pipe = wr s = ZEO.ClientStorage.ClientStorage(('', 1975), debug=0, client=cache) if hasattr(s, 'is_connected'): while not s.is_connected(): time.sleep(0.1) else: time.sleep(1.0) data=open(data).read() db=ZODB.DB(s, # disable cache deactivation cache_size=4000, cache_deactivate_after=6000,) db.open().root() results={1:0, 10:0, 100:0, 1000:0} for j in range(nrep): for r in 1, 10, 100, 1000: t = time.time() jar = db.open() get_transaction().begin() rt = jar.root() key = 's%s' % r if rt.has_key(key): p = rt[key] else: rt[key] = p =P() for i in range(r): v = getattr(p, str(i), P()) if compress is not None: v.d = compress(data) else: v.d = data setattr(p, str(i), v) get_transaction().commit() jar.close() t = time.time() - t if detailed: print "%s\t%s\t%.4f" % (j, r, t) results[r] = results[r] + t rt=d=p=v=None # release all references if minimize: time.sleep(3) jar.cacheMinimize(3) if zeo_pipe: os.write(zeo_pipe, "done") if detailed: print '-'*24 for r in 1, 10, 100, 1000: t=results[r]/nrep print "mean:\t%s\t%.4f\t%.4f (s/o)" % (r, t, t/r) ##def compress(s): ## c = zlib.compressobj() ## o = c.compress(s) ## return o + c.flush() if __name__=='__main__': main(sys.argv[1:]) --- Added File speed.py in package Packages/ZEO --- ############################################################################## # # Zope Public License (ZPL) Version 1.0 # ------------------------------------- # # Copyright (c) Digital Creations. All rights reserved. # # This license has been certified as Open Source(tm). # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions in source code must retain the above copyright # notice, this list of conditions, and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions, and the following disclaimer in # the documentation and/or other materials provided with the # distribution. # # 3. Digital Creations requests that attribution be given to Zope # in any manner possible. Zope includes a "Powered by Zope" # button that is installed by default. While it is not a license # violation to remove this button, it is requested that the # attribution remain. A significant investment has been put # into Zope, and this effort will continue if the Zope community # continues to grow. This is one way to assure that growth. # # 4. All advertising materials and documentation mentioning # features derived from or use of this software must display # the following acknowledgement: # # "This product includes software developed by Digital Creations # for use in the Z Object Publishing Environment # (http://www.zope.org/)." # # In the event that the product being advertised includes an # intact Zope distribution (with copyright and license included) # then this clause is waived. # # 5. Names associated with Zope or Digital Creations must not be used to # endorse or promote products derived from this software without # prior written permission from Digital Creations. # # 6. Modified redistributions of any form whatsoever must retain # the following acknowledgment: # # "This product includes software developed by Digital Creations # for use in the Z Object Publishing Environment # (http://www.zope.org/)." # # Intact (re-)distributions of any official Zope release do not # require an external acknowledgement. # # 7. Modifications are encouraged but must be packaged separately as # patches to official Zope releases. Distributions that do not # clearly separate the patches from the original work must be clearly # labeled as unofficial distributions. Modifications which do not # carry the name Zope may be packaged in any form, as long as they # conform to all of the clauses above. # # # Disclaimer # # THIS SOFTWARE IS PROVIDED BY DIGITAL CREATIONS ``AS IS'' AND ANY # EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL DIGITAL CREATIONS OR ITS # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF # USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND # ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT # OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF # SUCH DAMAGE. # # # This software consists of contributions made by Digital Creations and # many individuals on behalf of Digital Creations. Specific # attributions are listed in the accompanying credits file. # ############################################################################## usage="""Test speed of a ZODB storage Options: -d file The data file to use as input. The default is this script. -n n The number of repititions -s module A module that defines a 'Storage' attribute, which is an open storage. If not specified, a FileStorage will ne used. -z Test compressing data -D Run in debug mode -L Test loads as well as stores by minimizing the cache after eachrun -M Output means only -C Run with a persistent client cache """ import asyncore import sys, os, getopt, string, time ##sys.path.insert(0, os.getcwd()) import ZODB, ZODB.FileStorage import Persistence import ZEO.ClientStorage, ZEO.StorageServer class P(Persistence.Persistent): pass fs_name = "zeo-speed.fs" class ZEOExit(asyncore.file_dispatcher): """Used to exit ZEO.StorageServer when run is done""" def writable(self): return 0 def readable(self): return 1 def handle_read(self): buf = self.recv(4) assert buf == "done" self.delete_fs() os._exit(0) def handle_close(self): print "Parent process exited unexpectedly" self.delete_fs() os._exit(0) def delete_fs(self): os.unlink(fs_name) os.unlink(fs_name + ".lock") os.unlink(fs_name + ".tmp") def main(args): opts, args = getopt.getopt(args, 'zd:n:Ds:LM') s = None compress = None data=sys.argv[0] nrep=5 minimize=0 detailed=1 cache = None for o, v in opts: if o=='-n': nrep = int(v) elif o=='-d': data = v elif o=='-s': s = v elif o=='-z': import zlib compress = zlib.compress elif o=='-L': minimize=1 elif o=='-M': detailed=0 elif o=='-D': global debug os.environ['STUPID_LOG_FILE']='' os.environ['STUPID_LOG_SEVERITY']='-999' debug = 1 elif o == '-C': cache = 'speed' zeo_pipe = None if s: s = __import__(s, globals(), globals(), ('__doc__',)) s = s.Storage else: rd, wr = os.pipe() pid = os.fork() if pid: # in the child, run the storage server os.close(wr) import asyncore ZEOExit(rd) fs = ZODB.FileStorage.FileStorage(fs_name, create=1) serv = ZEO.StorageServer.StorageServer(('', 1975), {'1':fs}) asyncore.loop() else: os.close(rd) zeo_pipe = wr s = ZEO.ClientStorage.ClientStorage(('', 1975), debug=0, client=cache) if hasattr(s, 'is_connected'): while not s.is_connected(): time.sleep(0.1) else: time.sleep(1.0) data=open(data).read() db=ZODB.DB(s, # disable cache deactivation cache_size=4000, cache_deactivate_after=6000,) db.open().root() results={1:0, 10:0, 100:0, 1000:0} for j in range(nrep): for r in 1, 10, 100, 1000: t = time.time() jar = db.open() get_transaction().begin() rt = jar.root() key = 's%s' % r if rt.has_key(key): p = rt[key] else: rt[key] = p =P() for i in range(r): v = getattr(p, str(i), P()) if compress is not None: v.d = compress(data) else: v.d = data setattr(p, str(i), v) get_transaction().commit() jar.close() t = time.time() - t if detailed: print "%s\t%s\t%.4f" % (j, r, t) results[r] = results[r] + t rt=d=p=v=None # release all references if minimize: time.sleep(3) jar.cacheMinimize(3) if zeo_pipe: os.write(zeo_pipe, "done") if detailed: print '-'*24 for r in 1, 10, 100, 1000: t=results[r]/nrep print "mean:\t%s\t%.4f\t%.4f (s/o)" % (r, t, t/r) ##def compress(s): ## c = zlib.compressobj() ## o = c.compress(s) ## return o + c.flush() if __name__=='__main__': main(sys.argv[1:]) From jeremy at digicool.com Fri Apr 20 15:18:53 2001 From: jeremy at digicool.com (jeremy@digicool.com) Date: Sun Aug 10 16:31:15 2008 Subject: [ZEO-Checkins] CVS: Packages/ZEO - testZEO.py:1.1.2.5 Message-ID: <20010420191853.3DABC510E3@korak.digicool.com> Update of /cvs-repository/Packages/ZEO/tests In directory korak:/tmp/cvs-serv24694 Modified Files: Tag: ZEO-ZRPC-Dev testZEO.py Log Message: Replace zLOG with zeolog. Refactor GenericTests so that tearDown() calls delStorage(), which can clean up a file storage. --- Updated File testZEO.py in package Packages/ZEO -- --- testZEO.py 2001/04/19 18:26:05 1.1.2.4 +++ testZEO.py 2001/04/20 19:18:52 1.1.2.5 @@ -12,7 +12,7 @@ from ZEO.zrpc2 import ManagedServerConnection import ThreadedAsync, ZEO.trigger -import zLOG +from ZEO import zeolog # XXX The ZODB.tests package contains a grab bad things, including, # apparently, a collection of modules that define mixin classes @@ -24,7 +24,7 @@ def newConnection(self, sock, addr, nil): c = ManagedServerConnection(sock, addr, None, self) c.register_object(TestStorageProxy(self, c)) - zLOG.LOG("TSS", zLOG.INFO, "connected: %s" % c) + zeolog.LOG("TSS", zeolog.INFO, "connected: %s" % c) class TestStorageProxy(ZEO.StorageServer.StorageProxy): def shutdown(self): @@ -79,7 +79,6 @@ def _get_serial(self, r): """Return oid -> serialno dict from sequence of ZEO replies.""" d = {} - zLOG.LOG('ZTB', zLOG.BLATHER, 'reply: %s' % repr(r)) if r is None: return None if type(r) == types.StringType: @@ -123,12 +122,12 @@ def tearDown(self): """Try to cause the tests to halt""" - zLOG.LOG("test", zLOG.INFO, "tearDown()") self.running = 0 self._storage._server.rpc.callAsync('shutdown') self._storage._rpc_mgr.close() os.waitpid(self.__pid, 0) os.unlink(self.__sock) + self.delStorage() self.__super_tearDown() def startServer(self): @@ -139,7 +138,7 @@ self.__server = TestStorageServer(self.__sock, d) asyncore.loop() else: - zLOG.LOG("test", zLOG.INFO, "forked %s" % self.__pid) + zeolog.LOG("test", zeolog.INFO, "forked %s" % self.__pid) def checkFirst(self): self._storage.tpc_begin(self._transaction) @@ -147,7 +146,6 @@ class ZEOFileStorageTests(GenericTests): __super_setUp = GenericTests.setUp - __super_tearDown = GenericTests.tearDown from ZODB.FileStorage import FileStorage @@ -155,15 +153,14 @@ self.__fs_base = tempfile.mktemp() self.__super_setUp() - def tearDown(self): - self.__super_tearDown() + def getStorage(self): + return self.FileStorage(self.__fs_base, create=1) + + def delStorage(self): # file storage appears to create three files for ext in '', '.index', '.lock', '.tmp': path = self.__fs_base + ext os.unlink(path) - - def getStorage(self): - return self.FileStorage(self.__fs_base, create=1) def main(): tests = unittest.makeSuite(ZEOFileStorageTests, 'check') --- Updated File testZEO.py in package Packages/ZEO -- --- testZEO.py 2001/04/19 18:26:05 1.1.2.4 +++ testZEO.py 2001/04/20 19:18:52 1.1.2.5 @@ -12,7 +12,7 @@ from ZEO.zrpc2 import ManagedServerConnection import ThreadedAsync, ZEO.trigger -import zLOG +from ZEO import zeolog # XXX The ZODB.tests package contains a grab bad things, including, # apparently, a collection of modules that define mixin classes @@ -24,7 +24,7 @@ def newConnection(self, sock, addr, nil): c = ManagedServerConnection(sock, addr, None, self) c.register_object(TestStorageProxy(self, c)) - zLOG.LOG("TSS", zLOG.INFO, "connected: %s" % c) + zeolog.LOG("TSS", zeolog.INFO, "connected: %s" % c) class TestStorageProxy(ZEO.StorageServer.StorageProxy): def shutdown(self): @@ -79,7 +79,6 @@ def _get_serial(self, r): """Return oid -> serialno dict from sequence of ZEO replies.""" d = {} - zLOG.LOG('ZTB', zLOG.BLATHER, 'reply: %s' % repr(r)) if r is None: return None if type(r) == types.StringType: @@ -123,12 +122,12 @@ def tearDown(self): """Try to cause the tests to halt""" - zLOG.LOG("test", zLOG.INFO, "tearDown()") self.running = 0 self._storage._server.rpc.callAsync('shutdown') self._storage._rpc_mgr.close() os.waitpid(self.__pid, 0) os.unlink(self.__sock) + self.delStorage() self.__super_tearDown() def startServer(self): @@ -139,7 +138,7 @@ self.__server = TestStorageServer(self.__sock, d) asyncore.loop() else: - zLOG.LOG("test", zLOG.INFO, "forked %s" % self.__pid) + zeolog.LOG("test", zeolog.INFO, "forked %s" % self.__pid) def checkFirst(self): self._storage.tpc_begin(self._transaction) @@ -147,7 +146,6 @@ class ZEOFileStorageTests(GenericTests): __super_setUp = GenericTests.setUp - __super_tearDown = GenericTests.tearDown from ZODB.FileStorage import FileStorage @@ -155,15 +153,14 @@ self.__fs_base = tempfile.mktemp() self.__super_setUp() - def tearDown(self): - self.__super_tearDown() + def getStorage(self): + return self.FileStorage(self.__fs_base, create=1) + + def delStorage(self): # file storage appears to create three files for ext in '', '.index', '.lock', '.tmp': path = self.__fs_base + ext os.unlink(path) - - def getStorage(self): - return self.FileStorage(self.__fs_base, create=1) def main(): tests = unittest.makeSuite(ZEOFileStorageTests, 'check') From jeremy at digicool.com Tue Apr 24 16:21:09 2001 From: jeremy at digicool.com (jeremy@digicool.com) Date: Sun Aug 10 16:31:15 2008 Subject: [ZEO-Checkins] CVS: Packages/ZEO - __init__.py:1.2 Message-ID: <20010424202109.1E451510E3@korak.digicool.com> Update of /cvs-repository/Packages/ZEO/tests In directory korak:/tmp/cvs-serv1091 Added Files: __init__.py Log Message: placeholder for future tests (currently hiding in a branch) --- Updated File __init__.py in package Packages/ZEO -- --- Updated File __init__.py in package Packages/ZEO -- From jeremy at digicool.com Tue Apr 24 17:40:40 2001 From: jeremy at digicool.com (jeremy@digicool.com) Date: Sun Aug 10 16:31:15 2008 Subject: [ZEO-Checkins] CVS: Packages/ZEO - StorageServer.py:1.23 Message-ID: <20010424214040.11AA3510E3@korak.digicool.com> Update of /cvs-repository/Packages/ZEO In directory korak:/tmp/cvs-serv6121 Modified Files: StorageServer.py Log Message: fix two multi-argument appends for Python 2.0+ compatibility --- Updated File StorageServer.py in package Packages/ZEO -- --- StorageServer.py 2001/03/27 23:41:51 1.22 +++ StorageServer.py 2001/04/24 21:40:40 1.23 @@ -492,7 +492,7 @@ if storage._transaction is not None: try: waiting=storage.__waiting except: waiting=storage.__waiting=[] - waiting.append(self.unlock, ()) + waiting.append((self.unlock, ())) return 1 # Return a flag indicating a lock condition. self._transaction=t=Transaction() @@ -512,7 +512,7 @@ else: try: waiting=storage.__waiting except: waiting=storage.__waiting=[] - waiting.append(self.try_again_sync, (id, user, description, ext)) + waiting.append((self.try_again_sync, (id, user, description, ext))) return _noreturn --- Updated File StorageServer.py in package Packages/ZEO -- --- StorageServer.py 2001/03/27 23:41:51 1.22 +++ StorageServer.py 2001/04/24 21:40:40 1.23 @@ -492,7 +492,7 @@ if storage._transaction is not None: try: waiting=storage.__waiting except: waiting=storage.__waiting=[] - waiting.append(self.unlock, ()) + waiting.append((self.unlock, ())) return 1 # Return a flag indicating a lock condition. self._transaction=t=Transaction() @@ -512,7 +512,7 @@ else: try: waiting=storage.__waiting except: waiting=storage.__waiting=[] - waiting.append(self.try_again_sync, (id, user, description, ext)) + waiting.append((self.try_again_sync, (id, user, description, ext))) return _noreturn From jeremy at digicool.com Wed Apr 25 16:33:37 2001 From: jeremy at digicool.com (jeremy@digicool.com) Date: Sun Aug 10 16:31:15 2008 Subject: [ZEO-Checkins] CVS: Packages/ZEO - zeolog.py:1.1.2.2 Message-ID: <20010425203337.0E383510E3@korak.digicool.com> Update of /cvs-repository/Packages/ZEO In directory korak:/tmp/cvs-serv27942 Modified Files: Tag: ZEO-ZRPC-Dev zeolog.py Log Message: add support for printing tracebacks if the error argument is passsed --- Updated File zeolog.py in package Packages/ZEO -- --- zeolog.py 2001/04/20 19:14:09 1.1.2.1 +++ zeolog.py 2001/04/25 20:33:35 1.1.2.2 @@ -3,6 +3,7 @@ import os import sys import time +import traceback # the logging severity constants defined by zLOG from zLOG import TRACE, DEBUG, BLATHER, INFO, PROBLEM, WARNING, ERROR, PANIC @@ -25,6 +26,12 @@ def LOG(subsys, severity, summary, detail='', error=None, reraise=None): + """Print log info to current file. + + The error argument should be a 3-tuple as returned by + sys.exc_info(). A formatted traceback will be printed to the file + in addition to the summary. + """ if log_file is None or severity < log_level: return s = "%s %7s %s %s" % (now(), severity_strings[severity], subsys, summary) @@ -34,7 +41,11 @@ log_file.flush() if error: - pass # XXX haven't implemented traceback writing + lines = traceback.format_exception(*error) + # each line ends with a newline, hence the comma + print >> log_file, "".join(lines), + print >> log_file, "-" * 70 + log_file.flush() if reraise and error: raise error[0], error[1], error[2] --- Updated File zeolog.py in package Packages/ZEO -- --- zeolog.py 2001/04/20 19:14:09 1.1.2.1 +++ zeolog.py 2001/04/25 20:33:35 1.1.2.2 @@ -3,6 +3,7 @@ import os import sys import time +import traceback # the logging severity constants defined by zLOG from zLOG import TRACE, DEBUG, BLATHER, INFO, PROBLEM, WARNING, ERROR, PANIC @@ -25,6 +26,12 @@ def LOG(subsys, severity, summary, detail='', error=None, reraise=None): + """Print log info to current file. + + The error argument should be a 3-tuple as returned by + sys.exc_info(). A formatted traceback will be printed to the file + in addition to the summary. + """ if log_file is None or severity < log_level: return s = "%s %7s %s %s" % (now(), severity_strings[severity], subsys, summary) @@ -34,7 +41,11 @@ log_file.flush() if error: - pass # XXX haven't implemented traceback writing + lines = traceback.format_exception(*error) + # each line ends with a newline, hence the comma + print >> log_file, "".join(lines), + print >> log_file, "-" * 70 + log_file.flush() if reraise and error: raise error[0], error[1], error[2] From jeremy at digicool.com Wed Apr 25 16:36:22 2001 From: jeremy at digicool.com (jeremy@digicool.com) Date: Sun Aug 10 16:31:15 2008 Subject: [ZEO-Checkins] CVS: Packages/ZEO - zrpc2.py:1.1.2.10 Message-ID: <20010425203622.9B308510E3@korak.digicool.com> Update of /cvs-repository/Packages/ZEO In directory korak:/tmp/cvs-serv28083 Modified Files: Tag: ZEO-ZRPC-Dev zrpc2.py Log Message: Revised logging support. In module log(), pass error argument through to zeolog. Make some TRACE log message a bit more verbose to explain whether messages are being sent or received. If the method invoked via handle_request() raises an exception, log the traceback. If unexpected errors occur, e.g. async method raises exception or decoded error, log rather than printing to stdout. Other changes: import smac and trigger from ZEO package --- Updated File zrpc2.py in package Packages/ZEO -- --- zrpc2.py 2001/04/20 19:14:09 1.1.2.9 +++ zrpc2.py 2001/04/25 20:36:20 1.1.2.10 @@ -34,16 +34,16 @@ from cStringIO import StringIO from ZODB import POSException -import smac -import trigger +from ZEO import smac, trigger import zeolog import ThreadedAsync REPLY = ".reply" # message name used for replies ASYNC = 1 -def log(message, level=zeolog.BLATHER, label="zrpc:%s" % os.getpid()): - zeolog.LOG(label, level, message) +def log(message, level=zeolog.BLATHER, label="zrpc:%s" % os.getpid(), + error=None): + zeolog.LOG(label, level, message, error=error) class ZRPCError(POSException.StorageError): pass @@ -174,8 +174,8 @@ sys.exc_info()[1]) if __debug__: - log("message: %s, %s, %s, %s" % (msgid, flags, name, - repr(args)[:40]), + log("recv msg: %s, %s, %s, %s" % (msgid, flags, name, + repr(args)[:40]), level=zeolog.TRACE) if name == REPLY: self.handle_reply(msgid, flags, args) @@ -184,14 +184,15 @@ def handle_reply(self, msgid, flags, args): if __debug__: - log("reply: %s, %s, %s" % (msgid, flags, str(args)[:40]), + log("recv reply: %s, %s, %s" % (msgid, flags, str(args)[:40]), level=zeolog.TRACE) self.__reply = msgid, flags, args self.__reply_lock.release() # will fail if lock is unlocked def handle_request(self, msgid, flags, name, args): if __debug__: - log("%s%s" % (name, repr(args)[:40]), zeolog.TRACE) + log("call %s%s on %s" % (name, repr(args)[:40], repr(self.obj)), + zeolog.TRACE) if not self.check_method(name): raise ZRPCError("Invalid method name: %s" % name) @@ -207,30 +208,30 @@ ret = meth(*args) except (POSException.UndoError, POSException.VersionCommitError), msg: - return self.return_error(msgid, flags, sys.exc_info()[0], - sys.exc_info()[1]) + error = sys.exc_info() + log("%s() raised exception: %s" % (name, msg), zeolog.ERROR, error) + return self.return_error(msgid, flags, error[0], error[1]) except Exception, msg: - return self.return_error(msgid, flags, sys.exc_info()[0], - sys.exc_info()[1]) - + error = sys.exc_info() + log("%s() raised exception: %s" % (name, msg), zeolog.ERROR, error) + return self.return_error(msgid, flags, error[0], error[1]) + if flags & ASYNC: if ret is not None: + log("async method %s returned value %s" % (name, repr(ret)), + zeolog.ERROR) raise ZRPCError("async method returned value") else: if __debug__: - log("%s reply %s" % (name, repr(ret)[:40]), zeolog.TRACE) + log("%s return %s" % (name, repr(ret)[:40]), zeolog.TRACE) if isinstance(ret, Delay): ret.set_sender(msgid, self.send_reply) else: self.send_reply(msgid, ret) - def handle_error(self): - t, v, tb = sys.exc_info() - if type(v) == types.StringType: - print t, repr(v) - else: - print t, v - traceback.print_tb(tb) + def handle_error(self, msg="No error message supplied"): + error = sys.exc_info() + log(msg, zeolog.ERROR, error=error) def check_method(self, name): # XXX minimal security check should go here: Is name exported? @@ -242,12 +243,10 @@ def return_error(self, msgid, flags, err_type, err_value): if flags is None: - print "Exception raised during decoding" - self.handle_error() + self.handle_error("Exception raised during decoding") return if flags & ASYNC: - print "Asynchronous call raised exception:" - self.handle_error() + self.handle_error("Asynchronous call raised exception") return if type(err_value) is not types.InstanceType: err_value = err_type, err_value @@ -292,6 +291,8 @@ raise DisconnectedError("This action is temporarily unavailable") msgid = self.msgid self.msgid = self.msgid + 1 + if __debug__: + log("send msg: %d, 0, %s, ..." % (msgid, method)) self.message_output(self.marshal.encode(msgid, 0, method, args)) self.__reply = None @@ -313,6 +314,8 @@ raise DisconnectedError("This action is temporarily unavailable") msgid = self.msgid self.msgid = self.msgid + 1 + if __debug__: + log("send msg: %d, %d, %s, ..." % (msgid, ASYNC, method)) self.message_output(self.marshal.encode(msgid, ASYNC, method, args)) self._do_io() --- Updated File zrpc2.py in package Packages/ZEO -- --- zrpc2.py 2001/04/20 19:14:09 1.1.2.9 +++ zrpc2.py 2001/04/25 20:36:20 1.1.2.10 @@ -34,16 +34,16 @@ from cStringIO import StringIO from ZODB import POSException -import smac -import trigger +from ZEO import smac, trigger import zeolog import ThreadedAsync REPLY = ".reply" # message name used for replies ASYNC = 1 -def log(message, level=zeolog.BLATHER, label="zrpc:%s" % os.getpid()): - zeolog.LOG(label, level, message) +def log(message, level=zeolog.BLATHER, label="zrpc:%s" % os.getpid(), + error=None): + zeolog.LOG(label, level, message, error=error) class ZRPCError(POSException.StorageError): pass @@ -174,8 +174,8 @@ sys.exc_info()[1]) if __debug__: - log("message: %s, %s, %s, %s" % (msgid, flags, name, - repr(args)[:40]), + log("recv msg: %s, %s, %s, %s" % (msgid, flags, name, + repr(args)[:40]), level=zeolog.TRACE) if name == REPLY: self.handle_reply(msgid, flags, args) @@ -184,14 +184,15 @@ def handle_reply(self, msgid, flags, args): if __debug__: - log("reply: %s, %s, %s" % (msgid, flags, str(args)[:40]), + log("recv reply: %s, %s, %s" % (msgid, flags, str(args)[:40]), level=zeolog.TRACE) self.__reply = msgid, flags, args self.__reply_lock.release() # will fail if lock is unlocked def handle_request(self, msgid, flags, name, args): if __debug__: - log("%s%s" % (name, repr(args)[:40]), zeolog.TRACE) + log("call %s%s on %s" % (name, repr(args)[:40], repr(self.obj)), + zeolog.TRACE) if not self.check_method(name): raise ZRPCError("Invalid method name: %s" % name) @@ -207,30 +208,30 @@ ret = meth(*args) except (POSException.UndoError, POSException.VersionCommitError), msg: - return self.return_error(msgid, flags, sys.exc_info()[0], - sys.exc_info()[1]) + error = sys.exc_info() + log("%s() raised exception: %s" % (name, msg), zeolog.ERROR, error) + return self.return_error(msgid, flags, error[0], error[1]) except Exception, msg: - return self.return_error(msgid, flags, sys.exc_info()[0], - sys.exc_info()[1]) - + error = sys.exc_info() + log("%s() raised exception: %s" % (name, msg), zeolog.ERROR, error) + return self.return_error(msgid, flags, error[0], error[1]) + if flags & ASYNC: if ret is not None: + log("async method %s returned value %s" % (name, repr(ret)), + zeolog.ERROR) raise ZRPCError("async method returned value") else: if __debug__: - log("%s reply %s" % (name, repr(ret)[:40]), zeolog.TRACE) + log("%s return %s" % (name, repr(ret)[:40]), zeolog.TRACE) if isinstance(ret, Delay): ret.set_sender(msgid, self.send_reply) else: self.send_reply(msgid, ret) - def handle_error(self): - t, v, tb = sys.exc_info() - if type(v) == types.StringType: - print t, repr(v) - else: - print t, v - traceback.print_tb(tb) + def handle_error(self, msg="No error message supplied"): + error = sys.exc_info() + log(msg, zeolog.ERROR, error=error) def check_method(self, name): # XXX minimal security check should go here: Is name exported? @@ -242,12 +243,10 @@ def return_error(self, msgid, flags, err_type, err_value): if flags is None: - print "Exception raised during decoding" - self.handle_error() + self.handle_error("Exception raised during decoding") return if flags & ASYNC: - print "Asynchronous call raised exception:" - self.handle_error() + self.handle_error("Asynchronous call raised exception") return if type(err_value) is not types.InstanceType: err_value = err_type, err_value @@ -292,6 +291,8 @@ raise DisconnectedError("This action is temporarily unavailable") msgid = self.msgid self.msgid = self.msgid + 1 + if __debug__: + log("send msg: %d, 0, %s, ..." % (msgid, method)) self.message_output(self.marshal.encode(msgid, 0, method, args)) self.__reply = None @@ -313,6 +314,8 @@ raise DisconnectedError("This action is temporarily unavailable") msgid = self.msgid self.msgid = self.msgid + 1 + if __debug__: + log("send msg: %d, %d, %s, ..." % (msgid, ASYNC, method)) self.message_output(self.marshal.encode(msgid, ASYNC, method, args)) self._do_io() From jeremy at digicool.com Wed Apr 25 16:38:40 2001 From: jeremy at digicool.com (jeremy@digicool.com) Date: Sun Aug 10 16:31:15 2008 Subject: [ZEO-Checkins] CVS: Packages/ZEO - ServerStub.py:1.1.2.4 Message-ID: <20010425203840.4799C510E3@korak.digicool.com> Update of /cvs-repository/Packages/ZEO In directory korak:/tmp/cvs-serv28363 Modified Files: Tag: ZEO-ZRPC-Dev ServerStub.py Log Message: tpc_begin() is synchronous, not asynchronous --- Updated File ServerStub.py in package Packages/ZEO -- --- ServerStub.py 2001/03/30 22:55:01 1.1.2.3 +++ ServerStub.py 2001/04/25 20:38:38 1.1.2.4 @@ -42,9 +42,10 @@ self.rpc.callAsync('storea', oid, serial, data, version, id) def tpc_begin(self, id, user, descr, ext): - self.rpc.callAsync('tpc_begin', id, user, descr, ext) + return self.rpc.call('tpc_begin', id, user, descr, ext) def tpc_begin_sync(self, id, user, descr, ext): + # XXX nuked return self.rpc.call('tpc_begin_sync', id, user, descr, ext) def vote(self, trans_id): --- Updated File ServerStub.py in package Packages/ZEO -- --- ServerStub.py 2001/03/30 22:55:01 1.1.2.3 +++ ServerStub.py 2001/04/25 20:38:38 1.1.2.4 @@ -42,9 +42,10 @@ self.rpc.callAsync('storea', oid, serial, data, version, id) def tpc_begin(self, id, user, descr, ext): - self.rpc.callAsync('tpc_begin', id, user, descr, ext) + return self.rpc.call('tpc_begin', id, user, descr, ext) def tpc_begin_sync(self, id, user, descr, ext): + # XXX nuked return self.rpc.call('tpc_begin_sync', id, user, descr, ext) def vote(self, trans_id): From jeremy at digicool.com Wed Apr 25 16:43:54 2001 From: jeremy at digicool.com (jeremy@digicool.com) Date: Sun Aug 10 16:31:15 2008 Subject: [ZEO-Checkins] CVS: Packages/ZEO - StorageServer.py:1.21.4.7 Message-ID: <20010425204354.323AD510E3@korak.digicool.com> Update of /cvs-repository/Packages/ZEO In directory korak:/tmp/cvs-serv28767 Modified Files: Tag: ZEO-ZRPC-Dev StorageServer.py Log Message: Fixes to support multiple simultaneous clients of a single storage. Add _check_tid() helper that does check if the current call is using the current transaction id. Complain if not and raise exception if specified. Use _check_tid() helper for all interesting methods except tpc_begin(). Use _restart_delayed_transaction() to do correct bookkeeping about current transaction when one is restarted from the delayed queue. This new comment explains some of the details: # When multiple clients are using a single storage, there are several # different _transaction attributes to keep track of. Each # StorageProxy object has a single _transaction that refers to its # current transaction. The storage (self.__storage) has another # _transaction that is used for the *real* transaction. # The real trick comes with the __waiting queue for a storage. # When a StorageProxy pulls a new transaction from the queue, it # must inform the new transaction's proxy. (The two proxies may # be the same.) The new transaction's proxy sets its _transaction # and continues from there. Use handle_waiting() to check if a queue transaction exists. Remove old async tpc_begin() defs. Add __repr__() and _log() methods that provide per-instance labels for debugging purposes. --- Updated File StorageServer.py in package Packages/ZEO -- --- StorageServer.py 2001/04/20 19:14:08 1.21.4.6 +++ StorageServer.py 2001/04/25 20:43:52 1.21.4.7 @@ -59,8 +59,8 @@ # labeled as unofficial distributions. Modifications which do not # carry the name Zope may be packaged in any form, as long as they # conform to all of the clauses above. +# # -# # Disclaimer # # THIS SOFTWARE IS PROVIDED BY DIGITAL CREATIONS ``AS IS'' AND ANY @@ -86,12 +86,16 @@ This server acts as a front-end for one or more real storages, like file storage or Berkeley storage. + +XXX Need some basic access control-- a declaration of the methods +exported for invocation by the server. """ import cPickle import os import sys import threading +import types import ClientStub import zrpc2 @@ -110,8 +114,9 @@ pickler.fast = 1 # Don't use the memo dump = pickler.dump -def log(message, level=zeolog.INFO, label="zeoserver:%s" % os.getpid()): - zeolog.LOG(label, level, message) +def log(message, level=zeolog.INFO, label="ZEO Server:%s" % os.getpid(), + error=None): + zeolog.LOG(label, level, message, error=error) class StorageServerError(StorageError): pass @@ -165,6 +170,59 @@ self.__invalidated = [] self._transaction = None + def __repr__(self): + tid = self._transaction and repr(self._transaction.id) + if self.__storage: + stid = self.__storage._transaction and \ + repr(self.__storage._transaction.id) + else: + stid = None + return "" % (id(self), tid, + stid) + + def _log(self, msg, level=zeolog.INFO, error=None, pid=os.getpid()): + zeolog.LOG("ZEO Server %s %X" % (pid, id(self)), + level, msg, error=error) + + def setup_delegation(self): + """Delegate several methods to the storage""" + self.undoInfo = self.__storage.undoInfo + self.undoLog = self.__storage.undoLog + self.versionEmpty = self.__storage.versionEmpty + self.versions = self.__storage.versions + self.history = self.__storage.history + self.load = self.__storage.load + self.loadSerial = self.__storage.loadSerial + + def _check_tid(self, tid, exc=None): + caller = sys._getframe().f_back.f_code.co_name + if self._transaction is None: + self._log("no current transaction: %s()" % caller, + zeolog.PROBLEM) + if exc is not None: + raise exc(None, tid) + else: + return 0 + if self._transaction.id != tid: + self._log("%s(%s) invalid; current transaction = %s" % \ + (caller, repr(tid), repr(self._transaction.id)), + zeolog.PROBLEM) + if exc is not None: + raise exc(self._transaction.id, tid) + else: + return 0 + return 1 + + def _restart_delayed_transaction(self, delay, tinfo): + self._transaction = t = Transaction() + t.id = tinfo[0] + t.user = tinfo[1] + t.description = tinfo[2] + self.__storage.tpc_begin(t) + self.__invalidated = [] + assert self._transaction.id == self.__storage._transaction.id + delay.reply(None) + def register(self, storage_id): """Select the storage that this client will use @@ -172,25 +230,15 @@ """ storage = self.server.storages.get(storage_id) if storage is None: - log("unknown storage_id: %s" % storage_id) + self._log("unknown storage_id: %s" % storage_id) self.get_caller.close() self.__storage_id = storage_id self.__storage = storage self.setup_delegation() self.server.register(storage_id, self) - log("registered storage %s: %s" % (storage_id, storage)) + self._log("registered storage %s: %s" % (storage_id, storage)) - def setup_delegation(self): - """Delegate several methods to the storage""" - self.undoInfo = self.__storage.undoInfo - self.undoLog = self.__storage.undoLog - self.versionEmpty = self.__storage.versionEmpty - self.versions = self.__storage.versions - self.history = self.__storage.history - self.load = self.__storage.load - self.loadSerial = self.__storage.loadSerial - def get_info(self): return { 'length': len(self.__storage), @@ -247,9 +295,9 @@ try: self.__storage.pack(t, referencesf) except: - log('ZEO Server', zeolog.ERROR, - 'Pack failed for %s' % self.__storage_id, - error=sys.exc_info()) + self._log('ZEO Server', zeolog.ERROR, + 'Pack failed for %s' % self.__storage_id, + error=sys.exc_info()) if wait: raise else: @@ -259,19 +307,15 @@ self.get_size_info()) def abortVersion(self, src, id): - t = self._transaction - if t is None or id != t.id: - raise StorageTransactionError(self._transaction, id) - oids = self.__storage.abortVersion(src, t) + self._check_tid(id, exc=StorageTransactionError) + oids = self.__storage.abortVersion(src, self._transaction) for oid in oids: self.__invalidated.append((oid, src)) return oids def commitVersion(self, src, dest, id): - t = self._transaction - if t is None or id != t.id: - raise StorageTransactionError(self, id) - oids = self.__storage.commitVersion(src, dest, t) + self._check_tid(id, exc=StorageTransactionError) + oids = self.__storage.commitVersion(src, dest, self._transaction) for oid in oids: self.__invalidated.append((oid, dest)) if dest: @@ -279,26 +323,25 @@ return oids def storea(self, oid, serial, data, version, id): - t = self._transaction + self._check_tid(id, exc=StorageTransactionError) try: # XXX does this stmt need to be in the try/except? - if t is None or id != t.id: - raise StorageTransactionError(self, id) - newserial = self.__storage.store(oid, serial, data, version, t) + newserial = self.__storage.store(oid, serial, data, version, + self._transaction) except TransactionError, v: # This is a normal transaction error such as a conflict error # or a version lock or conflict error. It doesn't need to be # logged. - log("transaction error: %s" % repr(v)) + self._log("transaction error: %s" % repr(v)) newserial = v except: # all errors need to be serialized to prevent unexpected # returns, which would screw up the return handling. # IOW, Anything that ends up here is evil enough to be logged. error = sys.exc_info() - log('store error: %s: %s' % (error[0], error[1]). - zeolog.ERROR) + self._log('store error: %s: %s' % (error[0], error[1]), + zeolog.ERROR, error=error) newserial = sys.exc_info()[1] else: if serial != '\0\0\0\0\0\0\0\0': @@ -307,8 +350,8 @@ try: nil = dump(newserial, 1) except: - log("couldn't pickle newserial: %s" % repr(newserial), - zeolog.ERROR) + self._log("couldn't pickle newserial: %s" % repr(newserial), + zeolog.ERROR) dump('', 1) # clear pickler r = StorageServerError("Couldn't pickle exception %s" % \ `newserial`) @@ -316,11 +359,9 @@ self.client.serialno((oid, newserial)) - def vote(self, id): - t = self._transaction - if t is None or id != t.id: - raise StorageTransactionError(self._transaction, id) - return self.__storage.tpc_vote(t) + def vote(self, id): + self._check_tid(id, exc=StorageTransactionError) + return self.__storage.tpc_vote(self._transaction) def undo(self, transaction_id): oids = self.__storage.undo(transaction_id) @@ -330,38 +371,36 @@ return oids return () - def tpc_abort(self, id): - t = self._transaction - if t is None or id != t.id: - return - r = self.__storage.tpc_abort(t) - - waiting = self.__storage.__waiting - while waiting: - f, args = waiting.pop(0) - if apply(f, args): - break - - self._transaction = None - self.__invalidated = [] - def unlock(self): -# if self.__closed: -# return +## if self.__closed: +## return self.client.unlock() + # When multiple clients are using a single storage, there are several + # different _transaction attributes to keep track of. Each + # StorageProxy object has a single _transaction that refers to its + # current transaction. The storage (self.__storage) has another + # _transaction that is used for the *real* transaction. + + # The real trick comes with the __waiting queue for a storage. + # When a StorageProxy pulls a new transaction from the queue, it + # must inform the new transaction's proxy. (The two proxies may + # be the same.) The new transaction's proxy sets its _transaction + # and continues from there. + def tpc_begin(self, id, user, description, ext): - t = self._transaction - if t is not None: - if id == t.id: + if self._transaction is not None: + if self._transaction.id == id: + self._log("duplicate tpc_begin(%s)" % repr(id)) return else: - raise StorageServerError("Multiple simultaneous tpc_begin " - "requests from the same client.") + raise StorageTransactionError("Multiple simultaneous tpc_begin" + " requests from one client.") if self.__storage._transaction is not None: - self.__storage.__waiting.append((self.unlock, ())) - return 1 # Return a flag indicating a lock condition. - + d = zrpc2.Delay() + self.__storage.__waiting.append((d, self, (id, user, description))) + return d + self._transaction = t = Transaction() t.id = id t.user = user @@ -369,62 +408,52 @@ self.__storage.tpc_begin(t) self.__invalidated = [] - def tpc_begin_sync(self, id, user, description, ext): -# if self.__closed: -# return - t = self._transaction - if t is not None and id == t.id: + def tpc_finish(self, id, user, description, ext): + if not self._check_tid(id): return - if self.__storage._transaction is None: - return self.try_again_sync(id, user, description, ext) - else: - d = Delay() - self.__storage.__waiting.append((self.try_again_sync, - (id, user, description, ext, d))) - return d - - def try_again_sync(self, id, user, description, ext, delay=None): - if self.__storage._transaction is None: - self._transaction = t = Transaction() - t.id = id - t.user = user - t.description = description - self.__storage.tpc_begin(t) - self.__invalidated = [] - if delay is not None: - delay.reply(None) - else: - return None - return 1 - def tpc_finish(self, id, user, description, ext): + # XXX Why do we do this for the begin and the end? t = self._transaction - if id != t.id: - return t.user = user t.description = description t.ext = ext - r = self.__storage.tpc_finish(t) - - while self.__storage.__waiting: - f, args = self.__storage.__waiting.pop(0) - if apply(f,args): - break + r = self.__storage.tpc_finish(self._transaction) + assert self.__storage._transaction is None - self._transaction = None if self.__invalidated: self.server.invalidate(self, self.__storage_id, self.__invalidated, self.get_size_info()) + + if not self.handle_waiting(): + self._transaction = None + self.__invalidated = [] + assert self._transaction is None + + def tpc_abort(self, id): + if not self._check_tid(id): + return + r = self.__storage.tpc_abort(self._transaction) + assert self.__storage._transaction is None + + if not self.handle_waiting(): + self._transaction = None self.__invalidated = [] + assert self._transaction is None + def handle_waiting(self): + if self.__storage.__waiting: + d, proxy, tinfo = self.__storage.__waiting.pop(0) + proxy._restart_delayed_transaction(d, tinfo) + if self is proxy: + return 1 + def new_oids(self, n=100): """Return a sequence of n new oids, where n defaults to 100""" if n < 0: n = 1 return map(lambda x, self=self: self.__storage.new_oid(), range(n)) - def fixup_storage(storage): # backwards compatibility hack --- Updated File StorageServer.py in package Packages/ZEO -- --- StorageServer.py 2001/04/20 19:14:08 1.21.4.6 +++ StorageServer.py 2001/04/25 20:43:52 1.21.4.7 @@ -59,8 +59,8 @@ # labeled as unofficial distributions. Modifications which do not # carry the name Zope may be packaged in any form, as long as they # conform to all of the clauses above. +# # -# # Disclaimer # # THIS SOFTWARE IS PROVIDED BY DIGITAL CREATIONS ``AS IS'' AND ANY @@ -86,12 +86,16 @@ This server acts as a front-end for one or more real storages, like file storage or Berkeley storage. + +XXX Need some basic access control-- a declaration of the methods +exported for invocation by the server. """ import cPickle import os import sys import threading +import types import ClientStub import zrpc2 @@ -110,8 +114,9 @@ pickler.fast = 1 # Don't use the memo dump = pickler.dump -def log(message, level=zeolog.INFO, label="zeoserver:%s" % os.getpid()): - zeolog.LOG(label, level, message) +def log(message, level=zeolog.INFO, label="ZEO Server:%s" % os.getpid(), + error=None): + zeolog.LOG(label, level, message, error=error) class StorageServerError(StorageError): pass @@ -165,6 +170,59 @@ self.__invalidated = [] self._transaction = None + def __repr__(self): + tid = self._transaction and repr(self._transaction.id) + if self.__storage: + stid = self.__storage._transaction and \ + repr(self.__storage._transaction.id) + else: + stid = None + return "" % (id(self), tid, + stid) + + def _log(self, msg, level=zeolog.INFO, error=None, pid=os.getpid()): + zeolog.LOG("ZEO Server %s %X" % (pid, id(self)), + level, msg, error=error) + + def setup_delegation(self): + """Delegate several methods to the storage""" + self.undoInfo = self.__storage.undoInfo + self.undoLog = self.__storage.undoLog + self.versionEmpty = self.__storage.versionEmpty + self.versions = self.__storage.versions + self.history = self.__storage.history + self.load = self.__storage.load + self.loadSerial = self.__storage.loadSerial + + def _check_tid(self, tid, exc=None): + caller = sys._getframe().f_back.f_code.co_name + if self._transaction is None: + self._log("no current transaction: %s()" % caller, + zeolog.PROBLEM) + if exc is not None: + raise exc(None, tid) + else: + return 0 + if self._transaction.id != tid: + self._log("%s(%s) invalid; current transaction = %s" % \ + (caller, repr(tid), repr(self._transaction.id)), + zeolog.PROBLEM) + if exc is not None: + raise exc(self._transaction.id, tid) + else: + return 0 + return 1 + + def _restart_delayed_transaction(self, delay, tinfo): + self._transaction = t = Transaction() + t.id = tinfo[0] + t.user = tinfo[1] + t.description = tinfo[2] + self.__storage.tpc_begin(t) + self.__invalidated = [] + assert self._transaction.id == self.__storage._transaction.id + delay.reply(None) + def register(self, storage_id): """Select the storage that this client will use @@ -172,25 +230,15 @@ """ storage = self.server.storages.get(storage_id) if storage is None: - log("unknown storage_id: %s" % storage_id) + self._log("unknown storage_id: %s" % storage_id) self.get_caller.close() self.__storage_id = storage_id self.__storage = storage self.setup_delegation() self.server.register(storage_id, self) - log("registered storage %s: %s" % (storage_id, storage)) + self._log("registered storage %s: %s" % (storage_id, storage)) - def setup_delegation(self): - """Delegate several methods to the storage""" - self.undoInfo = self.__storage.undoInfo - self.undoLog = self.__storage.undoLog - self.versionEmpty = self.__storage.versionEmpty - self.versions = self.__storage.versions - self.history = self.__storage.history - self.load = self.__storage.load - self.loadSerial = self.__storage.loadSerial - def get_info(self): return { 'length': len(self.__storage), @@ -247,9 +295,9 @@ try: self.__storage.pack(t, referencesf) except: - log('ZEO Server', zeolog.ERROR, - 'Pack failed for %s' % self.__storage_id, - error=sys.exc_info()) + self._log('ZEO Server', zeolog.ERROR, + 'Pack failed for %s' % self.__storage_id, + error=sys.exc_info()) if wait: raise else: @@ -259,19 +307,15 @@ self.get_size_info()) def abortVersion(self, src, id): - t = self._transaction - if t is None or id != t.id: - raise StorageTransactionError(self._transaction, id) - oids = self.__storage.abortVersion(src, t) + self._check_tid(id, exc=StorageTransactionError) + oids = self.__storage.abortVersion(src, self._transaction) for oid in oids: self.__invalidated.append((oid, src)) return oids def commitVersion(self, src, dest, id): - t = self._transaction - if t is None or id != t.id: - raise StorageTransactionError(self, id) - oids = self.__storage.commitVersion(src, dest, t) + self._check_tid(id, exc=StorageTransactionError) + oids = self.__storage.commitVersion(src, dest, self._transaction) for oid in oids: self.__invalidated.append((oid, dest)) if dest: @@ -279,26 +323,25 @@ return oids def storea(self, oid, serial, data, version, id): - t = self._transaction + self._check_tid(id, exc=StorageTransactionError) try: # XXX does this stmt need to be in the try/except? - if t is None or id != t.id: - raise StorageTransactionError(self, id) - newserial = self.__storage.store(oid, serial, data, version, t) + newserial = self.__storage.store(oid, serial, data, version, + self._transaction) except TransactionError, v: # This is a normal transaction error such as a conflict error # or a version lock or conflict error. It doesn't need to be # logged. - log("transaction error: %s" % repr(v)) + self._log("transaction error: %s" % repr(v)) newserial = v except: # all errors need to be serialized to prevent unexpected # returns, which would screw up the return handling. # IOW, Anything that ends up here is evil enough to be logged. error = sys.exc_info() - log('store error: %s: %s' % (error[0], error[1]). - zeolog.ERROR) + self._log('store error: %s: %s' % (error[0], error[1]), + zeolog.ERROR, error=error) newserial = sys.exc_info()[1] else: if serial != '\0\0\0\0\0\0\0\0': @@ -307,8 +350,8 @@ try: nil = dump(newserial, 1) except: - log("couldn't pickle newserial: %s" % repr(newserial), - zeolog.ERROR) + self._log("couldn't pickle newserial: %s" % repr(newserial), + zeolog.ERROR) dump('', 1) # clear pickler r = StorageServerError("Couldn't pickle exception %s" % \ `newserial`) @@ -316,11 +359,9 @@ self.client.serialno((oid, newserial)) - def vote(self, id): - t = self._transaction - if t is None or id != t.id: - raise StorageTransactionError(self._transaction, id) - return self.__storage.tpc_vote(t) + def vote(self, id): + self._check_tid(id, exc=StorageTransactionError) + return self.__storage.tpc_vote(self._transaction) def undo(self, transaction_id): oids = self.__storage.undo(transaction_id) @@ -330,38 +371,36 @@ return oids return () - def tpc_abort(self, id): - t = self._transaction - if t is None or id != t.id: - return - r = self.__storage.tpc_abort(t) - - waiting = self.__storage.__waiting - while waiting: - f, args = waiting.pop(0) - if apply(f, args): - break - - self._transaction = None - self.__invalidated = [] - def unlock(self): -# if self.__closed: -# return +## if self.__closed: +## return self.client.unlock() + # When multiple clients are using a single storage, there are several + # different _transaction attributes to keep track of. Each + # StorageProxy object has a single _transaction that refers to its + # current transaction. The storage (self.__storage) has another + # _transaction that is used for the *real* transaction. + + # The real trick comes with the __waiting queue for a storage. + # When a StorageProxy pulls a new transaction from the queue, it + # must inform the new transaction's proxy. (The two proxies may + # be the same.) The new transaction's proxy sets its _transaction + # and continues from there. + def tpc_begin(self, id, user, description, ext): - t = self._transaction - if t is not None: - if id == t.id: + if self._transaction is not None: + if self._transaction.id == id: + self._log("duplicate tpc_begin(%s)" % repr(id)) return else: - raise StorageServerError("Multiple simultaneous tpc_begin " - "requests from the same client.") + raise StorageTransactionError("Multiple simultaneous tpc_begin" + " requests from one client.") if self.__storage._transaction is not None: - self.__storage.__waiting.append((self.unlock, ())) - return 1 # Return a flag indicating a lock condition. - + d = zrpc2.Delay() + self.__storage.__waiting.append((d, self, (id, user, description))) + return d + self._transaction = t = Transaction() t.id = id t.user = user @@ -369,62 +408,52 @@ self.__storage.tpc_begin(t) self.__invalidated = [] - def tpc_begin_sync(self, id, user, description, ext): -# if self.__closed: -# return - t = self._transaction - if t is not None and id == t.id: + def tpc_finish(self, id, user, description, ext): + if not self._check_tid(id): return - if self.__storage._transaction is None: - return self.try_again_sync(id, user, description, ext) - else: - d = Delay() - self.__storage.__waiting.append((self.try_again_sync, - (id, user, description, ext, d))) - return d - - def try_again_sync(self, id, user, description, ext, delay=None): - if self.__storage._transaction is None: - self._transaction = t = Transaction() - t.id = id - t.user = user - t.description = description - self.__storage.tpc_begin(t) - self.__invalidated = [] - if delay is not None: - delay.reply(None) - else: - return None - return 1 - def tpc_finish(self, id, user, description, ext): + # XXX Why do we do this for the begin and the end? t = self._transaction - if id != t.id: - return t.user = user t.description = description t.ext = ext - r = self.__storage.tpc_finish(t) - - while self.__storage.__waiting: - f, args = self.__storage.__waiting.pop(0) - if apply(f,args): - break + r = self.__storage.tpc_finish(self._transaction) + assert self.__storage._transaction is None - self._transaction = None if self.__invalidated: self.server.invalidate(self, self.__storage_id, self.__invalidated, self.get_size_info()) + + if not self.handle_waiting(): + self._transaction = None + self.__invalidated = [] + assert self._transaction is None + + def tpc_abort(self, id): + if not self._check_tid(id): + return + r = self.__storage.tpc_abort(self._transaction) + assert self.__storage._transaction is None + + if not self.handle_waiting(): + self._transaction = None self.__invalidated = [] + assert self._transaction is None + def handle_waiting(self): + if self.__storage.__waiting: + d, proxy, tinfo = self.__storage.__waiting.pop(0) + proxy._restart_delayed_transaction(d, tinfo) + if self is proxy: + return 1 + def new_oids(self, n=100): """Return a sequence of n new oids, where n defaults to 100""" if n < 0: n = 1 return map(lambda x, self=self: self.__storage.new_oid(), range(n)) - def fixup_storage(storage): # backwards compatibility hack From jeremy at digicool.com Wed Apr 25 17:02:59 2001 From: jeremy at digicool.com (jeremy@digicool.com) Date: Sun Aug 10 16:31:15 2008 Subject: [ZEO-Checkins] CVS: Packages/ZEO - ClientStorage.py:1.26.4.12 Message-ID: <20010425210259.4E9A3510E3@korak.digicool.com> Update of /cvs-repository/Packages/ZEO In directory korak:/tmp/cvs-serv30247 Modified Files: Tag: ZEO-ZRPC-Dev ClientStorage.py Log Message: Remove dependence on BaseStorage, which only define some locks. XXX What is the difference between thread.allocate_lock() and ThreadLock.allocate_lock() and bpthread.allocate_lock()? Add log2() helper that defines pid of ClientStorage. Replace call to BaseStorage.__init__ with _basic_init() method. XXX This should probably be refactor along with __init__. Revise load logic for versions that that it is in positive form. Simplify tpc_begin(): It is always synchronous, so there's no need for a while loop. Workaround possibility that we can an invalidation message before the DB is registered. --- Updated File ClientStorage.py in package Packages/ZEO -- --- ClientStorage.py 2001/04/20 19:14:08 1.26.4.11 +++ ClientStorage.py 2001/04/25 21:02:58 1.26.4.12 @@ -85,6 +85,7 @@ """Network ZODB storage client XXX support multiple outstanding requests up until the vote +XXX is_connected() vis ClientDisconnected error """ __version__='$Revision$'[11:-2] @@ -100,22 +101,24 @@ from types import TupleType, StringType from struct import pack, unpack -import ExtensionClass, Sync +import ExtensionClass, Sync, ThreadLock import ClientCache import zrpc2 import ServerStub from TransactionBuffer import TransactionBuffer -from ZODB import POSException, BaseStorage +from ZODB import POSException from ZODB.TimeStamp import TimeStamp from zeolog import LOG, PROBLEM, INFO, BLATHER +def log2(type, msg, subsys="ClientStorage %d" % os.getpid()): + LOG(subsys, type, msg) + try: from ZODB.ConflictResolution import ResolvedSerial except ImportError: ResolvedSerial = 'rs' - class ClientStorageError(POSException.StorageError): """An error occured in the ZEO Client Storage""" @@ -131,8 +134,7 @@ t = t.laterThan(prev_ts) return t -class ClientStorage(BaseStorage.BaseStorage): - __super_init = BaseStorage.BaseStorage.__init__ +class ClientStorage: _server = None @@ -160,7 +162,7 @@ self._serials = [] self._seriald = {} - self.__super_init(name) + self._basic_init(name) self._db = None self._cache = ClientCache.ClientCache(storage, cache_size, @@ -176,8 +178,32 @@ if not self._rpc_mgr.attempt_connect(): self._rpc_mgr.connect() + def _basic_init(self, name): + """Handle initialization activites of BaseStorage""" + + self.__name__ = name + + # Allocate locks: +## import debuglock +## commit_lock = debuglock.DebugLock() + commit_lock = thread.allocate_lock() + self._commit_lock_acquire = commit_lock.acquire + self._commit_lock_release = commit_lock.release + + # What's the difference between thread and ThreadLock? + l = ThreadLock.allocate_lock() + self._lock_acquire = l.acquire + self._lock_release = l.release + + t = time.time() + t = self._ts = apply(TimeStamp,(time.gmtime(t)[:5]+(t%60,))) + self._serial = `t` + self._oid='\0\0\0\0\0\0\0\0' + self._transaction = None + def registerDB(self, db, limit): """Register that the storage is controlled by the given DB.""" + log2(INFO, "registerDB(%s, %s)" % (repr(db), repr(limit))) self._db = db def is_connected(self): @@ -191,7 +217,7 @@ self._lock_release() def notifyConnected(self, c): - LOG("ClientStorage", INFO, "Connected to storage") + log2(INFO, "Connected to storage") self._lock_acquire() try: self._server = ServerStub.StorageServer(c) @@ -222,7 +248,7 @@ ### responsible for starting the thread that makes the connection. def notifyDisconnected(self, ignored): - LOG("ClientStorage", PROBLEM, "Disconnected from storage") + log2(PROBLEM, "Disconnected from storage") self._transaction = None try: self._commit_lock_release() @@ -303,11 +329,12 @@ p, s, v, pv, sv = self._server.zeoLoad(oid) self._cache.checkSize(0) self._cache.store(oid, p, s, v, pv, sv) - if not v or not version or version != v: + if v and version and v == version: + return pv, sv + else: if s: return p, s raise KeyError, oid # no non-version data for this - return pv, sv finally: self._lock_release() @@ -400,27 +427,24 @@ try: if self._transaction is transaction: return # can start the same transaction many times + self._lock_release() + self._commit_lock_acquire() + self._lock_acquire() - while 1: - self._lock_release() - self._commit_lock_acquire() - self._lock_acquire() - - self._ts = get_timestamp(self._ts) - id = `self._ts` - - try: - r = self._server.tpc_begin(id, - transaction.user, - transaction.description, - transaction._extension) - except: - self._commit_lock_release() - raise - - if r is None: - break + self._ts = get_timestamp(self._ts) + id = `self._ts` + + try: + r = self._server.tpc_begin(id, + transaction.user, + transaction.description, + transaction._extension) + except: + self._commit_lock_release() + raise + assert r is None + # We have *BOTH* the local and distributed commit # lock, now we can actually get ready to get started. self._serial = id @@ -462,7 +486,7 @@ oid, v, p = t s = self._seriald[oid] if type(s) != StringType: - LOG("ClientStorage", INFO, "bad serialno: %s for %s" % \ + log2(INFO, "bad serialno: %s for %s" % \ (repr(s), repr(oid))) assert type(s) == StringType, "bad serialno: %s" % repr(s) if s == ResolvedSerial: @@ -520,7 +544,9 @@ # below are methods invoked by the StorageServer def unlock(self): - self.commit_lock_release() + # XXX Don't believe this is used anymore... + log2(INFO, "unlock()") + self._commit_lock_release() def serialno(self, arg): self._serials.append(arg) @@ -555,6 +581,14 @@ self._db.invalidate(oid, version=version) def Invalidate(self, args): + # XXX _db could be None for oid, version in args: self._cache.invalidate(oid, version=version) - self._db.invalidate(oid, version=version) + try: + self._db.invalidate(oid, version=version) + except AttributeError, msg: + log2(PROBLEM, + "Invalidate(%s, %s) failed for _db: %s" % (repr(oid), + repr(version), + msg)) + --- Updated File ClientStorage.py in package Packages/ZEO -- --- ClientStorage.py 2001/04/20 19:14:08 1.26.4.11 +++ ClientStorage.py 2001/04/25 21:02:58 1.26.4.12 @@ -85,6 +85,7 @@ """Network ZODB storage client XXX support multiple outstanding requests up until the vote +XXX is_connected() vis ClientDisconnected error """ __version__='$Revision$'[11:-2] @@ -100,22 +101,24 @@ from types import TupleType, StringType from struct import pack, unpack -import ExtensionClass, Sync +import ExtensionClass, Sync, ThreadLock import ClientCache import zrpc2 import ServerStub from TransactionBuffer import TransactionBuffer -from ZODB import POSException, BaseStorage +from ZODB import POSException from ZODB.TimeStamp import TimeStamp from zeolog import LOG, PROBLEM, INFO, BLATHER +def log2(type, msg, subsys="ClientStorage %d" % os.getpid()): + LOG(subsys, type, msg) + try: from ZODB.ConflictResolution import ResolvedSerial except ImportError: ResolvedSerial = 'rs' - class ClientStorageError(POSException.StorageError): """An error occured in the ZEO Client Storage""" @@ -131,8 +134,7 @@ t = t.laterThan(prev_ts) return t -class ClientStorage(BaseStorage.BaseStorage): - __super_init = BaseStorage.BaseStorage.__init__ +class ClientStorage: _server = None @@ -160,7 +162,7 @@ self._serials = [] self._seriald = {} - self.__super_init(name) + self._basic_init(name) self._db = None self._cache = ClientCache.ClientCache(storage, cache_size, @@ -176,8 +178,32 @@ if not self._rpc_mgr.attempt_connect(): self._rpc_mgr.connect() + def _basic_init(self, name): + """Handle initialization activites of BaseStorage""" + + self.__name__ = name + + # Allocate locks: +## import debuglock +## commit_lock = debuglock.DebugLock() + commit_lock = thread.allocate_lock() + self._commit_lock_acquire = commit_lock.acquire + self._commit_lock_release = commit_lock.release + + # What's the difference between thread and ThreadLock? + l = ThreadLock.allocate_lock() + self._lock_acquire = l.acquire + self._lock_release = l.release + + t = time.time() + t = self._ts = apply(TimeStamp,(time.gmtime(t)[:5]+(t%60,))) + self._serial = `t` + self._oid='\0\0\0\0\0\0\0\0' + self._transaction = None + def registerDB(self, db, limit): """Register that the storage is controlled by the given DB.""" + log2(INFO, "registerDB(%s, %s)" % (repr(db), repr(limit))) self._db = db def is_connected(self): @@ -191,7 +217,7 @@ self._lock_release() def notifyConnected(self, c): - LOG("ClientStorage", INFO, "Connected to storage") + log2(INFO, "Connected to storage") self._lock_acquire() try: self._server = ServerStub.StorageServer(c) @@ -222,7 +248,7 @@ ### responsible for starting the thread that makes the connection. def notifyDisconnected(self, ignored): - LOG("ClientStorage", PROBLEM, "Disconnected from storage") + log2(PROBLEM, "Disconnected from storage") self._transaction = None try: self._commit_lock_release() @@ -303,11 +329,12 @@ p, s, v, pv, sv = self._server.zeoLoad(oid) self._cache.checkSize(0) self._cache.store(oid, p, s, v, pv, sv) - if not v or not version or version != v: + if v and version and v == version: + return pv, sv + else: if s: return p, s raise KeyError, oid # no non-version data for this - return pv, sv finally: self._lock_release() @@ -400,27 +427,24 @@ try: if self._transaction is transaction: return # can start the same transaction many times + self._lock_release() + self._commit_lock_acquire() + self._lock_acquire() - while 1: - self._lock_release() - self._commit_lock_acquire() - self._lock_acquire() - - self._ts = get_timestamp(self._ts) - id = `self._ts` - - try: - r = self._server.tpc_begin(id, - transaction.user, - transaction.description, - transaction._extension) - except: - self._commit_lock_release() - raise - - if r is None: - break + self._ts = get_timestamp(self._ts) + id = `self._ts` + + try: + r = self._server.tpc_begin(id, + transaction.user, + transaction.description, + transaction._extension) + except: + self._commit_lock_release() + raise + assert r is None + # We have *BOTH* the local and distributed commit # lock, now we can actually get ready to get started. self._serial = id @@ -462,7 +486,7 @@ oid, v, p = t s = self._seriald[oid] if type(s) != StringType: - LOG("ClientStorage", INFO, "bad serialno: %s for %s" % \ + log2(INFO, "bad serialno: %s for %s" % \ (repr(s), repr(oid))) assert type(s) == StringType, "bad serialno: %s" % repr(s) if s == ResolvedSerial: @@ -520,7 +544,9 @@ # below are methods invoked by the StorageServer def unlock(self): - self.commit_lock_release() + # XXX Don't believe this is used anymore... + log2(INFO, "unlock()") + self._commit_lock_release() def serialno(self, arg): self._serials.append(arg) @@ -555,6 +581,14 @@ self._db.invalidate(oid, version=version) def Invalidate(self, args): + # XXX _db could be None for oid, version in args: self._cache.invalidate(oid, version=version) - self._db.invalidate(oid, version=version) + try: + self._db.invalidate(oid, version=version) + except AttributeError, msg: + log2(PROBLEM, + "Invalidate(%s, %s) failed for _db: %s" % (repr(oid), + repr(version), + msg)) + From jeremy at digicool.com Wed Apr 25 17:05:43 2001 From: jeremy at digicool.com (jeremy@digicool.com) Date: Sun Aug 10 16:31:15 2008 Subject: [ZEO-Checkins] CVS: Packages/ZEO - multi.py:1.1.2.1 testZEO.py:1.1.2.6 Message-ID: <20010425210543.CFFAB510E3@korak.digicool.com> Update of /cvs-repository/Packages/ZEO/tests In directory korak:/tmp/cvs-serv30414 Modified Files: Tag: ZEO-ZRPC-Dev testZEO.py Added Files: Tag: ZEO-ZRPC-Dev multi.py Log Message: Add optional argument to testZEO that allows you to limit the tests that are run. A bit of a hack. Add multi.py a test of multiple clients using the same server (and the same storage on that server) --- Added File multi.py in package Packages/ZEO --- """A multi-client test of the ZEO storage server""" import ZODB, ZODB.DB, ZODB.FileStorage, ZODB.POSException import Persistence import PersistentMapping import asyncore import os import tempfile import time import types VERBOSE = 1 CLIENTS = 4 RECORDS_PER_CLIENT = 100 CONFLICT_DELAY = 0.1 CONNECT_DELAY = 0.1 CLIENT_CACHE = '' # use temporary cache class Record(Persistence.Persistent): def __init__(self, client=None, value=None): self.client = client self.value = None self.next = None def set_next(self, next): self.next = next class Stats(Persistence.Persistent): def __init__(self): self.begin = time.time() self.end = None def done(self): self.end = time.time() def init_storage(): path = tempfile.mktemp() if VERBOSE: print "FileStorage path:", path fs = ZODB.FileStorage.FileStorage(path) db = ZODB.DB(fs) root = db.open().root() root["multi"] = PersistentMapping.PersistentMapping() get_transaction().commit() return fs def start_server(addr): pid = os.fork() if pid == 0: import ZEO.StorageServer storage = init_storage() server = ZEO.StorageServer.StorageServer(addr, {'1': storage}) print "Server ready:", os.getpid() asyncore.loop() os._exit(0) else: return pid def start_client(addr): pid = os.fork() if pid == 0: import ZEO.ClientStorage if VERBOSE: print "Client process started:", os.getpid() cli = ZEO.ClientStorage.ClientStorage(addr, client=CLIENT_CACHE) run(cli) cli.close() os._exit(0) else: return pid def run(storage): if hasattr(storage, 'is_connected'): while not storage.is_connected(): time.sleep(CONNECT_DELAY) pid = os.getpid() print "Client process connected:", pid, storage db = ZODB.DB(storage) root = db.open().root() while 1: try: s = root[pid] = Stats() get_transaction().commit() except ZODB.POSException.ConflictError: get_transaction().abort() time.sleep(CONFLICT_DELAY) else: break dict = root["multi"] prev = None i = 0 while i < RECORDS_PER_CLIENT: time.sleep(CONFLICT_DELAY) try: size = len(dict) r = dict[size] = Record(pid, size) if prev: prev.set_next(r) get_transaction().commit() except ZODB.POSException.ConflictError, err: ## print "ConflictError: %s: %s" % (pid, err) get_transaction().abort() time.sleep(CONFLICT_DELAY) else: i = i + 1 if VERBOSE and (i < 5 or i % 10 == 0): print "Client %s: %s of %s" % (pid, i, RECORDS_PER_CLIENT) s.done() get_transaction().commit() print "Client completed:", pid def shutdown_server(addr): import ZEO.ClientStorage cli = ZEO.ClientStorage.ClientStorage(addr) cli._server.rpc.callAsync('shutdown') def main(): if VERBOSE: print "Main process:", os.getpid() addr = tempfile.mktemp() t0 = time.time() server_pid = start_server(addr) t1 = time.time() pids = [start_client(addr) for i in range(CLIENTS)] for pid in pids: assert type(pid) == types.IntType, "invalid pid type: %s (%s)" % \ (repr(pid), type(pid)) try: if VERBOSE: print "waitpid(%s)" % repr(pid) os.waitpid(pid, 0) except os.error, err: print "waitpid(%s) failed: %s" % (repr(pid), err) t2 = time.time() shutdown_server(addr) os.waitpid(server_pid, 0) print "Total time:", t2 - t0 print "Server start time", t1 - t0 print "Client time:", t2 - t1 if __name__ == "__main__": main() --- Updated File testZEO.py in package Packages/ZEO -- --- testZEO.py 2001/04/20 19:18:52 1.1.2.5 +++ testZEO.py 2001/04/25 21:05:42 1.1.2.6 @@ -2,7 +2,6 @@ import asyncore import os -import sys import tempfile import time import types @@ -163,7 +162,20 @@ os.unlink(path) def main(): - tests = unittest.makeSuite(ZEOFileStorageTests, 'check') + import sys, getopt + + name_of_test = '' + + opts, args = getopt.getopt(sys.argv[1:], 'n:') + for flag, val in opts: + if flag == '-n': + name_of_test = val + + if args: + print >> sys.stderr, "Did not expect arguments. Got %s" % args + return 0 + + tests = unittest.makeSuite(ZEOFileStorageTests, 'check' + name_of_test) runner = unittest.TextTestRunner() runner.run(tests) --- Added File multi.py in package Packages/ZEO --- """A multi-client test of the ZEO storage server""" import ZODB, ZODB.DB, ZODB.FileStorage, ZODB.POSException import Persistence import PersistentMapping import asyncore import os import tempfile import time import types VERBOSE = 1 CLIENTS = 4 RECORDS_PER_CLIENT = 100 CONFLICT_DELAY = 0.1 CONNECT_DELAY = 0.1 CLIENT_CACHE = '' # use temporary cache class Record(Persistence.Persistent): def __init__(self, client=None, value=None): self.client = client self.value = None self.next = None def set_next(self, next): self.next = next class Stats(Persistence.Persistent): def __init__(self): self.begin = time.time() self.end = None def done(self): self.end = time.time() def init_storage(): path = tempfile.mktemp() if VERBOSE: print "FileStorage path:", path fs = ZODB.FileStorage.FileStorage(path) db = ZODB.DB(fs) root = db.open().root() root["multi"] = PersistentMapping.PersistentMapping() get_transaction().commit() return fs def start_server(addr): pid = os.fork() if pid == 0: import ZEO.StorageServer storage = init_storage() server = ZEO.StorageServer.StorageServer(addr, {'1': storage}) print "Server ready:", os.getpid() asyncore.loop() os._exit(0) else: return pid def start_client(addr): pid = os.fork() if pid == 0: import ZEO.ClientStorage if VERBOSE: print "Client process started:", os.getpid() cli = ZEO.ClientStorage.ClientStorage(addr, client=CLIENT_CACHE) run(cli) cli.close() os._exit(0) else: return pid def run(storage): if hasattr(storage, 'is_connected'): while not storage.is_connected(): time.sleep(CONNECT_DELAY) pid = os.getpid() print "Client process connected:", pid, storage db = ZODB.DB(storage) root = db.open().root() while 1: try: s = root[pid] = Stats() get_transaction().commit() except ZODB.POSException.ConflictError: get_transaction().abort() time.sleep(CONFLICT_DELAY) else: break dict = root["multi"] prev = None i = 0 while i < RECORDS_PER_CLIENT: time.sleep(CONFLICT_DELAY) try: size = len(dict) r = dict[size] = Record(pid, size) if prev: prev.set_next(r) get_transaction().commit() except ZODB.POSException.ConflictError, err: ## print "ConflictError: %s: %s" % (pid, err) get_transaction().abort() time.sleep(CONFLICT_DELAY) else: i = i + 1 if VERBOSE and (i < 5 or i % 10 == 0): print "Client %s: %s of %s" % (pid, i, RECORDS_PER_CLIENT) s.done() get_transaction().commit() print "Client completed:", pid def shutdown_server(addr): import ZEO.ClientStorage cli = ZEO.ClientStorage.ClientStorage(addr) cli._server.rpc.callAsync('shutdown') def main(): if VERBOSE: print "Main process:", os.getpid() addr = tempfile.mktemp() t0 = time.time() server_pid = start_server(addr) t1 = time.time() pids = [start_client(addr) for i in range(CLIENTS)] for pid in pids: assert type(pid) == types.IntType, "invalid pid type: %s (%s)" % \ (repr(pid), type(pid)) try: if VERBOSE: print "waitpid(%s)" % repr(pid) os.waitpid(pid, 0) except os.error, err: print "waitpid(%s) failed: %s" % (repr(pid), err) t2 = time.time() shutdown_server(addr) os.waitpid(server_pid, 0) print "Total time:", t2 - t0 print "Server start time", t1 - t0 print "Client time:", t2 - t1 if __name__ == "__main__": main() --- Updated File testZEO.py in package Packages/ZEO -- --- testZEO.py 2001/04/20 19:18:52 1.1.2.5 +++ testZEO.py 2001/04/25 21:05:42 1.1.2.6 @@ -2,7 +2,6 @@ import asyncore import os -import sys import tempfile import time import types @@ -163,7 +162,20 @@ os.unlink(path) def main(): - tests = unittest.makeSuite(ZEOFileStorageTests, 'check') + import sys, getopt + + name_of_test = '' + + opts, args = getopt.getopt(sys.argv[1:], 'n:') + for flag, val in opts: + if flag == '-n': + name_of_test = val + + if args: + print >> sys.stderr, "Did not expect arguments. Got %s" % args + return 0 + + tests = unittest.makeSuite(ZEOFileStorageTests, 'check' + name_of_test) runner = unittest.TextTestRunner() runner.run(tests) From jeremy at digicool.com Wed Apr 25 18:15:23 2001 From: jeremy at digicool.com (jeremy@digicool.com) Date: Sun Aug 10 16:31:15 2008 Subject: [ZEO-Checkins] CVS: Packages/ZEO - zrpc2.py:1.1.2.11 Message-ID: <20010425221523.C7839510E3@korak.digicool.com> Update of /cvs-repository/Packages/ZEO In directory korak:/tmp/cvs-serv3129 Modified Files: Tag: ZEO-ZRPC-Dev zrpc2.py Log Message: Make the low-level zrpc log calls use DEBUG level, so that they are distinguished from the lower-level smac calls that use TRACE. --- Updated File zrpc2.py in package Packages/ZEO -- --- zrpc2.py 2001/04/25 20:36:20 1.1.2.10 +++ zrpc2.py 2001/04/25 22:15:22 1.1.2.11 @@ -176,7 +176,7 @@ if __debug__: log("recv msg: %s, %s, %s, %s" % (msgid, flags, name, repr(args)[:40]), - level=zeolog.TRACE) + level=zeolog.DEBUG) if name == REPLY: self.handle_reply(msgid, flags, args) else: @@ -185,14 +185,14 @@ def handle_reply(self, msgid, flags, args): if __debug__: log("recv reply: %s, %s, %s" % (msgid, flags, str(args)[:40]), - level=zeolog.TRACE) + level=zeolog.DEBUG) self.__reply = msgid, flags, args self.__reply_lock.release() # will fail if lock is unlocked def handle_request(self, msgid, flags, name, args): if __debug__: log("call %s%s on %s" % (name, repr(args)[:40], repr(self.obj)), - zeolog.TRACE) + zeolog.DEBUG) if not self.check_method(name): raise ZRPCError("Invalid method name: %s" % name) @@ -223,7 +223,7 @@ raise ZRPCError("async method returned value") else: if __debug__: - log("%s return %s" % (name, repr(ret)[:40]), zeolog.TRACE) + log("%s return %s" % (name, repr(ret)[:40]), zeolog.DEBUG) if isinstance(ret, Delay): ret.set_sender(msgid, self.send_reply) else: @@ -345,7 +345,7 @@ if __debug__: log("_do_io(wait=%d), async=%d" % (wait, self.is_async()), - level=zeolog.TRACE) + level=zeolog.DEBUG) if self.is_async(): self.trigger.pull_trigger() if wait: --- Updated File zrpc2.py in package Packages/ZEO -- --- zrpc2.py 2001/04/25 20:36:20 1.1.2.10 +++ zrpc2.py 2001/04/25 22:15:22 1.1.2.11 @@ -176,7 +176,7 @@ if __debug__: log("recv msg: %s, %s, %s, %s" % (msgid, flags, name, repr(args)[:40]), - level=zeolog.TRACE) + level=zeolog.DEBUG) if name == REPLY: self.handle_reply(msgid, flags, args) else: @@ -185,14 +185,14 @@ def handle_reply(self, msgid, flags, args): if __debug__: log("recv reply: %s, %s, %s" % (msgid, flags, str(args)[:40]), - level=zeolog.TRACE) + level=zeolog.DEBUG) self.__reply = msgid, flags, args self.__reply_lock.release() # will fail if lock is unlocked def handle_request(self, msgid, flags, name, args): if __debug__: log("call %s%s on %s" % (name, repr(args)[:40], repr(self.obj)), - zeolog.TRACE) + zeolog.DEBUG) if not self.check_method(name): raise ZRPCError("Invalid method name: %s" % name) @@ -223,7 +223,7 @@ raise ZRPCError("async method returned value") else: if __debug__: - log("%s return %s" % (name, repr(ret)[:40]), zeolog.TRACE) + log("%s return %s" % (name, repr(ret)[:40]), zeolog.DEBUG) if isinstance(ret, Delay): ret.set_sender(msgid, self.send_reply) else: @@ -345,7 +345,7 @@ if __debug__: log("_do_io(wait=%d), async=%d" % (wait, self.is_async()), - level=zeolog.TRACE) + level=zeolog.DEBUG) if self.is_async(): self.trigger.pull_trigger() if wait: From jeremy at digicool.com Wed Apr 25 18:33:51 2001 From: jeremy at digicool.com (jeremy@digicool.com) Date: Sun Aug 10 16:31:15 2008 Subject: [ZEO-Checkins] CVS: Packages/ZEO - StorageServer.py:1.21.4.8 Message-ID: <20010425223351.E6398510E3@korak.digicool.com> Update of /cvs-repository/Packages/ZEO In directory korak:/tmp/cvs-serv4553 Modified Files: Tag: ZEO-ZRPC-Dev StorageServer.py Log Message: Reorganize handling of transaction metadata Create a Transaction object immediately when tpc_begin() is called and populate it with the tpc_begin() arguments. Note that the extension should be stored as _extension, not ext! Place the Transaction object in the store's __waiting queue and use it when the transaction is finally started. Don't do anything with the arguments to tpc_finish(). They should match tpc_begin(). Move _restart_delayed_transaction() to a better location. --- Updated File StorageServer.py in package Packages/ZEO -- --- StorageServer.py 2001/04/25 20:43:52 1.21.4.7 +++ StorageServer.py 2001/04/25 22:33:50 1.21.4.8 @@ -213,16 +213,6 @@ return 0 return 1 - def _restart_delayed_transaction(self, delay, tinfo): - self._transaction = t = Transaction() - t.id = tinfo[0] - t.user = tinfo[1] - t.description = tinfo[2] - self.__storage.tpc_begin(t) - self.__invalidated = [] - assert self._transaction.id == self.__storage._transaction.id - delay.reply(None) - def register(self, storage_id): """Select the storage that this client will use @@ -396,15 +386,19 @@ else: raise StorageTransactionError("Multiple simultaneous tpc_begin" " requests from one client.") - if self.__storage._transaction is not None: - d = zrpc2.Delay() - self.__storage.__waiting.append((d, self, (id, user, description))) - return d - self._transaction = t = Transaction() + t = Transaction() t.id = id t.user = user t.description = description + t._extension = ext + + if self.__storage._transaction is not None: + d = zrpc2.Delay() + self.__storage.__waiting.append((d, self, t)) + return d + + self._transaction = t self.__storage.tpc_begin(t) self.__invalidated = [] @@ -412,12 +406,6 @@ if not self._check_tid(id): return - # XXX Why do we do this for the begin and the end? - t = self._transaction - t.user = user - t.description = description - t.ext = ext - r = self.__storage.tpc_finish(self._transaction) assert self.__storage._transaction is None @@ -426,7 +414,7 @@ self.__invalidated, self.get_size_info()) - if not self.handle_waiting(): + if not self._handle_waiting(): self._transaction = None self.__invalidated = [] assert self._transaction is None @@ -437,15 +425,22 @@ r = self.__storage.tpc_abort(self._transaction) assert self.__storage._transaction is None - if not self.handle_waiting(): + if not self._handle_waiting(): self._transaction = None self.__invalidated = [] assert self._transaction is None + + def _restart_delayed_transaction(self, delay, trans): + self._transaction = trans + self.__storage.tpc_begin(trans) + self.__invalidated = [] + assert self._transaction.id == self.__storage._transaction.id + delay.reply(None) - def handle_waiting(self): + def _handle_waiting(self): if self.__storage.__waiting: - d, proxy, tinfo = self.__storage.__waiting.pop(0) - proxy._restart_delayed_transaction(d, tinfo) + delay, proxy, trans = self.__storage.__waiting.pop(0) + proxy._restart_delayed_transaction(delay, trans) if self is proxy: return 1 --- Updated File StorageServer.py in package Packages/ZEO -- --- StorageServer.py 2001/04/25 20:43:52 1.21.4.7 +++ StorageServer.py 2001/04/25 22:33:50 1.21.4.8 @@ -213,16 +213,6 @@ return 0 return 1 - def _restart_delayed_transaction(self, delay, tinfo): - self._transaction = t = Transaction() - t.id = tinfo[0] - t.user = tinfo[1] - t.description = tinfo[2] - self.__storage.tpc_begin(t) - self.__invalidated = [] - assert self._transaction.id == self.__storage._transaction.id - delay.reply(None) - def register(self, storage_id): """Select the storage that this client will use @@ -396,15 +386,19 @@ else: raise StorageTransactionError("Multiple simultaneous tpc_begin" " requests from one client.") - if self.__storage._transaction is not None: - d = zrpc2.Delay() - self.__storage.__waiting.append((d, self, (id, user, description))) - return d - self._transaction = t = Transaction() + t = Transaction() t.id = id t.user = user t.description = description + t._extension = ext + + if self.__storage._transaction is not None: + d = zrpc2.Delay() + self.__storage.__waiting.append((d, self, t)) + return d + + self._transaction = t self.__storage.tpc_begin(t) self.__invalidated = [] @@ -412,12 +406,6 @@ if not self._check_tid(id): return - # XXX Why do we do this for the begin and the end? - t = self._transaction - t.user = user - t.description = description - t.ext = ext - r = self.__storage.tpc_finish(self._transaction) assert self.__storage._transaction is None @@ -426,7 +414,7 @@ self.__invalidated, self.get_size_info()) - if not self.handle_waiting(): + if not self._handle_waiting(): self._transaction = None self.__invalidated = [] assert self._transaction is None @@ -437,15 +425,22 @@ r = self.__storage.tpc_abort(self._transaction) assert self.__storage._transaction is None - if not self.handle_waiting(): + if not self._handle_waiting(): self._transaction = None self.__invalidated = [] assert self._transaction is None + + def _restart_delayed_transaction(self, delay, trans): + self._transaction = trans + self.__storage.tpc_begin(trans) + self.__invalidated = [] + assert self._transaction.id == self.__storage._transaction.id + delay.reply(None) - def handle_waiting(self): + def _handle_waiting(self): if self.__storage.__waiting: - d, proxy, tinfo = self.__storage.__waiting.pop(0) - proxy._restart_delayed_transaction(d, tinfo) + delay, proxy, trans = self.__storage.__waiting.pop(0) + proxy._restart_delayed_transaction(delay, trans) if self is proxy: return 1 From jeremy at digicool.com Wed Apr 25 18:50:39 2001 From: jeremy at digicool.com (jeremy@digicool.com) Date: Sun Aug 10 16:31:15 2008 Subject: [ZEO-Checkins] CVS: Packages/ZEO - multi.py:1.1.2.2 Message-ID: <20010425225039.B9C7B510E3@korak.digicool.com> Update of /cvs-repository/Packages/ZEO/tests In directory korak:/tmp/cvs-serv5828 Modified Files: Tag: ZEO-ZRPC-Dev multi.py Log Message: Remove the extra delay inside the main update loop. Add a couple of XXX comments. --- Updated File multi.py in package Packages/ZEO -- --- multi.py 2001/04/25 21:05:42 1.1.2.1 +++ multi.py 2001/04/25 22:50:38 1.1.2.2 @@ -94,7 +94,6 @@ prev = None i = 0 while i < RECORDS_PER_CLIENT: - time.sleep(CONFLICT_DELAY) try: size = len(dict) r = dict[size] = Record(pid, size) @@ -102,7 +101,6 @@ prev.set_next(r) get_transaction().commit() except ZODB.POSException.ConflictError, err: -## print "ConflictError: %s: %s" % (pid, err) get_transaction().abort() time.sleep(CONFLICT_DELAY) else: @@ -116,6 +114,7 @@ def shutdown_server(addr): import ZEO.ClientStorage + # XXX this doesn't work! cli = ZEO.ClientStorage.ClientStorage(addr) cli._server.rpc.callAsync('shutdown') @@ -139,6 +138,8 @@ t2 = time.time() shutdown_server(addr) os.waitpid(server_pid, 0) + + # XXX Should check that the results are consistent! print "Total time:", t2 - t0 print "Server start time", t1 - t0 --- Updated File multi.py in package Packages/ZEO -- --- multi.py 2001/04/25 21:05:42 1.1.2.1 +++ multi.py 2001/04/25 22:50:38 1.1.2.2 @@ -94,7 +94,6 @@ prev = None i = 0 while i < RECORDS_PER_CLIENT: - time.sleep(CONFLICT_DELAY) try: size = len(dict) r = dict[size] = Record(pid, size) @@ -102,7 +101,6 @@ prev.set_next(r) get_transaction().commit() except ZODB.POSException.ConflictError, err: -## print "ConflictError: %s: %s" % (pid, err) get_transaction().abort() time.sleep(CONFLICT_DELAY) else: @@ -116,6 +114,7 @@ def shutdown_server(addr): import ZEO.ClientStorage + # XXX this doesn't work! cli = ZEO.ClientStorage.ClientStorage(addr) cli._server.rpc.callAsync('shutdown') @@ -139,6 +138,8 @@ t2 = time.time() shutdown_server(addr) os.waitpid(server_pid, 0) + + # XXX Should check that the results are consistent! print "Total time:", t2 - t0 print "Server start time", t1 - t0 From jeremy at digicool.com Wed Apr 25 18:51:34 2001 From: jeremy at digicool.com (jeremy@digicool.com) Date: Sun Aug 10 16:31:15 2008 Subject: [ZEO-Checkins] CVS: Packages/ZEO - ClientStorage.py:1.26.4.13 ServerStub.py:1.1.2.5 StorageServer.py:1.21.4.9 Message-ID: <20010425225134.BEE4E510E3@korak.digicool.com> Update of /cvs-repository/Packages/ZEO In directory korak:/tmp/cvs-serv5894 Modified Files: Tag: ZEO-ZRPC-Dev ClientStorage.py ServerStub.py StorageServer.py Log Message: Change tpc_finish() to take just the transaction id. Remove tpc_begin_sync() from the ServerStub. --- Updated File ClientStorage.py in package Packages/ZEO -- --- ClientStorage.py 2001/04/25 21:02:58 1.26.4.12 +++ ClientStorage.py 2001/04/25 22:51:32 1.26.4.13 @@ -458,21 +458,20 @@ def tpc_finish(self, transaction, f=None): self._lock_acquire() try: - if transaction is not self._transaction: return - if f is not None: f() + if transaction is not self._transaction: + return + if f is not None: # XXX what is f()? + f() - # XXX what happens if the network dies RIGHT NOW? + self._server.tpc_finish(self._serial) - self._server.tpc_finish(self._serial, - transaction.user, - transaction.description, - transaction._extension) - r = self._check_serials() assert r is None or len(r) == 0, "unhandled serialnos: %s" % r self._cache.checkSize(self._tbuf.get_size()) + # Iterate over the objects in the transaction buffer and + # update or invalidate the cache. self._tbuf.begin_iterate() while 1: try: --- Updated File ServerStub.py in package Packages/ZEO -- --- ServerStub.py 2001/04/25 20:38:38 1.1.2.4 +++ ServerStub.py 2001/04/25 22:51:32 1.1.2.5 @@ -44,16 +44,11 @@ def tpc_begin(self, id, user, descr, ext): return self.rpc.call('tpc_begin', id, user, descr, ext) - def tpc_begin_sync(self, id, user, descr, ext): - # XXX nuked - return self.rpc.call('tpc_begin_sync', id, user, descr, ext) - def vote(self, trans_id): return self.rpc.call('vote', trans_id) - def tpc_finish(self, id, user, descr, ext): -## self.rpc.callAsync('tpc_finish', id, user, descr, ext) - return self.rpc.call('tpc_finish', id, user, descr, ext) + def tpc_finish(self, id): + return self.rpc.call('tpc_finish', id) def tpc_abort(self, id): self.rpc.callAsync('tpc_abort', id) --- Updated File StorageServer.py in package Packages/ZEO -- --- StorageServer.py 2001/04/25 22:33:50 1.21.4.8 +++ StorageServer.py 2001/04/25 22:51:32 1.21.4.9 @@ -402,7 +402,7 @@ self.__storage.tpc_begin(t) self.__invalidated = [] - def tpc_finish(self, id, user, description, ext): + def tpc_finish(self, id): if not self._check_tid(id): return --- Updated File ClientStorage.py in package Packages/ZEO -- --- ClientStorage.py 2001/04/25 21:02:58 1.26.4.12 +++ ClientStorage.py 2001/04/25 22:51:32 1.26.4.13 @@ -458,21 +458,20 @@ def tpc_finish(self, transaction, f=None): self._lock_acquire() try: - if transaction is not self._transaction: return - if f is not None: f() + if transaction is not self._transaction: + return + if f is not None: # XXX what is f()? + f() - # XXX what happens if the network dies RIGHT NOW? + self._server.tpc_finish(self._serial) - self._server.tpc_finish(self._serial, - transaction.user, - transaction.description, - transaction._extension) - r = self._check_serials() assert r is None or len(r) == 0, "unhandled serialnos: %s" % r self._cache.checkSize(self._tbuf.get_size()) + # Iterate over the objects in the transaction buffer and + # update or invalidate the cache. self._tbuf.begin_iterate() while 1: try: --- Updated File ServerStub.py in package Packages/ZEO -- --- ServerStub.py 2001/04/25 20:38:38 1.1.2.4 +++ ServerStub.py 2001/04/25 22:51:32 1.1.2.5 @@ -44,16 +44,11 @@ def tpc_begin(self, id, user, descr, ext): return self.rpc.call('tpc_begin', id, user, descr, ext) - def tpc_begin_sync(self, id, user, descr, ext): - # XXX nuked - return self.rpc.call('tpc_begin_sync', id, user, descr, ext) - def vote(self, trans_id): return self.rpc.call('vote', trans_id) - def tpc_finish(self, id, user, descr, ext): -## self.rpc.callAsync('tpc_finish', id, user, descr, ext) - return self.rpc.call('tpc_finish', id, user, descr, ext) + def tpc_finish(self, id): + return self.rpc.call('tpc_finish', id) def tpc_abort(self, id): self.rpc.callAsync('tpc_abort', id) --- Updated File StorageServer.py in package Packages/ZEO -- --- StorageServer.py 2001/04/25 22:33:50 1.21.4.8 +++ StorageServer.py 2001/04/25 22:51:32 1.21.4.9 @@ -402,7 +402,7 @@ self.__storage.tpc_begin(t) self.__invalidated = [] - def tpc_finish(self, id, user, description, ext): + def tpc_finish(self, id): if not self._check_tid(id): return From jeremy at digicool.com Wed Apr 25 18:55:05 2001 From: jeremy at digicool.com (jeremy@digicool.com) Date: Sun Aug 10 16:31:15 2008 Subject: [ZEO-Checkins] CVS: Packages/ZEO - ClientStorage.py:1.26.4.14 ClientStub.py:1.1.2.2 StorageServer.py:1.21.4.10 Message-ID: <20010425225505.8384C51014@korak.digicool.com> Update of /cvs-repository/Packages/ZEO In directory korak:/tmp/cvs-serv6221 Modified Files: Tag: ZEO-ZRPC-Dev ClientStorage.py ClientStub.py StorageServer.py Log Message: Remove the unlock() message sent from server to client. --- Updated File ClientStorage.py in package Packages/ZEO -- --- ClientStorage.py 2001/04/25 22:51:32 1.26.4.13 +++ ClientStorage.py 2001/04/25 22:55:03 1.26.4.14 @@ -542,11 +542,6 @@ # below are methods invoked by the StorageServer - def unlock(self): - # XXX Don't believe this is used anymore... - log2(INFO, "unlock()") - self._commit_lock_release() - def serialno(self, arg): self._serials.append(arg) --- Updated File ClientStub.py in package Packages/ZEO -- --- ClientStub.py 2001/03/17 00:14:51 1.1.2.1 +++ ClientStub.py 2001/04/25 22:55:03 1.1.2.2 @@ -18,9 +18,6 @@ def endVerify(self): self.rpc.callAsync('end') - def unlock(self): - self.rpc.callAsync('unlock') - def serialno(self, arg): self.rpc.callAsync('serialno', arg) --- Updated File StorageServer.py in package Packages/ZEO -- --- StorageServer.py 2001/04/25 22:51:32 1.21.4.9 +++ StorageServer.py 2001/04/25 22:55:03 1.21.4.10 @@ -361,11 +361,6 @@ return oids return () - def unlock(self): -## if self.__closed: -## return - self.client.unlock() - # When multiple clients are using a single storage, there are several # different _transaction attributes to keep track of. Each # StorageProxy object has a single _transaction that refers to its --- Updated File ClientStorage.py in package Packages/ZEO -- --- ClientStorage.py 2001/04/25 22:51:32 1.26.4.13 +++ ClientStorage.py 2001/04/25 22:55:03 1.26.4.14 @@ -542,11 +542,6 @@ # below are methods invoked by the StorageServer - def unlock(self): - # XXX Don't believe this is used anymore... - log2(INFO, "unlock()") - self._commit_lock_release() - def serialno(self, arg): self._serials.append(arg) --- Updated File ClientStub.py in package Packages/ZEO -- --- ClientStub.py 2001/03/17 00:14:51 1.1.2.1 +++ ClientStub.py 2001/04/25 22:55:03 1.1.2.2 @@ -18,9 +18,6 @@ def endVerify(self): self.rpc.callAsync('end') - def unlock(self): - self.rpc.callAsync('unlock') - def serialno(self, arg): self.rpc.callAsync('serialno', arg) --- Updated File StorageServer.py in package Packages/ZEO -- --- StorageServer.py 2001/04/25 22:51:32 1.21.4.9 +++ StorageServer.py 2001/04/25 22:55:03 1.21.4.10 @@ -361,11 +361,6 @@ return oids return () - def unlock(self): -## if self.__closed: -## return - self.client.unlock() - # When multiple clients are using a single storage, there are several # different _transaction attributes to keep track of. Each # StorageProxy object has a single _transaction that refers to its From jeremy at digicool.com Fri Apr 27 00:01:03 2001 From: jeremy at digicool.com (jeremy@digicool.com) Date: Sun Aug 10 16:31:15 2008 Subject: [ZEO-Checkins] CVS: Packages/ZEO - StorageServer.py:1.24 Message-ID: <20010427040103.112EF510E3@korak.digicool.com> Update of /cvs-repository/Packages/ZEO In directory korak:/tmp/cvs-serv3060 Modified Files: StorageServer.py Log Message: tpc_begin(): Assign the ext arg to the right attribute on the Transaction instance. tpc_finish(): Ignore the arguments other than id. They were handled in the tpc_begin(). --- Updated File StorageServer.py in package Packages/ZEO -- --- StorageServer.py 2001/04/24 21:40:40 1.23 +++ StorageServer.py 2001/04/27 04:01:01 1.24 @@ -441,7 +441,7 @@ r=StorageServerError("Couldn't pickle exception %s" % `newserial`) dump('',1) # clear pickler r=dump((oid, r),1) - + self.message_output('s'+r) return _noreturn @@ -499,6 +499,7 @@ t.id=id t.user=user t.description=description + t._extension=ext storage.tpc_begin(t) self.__invalidated=[] @@ -532,9 +533,6 @@ def tpc_finish(self, id, user, description, ext): t=self._transaction if id != t.id: return - t.user=user - t.description=description - t.ext=ext storage=self.__storage r=storage.tpc_finish(t) --- Updated File StorageServer.py in package Packages/ZEO -- --- StorageServer.py 2001/04/24 21:40:40 1.23 +++ StorageServer.py 2001/04/27 04:01:01 1.24 @@ -441,7 +441,7 @@ r=StorageServerError("Couldn't pickle exception %s" % `newserial`) dump('',1) # clear pickler r=dump((oid, r),1) - + self.message_output('s'+r) return _noreturn @@ -499,6 +499,7 @@ t.id=id t.user=user t.description=description + t._extension=ext storage.tpc_begin(t) self.__invalidated=[] @@ -532,9 +533,6 @@ def tpc_finish(self, id, user, description, ext): t=self._transaction if id != t.id: return - t.user=user - t.description=description - t.ext=ext storage=self.__storage r=storage.tpc_finish(t) From jeremy at digicool.com Fri Apr 27 13:30:01 2001 From: jeremy at digicool.com (jeremy@digicool.com) Date: Sun Aug 10 16:31:15 2008 Subject: [ZEO-Checkins] CVS: Packages/ZEO - ClientStorage.py:1.28 Message-ID: <20010427173001.71C00510E3@korak.digicool.com> Update of /cvs-repository/Packages/ZEO In directory korak:/tmp/cvs-serv29870 Modified Files: ClientStorage.py Log Message: Add default arguments to undoInfo(), since the storage API (the tests at least) seems to imply that they are optional. Fix typo in long attribute name. --- Updated File ClientStorage.py in package Packages/ZEO -- --- ClientStorage.py 2001/04/02 21:34:01 1.27 +++ ClientStorage.py 2001/04/27 17:29:59 1.28 @@ -120,7 +120,7 @@ def __init__(self, connection, storage='1', cache_size=20000000, name='', client='', debug=0, var=None, min_disconnect_poll=5, max_disconnect_poll=300, - wait_for_server_on_starup=1): + wait_for_server_on_startup=1): # Decide whether to use non-temporary files client=client or os.environ.get('ZEO_CLIENT','') @@ -128,7 +128,7 @@ self._connection=connection self._storage=storage self._debug=debug - self._wait_for_server_on_starup=wait_for_server_on_starup + self._wait_for_server_on_startup=wait_for_server_on_startup self._info={'length': 0, 'size': 0, 'name': 'ZEO Client', 'supportsUndo':0, 'supportsVersions': 0, @@ -198,7 +198,7 @@ def _startup(self): - if not self._call.connect(not self._wait_for_server_on_starup): + if not self._call.connect(not self._wait_for_server_on_startup): # If we can't connect right away, go ahead and open the cache # and start a separate thread to try and reconnect. @@ -542,7 +542,7 @@ finally: self._lock_release() - def undoInfo(self, first, last, specification): + def undoInfo(self, first=0, last=-20, specification=None): self._lock_acquire() try: return self._call('undoInfo', first, last, specification) --- Updated File ClientStorage.py in package Packages/ZEO -- --- ClientStorage.py 2001/04/02 21:34:01 1.27 +++ ClientStorage.py 2001/04/27 17:29:59 1.28 @@ -120,7 +120,7 @@ def __init__(self, connection, storage='1', cache_size=20000000, name='', client='', debug=0, var=None, min_disconnect_poll=5, max_disconnect_poll=300, - wait_for_server_on_starup=1): + wait_for_server_on_startup=1): # Decide whether to use non-temporary files client=client or os.environ.get('ZEO_CLIENT','') @@ -128,7 +128,7 @@ self._connection=connection self._storage=storage self._debug=debug - self._wait_for_server_on_starup=wait_for_server_on_starup + self._wait_for_server_on_startup=wait_for_server_on_startup self._info={'length': 0, 'size': 0, 'name': 'ZEO Client', 'supportsUndo':0, 'supportsVersions': 0, @@ -198,7 +198,7 @@ def _startup(self): - if not self._call.connect(not self._wait_for_server_on_starup): + if not self._call.connect(not self._wait_for_server_on_startup): # If we can't connect right away, go ahead and open the cache # and start a separate thread to try and reconnect. @@ -542,7 +542,7 @@ finally: self._lock_release() - def undoInfo(self, first, last, specification): + def undoInfo(self, first=0, last=-20, specification=None): self._lock_acquire() try: return self._call('undoInfo', first, last, specification) From jeremy at digicool.com Fri Apr 27 13:32:20 2001 From: jeremy at digicool.com (jeremy@digicool.com) Date: Sun Aug 10 16:31:15 2008 Subject: [ZEO-Checkins] CVS: Packages/ZEO - forker.py:1.1 testZEO.py:1.2 Message-ID: <20010427173220.B00A9510E3@korak.digicool.com> Update of /cvs-repository/Packages/ZEO/tests In directory korak:/tmp/cvs-serv30049 Added Files: forker.py testZEO.py Log Message: Add ZEO tests using Python 2.1 unittest framework. All tests except checkLen() pass. --- Added File forker.py in package Packages/ZEO --- """Library for forking storage server and connecting client storage""" import asyncore import os import sys import ThreadedAsync import ZEO.ClientStorage, ZEO.StorageServer class ZEOServerExit(asyncore.file_dispatcher): """Used to exit ZEO.StorageServer when run is done""" def writable(self): return 0 def readable(self): return 1 def handle_read(self): buf = self.recv(4) if buf: assert buf == "done" asyncore.socket_map.clear() def handle_close(self): asyncore.socket_map.clear() class ZEOClientExit: """Used by client to cause server to exit""" def __init__(self, pipe): self.pipe = pipe def close(self): os.write(self.pipe, "done") def start_zeo(storage, cache=None, cleanup=None, domain="AF_INET"): """Setup ZEO client-server for storage. Returns a ClientStorage instance and a ZEOClientExit instance. XXX Don't know if os.pipe() will work on Windows. """ if domain == "AF_INET": import random addr = '', random.randrange(2000, 3000) elif domain == "AF_UNIX": import tempfile addr = tempfile.mktemp() else: raise ValueError, "bad domain: %s" % domain rd, wr = os.pipe() pid = os.fork() if pid == 0: # in the child, run the storage server try: os.close(wr) ZEOServerExit(rd) serv = ZEO.StorageServer.StorageServer(addr, {'1':storage}) asyncore.loop() storage.close() if domain == "AF_UNIX": os.unlink(addr) if cleanup: cleanup() finally: os._exit(0) else: os.close(rd) s = ZEO.ClientStorage.ClientStorage(addr, debug=1, client=cache) return s, ZEOClientExit(wr), pid --- Updated File testZEO.py in package Packages/ZEO -- --- Added File forker.py in package Packages/ZEO --- """Library for forking storage server and connecting client storage""" import asyncore import os import sys import ThreadedAsync import ZEO.ClientStorage, ZEO.StorageServer class ZEOServerExit(asyncore.file_dispatcher): """Used to exit ZEO.StorageServer when run is done""" def writable(self): return 0 def readable(self): return 1 def handle_read(self): buf = self.recv(4) if buf: assert buf == "done" asyncore.socket_map.clear() def handle_close(self): asyncore.socket_map.clear() class ZEOClientExit: """Used by client to cause server to exit""" def __init__(self, pipe): self.pipe = pipe def close(self): os.write(self.pipe, "done") def start_zeo(storage, cache=None, cleanup=None, domain="AF_INET"): """Setup ZEO client-server for storage. Returns a ClientStorage instance and a ZEOClientExit instance. XXX Don't know if os.pipe() will work on Windows. """ if domain == "AF_INET": import random addr = '', random.randrange(2000, 3000) elif domain == "AF_UNIX": import tempfile addr = tempfile.mktemp() else: raise ValueError, "bad domain: %s" % domain rd, wr = os.pipe() pid = os.fork() if pid == 0: # in the child, run the storage server try: os.close(wr) ZEOServerExit(rd) serv = ZEO.StorageServer.StorageServer(addr, {'1':storage}) asyncore.loop() storage.close() if domain == "AF_UNIX": os.unlink(addr) if cleanup: cleanup() finally: os._exit(0) else: os.close(rd) s = ZEO.ClientStorage.ClientStorage(addr, debug=1, client=cache) return s, ZEOClientExit(wr), pid --- Updated File testZEO.py in package Packages/ZEO -- From jeremy at digicool.com Fri Apr 27 16:57:20 2001 From: jeremy at digicool.com (jeremy@digicool.com) Date: Sun Aug 10 16:31:15 2008 Subject: [ZEO-Checkins] CVS: Packages/ZEO - ClientStorage.py:1.26.4.15 ServerStub.py:1.1.2.6 StorageServer.py:1.21.4.11 Message-ID: <20010427205720.BB1DB510E3@korak.digicool.com> Update of /cvs-repository/Packages/ZEO In directory korak:/tmp/cvs-serv20312 Modified Files: Tag: ZEO-ZRPC-Dev ClientStorage.py ServerStub.py StorageServer.py Log Message: Support for transactionalUndo Add default args to undoInfo(). Define supportsTransactionalUndo() via get_info() call. XXX In transactionalUndo() on client, invalidate objects immediately. This may not be right, because it assumes that transaction will commit. To do it right, I think the TransactionBuffer needs to be updated with a second kind record -- the undo record, which only invalidates the cache. Also, modify _check_serials so test isn't "not StringType" but "is Exception." --- Updated File ClientStorage.py in package Packages/ZEO -- --- ClientStorage.py 2001/04/25 22:55:03 1.26.4.14 +++ ClientStorage.py 2001/04/27 20:57:18 1.26.4.15 @@ -183,9 +183,6 @@ self.__name__ = name - # Allocate locks: -## import debuglock -## commit_lock = debuglock.DebugLock() commit_lock = thread.allocate_lock() self._commit_lock_acquire = commit_lock.acquire self._commit_lock_release = commit_lock.release @@ -376,6 +373,8 @@ r = self._serials[:l] del self._serials[:l] for oid, s in r: + if isinstance(s, Exception): + raise s self._seriald[oid] = s return r @@ -405,6 +404,9 @@ def supportsVersions(self): return self._info['supportsVersions'] + + def supportsTransactionalUndo(self): + return self._info['supportsTransactionalUndo'] def tpc_abort(self, transaction): self._lock_acquire() @@ -498,6 +500,19 @@ self._commit_lock_release() finally: self._lock_release() + def transactionalUndo(self, trans_id, trans): + if trans is not self._transaction: + raise POSException.StorageTransactionError(self._transaction, + transaction) + self._lock_acquire() + try: + oids = self._server.transactionalUndo(trans_id, self._serial) + for oid in oids: + self._cache.invalidate(oid, '') + return oids + finally: + self._lock_release() + def undo(self, transaction_id): self._lock_acquire() try: @@ -509,7 +524,7 @@ finally: self._lock_release() - def undoInfo(self, first, last, specification): + def undoInfo(self, first=0, last=-20, specification=None): self._lock_acquire() try: return self._server.undoInfo(first, last, specification) --- Updated File ServerStub.py in package Packages/ZEO -- --- ServerStub.py 2001/04/25 22:51:32 1.1.2.5 +++ ServerStub.py 2001/04/27 20:57:19 1.1.2.6 @@ -83,6 +83,9 @@ def store(self, oid, serial, data, version, trans): return self.rpc.call('store', oid, serial, data, version, trans) + def transactionalUndo(self, trans_id, trans): + return self.rpc.call('transactionalUndo', trans_id, trans) + def undo(self, trans_id): return self.rpc.call('undo', trans_id) --- Updated File StorageServer.py in package Packages/ZEO -- --- StorageServer.py 2001/04/25 22:55:03 1.21.4.10 +++ StorageServer.py 2001/04/27 20:57:19 1.21.4.11 @@ -230,19 +230,19 @@ self._log("registered storage %s: %s" % (storage_id, storage)) def get_info(self): - return { - 'length': len(self.__storage), - 'size': self.__storage.getSize(), - 'name': self.__storage.getName(), - 'supportsUndo': self.__storage.supportsUndo(), - 'supportsVersions': self.__storage.supportsVersions(), - } + return {'length': len(self.__storage), + 'size': self.__storage.getSize(), + 'name': self.__storage.getName(), + 'supportsUndo': self.__storage.supportsUndo(), + 'supportsVersions': self.__storage.supportsVersions(), + 'supportsTransactionalUndo': + self.__storage.supportsTransactionalUndo(), + } def get_size_info(self): - return { - 'length': len(self.__storage), - 'size': self.__storage.getSize(), - } + return {'length': len(self.__storage), + 'size': self.__storage.getSize(), + } def zeoLoad(self, oid): v = self.__storage.modifiedInVersion(oid) @@ -352,6 +352,10 @@ def vote(self, id): self._check_tid(id, exc=StorageTransactionError) return self.__storage.tpc_vote(self._transaction) + + def transactionalUndo(self, trans_id, id): + self._check_tid(id, exc=StorageTransactionError) + return self.__storage.transactionalUndo(trans_id, self._transaction) def undo(self, transaction_id): oids = self.__storage.undo(transaction_id) --- Updated File ClientStorage.py in package Packages/ZEO -- --- ClientStorage.py 2001/04/25 22:55:03 1.26.4.14 +++ ClientStorage.py 2001/04/27 20:57:18 1.26.4.15 @@ -183,9 +183,6 @@ self.__name__ = name - # Allocate locks: -## import debuglock -## commit_lock = debuglock.DebugLock() commit_lock = thread.allocate_lock() self._commit_lock_acquire = commit_lock.acquire self._commit_lock_release = commit_lock.release @@ -376,6 +373,8 @@ r = self._serials[:l] del self._serials[:l] for oid, s in r: + if isinstance(s, Exception): + raise s self._seriald[oid] = s return r @@ -405,6 +404,9 @@ def supportsVersions(self): return self._info['supportsVersions'] + + def supportsTransactionalUndo(self): + return self._info['supportsTransactionalUndo'] def tpc_abort(self, transaction): self._lock_acquire() @@ -498,6 +500,19 @@ self._commit_lock_release() finally: self._lock_release() + def transactionalUndo(self, trans_id, trans): + if trans is not self._transaction: + raise POSException.StorageTransactionError(self._transaction, + transaction) + self._lock_acquire() + try: + oids = self._server.transactionalUndo(trans_id, self._serial) + for oid in oids: + self._cache.invalidate(oid, '') + return oids + finally: + self._lock_release() + def undo(self, transaction_id): self._lock_acquire() try: @@ -509,7 +524,7 @@ finally: self._lock_release() - def undoInfo(self, first, last, specification): + def undoInfo(self, first=0, last=-20, specification=None): self._lock_acquire() try: return self._server.undoInfo(first, last, specification) --- Updated File ServerStub.py in package Packages/ZEO -- --- ServerStub.py 2001/04/25 22:51:32 1.1.2.5 +++ ServerStub.py 2001/04/27 20:57:19 1.1.2.6 @@ -83,6 +83,9 @@ def store(self, oid, serial, data, version, trans): return self.rpc.call('store', oid, serial, data, version, trans) + def transactionalUndo(self, trans_id, trans): + return self.rpc.call('transactionalUndo', trans_id, trans) + def undo(self, trans_id): return self.rpc.call('undo', trans_id) --- Updated File StorageServer.py in package Packages/ZEO -- --- StorageServer.py 2001/04/25 22:55:03 1.21.4.10 +++ StorageServer.py 2001/04/27 20:57:19 1.21.4.11 @@ -230,19 +230,19 @@ self._log("registered storage %s: %s" % (storage_id, storage)) def get_info(self): - return { - 'length': len(self.__storage), - 'size': self.__storage.getSize(), - 'name': self.__storage.getName(), - 'supportsUndo': self.__storage.supportsUndo(), - 'supportsVersions': self.__storage.supportsVersions(), - } + return {'length': len(self.__storage), + 'size': self.__storage.getSize(), + 'name': self.__storage.getName(), + 'supportsUndo': self.__storage.supportsUndo(), + 'supportsVersions': self.__storage.supportsVersions(), + 'supportsTransactionalUndo': + self.__storage.supportsTransactionalUndo(), + } def get_size_info(self): - return { - 'length': len(self.__storage), - 'size': self.__storage.getSize(), - } + return {'length': len(self.__storage), + 'size': self.__storage.getSize(), + } def zeoLoad(self, oid): v = self.__storage.modifiedInVersion(oid) @@ -352,6 +352,10 @@ def vote(self, id): self._check_tid(id, exc=StorageTransactionError) return self.__storage.tpc_vote(self._transaction) + + def transactionalUndo(self, trans_id, id): + self._check_tid(id, exc=StorageTransactionError) + return self.__storage.transactionalUndo(trans_id, self._transaction) def undo(self, transaction_id): oids = self.__storage.undo(transaction_id) From jeremy at digicool.com Fri Apr 27 17:03:37 2001 From: jeremy at digicool.com (jeremy@digicool.com) Date: Sun Aug 10 16:31:15 2008 Subject: [ZEO-Checkins] CVS: Packages/ZEO - forker.py:1.1.2.1 testZEO.py:1.1.2.7 Message-ID: <20010427210337.0D857510E3@korak.digicool.com> Update of /cvs-repository/Packages/ZEO/tests In directory korak:/tmp/cvs-serv20840 Modified Files: Tag: ZEO-ZRPC-Dev testZEO.py Added Files: Tag: ZEO-ZRPC-Dev forker.py Log Message: Update test to use all tests defined in ZODB.tests against FileStorage Remove TestStorage subclasses in favor of forker module. Updated _dostore() to handle the already_pickled arg. Add new tests to GenericTests. --- Added File forker.py in package Packages/ZEO --- """Library for forking storage server and connecting client storage""" import asyncore import os import sys import ThreadedAsync import ZEO.ClientStorage, ZEO.StorageServer class ZEOServerExit(asyncore.file_dispatcher): """Used to exit ZEO.StorageServer when run is done""" def writable(self): return 0 def readable(self): return 1 def handle_read(self): buf = self.recv(4) if buf: assert buf == "done" asyncore.socket_map.clear() def handle_close(self): asyncore.socket_map.clear() class ZEOClientExit: """Used by client to cause server to exit""" def __init__(self, pipe): self.pipe = pipe def close(self): os.write(self.pipe, "done") def start_zeo(storage, cache=None, cleanup=None, domain="AF_INET"): """Setup ZEO client-server for storage. Returns a ClientStorage instance and a ZEOClientExit instance. XXX Don't know if os.pipe() will work on Windows. """ if domain == "AF_INET": import random addr = '', random.randrange(2000, 3000) elif domain == "AF_UNIX": import tempfile addr = tempfile.mktemp() else: raise ValueError, "bad domain: %s" % domain rd, wr = os.pipe() pid = os.fork() if pid == 0: # in the child, run the storage server try: os.close(wr) ZEOServerExit(rd) serv = ZEO.StorageServer.StorageServer(addr, {'1':storage}) asyncore.loop() storage.close() if domain == "AF_UNIX": os.unlink(addr) if cleanup: cleanup() finally: os._exit(0) else: os.close(rd) s = ZEO.ClientStorage.ClientStorage(addr, debug=1, client=cache) return s, ZEOClientExit(wr), pid --- Updated File testZEO.py in package Packages/ZEO -- --- testZEO.py 2001/04/25 21:05:42 1.1.2.6 +++ testZEO.py 2001/04/27 21:03:36 1.1.2.7 @@ -8,27 +8,15 @@ import unittest import ZEO.ClientStorage, ZEO.StorageServer -from ZEO.zrpc2 import ManagedServerConnection import ThreadedAsync, ZEO.trigger from ZEO import zeolog +from ZEO.tests import forker -# XXX The ZODB.tests package contains a grab bad things, including, -# apparently, a collection of modules that define mixin classes -# containing tests cases. - -from ZODB.tests import StorageTestBase, BasicStorage, VersionStorage - -class TestStorageServer(ZEO.StorageServer.StorageServer): - def newConnection(self, sock, addr, nil): - c = ManagedServerConnection(sock, addr, None, self) - c.register_object(TestStorageProxy(self, c)) - zeolog.LOG("TSS", zeolog.INFO, "connected: %s" % c) - -class TestStorageProxy(ZEO.StorageServer.StorageProxy): - def shutdown(self): - self._StorageProxy__storage.close() - os._exit(0) +# Sorry Jim... +from ZODB.tests import StorageTestBase, BasicStorage, VersionStorage, \ + TransactionalUndoStorage, TransactionalUndoVersionStorage, \ + PackableStorage ZERO = '\0'*8 import pickle @@ -41,7 +29,8 @@ will get no later than the return value from vote. """ - def _dostore(self, oid=None, revid=None, data=None, version=None): + def _dostore(self, oid=None, revid=None, data=None, version=None, + already_pickled=0): """Do a complete storage transaction. The defaults are: @@ -57,8 +46,8 @@ if revid is None: revid = ZERO if data is None: - data = pickle.dumps(7) - else: + data = 7 + if not already_pickled: data = pickle.dumps(data) if version is None: version = '' @@ -84,15 +73,15 @@ raise RuntimeError, "unexpected ZEO response: no oid" else: for oid, serial in r: - if type(serial) != types.StringType: - raise serial - else: - d[oid] = serial + d[oid] = serial return d class GenericTests(ZEOTestBase, BasicStorage.BasicStorage, - VersionStorage.VersionStorage + VersionStorage.VersionStorage, + TransactionalUndoStorage.TransactionalUndoStorage, + PackableStorage.PackableStorage, + TransactionalUndoVersionStorage.TransactionalUndoVersionStorage, ): """An abstract base class for ZEO tests @@ -112,9 +101,10 @@ getStorage() method. """ self.running = 1 - self.__sock = tempfile.mktemp() - self.startServer() - self._storage = ZEO.ClientStorage.ClientStorage(self.__sock) + client, exit, pid = forker.start_zeo(self.getStorage()) + self._pid = pid + self._server = exit + self._storage = client while not self._storage.is_connected(): time.sleep(0.1) self.__super_setUp() @@ -122,26 +112,9 @@ def tearDown(self): """Try to cause the tests to halt""" self.running = 0 - self._storage._server.rpc.callAsync('shutdown') - self._storage._rpc_mgr.close() - os.waitpid(self.__pid, 0) - os.unlink(self.__sock) - self.delStorage() + self._server.close() + os.waitpid(self._pid, 0) self.__super_tearDown() - - def startServer(self): - self.__pid = os.fork() - if self.__pid == 0: - self.__storage = self.getStorage() - d = {'1': self.__storage} - self.__server = TestStorageServer(self.__sock, d) - asyncore.loop() - else: - zeolog.LOG("test", zeolog.INFO, "forked %s" % self.__pid) - - def checkFirst(self): - self._storage.tpc_begin(self._transaction) - self._storage.tpc_abort(self._transaction) class ZEOFileStorageTests(GenericTests): __super_setUp = GenericTests.setUp --- Added File forker.py in package Packages/ZEO --- """Library for forking storage server and connecting client storage""" import asyncore import os import sys import ThreadedAsync import ZEO.ClientStorage, ZEO.StorageServer class ZEOServerExit(asyncore.file_dispatcher): """Used to exit ZEO.StorageServer when run is done""" def writable(self): return 0 def readable(self): return 1 def handle_read(self): buf = self.recv(4) if buf: assert buf == "done" asyncore.socket_map.clear() def handle_close(self): asyncore.socket_map.clear() class ZEOClientExit: """Used by client to cause server to exit""" def __init__(self, pipe): self.pipe = pipe def close(self): os.write(self.pipe, "done") def start_zeo(storage, cache=None, cleanup=None, domain="AF_INET"): """Setup ZEO client-server for storage. Returns a ClientStorage instance and a ZEOClientExit instance. XXX Don't know if os.pipe() will work on Windows. """ if domain == "AF_INET": import random addr = '', random.randrange(2000, 3000) elif domain == "AF_UNIX": import tempfile addr = tempfile.mktemp() else: raise ValueError, "bad domain: %s" % domain rd, wr = os.pipe() pid = os.fork() if pid == 0: # in the child, run the storage server try: os.close(wr) ZEOServerExit(rd) serv = ZEO.StorageServer.StorageServer(addr, {'1':storage}) asyncore.loop() storage.close() if domain == "AF_UNIX": os.unlink(addr) if cleanup: cleanup() finally: os._exit(0) else: os.close(rd) s = ZEO.ClientStorage.ClientStorage(addr, debug=1, client=cache) return s, ZEOClientExit(wr), pid --- Updated File testZEO.py in package Packages/ZEO -- --- testZEO.py 2001/04/25 21:05:42 1.1.2.6 +++ testZEO.py 2001/04/27 21:03:36 1.1.2.7 @@ -8,27 +8,15 @@ import unittest import ZEO.ClientStorage, ZEO.StorageServer -from ZEO.zrpc2 import ManagedServerConnection import ThreadedAsync, ZEO.trigger from ZEO import zeolog +from ZEO.tests import forker -# XXX The ZODB.tests package contains a grab bad things, including, -# apparently, a collection of modules that define mixin classes -# containing tests cases. - -from ZODB.tests import StorageTestBase, BasicStorage, VersionStorage - -class TestStorageServer(ZEO.StorageServer.StorageServer): - def newConnection(self, sock, addr, nil): - c = ManagedServerConnection(sock, addr, None, self) - c.register_object(TestStorageProxy(self, c)) - zeolog.LOG("TSS", zeolog.INFO, "connected: %s" % c) - -class TestStorageProxy(ZEO.StorageServer.StorageProxy): - def shutdown(self): - self._StorageProxy__storage.close() - os._exit(0) +# Sorry Jim... +from ZODB.tests import StorageTestBase, BasicStorage, VersionStorage, \ + TransactionalUndoStorage, TransactionalUndoVersionStorage, \ + PackableStorage ZERO = '\0'*8 import pickle @@ -41,7 +29,8 @@ will get no later than the return value from vote. """ - def _dostore(self, oid=None, revid=None, data=None, version=None): + def _dostore(self, oid=None, revid=None, data=None, version=None, + already_pickled=0): """Do a complete storage transaction. The defaults are: @@ -57,8 +46,8 @@ if revid is None: revid = ZERO if data is None: - data = pickle.dumps(7) - else: + data = 7 + if not already_pickled: data = pickle.dumps(data) if version is None: version = '' @@ -84,15 +73,15 @@ raise RuntimeError, "unexpected ZEO response: no oid" else: for oid, serial in r: - if type(serial) != types.StringType: - raise serial - else: - d[oid] = serial + d[oid] = serial return d class GenericTests(ZEOTestBase, BasicStorage.BasicStorage, - VersionStorage.VersionStorage + VersionStorage.VersionStorage, + TransactionalUndoStorage.TransactionalUndoStorage, + PackableStorage.PackableStorage, + TransactionalUndoVersionStorage.TransactionalUndoVersionStorage, ): """An abstract base class for ZEO tests @@ -112,9 +101,10 @@ getStorage() method. """ self.running = 1 - self.__sock = tempfile.mktemp() - self.startServer() - self._storage = ZEO.ClientStorage.ClientStorage(self.__sock) + client, exit, pid = forker.start_zeo(self.getStorage()) + self._pid = pid + self._server = exit + self._storage = client while not self._storage.is_connected(): time.sleep(0.1) self.__super_setUp() @@ -122,26 +112,9 @@ def tearDown(self): """Try to cause the tests to halt""" self.running = 0 - self._storage._server.rpc.callAsync('shutdown') - self._storage._rpc_mgr.close() - os.waitpid(self.__pid, 0) - os.unlink(self.__sock) - self.delStorage() + self._server.close() + os.waitpid(self._pid, 0) self.__super_tearDown() - - def startServer(self): - self.__pid = os.fork() - if self.__pid == 0: - self.__storage = self.getStorage() - d = {'1': self.__storage} - self.__server = TestStorageServer(self.__sock, d) - asyncore.loop() - else: - zeolog.LOG("test", zeolog.INFO, "forked %s" % self.__pid) - - def checkFirst(self): - self._storage.tpc_begin(self._transaction) - self._storage.tpc_abort(self._transaction) class ZEOFileStorageTests(GenericTests): __super_setUp = GenericTests.setUp From jeremy at digicool.com Sun Apr 29 13:08:25 2001 From: jeremy at digicool.com (jeremy@digicool.com) Date: Sun Aug 10 16:31:15 2008 Subject: [ZEO-Checkins] CVS: Packages/ZEO - ClientStorage.py:1.29 Message-ID: <20010429170825.8F2E9510E3@korak.digicool.com> Update of /cvs-repository/Packages/ZEO In directory korak:/tmp/cvs-serv15752 Modified Files: ClientStorage.py Log Message: Treat tpc_vote() as a noop if the transaction arg isn't the current transaction. --- Updated File ClientStorage.py in package Packages/ZEO -- --- ClientStorage.py 2001/04/27 17:29:59 1.28 +++ ClientStorage.py 2001/04/29 17:08:23 1.29 @@ -402,10 +402,10 @@ finally: self._lock_release() def tpc_vote(self, transaction): - if transaction is not self._transaction: - raise POSException.StorageTransactionError(self, transaction) self._lock_acquire() try: + if transaction is not self._transaction: + return self._call('vote', self._serial) if self._serials: --- Updated File ClientStorage.py in package Packages/ZEO -- --- ClientStorage.py 2001/04/27 17:29:59 1.28 +++ ClientStorage.py 2001/04/29 17:08:23 1.29 @@ -402,10 +402,10 @@ finally: self._lock_release() def tpc_vote(self, transaction): - if transaction is not self._transaction: - raise POSException.StorageTransactionError(self, transaction) self._lock_acquire() try: + if transaction is not self._transaction: + return self._call('vote', self._serial) if self._serials: