[Zope-Checkins] SVN: Zope/trunk/src/Zope2/utilities/ Removed a couple of very old scripts written in 2002-2004

Hanno Schlichting hannosch at hannosch.eu
Sat Jul 2 11:58:58 EDT 2011


Log message for revision 122058:
  Removed a couple of very old scripts written in 2002-2004
  

Changed:
  D   Zope/trunk/src/Zope2/utilities/README.txt
  U   Zope/trunk/src/Zope2/utilities/__init__.py
  D   Zope/trunk/src/Zope2/utilities/check_catalog.py
  D   Zope/trunk/src/Zope2/utilities/compilezpy.py
  D   Zope/trunk/src/Zope2/utilities/decompilezpy.py
  D   Zope/trunk/src/Zope2/utilities/fixheaders.py
  D   Zope/trunk/src/Zope2/utilities/load_site.py
  D   Zope/trunk/src/Zope2/utilities/requestprofiler.py
  D   Zope/trunk/src/Zope2/utilities/tracelog.py

-=-
Deleted: Zope/trunk/src/Zope2/utilities/README.txt
===================================================================
--- Zope/trunk/src/Zope2/utilities/README.txt	2011-07-02 15:52:15 UTC (rev 122057)
+++ Zope/trunk/src/Zope2/utilities/README.txt	2011-07-02 15:58:58 UTC (rev 122058)
@@ -1,28 +0,0 @@
-This directory contains utility scripts and modules that augment Zope.
-
-To get detailed usage information, run any of these scripts without arguments:
-
-  load_site.py -- Load a Zope site from files and directories
-
-    This script illustrates used of the Zope RPC mechanism,
-    ZPublisher.Client.  It provides some examples of pitfalls
-    and their work-arounds.
-
-  check_catalog.py -- Perform some consistency tests on a ZCatalog instance
-
-  mkzopeinstance.py -- create a Zope instance home
-
-  copyzopeskel.py -- copy a Zope instance home skeleton directory to target
-
-  mkzeoinstance.py -- create a ZEO instance home
-
-  requestprofiler.py -- parse and analyze the Zope "detailed" log file
-
-  zpasswd.py -- generate "access" or "inituser" files for use with Zope
-
-  compilezpy.py -- compile all .py files to .pyc files in the current
-     directory and below
-
-  decompilezpy.py -- remove all .py[co] files in the current directory
-     and below
-

Modified: Zope/trunk/src/Zope2/utilities/__init__.py
===================================================================
--- Zope/trunk/src/Zope2/utilities/__init__.py	2011-07-02 15:52:15 UTC (rev 122057)
+++ Zope/trunk/src/Zope2/utilities/__init__.py	2011-07-02 15:58:58 UTC (rev 122058)
@@ -1 +1 @@
-# placeholder
+#
\ No newline at end of file

Deleted: Zope/trunk/src/Zope2/utilities/check_catalog.py
===================================================================
--- Zope/trunk/src/Zope2/utilities/check_catalog.py	2011-07-02 15:52:15 UTC (rev 122057)
+++ Zope/trunk/src/Zope2/utilities/check_catalog.py	2011-07-02 15:58:58 UTC (rev 122058)
@@ -1,143 +0,0 @@
-##############################################################################
-#
-# Copyright (c) 2002 Zope Foundation and Contributors.
-#
-# This software is subject to the provisions of the Zope Public License,
-# Version 2.1 (ZPL).  A copy of the ZPL should accompany this distribution.
-# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
-# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
-# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
-# FOR A PARTICULAR PURPOSE
-#
-##############################################################################
-"""Script to check consistency of a ZCatalog
-"""
-
-import Zope2
-import os,sys,re,getopt
-from types import IntType
-from BTrees.IIBTree import IISet,difference,intersection
-
-def checkCatalog(path,indexes):
-    """ perform some consistency checks on a ZCatalog instance"""
-
-    root = Zope2.app()
-
-    try:
-        catalog = root.unrestrictedTraverse(path)
-    except AttributeError:
-        print 'Error: catalog object not found'
-        sys.exit(1)
-
-    # get Catalog instance
-    _cat = catalog._catalog
-
-    # check Catalog internal BTrees
-    l_data  = list(_cat.data.keys())
-    l_data.sort()
-    l_uids  = list(_cat.uids.values())
-    l_uids.sort()
-    l_paths = list(_cat.data.keys())
-    l_paths.sort()
-
-    print "Checking catalog internal BTrees"
-    print "\tINFO: Mapping data:  %d entries" % len(l_data)
-    print "\tINFO: Mapping uids:  %d entries" % len(l_uids)
-    print "\tINFO: Mapping paths: %d entries" % len(l_paths)
-
-    if l_data == l_uids:
-        print "\tOK:  Mapping data equals Mapping uids"
-    else:
-        print "\tERR: Mapping data does not equal Mapping uids"
-
-    if l_data == l_paths:
-        print "\tOK:  Mapping data equals Maaping paths"
-    else:
-        print "\tERR: Mapping data does not equal Maaping paths"
-
-
-    # check BTrees of indexes
-
-    for id,idx in _cat.indexes.items():
-
-        if indexes and not idx.meta_type in indexes: continue
-
-        print "Checking index '%s' (type: %s)" % (id, idx.meta_type)
-
-        if idx.meta_type in ['FieldIndex','KeywordIndex']:
-
-            # check forward entries
-            RIDS = IISet()
-            for key, rids in idx._index.items():
-                if isinstance(rids,IntType):
-                    RIDS.insert(  rids  )
-                else:
-                    map(RIDS.insert , rids.keys())
-
-            diff = difference(RIDS, IISet(_cat.data.keys()))
-            if len(diff)!=0:
-                print '\tERR: Problem with forward entries'
-                print '\tERR: too much forward entries:', diff
-            else:
-                print '\tOK:  Forward entries (%d entries)'  % (len(RIDS))
-
-
-        elif idx.meta_type in ['PathIndex']:
-
-            RIDS = IISet()
-
-            for rids in map(None,idx._index.values()):
-                map(RIDS.insert , rids.values()[0])
-
-            diff = difference(RIDS, IISet(_cat.data.keys()))
-            if len(diff)!=0:
-                print '\tERR: Problem with forward entries'
-                print '\tERR: too much forward entries:', diff
-            else:
-                print '\tOK:  Forward entries (%d entries)'  % (len(RIDS))
-
-
-        if idx.meta_type in ['FieldIndex','KeywordIndex','PathIndex']:
-
-            # check backward entries
-            RIDS = IISet(idx._unindex.keys())
-            diff = difference(RIDS, IISet(_cat.data.keys()))
-            if len(diff)!=0:
-                print '\tERR: Problem with backward entries'
-                print '\tERR: too much backward entries:', diff
-            else:
-                print '\tOK:  Backward entries (%d entries)'  % (len(RIDS))
-
-
-
-
-
-
-
-def usage():
-    print "Usage: %s [--FieldIndex|KeywordIndex|PathIndex] /path/to/ZCatalog" % \
-                os.path.basename(sys.argv[0])
-    print
-    print "This scripts checks the consistency of the internal"
-    print "BTrees of a ZCatalog and its indexes."
-    sys.exit(1)
-
-
-def main():
-
-    opts,args = getopt.getopt(sys.argv[1:],'h',\
-            ['help','FieldIndex','KeywordIndex','PathIndex'])
-
-    indexes = []
-
-    for o,v in opts:
-
-        if o in ['-h','--help']: usage()
-        if o in ['--FieldIndex','--KeywordIndex','--PathIndex']:
-            indexes.append(o[2:])
-
-    checkCatalog(args,indexes)
-
-
-if __name__=='__main__':
-    main()

Deleted: Zope/trunk/src/Zope2/utilities/compilezpy.py
===================================================================
--- Zope/trunk/src/Zope2/utilities/compilezpy.py	2011-07-02 15:52:15 UTC (rev 122057)
+++ Zope/trunk/src/Zope2/utilities/compilezpy.py	2011-07-02 15:58:58 UTC (rev 122058)
@@ -1,68 +0,0 @@
-##############################################################################
-#
-# Copyright (c) 2002 Zope Foundation and Contributors.
-#
-# This software is subject to the provisions of the Zope Public License,
-# Version 2.1 (ZPL).  A copy of the ZPL should accompany this distribution.
-# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
-# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
-# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
-# FOR A PARTICULAR PURPOSE
-#
-##############################################################################
-
-import compileall, os, sys
-
-class Shutup:
-    def write(*args): pass # :)
-
-class NoteErr:
-    wrote = 0
-    def write(self, *args):
-        self.wrote = 1
-        apply(stderr.write, args)
-
-def compile_non_test(dir):
-    """Byte-compile all modules except those in test directories."""
-    success = compileall.compile_dir(dir, maxlevels=0)
-    try:
-        names = os.listdir(dir)
-    except os.error:
-        print "Can't list", dir
-        names = []
-    names.sort()
-    for name in names:
-        fullname = os.path.join(dir, name)
-        if (name != os.curdir and name != os.pardir and
-            os.path.isdir(fullname) and not os.path.islink(fullname) and
-            name != 'test' and name != 'tests' and name != 'skins'):
-            success = success and compile_non_test(fullname)
-    return success
-
-print
-print '-'*78
-print 'Compiling python modules'
-stdout = sys.stdout
-stderr = sys.stderr
-try:
-    try:
-        success = 0
-        sys.stdout = Shutup()
-        sys.stderr = NoteErr()
-        success = compile_non_test(os.getcwd())
-    finally:
-        success = success and not sys.stderr.wrote
-        sys.stdout = stdout
-        sys.stderr = stderr
-except:
-    success = 0
-    import traceback
-    traceback.print_exc()
-
-if not success:
-    print
-    print '!' * 78
-    print 'There were errors during Python module compilation.'
-    print '!' * 78
-    print
-    sys.exit(1)

Deleted: Zope/trunk/src/Zope2/utilities/decompilezpy.py
===================================================================
--- Zope/trunk/src/Zope2/utilities/decompilezpy.py	2011-07-02 15:52:15 UTC (rev 122057)
+++ Zope/trunk/src/Zope2/utilities/decompilezpy.py	2011-07-02 15:58:58 UTC (rev 122058)
@@ -1,27 +0,0 @@
-##############################################################################
-#
-# Copyright (c) 2002 Zope Foundation and Contributors.
-#
-# This software is subject to the provisions of the Zope Public License,
-# Version 2.1 (ZPL).  A copy of the ZPL should accompany this distribution.
-# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
-# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
-# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
-# FOR A PARTICULAR PURPOSE
-#
-##############################################################################
-import os
-import sys
-
-def main(dirname):
-    os.path.walk(dirname, rmpycs, None)
-
-def rmpycs(arg, dirname, names):
-    for name in names:
-        path = os.path.join(dirname, name)
-        if ( name.endswith('.pyc') or name.endswith('.pyo') and
-             os.path.isfile(path) ):
-            os.unlink(path)
-
-if __name__ == '__main__':
-    main(sys.argv[1])

Deleted: Zope/trunk/src/Zope2/utilities/fixheaders.py
===================================================================
--- Zope/trunk/src/Zope2/utilities/fixheaders.py	2011-07-02 15:52:15 UTC (rev 122057)
+++ Zope/trunk/src/Zope2/utilities/fixheaders.py	2011-07-02 15:58:58 UTC (rev 122058)
@@ -1,26 +0,0 @@
-
-# Script to fix the header files to ZPL 2.1
-
-import os
-
-for dirpath, dirnames, filenames in os.walk('.'):
-
-    for fname in filenames:
-        base,ext = os.path.splitext(fname)
-        if not ext in ('.py', '.c', '.h'): continue
-        fullname = os.path.join(dirpath, fname)
-        if '.svn' in fullname: continue
-        data = open(fullname).read()
-
-        changed = False
-        if 'Version 2.1 (ZPL)' in data:
-            data = data.replace('Version 2.1 (ZPL)', 'Version 2.1 (ZPL)')
-            changed = True
-
-        if '(c) 2002 Zope Corporation' in data:
-            data = data.replace('(c) 2002 Zope Corporation', '(c) 2002 Zope Corporation')
-            changed = True
-
-        print fullname, changed
-        if changed:
-            open(fullname, 'w').write(data)

Deleted: Zope/trunk/src/Zope2/utilities/load_site.py
===================================================================
--- Zope/trunk/src/Zope2/utilities/load_site.py	2011-07-02 15:52:15 UTC (rev 122057)
+++ Zope/trunk/src/Zope2/utilities/load_site.py	2011-07-02 15:58:58 UTC (rev 122058)
@@ -1,303 +0,0 @@
-##############################################################################
-#
-# Copyright (c) 2002 Zope Foundation and Contributors.
-#
-# This software is subject to the provisions of the Zope Public License,
-# Version 2.1 (ZPL).  A copy of the ZPL should accompany this distribution.
-# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
-# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
-# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
-# FOR A PARTICULAR PURPOSE
-#
-##############################################################################
-"""Load a Zope site from a collection of files or directories
-"""
-
-usage=""" [options] url file .....
-
-    where options are:
-
-      -D
-
-         For HTML documents, replace the start of the content, up to
-         and including the opening body tag with a DTML var tag that
-         inserts the standard header. Also replace the closing body
-         and html tag with a DTML var tag that inserts the standard
-         footer.
-
-      -I
-
-         For each index.html, add an index_html that redirects.
-
-      -p path
-
-         Path to ZPublisher.  If not provided, load_site will
-         make an attempt to figure it out.
-
-      -u user:password
-
-         Credentials
-
-      -v
-
-         Run in verbose mode.
-
-      -9
-
-         Use *old* zope method names.
-"""
-
-import sys, getopt, os, string
-ServerError=''
-verbose=0
-old=0
-doctor=0
-index_html=0
-
-def main():
-    user, password = 'superuser', '123'
-    opts, args = getopt.getopt(sys.argv[1:], 'p:u:DIv9')
-    global verbose
-    global old
-    global doctor
-    global index_html
-    havepath=None
-    for o, v in opts:
-        if o=='-p':
-            d, f = os.path.split(v)
-            if f=='ZPublisher': sys.path.insert(0,d)
-            else: sys.path.insert(0,v)
-            havepath=1
-        elif o=='-u':
-            v = string.split(v,':')
-            user, password = v[0], string.join(v[1:],':')
-        elif o=='-D': doctor=1
-        elif o=='-I': index_html=1
-        elif o=='-v': verbose=1
-        elif o=='-9': old=1
-
-    if not args:
-        print sys.argv[0]+usage
-        sys.exit(1)
-
-    if not havepath:
-        here=os.path.split(sys.argv[0])[0]
-        if os.path.exists(os.path.join(here,'ZPublisher')):
-            sys.path.insert(0,here)
-        else:
-            here=os.path.split(here)[0]
-            here=os.path.join(here,'lib','python')
-            if os.path.exists(os.path.join(here,'ZPublisher')):
-                sys.path.insert(0,here)
-
-
-    url=args[0]
-    files=args[1:]
-
-    import ZPublisher.Client
-    global ServerError
-    ServerError=ZPublisher.Client.ServerError
-    object=ZPublisher.Client.Object(url, username=user, password=password)
-
-    for f in files: upload_file(object, f)
-
-def call(f, *args, **kw):
-    # Call a function ignoring redirect bci errors.
-    try: apply(f,args, kw)
-    except ServerError, v:
-        if str(v)[:1] != '3':
-            raise sys.exc_info()[0], sys.exc_info()[1], sys.exc_info()[2]
-
-def upload_file(object, f):
-    if os.path.isdir(f): return upload_dir(object, f)
-    dir, name = os.path.split(f)
-    root, ext = os.path.splitext(name)
-    if ext in ('file', 'dir'): ext=''
-    else:
-        ext=string.lower(ext)
-        if ext and ext[0] in '.': ext=ext[1:]
-    if ext and globals().has_key('upload_'+ext):
-        if verbose: print 'upload_'+ext, f
-        return globals()['upload_'+ext](object, f)
-
-    if verbose: print 'upload_file', f, ext
-    call(object.manage_addFile, id=name, file=open(f,'rb'))
-
-def upload_dir(object, f):
-    if verbose: print 'upload_dir', f
-    dir, name = os.path.split(f)
-    call(object.manage_addFolder, id=name)
-    object=object.__class__(object.url+'/'+name,
-                            username=object.username,
-                            password=object.password)
-    for n in os.listdir(f):
-        upload_file(object, os.path.join(f,n))
-
-# ----- phd -----
-# Modified by Oleg Broytmann <phd2 at earthling.net>
-
-from sgmllib import SGMLParser
-
-def join_attrs(attrs):
-    attr_list = []
-    for attrname, value in attrs:
-        attr_list.append('%s="%s"' % (attrname, string.strip(value)))
-
-    if attr_list:
-        s = " " + string.join(attr_list, " ")
-    else:
-        s = ""
-
-    return s
-
-
-class HeadParser(SGMLParser):
-    def __init__(self):
-        SGMLParser.__init__(self)
-
-        self.seen_starthead = 0
-        self.seen_endhead   = 0
-        self.seen_startbody = 0
-
-        self.head = ""
-        self.title = ""
-        self.accumulator = ""
-
-
-    def handle_data(self, data):
-        if data:
-            self.accumulator = self.accumulator + data
-
-    def handle_charref(self, ref):
-        self.handle_data("&#%s;" % ref)
-
-    def handle_entityref(self, ref):
-        self.handle_data("&%s;" % ref)
-
-    def handle_comment(self, data):
-        if data:
-            self.accumulator = self.accumulator + "<!--%s-->" % data
-
-
-    def start_head(self, attrs):
-        if not self.seen_starthead:
-            self.seen_starthead = 1
-            self.head = ""
-            self.title = ""
-            self.accumulator = ""
-
-    def end_head(self):
-        if not self.seen_endhead:
-            self.seen_endhead = 1
-            self.head = self.head + self.accumulator
-            self.accumulator = ""
-
-
-    def start_title(self, attrs):
-        self.head = self.head + self.accumulator
-        self.accumulator = ""
-
-    def end_title(self):
-        self.title = self.accumulator
-        self.accumulator = ""
-
-
-    def start_body(self, attrs):
-        if not self.seen_startbody:
-            self.seen_startbody = 1
-            self.accumulator = ""
-
-    def end_body(self): pass # Do not put </BODY> and </HTML>
-    def end_html(self): pass # into output stream
-
-
-    # Pass other tags unmodified
-    def unknown_starttag(self, tag, attrs):
-        self.accumulator = self.accumulator + "<%s%s>" % (string.upper(tag), join_attrs(attrs))
-
-    def unknown_endtag(self, tag):
-        self.accumulator = self.accumulator + "</%s>" % string.upper(tag)
-
-
-
-def parse_html(infile):
-    parser = HeadParser()
-
-    while 1:
-        line = infile.readline()
-        if not line: break
-        parser.feed(line)
-
-    parser.close()
-    infile.close()
-
-    return (string.strip(parser.title), string.strip(parser.head),
-            string.strip(parser.accumulator))
-
-
-def upload_html(object, f):
-    dir, name = os.path.split(f)
-    f=open(f)
-
-    if doctor:
-        title, head, body = parse_html(f)
-        if old:
-            body = ("<!--#var standard_html_header-->\n\n" +
-                    body + "\n\n<!--#var standard_html_footer-->")
-        else:
-            body = ("<html><head><title><dtml-var title_or_id></title>"
-                    "</head><body bgcolor=\"#FFFFFF\">\n\n" +
-                     body + "\n\n</body></html>")
-
-    else:
-        if old: f=f.read()
-        title, head, body = '', '', f
-
-    if old:
-        call(object.manage_addDocument, id=name, file=body)
-        if index_html and name in ('index.html', 'index.htm'):
-            call(object.manage_addDocument, id='index_html',
-                 file=('<!--#raise Redirect-->'
-                       '<!--#var URL1-->/%s'
-                       '<!--#/raise-->' % name
-                       ))
-    else:
-        call(object.manage_addDTMLDocument, id=name, title=title, file=body)
-        if index_html and name in ('index.html', 'index.htm'):
-            call(object.manage_addDTMLMethod, id='index_html',
-                 file=('<dtml-raise Redirect>'
-                       '<dtml-var URL1>/%s'
-                       '</dtml-raise>' % name
-                       ))
-
-    # Now add META and other tags as property
-    if head:
-        object=object.__class__(object.url+'/'+name,
-                              username=object.username,
-                              password=object.password)
-        call(object.manage_addProperty,
-             id="loadsite-head", type="text", value=head)
-
-# ----- /phd -----
-
-upload_htm=upload_html
-
-def upload_dtml(object, f):
-    dir, name = os.path.split(f)
-    f=open(f)
-
-    if old:
-        f=f.read()
-        call(object.manage_addDocument, id=name, file=f)
-    else:
-        call(object.manage_addDTMLMethod, id=name, file=f)
-
-
-def upload_gif(object, f):
-    dir, name = os.path.split(f)
-    call(object.manage_addImage, id=name, file=open(f,'rb'))
-
-upload_jpg=upload_gif
-upload_png=upload_gif
-
-if __name__=='__main__': main()

Deleted: Zope/trunk/src/Zope2/utilities/requestprofiler.py
===================================================================
--- Zope/trunk/src/Zope2/utilities/requestprofiler.py	2011-07-02 15:52:15 UTC (rev 122057)
+++ Zope/trunk/src/Zope2/utilities/requestprofiler.py	2011-07-02 15:58:58 UTC (rev 122058)
@@ -1,841 +0,0 @@
-##############################################################################
-#
-# Copyright (c) 2002 Zope Foundation and Contributors.
-#
-# This software is subject to the provisions of the Zope Public License,
-# Version 2.1 (ZPL).  A copy of the ZPL should accompany this distribution.
-# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
-# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
-# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
-# FOR A PARTICULAR PURPOSE
-#
-##############################################################################
-"""Request log profiler script
-"""
-
-import sys, time, getopt, math, cPickle
-from types import StringType
-try: import gzip
-except: pass
-
-class ProfileException(Exception): pass
-
-class Request:
-    def __init__(self):
-        self.url = None
-        self.start = None
-        self.method = None
-        self.t_recdinput = None
-        self.isize = None
-        self.t_recdoutput = None
-        self.osize = None
-        self.httpcode = None
-        self.t_end = None
-        self.elapsed = "I"
-        self.active = 0
-
-    def put(self, code, t, desc):
-        if code not in ('A', 'B', 'I', 'E'):
-            raise "unknown request code %s" % code
-        if code == 'B':
-            self.start = t
-            self.method, self.url = desc.strip().split()
-        elif code == "I":
-            self.t_recdinput = t
-            self.isize = desc.strip()
-        elif code == "A":
-            self.t_recdoutput = t
-            self.httpcode, self.osize = desc.strip().split()
-        elif code == 'E':
-            self.t_end = t
-            self.elapsed = int(self.t_end - self.start)
-
-    def isfinished(self):
-        return not self.elapsed == "I"
-
-    def prettystart(self):
-        if self.start is not None:
-            t = time.localtime(self.start)
-            return time.strftime('%Y-%m-%dT%H:%M:%S', t)
-        else:
-            return "NA"
-
-    def shortprettystart(self):
-        if self.start is not None:
-            t = time.localtime(self.start)
-            return time.strftime('%H:%M:%S', t)
-        else:
-            return "NA"
-
-    def win(self):
-        if self.t_recdinput is not None and self.start is not None:
-            return self.t_recdinput - self.start
-        else:
-            return "NA"
-
-    def wout(self):
-        if self.t_recdoutput is not None and self.t_recdinput is not None:
-            return self.t_recdoutput - self.t_recdinput
-        else:
-            return "NA"
-
-    def wend(self):
-        if self.t_end is not None and self.t_recdoutput is not None:
-            return self.t_end - self.t_recdoutput
-        else:
-            return "NA"
-
-    def endstage(self):
-        if self.t_end is not None:
-            stage = "E"
-        elif self.t_recdoutput is not None:
-            stage = "A"
-        elif self.t_recdinput is not None:
-            stage = "I"
-        else:
-            stage = "B"
-        return stage
-
-    def total(self):
-        stage = self.endstage()
-        if stage == "B": return 0
-        if stage == "I": return self.t_recdinput - self.start
-        if stage == "A": return self.t_recdoutput - self.start
-        if stage == "E": return self.elapsed
-
-    def prettyisize(self):
-        if self.isize is not None:
-            return self.isize
-        else:
-            return "NA"
-
-    def prettyosize(self):
-        if self.osize is not None:
-            return self.osize
-        else:
-            return "NA"
-
-    def prettyhttpcode(self):
-        if self.httpcode is not None:
-            return self.httpcode
-        else:
-            return "NA"
-
-    def __str__(self):
-        body = (
-            self.prettystart(), self.win(), self.wout(), self.wend(),
-            self.total(), self.endstage(), self.prettyosize(),
-            self.prettyhttpcode(), self.active, self.url
-            )
-        return self.fmt % body
-
-    fmt = "%19s %4s %4s %4s %3s %1s %7s %4s %4s %s"
-
-    def getheader(self):
-        body = ('Start', 'WIn', 'WOut', 'WEnd', 'Tot', 'S', 'OSize',
-                'Code', 'Act', 'URL')
-        return self.fmt % body
-
-class StartupRequest(Request):
-    def endstage(self):
-        return "U"
-
-    def total(self):
-        return 0
-
-class Cumulative:
-    def __init__(self, url):
-        self.url = url
-        self.times = []
-        self.hangs = 0
-        self.allelapsed = None
-
-    def put(self, request):
-        elapsed = request.elapsed
-        if elapsed == "I": self.hangs = self.hangs + 1
-        self.times.append(elapsed)
-
-    def all(self):
-        if self.allelapsed == None:
-            self.allelapsed = []
-            for elapsed in self.times:
-                self.allelapsed.append(elapsed)
-            self.allelapsed.sort()
-        return self.allelapsed
-
-    def __str__(self):
-        body = (
-            self.hangs, self.hits(), self.total(), self.max(), self.min(),
-            self.median(), self.mean(), self.url
-            )
-        return self.fmt % body
-
-    def getheader(self):
-        return self.fmt % ('Hangs', 'Hits', 'Total', 'Max', 'Min', 'Median',
-                           'Mean', 'URL')
-
-    fmt = "%5s %5s %5s %5s %5s %6s %5s %s"
-
-    def hits(self):
-        return len(self.times)
-
-    def max(self):
-        return max(self.all())
-
-    def min(self):
-        return min(self.all())
-
-    def mean(self):
-        l = len(self.times)
-        if l == 0:
-            return "I"
-        else:
-            t = self.total()
-            if t == "I": return "I"
-            return t/l
-
-    def median(self):
-        all = self.all()
-        l = len(all)
-        if l == 0:
-            return "I"
-        else:
-            if l == 1:
-                return all[0]
-            elif l % 2 != 0:
-                i = l/2 + 1
-                return all[i]
-            else:
-                i = l/2 - 1
-                i2 = i + 1
-                v1 = all[i]
-                v2 = all[i2]
-                if isinstance(v1, StringType) or isinstance(v2, StringType):
-                    return "I"
-                else: return (v1 + v2) / 2
-
-    def total(self):
-        t = 0
-        all = self.all()
-        for elapsed in all:
-            if elapsed == "I": continue
-            t = t + elapsed
-        return t
-
-def parsebigmlogline(line):
-    tup = line.split(None, 3)
-    if len(tup) == 3:
-        code, id, timestr = tup
-        return code, id, timestr, ''
-    elif len(tup) == 4:
-        return tup
-    else:
-        return None
-
-def get_earliest_file_data(files):
-    temp = {}
-    earliest_fromepoch = 0
-    earliest = None
-    retn = None
-    for file in files:
-        line = file.readline()
-        if not line:
-            continue
-        linelen = len(line)
-        line = line.strip()
-        tup = parsebigmlogline(line)
-        if tup is None:
-            print "Could not interpret line: %s" % line
-            continue
-        code, id, timestr, desc = tup
-        timestr = timestr.strip()
-        fromepoch = getdate(timestr)
-        temp[file] = linelen
-        if earliest_fromepoch == 0 or fromepoch < earliest_fromepoch:
-            earliest_fromepoch = fromepoch
-            earliest = file
-            retn = [code, id, fromepoch, desc]
-
-    for file, linelen in temp.items():
-        if file is not earliest:
-            file.seek(file.tell() - linelen)
-
-    return retn
-
-def get_requests(files, start=None, end=None, statsfname=None,
-                 writestats=None, readstats=None):
-    finished = []
-    unfinished = {}
-    if readstats:
-        fp = open(statsfname, 'r')
-        u = cPickle.Unpickler(fp)
-        requests = u.load()
-        fp.close()
-        del u
-        del fp
-    else:
-        while 1:
-            tup = get_earliest_file_data(files)
-            if tup is None:
-                break
-            code, id, fromepoch, desc = tup
-            if start is not None and fromepoch < start: continue
-            if end is not None and fromepoch > end: break
-            if code == 'U':
-                finished.extend(unfinished.values())
-                unfinished.clear()
-                request = StartupRequest()
-                request.url = desc
-                request.start = int(fromepoch)
-                finished.append(request)
-                continue
-            request = unfinished.get(id)
-            if request is None:
-                if code != "B": continue # garbage at beginning of file
-                request = Request()
-                for pending_req in unfinished.values():
-                    pending_req.active = pending_req.active + 1
-                unfinished[id] = request
-            t = int(fromepoch)
-            try:
-                request.put(code, t, desc)
-            except:
-                print "Unable to handle entry: %s %s %s"%(code, t, desc)
-            if request.isfinished():
-                del unfinished[id]
-                finished.append(request)
-
-        finished.extend(unfinished.values())
-        requests = finished
-
-        if writestats:
-            fp = open(statsfname, 'w')
-            p = cPickle.Pickler(fp)
-            p.dump(requests)
-            fp.close()
-            del p
-            del fp
-
-    return requests
-
-def analyze(requests, top, sortf, start=None, end=None, mode='cumulative',
-            resolution=60, urlfocusurl=None, urlfocustime=60):
-
-    if mode == 'cumulative':
-        cumulative = {}
-        for request in requests:
-            url = request.url
-            stats = cumulative.get(url)
-            if stats is None:
-                stats = Cumulative(url)
-                cumulative[url] = stats
-            stats.put(request)
-        requests = cumulative.values()
-        requests.sort(sortf)
-        write(requests, top)
-
-    elif mode=='timed':
-        computed_start = requests[0].start
-        computed_end = requests[-1].t_end
-        if start and end:
-            timewrite(requests,start,end,resolution)
-        if start and not end:
-            timewrite(requests,start,computed_end,resolution)
-        if end and not start:
-            timewrite(requests,computed_start,end,resolution)
-        if not end and not start:
-            timewrite(requests,computed_start,computed_end,resolution)
-
-    elif mode == 'urlfocus':
-        requests.sort(sortf)
-        urlfocuswrite(requests, urlfocusurl, urlfocustime)
-
-    else:
-        requests.sort(sortf)
-        write(requests, top)
-
-def urlfocuswrite(requests, url, t):
-    l = []
-    i = 0
-    for request in requests:
-        if request.url == url: l.append(i)
-        i = i + 1
-    before = {}
-    after = {}
-    x = 0
-    for n in l:
-        x = x + 1
-        r = requests[n]
-        start = r.start
-        earliest = start - t
-        latest = start + t
-        print 'URLs invoked %s seconds before and after %s (#%s, %s)' % \
-              (t, url, x, r.shortprettystart())
-        print '---'
-        i = -1
-        for request in requests:
-            i = i + 1
-            if request.start < earliest: continue
-            if request.start > latest: break
-            if n == i: # current request
-                print '%3d' % (request.start - start),
-                print '%s' % (request.shortprettystart()),
-                print request.url
-                continue
-            if request.start <= start:
-                if before.get(i):
-                    before[i] = before[i] + 1
-                else:
-                    before[i] = 1
-            if request.start > start:
-                if after.get(i):
-                    after[i] = after[i] + 1
-                else:
-                    after[i] = 1
-            print '%3d' % (request.start - start),
-            print '%s' % (request.shortprettystart()),
-            print request.url
-        print
-    print ('Summary of URLs invoked before (and at the same time as) %s '
-           '(times, url)' % url)
-    before = before.items()
-    before.sort()
-    for k,v in before:
-        print v, requests[k].url
-    print
-    print 'Summary of URLs invoked after %s (times, url)' % url
-    after = after.items()
-    after.sort()
-    for k,v in after:
-        print v, requests[k].url
-
-def write(requests, top=0):
-    if len(requests) == 0:
-        print "No data.\n"
-        return
-    i = 0
-    header = requests[0].getheader()
-    print header
-    for stat in requests:
-        i = i + 1
-        if verbose:
-            print str(stat)
-        else:
-            print str(stat)[:78]
-        if i == top:
-            break
-
-def getdate(val):
-    try:
-        val = val.strip()
-        year, month, day = int(val[:4]), int(val[5:7]), int(val[8:10])
-        hour,minute,second=int(val[11:13]),int(val[14:16]),int(val[17:19])
-        t = time.mktime((year, month, day, hour, minute, second, 0, 0, -1))
-        return t
-    except:
-        raise ProfileException, "bad date %s" % val
-
-def getTimeslice(period, utime):
-    low = int(math.floor(utime)) - period + 1
-    high = int(math.ceil(utime)) + 1
-    for x in range(low, high):
-        if x % period == 0:
-            return x
-
-def timewrite(requests, start, end, resolution):
-    print "Start: %s    End: %s   Resolution: %d secs" % \
-        (tick2str(start), tick2str(end), resolution)
-    print "-" * 78
-    print
-    print "Date/Time                #requests requests/second"
-
-    d = {}
-    max = 0
-    min = None
-    for r in requests:
-        t = r.start
-        slice = getTimeslice(resolution,t)
-        if slice > max: max = slice
-        if (min is None) or (slice < min): min = slice
-        if d.has_key(slice):
-            d[slice] = d[slice] + 1
-        else:
-            d[slice] = 1
-
-    num = 0
-    hits = 0
-    avg_requests = None
-    max_requests = 0
-    for slice in range(min, max, resolution):
-        num = d.get(slice, 0)
-        if num>max_requests: max_requests = num
-        hits = hits + num
-
-        if avg_requests is None:
-            avg_requests = num
-        else:
-            avg_requests = (avg_requests + num) / 2
-
-        s = tick2str(slice)
-        s = s + "     %6d         %4.2lf" % (num,num*1.0/resolution)
-        print s
-
-    print '='*78
-    print " Peak:                  %6d         %4.2lf" % \
-        (max_requests,max_requests*1.0/resolution)
-    print "  Avg:                  %6d         %4.2lf" % \
-        (avg_requests,avg_requests*1.0/resolution)
-    print "Total:                  %6d          n/a " % (hits)
-
-def tick2str(t):
-    return time.strftime('%Y-%m-%dT%H:%M:%S', time.localtime(t))
-
-def codesort(v1, v2):
-    v1 = v1.endstage()
-    v2 = v2.endstage()
-    if v1 == v2:
-        return 0
-
-    if v1 == "B":
-        return -1 # v1 is smaller than v2
-    if v1 == "I":
-        if v2 == "B": return 1 # v1 is larger than v2
-        else: return -1
-    if v1 == "A":
-        if v2 in ['B', 'I']: return 1
-        else: return -1
-    if v1 == "E":
-        return 1
-
-class Sort:
-    def __init__(self, fname, ascending=0):
-        self.fname = fname
-        self.ascending = ascending
-
-    def __call__(self, i1, i2):
-        f1 = getattr(i1, self.fname)
-        f2 = getattr(i2, self.fname)
-        if callable(f1): f1 = f1()
-        if callable(f2): f2 = f2()
-        if f1 < f2:
-            if self.ascending: return -1
-            else: return 1
-        elif f1 == f2:
-            return 0
-        else:
-            if self.ascending: return 1
-            else: return -1
-
-def detailedusage():
-    details = usage(0)
-    pname = sys.argv[0]
-    details = details + """
-Reports are of four types: cumulative, detailed, timed, or urlfocus.  The
-default is cumulative. Data is taken from one or more Zope detailed request
-logs (-M logs, aka 'big M' logs) or from a preprocessed statistics file.
-
-For cumulative reports, each line in the profile indicates information
-about a Zope method (URL) collected via a detailed request log.
-
-For detailed reports, each line in the profile indicates information about
-a single request.
-
-For timed reports, each line in the profile indicates informations about
-the number of requests and the number of requests/second for a period of time.
-
-For urlfocus reports, ad-hoc information about requests surrounding the
-specified url is given.
-
-Each 'filename' is a path to a '-M' log that contains detailed request data.
-Multiple input files can be analyzed at the same time by providing the path
-to each file.  (Analyzing  multiple big M log files at once is useful if you
-have more than one Zope client on a single machine and you'd like to
-get an overview of all Zope activity on that machine).
-
-If you wish to make multiple analysis runs against the same input data, you
-may want to use the --writestats option.  The --writestats option creates a
-file which holds preprocessed data representing the specfified input files.
-Subsequent runs (for example with a different sort spec) will be much
-faster if the  --readstats option is used to specify a preprocessed stats
-file instead of actual input files because the logfile parse step is skipped.
-
-If a 'sort' value is specified, sort the profile info by the spec.  The sort
-order is descending unless indicated.    The default cumulative sort spec is
-'total'.  The default detailed sort spec is 'start'.
-
-For cumulative reports, the following sort specs are accepted:
-
-  'hits'        -- the number of hits against the method
-  'hangs'       -- the number of unfinished requests to the method
-  'max'         -- the maximum time in secs taken by a request to this method
-  'min'         -- the minimum time in secs taken by a request to this method
-  'mean'        -- the mean time in secs taken by a request to this method
-  'median'      -- the median time in secs taken by a request to this method
-  'total'       -- the total time in secs across all requests to this method
-  'url'         -- the URL/method name (ascending)
-
-For detailed (non-cumulative) reports, the following sort specs are accepted:
-
-  'start'       -- the start time of the request to ZServer (ascending)
-  'win'         -- the num of secs ZServer spent waiting for input from client
-  'wout'        -- the secs ZServer spent waiting for output from ZPublisher
-  'wend'        -- the secs ZServer spent sending data to the client
-  'total'       -- the secs taken for the request from begin to end
-  'endstage'    -- the last successfully completed request stage (B, I, A, E)
-  'osize'       -- the size in bytes of output provided by ZPublisher
-  'httpcode'    -- the HTTP response code provided by ZPublisher (ascending)
-  'active'      -- total num of requests pending at the end of this request
-  'url'         -- the URL/method name (ascending)
-
-For timed and urlfocus reports, there are no sort specs allowed.
-
-If the 'top' argument is specified, only report on the top 'n' entries in
-the profile (as per the sort). The default is to show all data in the profile.
-
-If the 'verbose' argument is specified, do not trim url to fit into 80 cols.
-
-If the 'today' argument is specified, limit results to hits received today.
-
-If the 'daysago' argument is specified, limit results to hits received n days ago.
-
-The 'resolution' argument is used only for timed reports and specifies the
-number of seconds between consecutive lines in the report
-(default is 60 seconds).
-
-The 'urlfocustime' argument is used only for urlfocus reports and specifies the
-number of seconds to target before and after the URL provided in urlfocus mode.
-(default is 10 seconds).
-
-If the 'start' argument is specified in the form 'DD/MM/YYYY HH:MM:SS' (UTC),
-limit results to hits received after this date/time.
-
-If the 'end' argument is specified in the form 'DD/MM/YYYY HH:MM:SS' (UTC),
-limit results to hits received before this date/time.
-
-'start' and 'end' arguments are not honored when request stats are obtained
-via the --readstats argument.
-
-Examples:
-
-  %(pname)s debug.log
-
-    Show cumulative report statistics for information in the file 'debug.log',
-    by default sorted by 'total'.
-
-  %(pname)s debug.log --detailed
-
-    Show detailed report statistics sorted by 'start' (by default).
-
-  %(pname)s debug.log debug2.log --detailed
-
-    Show detailed report statistics for both logs sorted by 'start'
-    (by default).
-
-  %(pname)s debug.log --cumulative --sort=mean --today --verbose
-
-    Show cumulative report statistics sorted by mean for entries in the log
-    which happened today, and do not trim the URL in the resulting report.
-
-  %(pname)s debug.log --cumulative --sort=mean --daysago=3 --verbose
-
-    Show cumulative report statistics sorted by mean for entries in the log
-    which happened three days ago, and do not trim the URL in the resulting report.
-
-  %(pname)s debug.log --urlfocus='/manage_main' --urlfocustime=60
-
-    Show 'urlfocus' report which displays statistics about requests
-    surrounding the invocation of '/manage_main'.  Focus on the time periods
-    60 seconds before and after each invocation of the '/manage_main' URL.
-
-  %(pname)s debug.log --detailed --start='2001/05/10 06:00:00'
-    --end='2001/05/11 23:00:00'
-
-    Show detailed report statistics for entries in 'debug.log' which
-    begin after 6am UTC on May 10, 2001 and which end before
-    11pm UTC on May 11, 2001.
-
-  %(pname)s debug.log --timed --resolution=300 --start='2001/05/10 06:00:00'
-    --end='2001/05/11 23:00:00'
-
-    Show timed report statistics for entries in the log for one day
-    with a resolution of 5 minutes
-
-  %(pname)s debug.log --top=100 --sort=max
-
-    Show cumulative report of the the 'top' 100 methods sorted by maximum
-    elapsed time.
-
-  %(pname)s debug.log debug2.log --writestats='requests.stat'
-
-    Write stats file for debug.log and debug2.log into 'requests.stat' and
-    show default report.
-
-  %(pname)s --readstats='requests.stat' --detailed
-
-    Read from 'requests.stat' stats file (instead of actual -M log files)
-    and show detailed report against this data.""" % {'pname':pname}
-    return details
-
-def usage(basic=1):
-    usage = (
-        """
-Usage: %s filename1 [filename2 ...]
-          [--cumulative | --detailed | [--timed --resolution=seconds]]
-          [--sort=spec]
-          [--top=n]
-          [--verbose]
-          [--today | [--start=date] [--end=date] | --daysago=n ]
-          [--writestats=filename | --readstats=filename]
-          [--urlfocus=url]
-          [--urlfocustime=seconds]
-          [--help]
-
-Provides a profile of one or more Zope "-M" request log files.
-""" % sys.argv[0]
-        )
-    if basic == 1:
-        usage = usage + """
-If the --help argument is given, detailed usage docs are provided."""
-    return usage
-
-
-def main():
-    if len(sys.argv) == 1:
-        print usage()
-        sys.exit(0)
-    if sys.argv[1] == '--help': print detailedusage(); sys.exit(0)
-    mode = 'cumulative'
-    sortby = None
-    trim = 0
-    top = 0
-    verbose = 0
-    start = None
-    end = None
-    resolution=60
-    urlfocustime=10
-    urlfocusurl=None
-    statsfname = None
-    readstats = 0
-    writestats = 0
-
-    files = []
-    i = 1
-    for arg in sys.argv[1:]:
-        if arg[:2] != '--':
-            if arg[-3:] == '.gz' and globals().has_key('gzip'):
-                files.append(gzip.GzipFile(arg,'r'))
-            else:
-                files.append(open(arg))
-            sys.argv.remove(arg)
-            i = i + 1
-
-    try:
-        opts, extra = getopt.getopt(
-            sys.argv[1:], '', ['sort=', 'top=', 'help', 'verbose', 'today',
-                               'cumulative', 'detailed', 'timed','start=',
-                               'end=','resolution=', 'writestats=','daysago=',
-                               'readstats=','urlfocus=','urlfocustime=']
-            )
-        for opt, val in opts:
-
-            if opt=='--readstats':
-                statsfname = val
-                readstats = 1
-            elif opt=='--writestats':
-                statsfname = val
-                writestats = 1
-            if opt=='--sort': sortby = val
-            if opt=='--top': top=int(val)
-            if opt=='--help': print detailedusage(); sys.exit(0)
-            if opt=='--verbose':
-                verbose = 1
-            if opt=='--resolution':
-                resolution=int(val)
-            if opt=='--today':
-                now = time.localtime(time.time())
-                # for testing - now = (2001, 04, 19, 0, 0, 0, 0, 0, -1)
-                start = list(now)
-                start[3] = start[4] = start[5] = 0
-                start = time.mktime(start)
-                end = list(now)
-                end[3] = 23; end[4] = 59; end[5] = 59
-                end = time.mktime(end)
-            if opt=='--daysago':
-                now = time.localtime(time.time() - int(val)*3600*24 )
-                # for testing - now = (2001, 04, 19, 0, 0, 0, 0, 0, -1)
-                start = list(now)
-                start[3] = start[4] = start[5] = 0
-                start = time.mktime(start)
-                end = list(now)
-                end[3] = 23; end[4] = 59; end[5] = 59
-                end = time.mktime(end)
-            if opt=='--start':
-                start = getdate(val)
-            if opt=='--end':
-                end = getdate(val)
-            if opt=='--detailed':
-                mode='detailed'
-                d_sortby = sortby
-            if opt=='--cumulative':
-                mode='cumulative'
-            if opt=='--timed':
-                mode='timed'
-            if opt=='--urlfocus':
-                mode='urlfocus'
-                urlfocusurl = val
-            if opt=='--urlfocustime':
-                urlfocustime=int(val)
-
-        validcumsorts = ['url', 'hits', 'hangs', 'max', 'min', 'median',
-                         'mean', 'total']
-        validdetsorts = ['start', 'win', 'wout', 'wend', 'total',
-                         'endstage', 'isize', 'osize', 'httpcode',
-                         'active', 'url']
-
-        if mode == 'cumulative':
-            if sortby is None: sortby = 'total'
-            assert sortby in validcumsorts, (sortby, mode, validcumsorts)
-            if sortby in ['url']:
-                sortf = Sort(sortby, ascending=1)
-            else:
-                sortf = Sort(sortby)
-        elif mode == 'detailed':
-            if sortby is None: sortby = 'start'
-            assert sortby in validdetsorts, (sortby, mode, validdetsorts)
-            if sortby in ['start', 'url', 'httpcode']:
-                sortf = Sort(sortby, ascending=1)
-            elif sortby == 'endstage':
-                sortf = codesort
-            else:
-                sortf = Sort(sortby)
-        elif mode=='timed':
-            sortf = None
-        elif mode=='urlfocus':
-            sortf = Sort('start', ascending=1)
-        else:
-            raise 'Invalid mode'
-
-        req=get_requests(files, start, end, statsfname, writestats, readstats)
-        analyze(req, top, sortf, start, end, mode, resolution, urlfocusurl,
-                urlfocustime)
-
-    except AssertionError, val:
-        a = "%s is not a valid %s sort spec, use one of %s"
-        print a % (val[0], val[1], val[2])
-        sys.exit(0)
-    except getopt.error, val:
-        print val
-        sys.exit(0)
-    except ProfileException, val:
-        print val
-        sys.exit(0)
-    except SystemExit:
-        sys.exit(0)
-    except:
-        import traceback
-        traceback.print_exc()
-        print usage()
-        sys.exit(0)
-
-if __name__ == '__main__':
-    main()

Deleted: Zope/trunk/src/Zope2/utilities/tracelog.py
===================================================================
--- Zope/trunk/src/Zope2/utilities/tracelog.py	2011-07-02 15:52:15 UTC (rev 122057)
+++ Zope/trunk/src/Zope2/utilities/tracelog.py	2011-07-02 15:58:58 UTC (rev 122058)
@@ -1,453 +0,0 @@
-##############################################################################
-#
-# Copyright (c) 2006 Zope Foundation and Contributors.
-# All Rights Reserved.
-#
-# This software is subject to the provisions of the Zope Public License,
-# Version 2.1 (ZPL).  A copy of the ZPL should accompany this distribution.
-# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
-# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
-# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
-# FOR A PARTICULAR PURPOSE.
-#
-##############################################################################
-"""Yet another lag analysis tool
-"""
-
-import datetime, optparse, sys
-
-class Request:
-
-    output_bytes = '-'
-
-    def __init__(self, start, method, url):
-        self.method = method
-        self.url = url
-        self.start = start
-        self.state = 'input'
-
-    def I(self, input_time, input_bytes):
-        self.input_time = input_time
-        self.input_bytes = input_bytes
-        self.state = 'app'
-
-    def A(self, app_time, response, output_bytes):
-        self.app_time = app_time
-        self.response = response
-        self.output_bytes = output_bytes
-        self.state = 'output'
-
-    def E(self, end):
-        self.end = end
-
-    @property
-    def app_seconds(self):
-        return (self.app_time - self.input_time).seconds
-
-    @property
-    def total_seconds(self):
-        return (self.end - self.start).seconds
-
-class Times:
-
-    tid = 1l
-
-    def __init__(self):
-        self.times = []
-        self.hangs = 0
-        Times.tid += 1
-        self.tid = Times.tid # generate a unique id
-
-    def finished(self, request):
-        self.times.append(request.app_seconds)
-
-    def hung(self):
-        self.hangs += 1
-
-    def impact(self):
-        times = self.times
-        if not times:
-            self.median = self.mean = self.impact = 0
-            return 0
-        self.times.sort()
-        n = len(times) 
-        if n % 2:
-            m = times[(n+1)/2-1]
-        else:
-            m = .5 * (times[n/2]+times[n/2-1])
-        self.median = m
-        self.mean = sum(times)/n
-        self.impact = self.mean * (n+self.hangs)
-        return self.impact
-
-    def __str__(self):
-        times = self.times
-        if not times:
-            return "              0                             %5d" % (
-                self.hangs)
-            
-        n = len(times)
-        m = self.median
-        return "%9.1f %5d %6.0f %6.2f %6.2f %6.0f %5d" % (
-            self.impact, n, times[0], m, self.mean, times[-1], self.hangs)
-
-    def html(self):
-        times = self.times
-        if not times:
-            print td('', 0, '', '', '', '', self.hangs)
-        else:
-            n = len(times)
-            m = self.median
-            impact = '<a name="u%s">%s' % (self.tid, self.impact)
-            print td(impact, n, times[0], m, self.mean, times[-1],
-                     self.hangs)
-
-def parsedt(s):
-    date, time = s.split('T')
-    return datetime.datetime(*(
-        map(int, date.split('-')) 
-        +
-        map(int, time.split(':'))         
-        ))
-
-def main(args=None):
-    if args is None:
-        args = sys.argv[1:]
-
-    options, args = parser.parse_args(args)
-    if options.event_log:
-        restarts = find_restarts(options.event_log)
-    else:
-        restarts = []
-
-    restarts.append(datetime.datetime.utcnow()+datetime.timedelta(1000))
-
-    if options.html:
-        print_app_requests = print_app_requests_html
-        output_minute = output_minute_html
-        output_stats = output_stats_html
-        minutes_header = minutes_header_html
-        minutes_footer = minutes_footer_html
-        print '<html title="trace log statistics"><body>'
-    else:
-        print_app_requests = print_app_requests_text
-        output_minute = output_minute_text
-        output_stats = output_stats_text
-        minutes_header = minutes_header_text
-        minutes_footer = minutes_footer_text
-        
-        
-    urls = {}
-    [file] = args
-    lmin = ldt = None
-    requests = {}
-    input = apps = output = n = 0
-    spr = spa = 0.0
-    restart = restarts.pop(0)
-    minutes_header()
-    remove_prefix = options.remove_prefix
-    for record in open(file):
-        record = record.split()
-        typ, rid, dt = record[:3]
-        min = dt[:-3]
-        dt = parsedt(dt)
-        if dt == restart:
-            continue
-        while dt > restart:
-            print_app_requests(requests, ldt,
-                               options.old_requests,
-                               options.app_requests,
-                               urls,
-                               "\nLeft over:")
-            record_hung(urls, requests)
-            requests = {}
-            input = apps = output = n = 0
-            spr = spa = 0.0
-            restart = restarts.pop(0)
-        ldt = dt
-        
-        if min != lmin:
-            if lmin is not None:
-                output_minute(lmin, requests, input, apps, output, n, spr, spa)
-                if apps > options.apps:
-                    print_app_requests(requests, dt,
-                                       options.old_requests,
-                                       options.app_requests,
-                                       urls,
-                                       )
-            lmin = min
-            spr = 0.0
-            spa = 0.0
-            n = 0
-            
-        if typ == 'B':
-            if rid in requests:
-                request = requests[rid]
-                if request.state == 'output':
-                    output -= 1
-                elif request.state == 'app':
-                    apps -= 1
-                else:
-                    input -= 1
-            
-            input += 1
-            request = Request(dt, *record[3:5])
-            if remove_prefix and request.url.startswith(remove_prefix):
-                request.url = request.url[len(remove_prefix):]
-            requests[rid] = request
-            times = urls.get(request.url)
-            if times is None:
-                times = urls[request.url] = Times()
-        elif typ == 'I':
-            if rid in requests:
-                input -= 1
-                apps += 1
-                requests[rid].I(dt, record[3])
-        elif typ == 'A':
-            if rid in requests:
-                apps -= 1
-                output += 1
-                requests[rid].A(dt, *record[3:5])
-        elif typ == 'E':
-            if rid in requests:
-                output -= 1
-                request = requests.pop(rid)
-                request.E(dt)
-                spr += request.total_seconds
-                spa += request.app_seconds
-                n += 1
-                times = urls[request.url]
-                times.finished(request)
-        else:
-            print 'WTF', record
-
-    print_app_requests(requests, dt,
-                       options.old_requests,
-                       options.app_requests,
-                       urls,
-                       "Left over:")
-
-    minutes_footer()
-
-    output_stats(urls)
-
-    if options.html:
-        print '</body></html>'
-
-def output_stats_text(urls):
-    print
-    print 'URL statistics:'
-    print "   Impact count    min median   mean    max hangs"
-    print "========= ===== ====== ====== ====== ====== ====="
-    urls = [(times.impact(), url, times)
-            for (url, times) in urls.iteritems()
-            ]
-    urls.sort()
-    urls.reverse()
-    for (_, url, times) in urls:
-        if times.impact > 0 or times.hangs:
-            print times, url
-
-def output_stats_html(urls):
-    print
-    print 'URL statistics:'
-    print '<table border="1">'
-    print '<tr><th>Impact</th><th>count</th><th>min</th>'
-    print     '<th>median</th><th>mean</th><th>max</th><th>hangs</th></tr>'
-    urls = [(times.impact(), url, times)
-            for (url, times) in urls.iteritems()
-            ]
-    urls.sort()
-    urls.reverse()
-    for (_, url, times) in urls:
-        if times.impact > 0 or times.hangs:
-            print '<tr>'
-            times.html()
-            print td(url)
-            print '</tr>'
-    print '</table>'
-
-def minutes_header_text():
-    print
-    print "          minute   req input   app output"
-    print "================ ===== ===== ===== ======"
-
-def minutes_footer_text():
-    print
-
-def minutes_header_html():
-    print '<table border="2">'
-    print "<tr>"
-    print '<th>Minute</th>'
-    print '<th>Requests</th>'
-    print '<th>Requests inputing</th>'
-    print '<th>Requests executing or waiting</th>'
-    print '<th>Requests outputing</th>'
-    print '<th>Requests completed</th>'
-    print '<th>Mean Seconds Per Request Total</th>'
-    print '<th>Mean Seconds Per Request in App</th>'
-    print "</tr>"
-
-def minutes_footer_html():
-    print '</table>'
-
-def output_minute_text(lmin, requests, input, apps, output, n, spr, spa):
-    print lmin.replace('T', ' '), "%5d I=%3d A=%3d O=%5d " % (
-        len(requests), input, apps, output),
-    if n:
-        print "N=%4d %10.2f %10.2f" % (n, spr/n, spa/n)
-    else:
-        print
-
-def td(*values):
-    return ''.join([("<td>%s</td>" % s) for s in values])
-
-def output_minute_html(lmin, requests, input, apps, output, n, spr, spa):
-    print '<tr>'
-    apps = '<font size="+2"><strong>%s</strong></font>' % apps
-    print td(lmin.replace('T', ' '), len(requests), input, apps, output)
-    if n:
-        print td(n, "%10.2f" % (spr/n), "%10.2f" % (spa/n))
-    print '</tr>'
-
-def find_restarts(event_log):
-    result = []
-    for l in open(event_log):
-        if l.strip().endswith("Zope Ready to handle requests"):
-            result.append(parsedt(l.split()[0]))
-    return result
-
-def record_hung(urls, requests):
-    for request in requests.itervalues():
-        times = urls.get(request.url)
-        if times is None:
-            times = urls[request.url] = Times()
-        times.hung()
-
-def print_app_requests_text(requests, dt, min_seconds, max_requests, urls,
-                       label=''):
-    requests = [
-        ((dt-request.input_time).seconds, request)
-        for request in requests.values()
-        if request.state == 'app'
-    ]
-
-    urls = {}
-    for s, request in requests:
-        urls[request.url] = urls.get(request.url, 0) + 1
-    
-    requests.sort()
-    requests.reverse()
-    for s, request in requests[:max_requests]:
-        if s < min_seconds:
-            continue
-        if label:
-            print label
-            label = ''
-        url = request.url
-        repeat = urls[url]
-        if repeat > 1:
-            print s, "R=%d" % repeat, url
-        else:
-            print s, url
-
-def print_app_requests_html(requests, dt, min_seconds, max_requests, allurls,
-                            label=''):
-    requests = [
-        ((dt-request.input_time).seconds, request)
-        for request in requests.values()
-        if request.state == 'app'
-    ]
-
-    urls = {}
-    for s, request in requests:
-        urls[request.url] = urls.get(request.url, 0) + 1
-    
-    requests.sort()
-    requests.reverse()
-    printed = False
-    for s, request in requests[:max_requests]:
-        if s < min_seconds:
-            continue
-        if label:
-            print label
-            label = ''
-        if not printed:
-            minutes_footer_html()
-            print '<table border="1">'
-            print '<tr><th>age</th><th>R</th><th>url</th></tr>'
-            printed = True
-        url = request.url
-        repeat = urls[url]
-        print '<tr>'
-        if repeat <= 1:
-            repeat = ''
-        url = '<a href="#u%s">%s</a>' % (allurls[url].tid, url)
-        print td(s, repeat, url)
-        print '</tr>'
-
-    if printed:
-        print '</table>'
-        minutes_header_html()
-
-parser = optparse.OptionParser("""
-Usage: %prog [options] trace_log_file
-
-Output trace log data showing:
-
-- number of active requests,
-- number of input requests (requests gathering input),
-- number of application requests,
-- number of output requests,
-- number of requests completed in the minute shown,
-- mean seconds per request and
-- mean application seconds per request.
-
-Note that we don't seem to be logging when a connection to the client
-is broken, so the number of active requests, and especially the number
-of outputing requests tends to grow over time. This is spurious.
-
-Also, note that, unfortunately, application requests include requests
-that are running in application threads and requests waiting to get an
-application thread.
-
-When application threads get above the app request threshold, then we
-show the requests that have been waiting the longest.
-
-""")
-
-parser.add_option("--app-request-threshold", "-a", dest='apps',
-                  type="int", default=10,
-                  help="""
-Number of pending application requests at which detailed request information
-if printed.
-""")
-parser.add_option("--app-requests", "-r", dest='app_requests',
-                  type="int", default=10,
-                  help="""
-How many requests to show when the maximum number of pending
-apps is exceeded.
-""")
-parser.add_option("--old-requests", "-o", dest='old_requests',
-                  type="int", default=10,
-                  help="""
-Number of seconds beyond which a request is considered old.
-""")
-parser.add_option("--event-log", "-e", dest='event_log',
-                  help="""
-The name of an event log that goes with the trace log.  This is used
-to determine when the server is restarted, so that the running trace data structures can be reinitialized.
-""")
-parser.add_option("--html", dest='html', action='store_true',
-                  help="""
-Generate HTML output.
-""")
-parser.add_option("--remove-prefix", dest='remove_prefix',
-                  help="""
-A prefex to be removed from URLS.
-""")
-                  
-
-            
-if __name__ == '__main__':
-    main()



More information about the Zope-Checkins mailing list