[Checkins] SVN: Sandbox/ctheune/testsummarizer/ Add test summarizer code as send to me by Stefan Holek.
Christian Theune
ct at gocept.com
Tue Aug 24 02:24:29 EDT 2010
Log message for revision 115903:
Add test summarizer code as send to me by Stefan Holek.
Changed:
A Sandbox/ctheune/testsummarizer/
A Sandbox/ctheune/testsummarizer/summarizer.py
-=-
Added: Sandbox/ctheune/testsummarizer/summarizer.py
===================================================================
--- Sandbox/ctheune/testsummarizer/summarizer.py (rev 0)
+++ Sandbox/ctheune/testsummarizer/summarizer.py 2010-08-24 06:24:28 UTC (rev 115903)
@@ -0,0 +1,354 @@
+#!/usr/bin/env python
+
+##############################################################################
+#
+# Copyright (c) 2003 Zope Corporation and Contributors.
+# All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.0 (ZPL). A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE.
+#
+##############################################################################
+"""Script to check pipermail archive for recent messages, and post a summary.
+
+$Id: list_summarizer.py,v 1.3 2009-07-16 17:41:33 stefan Exp $
+"""
+
+import sys
+import getopt
+import urllib2
+import re
+import datetime
+import pprint
+import StringIO
+import rfc822
+import smtplib
+from email.MIMEText import MIMEText
+from email.Utils import parseaddr
+
+from config import *
+
+__metaclass__ = type
+
+# Settings used by the script. You'll want to customize some of these.
+archive_url = 'http://mail.zope.org/pipermail/zope-tests/'
+archive_user = 'fred'
+archive_pass = 'secret'
+mailfrom = 'Zope tests summarizer <zopetests at z3u.com>'
+mailto = 'Zope-dev list <zope-dev at zope.org>'
+smtpserver = 'mail.z3u.com'
+
+# used when debugging
+print_not_email = True
+
+months = ("January February March April May June July August September "
+ "October November December").split()
+
+
+def create_subject_regex():
+ """Create a regex that parses subjects like these:
+
+ OK -- anything at all -- Description
+ FAIL -- anything -- Descrcription
+ FAIL (failures=1) -- anything -- description
+ FAIL (errors=10) -- anything -- description
+ FAIL (errors=23, failures=3) -- anything -- description
+ FAIL (failures=3, errors=2) -- anything -- description
+ FAIL (errors=23 failures=3) -- anything -- description
+ FAIL (failures=3 errors=2) -- anything -- description
+ FAIL (errors:23, failures:3) -- anything -- description
+ FAIL (failures:3, errors:2) -- anything -- description
+ FAILED (errors=1): Test ZODB MVCC / Python 2.3 / Linux
+ OK: Test Zope 2.7 / Python 2.3 / Linux
+
+ TODO: Write these examples as a DocTest.
+ """
+ ok_or_fail = r"(?P<success>OK|FAIL(ED)?)"
+ failures_errors = (r"failures[=:](?P<failures1>\d+)"
+ r"(,?\s*errors[=:](?P<errors1>\d+))?")
+ errors_failures = (r"errors[=:](?P<errors2>\d+)"
+ r"(,?\s*failures[=:](?P<failures2>\d+))?")
+ success = (r"%(ok_or_fail)s"
+ r"(\s*[(]"
+ r"(%(failures_errors)s|%(errors_failures)s)"
+ r"[)])?"
+ ) % vars()
+ anything = r"(?P<anything>.*?)"
+ description = r"(?P<description1>.*?)"
+
+ full_regex = (r"^%(success)s"
+ r":?\s+(?P<description2>.*)|"
+ r"\s+--\s+"
+ r"%(anything)s"
+ r"\s+--\s+"
+ r"%(description)s$"
+ ) % vars()
+ return re.compile(full_regex)
+
+subject_regex = create_subject_regex()
+
+
+def get_archive(year, month):
+ """Returns a list of the URLs archived for the given year and month.
+
+ If there is nothing at the appropriate URL for that year and month,
+ returns an empty list.
+ """
+ if not (1 <= month <= 12):
+ raise ValueError, month
+ stem = archive_url + ('%s-%s' % (year, months[month-1]))
+ url = '%s/date.html' % stem
+ try:
+ f = urllib2.urlopen(url)
+ except urllib2.HTTPError, eee:
+ if eee.code == 404:
+ return []
+ else:
+ raise
+ data = f.read()
+ results = re.compile(r'(\d{6}.html)', re.M).findall(data)
+ return ['%s/%s' % (stem, result) for result in results]
+
+
+class Message:
+ """Represents a single message, scraped from the mail archive."""
+
+ def __init__(self, url, datetext, subject, fromaddr):
+ self.url = url
+ self.datetext = datetext
+ self.date = datetime.datetime.utcfromtimestamp(
+ rfc822.mktime_tz(rfc822.parsedate_tz(self.datetext))
+ )
+ self.fromaddr = fromaddr
+ self.subject = ' '.join(subject.split())
+ subject_search = subject_regex.search(self.subject)
+ self.subjectparsed = bool(subject_search)
+ if subject_search:
+ groups = subject_search.groupdict()
+ self.failed = groups['success'] != 'OK'
+ self.description = (groups['description1'] or
+ groups['description2'])
+ self.anything = groups['anything']
+ self.failures = int(groups['failures1'] or
+ groups['failures2'] or 0)
+ self.errors = int(groups['errors1'] or groups['errors2'] or 0)
+
+ def printToStream(self, stream):
+ print >>stream, "Subject: %s" % self.subject
+ print >>stream, "From: %s" % self.fromaddr
+ print >>stream, "Date: %s" % self.datetext
+ print >>stream, "URL: %s" % self.url
+
+
+def get_message(url):
+ """Returns a Message object from the message archived at the given URL."""
+ f = urllib2.urlopen(url)
+ data = f.read()
+
+ # Although the data on the web has lower-case tag names, for some reason
+ # these become upper-cased when retrieved using urllib2.
+
+ # There should be only one date, between <I> tags.
+ dates = re.compile(r'<I>([^<]*)</I>', re.M|re.I).findall(data)
+ if len(dates) != 1:
+ print "ERROR", dates
+ if not dates:
+ raise RuntimeError('Cannot find date')
+ datetext = dates[0]
+
+ # The subject and from-address should look like this:
+ # <H1>[Zope-tests] subject line</H1> <B>from address</B>
+ subjects = re.compile(r'<H1>\[%s\] ([^<]*)</H1>\s*'
+ '<B>([^>]*)</B>' % list_name,
+ re.M|re.I).findall(data)
+ if len(subjects) != 1:
+ print "ERROR", subjects
+ if subjects:
+ subject, fromaddr = subjects[0]
+ else:
+ subject, fromaddr = ['ERROR IN TEST AGGREGATOR'] * 2
+ else:
+ subject, fromaddr = subjects[0]
+ return Message(url, datetext, subject, fromaddr)
+
+
+def monthMinusOne(year, month):
+ """Takes a year and a 1-based month.
+
+ Returns as a two-tuple (year, month) the year and 1-based month of
+ the previous month.
+ """
+ if not (1 <= month <= 12):
+ raise ValueError, month
+ months = year * 12 + month - 1
+ y, m = divmod(months - 1, 12)
+ return y, m + 1
+
+
+def err_exit(msg, rc=1):
+ """Bails out."""
+ print >>sys.stderr, msg
+ sys.exit(rc)
+
+
+def main(argv):
+ """Do the work!
+
+ Get the list of URLs, get the appropriate messages, compose an email,
+ send it to the mailing list.
+ """
+ usage = 'Usage: list_summarizer.py -C zope|cmf|plone [-T isodate] [-D]'
+ selected_config = ''
+ selected_date = ''
+ debug_mode = 0
+
+ try:
+ options, arg = getopt.getopt(argv, 'hC:T:D')
+ except getopt.GetoptError, e:
+ err_exit('%s\n%s' % (e.msg, usage))
+
+ for name, value in options:
+ if name == '-C':
+ selected_config = value.strip()+'_summarizer'
+ elif name == '-T':
+ selected_date = value.strip()
+ elif name == '-D':
+ debug_mode = 1
+ elif name == '-h':
+ err_exit(usage, 0)
+ else:
+ err_exit(usage)
+
+ if not configs.has_key(selected_config):
+ err_exit(usage)
+
+ config = configs[selected_config]
+ globals().update(config)
+
+ if debug_mode:
+ global mailto
+ mailto = debug_mailto
+
+ # All dates used are naive dates (no explicit tz).
+ now = datetime.datetime.utcnow()
+ now = now.replace(second=0)
+
+ if selected_date:
+ date = selected_date.replace('-', '')
+ y = int(date[0:4])
+ m = int(date[4:6])
+ d = int(date[6:8])
+ now = now.replace(year=y, month=m, day=d)
+
+ this_month_urls = get_archive(now.year, now.month)
+ last_month_year, last_month = monthMinusOne(now.year, now.month)
+ last_month_urls = get_archive(last_month_year, last_month)
+
+ # urls is a list of urls for this month an last month, most recent first.
+ urls = last_month_urls + this_month_urls
+ urls.reverse()
+
+ yesterday = now - datetime.timedelta(days=1)
+ tomorrow = now + datetime.timedelta(days=1)
+
+ # Get messages starting at the most recent message, and stopping when
+ # we run out of messages or when we get a message that was posted more
+ # than one day ago.
+ messages = []
+ for url in urls:
+ message = get_message(url)
+ if message.date >= tomorrow:
+ continue
+ if message.date < yesterday:
+ break
+ messages.append(message)
+
+ out = StringIO.StringIO()
+
+ print >>out, "Summary of messages to the %s list." % list_name
+ print >>out, "Period %s UTC to %s UTC." % (yesterday.ctime(), now.ctime())
+
+ # Nicely format the number of messages, and a summary of whom they are
+ # from.
+ if len(messages) == 0:
+ print >>out, "There were no messages."
+ elif len(messages) == 1:
+ print >>out, "There was 1 message, from %s" % messages[0].fromaddr
+ else:
+ print >>out, "There were %s messages:" % len(messages),
+ msgcount = {}
+ for message in messages:
+ addr = message.fromaddr
+ msgcount[addr] = msgcount.get(addr, 0) + 1
+ fromaddrs = msgcount.keys()
+ fromaddrs.sort()
+ print >>out, ', '.join(
+ ['%s from %s' % (msgcount[addr], addr) for addr in fromaddrs])+'.'
+
+ print >>out
+
+ # We want the messages to be oldest first, so reverse them.
+ messages.reverse()
+
+ fail_messages = []
+ ok_messages = []
+ unknown_messages = []
+
+ for message in messages:
+ if message.subjectparsed:
+ if message.failed:
+ fail_messages.append(message)
+ else:
+ ok_messages.append(message)
+ else:
+ unknown_messages.append(message)
+
+ def print_messages(title, message_list):
+ if message_list:
+ print >>out
+ print >>out, title
+ print >>out, '-' * len(title)
+ print >>out
+ for message in message_list:
+ message.printToStream(out)
+ print >>out
+
+ print_messages('Test failures', fail_messages)
+ print_messages('Unknown', unknown_messages)
+ print_messages('Tests passed OK', ok_messages)
+
+ subject_info = ['%s %s' % (val, txt)
+ for val, txt in (len(ok_messages), 'OK'),
+ (len(fail_messages), 'Failed'),
+ (len(unknown_messages), 'Unknown')
+ if val
+ ]
+ if not subject_info:
+ subject_info = ['No messages']
+ subject = '%s%s' % (subject_prefix, ', '.join(subject_info))
+
+ if print_not_email:
+ print "Not sending this email."
+ print
+ print "Subject:", subject
+ print "From:", mailfrom
+ print "To:", mailto
+ print
+ print out.getvalue()
+ else:
+ body = "From: %s\nTo: %s\nSubject: %s\n\n%s" % (mailfrom, mailto, subject, out.getvalue())
+
+ fromname, fromaddr = parseaddr(mailfrom)
+ toname, toaddr = parseaddr(mailto)
+
+ s = smtplib.SMTP(mailhost, mailport)
+ s.sendmail(fromaddr, toaddr, body)
+ s.quit()
+
+
+if __name__ == '__main__':
+ main(sys.argv[1:])
Property changes on: Sandbox/ctheune/testsummarizer/summarizer.py
___________________________________________________________________
Added: svn:eol-style
+ native
More information about the checkins
mailing list