[Checkins] SVN: zopetoolkit/doc/src/ztksphinx.py Query buildbots in parallel.

[Marius Gedminas]

Log message for revision 117372:
  Query buildbots in parallel.
  
  Helps somewhat (28s -> 17s), but not too much: winbot.zope.org takes 17s
  to process all XML-RPC requests.  It doesn't matter if they're sequential or
  concurrent; looks like Twisted serializes them all on the server side.
  
  http://winbot.zope.org/builders returns all build status in 0.6 seconds,
  so switching to screen-scraping HTML would be a win, latency-wise.
  
  

Changed:
  U   zopetoolkit/doc/src/ztksphinx.py

-=-
Modified: zopetoolkit/doc/src/ztksphinx.py
===================================================================
--- zopetoolkit/doc/src/ztksphinx.py	2010-10-07 18:51:40 UTC (rev 117371)
+++ zopetoolkit/doc/src/ztksphinx.py	2010-10-07 19:03:38 UTC (rev 117372)
@@ -1,11 +1,13 @@
-from random import random
 from docutils import nodes
-from sphinx.util.compat import Directive, make_admonition
+from sphinx.util.compat import Directive
 from xmlrpclib import ServerProxy
 
+import time
 import urllib
 import socket
+import threading
 
+
 def setup(app):
     app.add_config_value('buildbot_check', True, 'html')
     app.add_node(BuildbotColor,
@@ -13,6 +15,7 @@
     app.connect('doctree-resolved', process_buildbot_nodes)
     app.add_directive('buildbotresult', BuildbotDirective)
 
+
 class BuildbotDirective(Directive):
 
     has_content = True
@@ -29,37 +32,82 @@
 
         return [targetnode]
 
+
 def process_buildbot_nodes(app, doctree, fromdocname):
-    if app.config.buildbot_check:
-        socket_timeout = socket.getdefaulttimeout()
-        try:
-            socket.setdefaulttimeout(min(socket_timeout, 5))
-            for node in doctree.traverse(BuildbotColor):
-                buildbot_url = node.buildbot_url
-                try:
-                    buildbot_result = getBuildbotResult(buildbot_url)
-                except Exception, e:
-                    node.css_class = 'tests_could_not_determine'
-                    node.title = '%s: %s' % (e.__class__.__name__, e)
-                else:
-                    if buildbot_result:
-                        node.css_class = 'tests_passed'
-                    else:
-                        node.css_class = 'tests_not_passed'
-        finally:
-            socket.setdefaulttimeout(socket_timeout)
+    if not app.config.buildbot_check:
+        return
+    socket_timeout = socket.getdefaulttimeout()
+    try:
+        socket.setdefaulttimeout(min(socket_timeout, 5))
+        by_url = {}
+        for node in doctree.traverse(BuildbotColor):
+            url, builder = parse_builder_url(node.buildbot_url)
+            by_url.setdefault(url, []).append((node, builder))
+        jobs = []
+        for url, nodes in by_url.items():
+            thread = threading.Thread(target=update_buildbot_nodes,
+                                      args=(url, nodes),
+                                      name=url)
+            thread.start()
+            jobs.append(thread)
+        for thread in jobs:
+            thread.join()
+    finally:
+        socket.setdefaulttimeout(socket_timeout)
 
-def getBuildbotResult(url):
-    # url should end with '/buildbot/builders/$buildername'
+
+def parse_builder_url(url):
+    """Parse a builder URL into buildbot URL and builder name."""
     url = url.rstrip('/') # make sure trailing slashes don't cause failures
     xmlrpc_url = '/'.join(url.split('/')[:-2] + ['xmlrpc'])
     builder = urllib.unquote(url.split('/')[-1])
-    xmlrpc = ServerProxy(xmlrpc_url)
-    return xmlrpc.getLastBuildResults(builder) == 'success'
+    return xmlrpc_url, builder
 
+
+def update_buildbot_nodes(url, nodes_and_builders):
+    """Get build status of a number of builders and update document nodes.
+
+    ``nodes_and_builders`` is a list of tuples (node, builder_name).
+    """
+    results = get_buildbot_results(url, [b for n, b in nodes_and_builders])
+    for node, builder in nodes_and_builders:
+        result = results[builder]
+        if isinstance(result, Exception):
+            node.css_class = 'tests_could_not_determine'
+            node.title = '%s: %s' % (result.__class__.__name__, result)
+        elif result:
+            node.css_class = 'tests_passed'
+        else:
+            node.css_class = 'tests_not_passed'
+
+
+def get_buildbot_results(xmlrpc_url, builders):
+    """Return build status of a number of builders.
+
+    ``builders`` is a list of builder names.
+
+    Returns a dictionary mapping builder names to True/False or exception
+    objects, in case of errors.
+    """
+    start = time.time()
+    try:
+        xmlrpc = ServerProxy(xmlrpc_url)
+    except Exception, e:
+        return dict.fromkeys(builders, e)
+    results = {}
+    for builder in builders:
+        try:
+            results[builder] = (xmlrpc.getLastBuildResults(builder) == 'success')
+        except Exception, e:
+            results[builder] = e
+    print xmlrpc_url, '%.3fs' % (time.time() - start)
+    return results
+
+
 class BuildbotColor(nodes.Inline, nodes.TextElement):
     pass
 
+
 def visit_buildbot_node(self, node):
     kwargs = {'href' : node.buildbot_url}
     css_class = getattr(node, 'css_class', '')
@@ -71,6 +119,7 @@
     self.body.append(self.starttag(node, 'a', **kwargs))
     self.body.append(node.text)
 
+
 def depart_buildbot_node(self, node):
     self.body.append('</a>')