[Checkins] SVN: Sandbox/ulif/grok-reference-with-rest2/doc/grokref/ Added extensions to support standard python doc tags in documentation.

Uli Fouquet uli at gnufix.de
Fri Jan 4 07:33:17 EST 2008


Log message for revision 82665:
  Added extensions to support standard python doc tags in documentation.

Changed:
  A   Sandbox/ulif/grok-reference-with-rest2/doc/grokref/
  A   Sandbox/ulif/grok-reference-with-rest2/doc/grokref/__init__.py
  A   Sandbox/ulif/grok-reference-with-rest2/doc/grokref/extensions/
  A   Sandbox/ulif/grok-reference-with-rest2/doc/grokref/extensions/__init__.py
  A   Sandbox/ulif/grok-reference-with-rest2/doc/grokref/extensions/addnodes.py
  A   Sandbox/ulif/grok-reference-with-rest2/doc/grokref/extensions/directives.py
  A   Sandbox/ulif/grok-reference-with-rest2/doc/grokref/extensions/roles.py
  A   Sandbox/ulif/grok-reference-with-rest2/doc/grokref/grokref2html.py
  A   Sandbox/ulif/grok-reference-with-rest2/doc/grokref/translators.py

-=-
Added: Sandbox/ulif/grok-reference-with-rest2/doc/grokref/__init__.py
===================================================================

Added: Sandbox/ulif/grok-reference-with-rest2/doc/grokref/extensions/__init__.py
===================================================================

Added: Sandbox/ulif/grok-reference-with-rest2/doc/grokref/extensions/addnodes.py
===================================================================
--- Sandbox/ulif/grok-reference-with-rest2/doc/grokref/extensions/addnodes.py	                        (rev 0)
+++ Sandbox/ulif/grok-reference-with-rest2/doc/grokref/extensions/addnodes.py	2008-01-04 12:33:17 UTC (rev 82665)
@@ -0,0 +1,72 @@
+##############################################################################
+#
+# Copyright (c) 2008 Zope Corporation and Contributors.
+# All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.1 (ZPL).  A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE.
+#
+##############################################################################
+
+from docutils import nodes
+
+# index markup
+class index(nodes.Invisible, nodes.Inline, nodes.TextElement): pass
+
+# description units (classdesc, funcdesc etc.)
+class desc(nodes.Admonition, nodes.Element): pass
+class desc_content(nodes.General, nodes.Element): pass
+class desc_signature(nodes.Part, nodes.Inline, nodes.TextElement): pass
+class desc_classname(nodes.Part, nodes.Inline, nodes.TextElement): pass
+class desc_name(nodes.Part, nodes.Inline, nodes.TextElement): pass
+class desc_parameterlist(nodes.Part, nodes.Inline, nodes.TextElement): pass
+class desc_parameter(nodes.Part, nodes.Inline, nodes.TextElement): pass
+class desc_optional(nodes.Part, nodes.Inline, nodes.TextElement): pass
+
+# refcount annotation
+class refcount(nodes.emphasis): pass
+
+# \versionadded, \versionchanged, \deprecated
+class versionmodified(nodes.Admonition, nodes.TextElement): pass
+
+# seealso
+class seealso(nodes.Admonition, nodes.Element): pass
+
+# productionlist
+class productionlist(nodes.Admonition, nodes.Element): pass
+class production(nodes.Part, nodes.Inline, nodes.TextElement): pass
+
+# toc tree
+class toctree(nodes.General, nodes.Element): pass
+
+
+# centered
+class centered(nodes.Part, nodes.Element): pass
+
+# pending xref
+class pending_xref(nodes.Element): pass
+
+# compact paragraph -- never makes a <p>
+class compact_paragraph(nodes.paragraph): pass
+
+# sets the highlighting language for literal blocks
+class highlightlang(nodes.Element): pass
+
+# like emphasis, but doesn't apply further text processors, e.g. smartypants
+class literal_emphasis(nodes.emphasis): pass
+
+# glossary
+class glossary(nodes.Element): pass
+
+# make them known to docutils. this is needed, because the HTMl writer
+# will choke at some point if these are not added
+nodes._add_node_class_names("""index desc desc_content desc_signature
+      desc_classname desc_name desc_parameterlist desc_parameter desc_optional
+      centered versionmodified seealso productionlist production toctree
+      pending_xref compact_paragraph highlightlang literal_emphasis
+      glossary""".split())
+

Added: Sandbox/ulif/grok-reference-with-rest2/doc/grokref/extensions/directives.py
===================================================================
--- Sandbox/ulif/grok-reference-with-rest2/doc/grokref/extensions/directives.py	                        (rev 0)
+++ Sandbox/ulif/grok-reference-with-rest2/doc/grokref/extensions/directives.py	2008-01-04 12:33:17 UTC (rev 82665)
@@ -0,0 +1,139 @@
+##############################################################################
+#
+# Copyright (c) 2008 Zope Corporation and Contributors.
+# All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.1 (ZPL).  A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE.
+#
+##############################################################################
+"""
+Additional directives for reference documentation.
+"""
+import re
+from docutils.parsers.rst import directives, roles
+import addnodes
+
+# ------ functions to parse a Python or C signature and create desc_* nodes.
+# ------ also used for parameters like '*args', '**kw'.
+
+py_sig_re = re.compile(r'''^([\w.]*\.)?        # class names
+                           (\w+)  \s*          # thing name
+                           (?: \((.*)\) )? $   # optionally arguments
+                        ''', re.VERBOSE)
+py_paramlist_re = re.compile(r'([\[\],])')  # split at '[', ']' and ','
+
+
+def parse_py_signature(signode, sig, desctype):
+    """
+    Transform a python signature into RST nodes.
+    Return (fully qualified name of the thing, classname if any).
+
+    If inside a class, the current class name is handled intelligently:
+    * it is stripped from the displayed name if present
+    * it is added to the full name (return value) if not present
+    """
+    m = py_sig_re.match(sig)
+    if m is None: raise ValueError
+    classname, name, arglist = m.groups()
+    fullname = classname and classname + name or name
+
+    signode += addnodes.desc_classname(classname, classname)
+
+    signode += addnodes.desc_name(name, name)
+    if not arglist:
+        if desctype in ('function', 'method'):
+            # for callables, add an empty parameter list
+            signode += addnodes.desc_parameterlist()
+        return fullname, classname
+    signode += addnodes.desc_parameterlist()
+
+    stack = [signode[-1]]
+    for token in py_paramlist_re.split(arglist):
+        if token == '[':
+            opt = addnodes.desc_optional()
+            stack[-1] += opt
+            stack.append(opt)
+        elif token == ']':
+            try: stack.pop()
+            except IndexError: raise ValueError
+        elif not token or token == ',' or token.isspace():
+            pass
+        else:
+            token = token.strip()
+            stack[-1] += addnodes.desc_parameter(token, token)
+    if len(stack) != 1: raise ValueError
+    return fullname, classname
+
+
+
+def desc_directive(desctype, arguments, options, content, lineno,
+                   content_offset, block_text, state, state_machine):
+    node = addnodes.desc()
+    node['desctype'] = desctype
+
+    noindex = ('noindex' in options)
+    signatures = map(lambda s: s.strip(), arguments[0].split('\n'))
+    names = []
+    clsname = None
+    for i, sig in enumerate(signatures):
+        # add a signature node for each signature in the current unit
+        # and add a reference target for it
+        sig = sig.strip()
+        signode = addnodes.desc_signature(sig, '')
+        signode['first'] = False
+        node.append(signode)
+        try:
+            if desctype in ('function', 'data', 'class', 'exception',
+                            'method', 'attribute'):
+                name, clsname = parse_py_signature(signode, sig, desctype)
+            elif desctype in ('cfunction', 'cmember', 'cmacro', 'ctype',
+                              'cvar'):
+                name = parse_c_signature(signode, sig, desctype)
+            elif desctype == 'opcode':
+                name = parse_opcode_signature(signode, sig, desctype)
+            else:
+                # describe: use generic fallback
+                raise ValueError
+        except ValueError, err:
+            signode.clear()
+            signode += addnodes.desc_name(sig, sig)
+            continue
+        # we don't want an index entry here
+        # only add target and index entry if this is the first
+        # description of the function name in this desc block
+        if not noindex and name not in names:
+            fullname = name
+            # note target
+            if fullname not in state.document.ids:
+                signode['names'].append(fullname)
+                signode['ids'].append(fullname)
+                signode['first'] = (not names)
+                state.document.note_explicit_target(signode)
+            names.append(name)
+
+    subnode = addnodes.desc_content()
+    # needed for automatic qualification of members
+    clsname_set = False
+    if desctype == 'class' and names:
+        clsname_set = True
+    state.nested_parse(content, content_offset, subnode)
+    node.append(subnode)
+    return [node]
+
+desc_directive.content = 1
+desc_directive.arguments = (1, 0, 1)
+desc_directive.options = {'noindex': directives.flag}
+
+desctypes = [
+    # the Python ones
+    'function',
+]
+
+for name in desctypes:
+    directives.register_directive(name, desc_directive)
+

Added: Sandbox/ulif/grok-reference-with-rest2/doc/grokref/extensions/roles.py
===================================================================
--- Sandbox/ulif/grok-reference-with-rest2/doc/grokref/extensions/roles.py	                        (rev 0)
+++ Sandbox/ulif/grok-reference-with-rest2/doc/grokref/extensions/roles.py	2008-01-04 12:33:17 UTC (rev 82665)
@@ -0,0 +1,96 @@
+##############################################################################
+#
+# Copyright (c) 2008 Zope Corporation and Contributors.
+# All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.1 (ZPL).  A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE.
+#
+##############################################################################
+"""Additional roles for reference documentation.
+"""
+
+import re
+from docutils import nodes, utils
+from docutils.parsers.rst import roles
+
+import addnodes
+
+# default is `literal`
+innernodetypes = {
+    'ref': nodes.emphasis,
+    'term': nodes.emphasis,
+    'token': nodes.strong,
+}
+
+ws_re = re.compile(r'\s+')
+_litvar_re = re.compile('{([^}]+)}')
+
+def xfileref_role(typ, rawtext, text, lineno, inliner, options={}, content=[]):
+    text = utils.unescape(text)
+    if text[0:1] == '!':
+        text = text[1:]
+        return [innernodetypes.get(typ, nodes.literal)(
+            rawtext, text, classes=['xref'])], []
+    pnode = addnodes.pending_xref(rawtext)
+    pnode['reftype'] = typ
+    # if the first character is a dot, search more specific namespaces first
+    # else search builtins first
+    if text[0:1] == '.' and \
+       typ in ('data', 'exc', 'func', 'class', 'const', 'attr', 'meth'):
+        text = text[1:]
+        pnode['refspecific'] = True
+    pnode['reftarget'] = ws_re.sub((typ == 'term' and ' ' or ''), text)
+    pnode += innernodetypes.get(typ, nodes.literal)(rawtext, text,
+                                                    classes=['xref'])
+    return [pnode], []
+
+
+
+def emph_literal_role(typ, rawtext, text, lineno, inliner, options={},
+                      content=[]):
+    text = utils.unescape(text)
+    retnodes = []
+    pos = 0
+    for m in _litvar_re.finditer(text):
+        if m.start() > pos:
+            txt = text[pos:m.start()]
+            retnodes.append(nodes.literal(txt, txt))
+        retnodes.append(nodes.emphasis('', '', nodes.literal(m.group(1),
+                                                             m.group(1))))
+        pos = m.end()
+    if pos < len(text):
+        retnodes.append(nodes.literal(text[pos:], text[pos:]))
+    return retnodes, []
+
+
+specific_docroles = {
+    'data': xfileref_role,
+    'exc': xfileref_role,
+    'func': xfileref_role,
+    'class': xfileref_role,
+    'const': xfileref_role,
+    'attr': xfileref_role,
+    'meth': xfileref_role,
+
+    'cfunc' : xfileref_role,
+    'cdata' : xfileref_role,
+    'ctype' : xfileref_role,
+    'cmacro' : xfileref_role,
+
+    'mod' : xfileref_role,
+
+    'ref': xfileref_role,
+    'token' : xfileref_role,
+    'term': xfileref_role,
+
+    'file' : emph_literal_role,
+    'samp' : emph_literal_role,
+}
+
+for rolename, func in specific_docroles.iteritems():
+    roles.register_canonical_role(rolename, func)

Added: Sandbox/ulif/grok-reference-with-rest2/doc/grokref/grokref2html.py
===================================================================
--- Sandbox/ulif/grok-reference-with-rest2/doc/grokref/grokref2html.py	                        (rev 0)
+++ Sandbox/ulif/grok-reference-with-rest2/doc/grokref/grokref2html.py	2008-01-04 12:33:17 UTC (rev 82665)
@@ -0,0 +1,98 @@
+##############################################################################
+#
+# Copyright (c) 2008 Zope Corporation and Contributors.
+# All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.1 (ZPL).  A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE.
+#
+##############################################################################
+"""Create the grok reference in various output formats."
+
+import os
+import sys
+import codecs
+
+import docutils.core
+#from docutils import nodes
+from docutils.writers.html4css1 import Writer
+
+from grokref.translators import HTMLTranslator
+from grokref.extensions import addnodes, roles, directives
+
+#from zope.app.renderer.rest import ZopeTranslator
+
+class ReStructuredTextToHTMLRenderer:
+    """Convert from Restructured Text to HTML."""
+
+    def __init__(self, content):
+        self.content = content
+
+    def render(self):
+        settings_overrides = {
+            'halt_level': 6,
+            'input_encoding': 'utf8',
+            'output_encoding': 'utf8',
+            'initial_header_level': 2,
+            # don't try to include the stylesheet (docutils gets hiccups)
+            'stylesheet_path': '',
+        }
+
+        #docutils.nodes._add_node_class_names('desc')
+        writer = Writer()
+        #print docutils.nodes
+        #writer.translator_class = ZopeTranslator
+        writer.translator_class = HTMLTranslator
+        html = docutils.core.publish_string(
+                        self.content,
+                        writer=writer,
+                        settings_overrides=settings_overrides,)
+        html = codecs.decode(html, 'utf_8')
+        return html
+
+
+class ReSTFile(object):
+    source_text = None
+    file_path = None
+
+    def __init__(self, filename=None):
+        self.source_text = codecs.open(filename, "r", 'utf-8').read()
+        self.file_path = filename
+
+    def getHTML(self):
+        return ReStructuredTextToHTMLRenderer(self.source_text).render()
+        
+
+def getRestFiles(filepath):
+    if os.path.isdir(filepath):
+        return [os.path.join(filepath, filename)
+                for filename in os.listdir(filepath)
+                if filename.endswith('.rst')]
+    return [filepath]
+
+def main(argv=None):
+    if argv is None:
+        argv = sys.argv[1:]
+    
+    if not len(argv):
+        print "Usage: %s REFDIR|FILE" % (sys.argv[0])
+        sys.exit(1)
+
+    if not os.path.isdir(argv[0]) and not os.path.isfile(argv[0]):
+        print "%s: No such file or directory" % (argv[0])
+        sys.exit(1)
+
+    rest_files = getRestFiles(argv[0])
+    print "files: ", rest_files
+
+    print "Content: "
+    for filename in rest_files:
+        print ReSTFile(filename).getHTML()
+
+    
+if __name__ == '__main__':
+    main()

Added: Sandbox/ulif/grok-reference-with-rest2/doc/grokref/translators.py
===================================================================
--- Sandbox/ulif/grok-reference-with-rest2/doc/grokref/translators.py	                        (rev 0)
+++ Sandbox/ulif/grok-reference-with-rest2/doc/grokref/translators.py	2008-01-04 12:33:17 UTC (rev 82665)
@@ -0,0 +1,193 @@
+##############################################################################
+#
+# Copyright (c) 2008 Zope Corporation and Contributors.
+# All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.1 (ZPL).  A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE.
+#
+##############################################################################
+"""Translators, that can handle extensions in HTML and LaTeX.
+"""
+from docutils import nodes
+from docutils.writers.html4css1 import HTMLTranslator as BaseTranslator
+
+class HTMLTranslator(BaseTranslator):
+    """A custom HTML translator.
+    """
+    def __init__(self, *args, **kwds):
+        BaseTranslator.__init__(self, *args, **kwds)
+        self.highlightlang = 'python'
+
+    def visit_desc(self, node):
+        self.body.append(self.starttag(node, 'dl', CLASS=node['desctype']))
+
+    def depart_desc(self, node):
+        self.body.append('</dl>\n\n')
+
+    def visit_desc_signature(self, node):
+        # the id is set automatically
+        self.body.append(self.starttag(node, 'dt'))
+        # anchor for per-desc interactive data
+        if (node.parent['desctype'] != 'describe' and node['ids']
+            and node['first']):
+            self.body.append('<!--#%s#-->' % node['ids'][0])
+        if node.parent['desctype'] in ('class', 'exception'):
+            self.body.append('%s ' % node.parent['desctype'])
+    def depart_desc_signature(self, node):
+        self.body.append('</dt>\n')
+
+    def visit_desc_classname(self, node):
+        print "NODE: ", dir(node)
+        self.body.append(self.starttag(node, 'tt', '', CLASS='descclassname'))
+    def depart_desc_classname(self, node):
+        self.body.append('</tt>')
+
+    def visit_desc_name(self, node):
+        self.body.append(self.starttag(node, 'tt', '', CLASS='descname'))
+    def depart_desc_name(self, node):
+        self.body.append('</tt>')
+
+    def visit_desc_parameterlist(self, node):
+        self.body.append('<big>(</big>')
+        self.first_param = 1
+    def depart_desc_parameterlist(self, node):
+        self.body.append('<big>)</big>')
+
+    def visit_desc_parameter(self, node):
+        if not self.first_param:
+            self.body.append(', ')
+        else:
+            self.first_param = 0
+        if not node.hasattr('noemph'):
+            self.body.append('<em>')
+    def depart_desc_parameter(self, node):
+        if not node.hasattr('noemph'):
+            self.body.append('</em>')
+    def visit_desc_optional(self, node):
+        self.body.append('<span class="optional">[</span>')
+    def depart_desc_optional(self, node):
+        self.body.append('<span class="optional">]</span>')
+
+    def visit_desc_content(self, node):
+        self.body.append(self.starttag(node, 'dd', ''))
+    def depart_desc_content(self, node):
+        self.body.append('</dd>')
+
+    def visit_refcount(self, node):
+        self.body.append(self.starttag(node, 'em', '', CLASS='refcount'))
+    def depart_refcount(self, node):
+        self.body.append('</em>')
+
+    def visit_versionmodified(self, node):
+        self.body.append(self.starttag(node, 'p'))
+        text = version_text[node['type']] % node['version']
+        if len(node):
+            text += ': '
+        else:
+            text += '.'
+        self.body.append('<span class="versionmodified">%s</span>' % text)
+    def depart_versionmodified(self, node):
+        self.body.append('</p>\n')
+
+    # overwritten -- we don't want source comments to show up in the HTML
+    def visit_comment(self, node):
+        raise nodes.SkipNode
+
+
+        # overwritten
+    def visit_admonition(self, node, name=''):
+        self.body.append(self.start_tag_with_title(
+            node, 'div', CLASS=('admonition ' + name)))
+        if name and name != 'seealso':
+            node.insert(0, nodes.title(name, self.language.labels[name]))
+        self.set_first_last(node)
+
+    def visit_seealso(self, node):
+        self.visit_admonition(node, 'seealso')
+    def depart_seealso(self, node):
+        self.depart_admonition(node)
+
+    # overwritten
+    def visit_title(self, node, move_ids=1):
+        # if we have a section we do our own processing in order
+        # to have ids in the hN-tags and not in additional a-tags
+        if isinstance(node.parent, nodes.section):
+            h_level = self.section_level + self.initial_header_level - 1
+            if node.parent.get('ids'):
+                attrs = {'ids': node.parent['ids']}
+            else:
+                attrs = {}
+            self.body.append(self.starttag(node, 'h%d' % h_level, '', **attrs))
+            self.context.append('</h%d>\n' % h_level)
+        else:
+            BaseTranslator.visit_title(self, node, move_ids)
+
+    # overwritten
+    def visit_literal_block(self, node):
+        #from .highlighting import highlight_block
+        #self.body.append(highlight_block(node.rawsource, self.highlightlang))
+        raise nodes.SkipNode
+
+    def visit_productionlist(self, node):
+        self.body.append(self.starttag(node, 'pre'))
+        names = []
+        for production in node:
+            names.append(production['tokenname'])
+        maxlen = max(len(name) for name in names)
+        for production in node:
+            if production['tokenname']:
+                self.body.append(self.starttag(production, 'strong', ''))
+                self.body.append(production['tokenname'].ljust(maxlen) +
+                                 '</strong> ::= ')
+                lastname = production['tokenname']
+            else:
+                self.body.append('%s     ' % (' '*len(lastname)))
+            production.walkabout(self)
+            self.body.append('\n')
+        self.body.append('</pre>\n')
+        raise nodes.SkipNode
+    def depart_productionlist(self, node):
+        pass
+
+    def visit_production(self, node):
+        pass
+    def depart_production(self, node):
+        pass
+
+    def visit_centered(self, node):
+        self.body.append(self.starttag(node, 'center') + '<strong>')
+    def depart_centered(self, node):
+        self.body.append('</strong></center>')
+
+    def visit_compact_paragraph(self, node):
+        pass
+    def depart_compact_paragraph(self, node):
+        pass
+
+    def visit_highlightlang(self, node):
+        self.highlightlang = node['lang']
+    def depart_highlightlang(self, node):
+        pass
+
+    def visit_toctree(self, node):
+        # this only happens when formatting a toc from env.tocs -- in this
+        # case we don't want to include the subtree
+        raise nodes.SkipNode
+
+    def visit_index(self, node):
+        raise nodes.SkipNode
+
+    #def unknown_visit(self, node):
+    #    print "UNKNOWN: ", node
+    #def unknown_departure(self, node):
+    #    print "UNKNOWN DEP: ", node
+
+    def visit_pending_xref(self, node):
+        #self.parent.append(nodes.literal(node['reftarget'], node['reftarget']))
+        raise nodes.SkipNode
+



More information about the Checkins mailing list