[Checkins] SVN: Sandbox/malthe/chameleon.core/ Use sourcecodegen package to generate Python-code from AST as a way to avoid using the compiler.pycodegen module for byte-code compilation. Compilation-performance is up by 50%.
Malthe Borch
mborch at gmail.com
Wed Nov 26 20:58:44 EST 2008
Log message for revision 93377:
Use sourcecodegen package to generate Python-code from AST as a way to avoid using the compiler.pycodegen module for byte-code compilation. Compilation-performance is up by 50%.
Changed:
U Sandbox/malthe/chameleon.core/CHANGES.txt
U Sandbox/malthe/chameleon.core/setup.py
U Sandbox/malthe/chameleon.core/src/chameleon/core/codegen.py
U Sandbox/malthe/chameleon.core/src/chameleon/core/codegen.txt
U Sandbox/malthe/chameleon.core/src/chameleon/core/generation.py
U Sandbox/malthe/chameleon.core/src/chameleon/core/translation.py
U Sandbox/malthe/chameleon.core/src/chameleon/core/utils.py
-=-
Modified: Sandbox/malthe/chameleon.core/CHANGES.txt
===================================================================
--- Sandbox/malthe/chameleon.core/CHANGES.txt 2008-11-27 00:49:59 UTC (rev 93376)
+++ Sandbox/malthe/chameleon.core/CHANGES.txt 2008-11-27 01:58:44 UTC (rev 93377)
@@ -4,6 +4,11 @@
HEAD
~~~~
+- Use ``sourcecodegen`` package to generate Python-code from AST; this
+ is done to avoid generating code using the ``compiler.pycodegen``
+ module, which has serious issues. Interestingly, this approach cuts
+ compilation time in half. [malthe]
+
- Bind code-generation utilities in closure as well. [malthe]
- Reworked global scope initialization; we now bind these using a
Modified: Sandbox/malthe/chameleon.core/setup.py
===================================================================
--- Sandbox/malthe/chameleon.core/setup.py 2008-11-27 00:49:59 UTC (rev 93376)
+++ Sandbox/malthe/chameleon.core/setup.py 2008-11-27 01:58:44 UTC (rev 93377)
@@ -8,6 +8,7 @@
'zope.component',
'zope.i18n >= 3.5',
'lxml>=2.1.1',
+ 'sourcecodegen',
]
if sys.version_info[:3] < (2,5,0):
Modified: Sandbox/malthe/chameleon.core/src/chameleon/core/codegen.py
===================================================================
--- Sandbox/malthe/chameleon.core/src/chameleon/core/codegen.py 2008-11-27 00:49:59 UTC (rev 93376)
+++ Sandbox/malthe/chameleon.core/src/chameleon/core/codegen.py 2008-11-27 01:58:44 UTC (rev 93377)
@@ -1,5 +1,5 @@
from compiler import ast, parse
-from compiler.pycodegen import ModuleCodeGenerator
+from sourcecodegen import ModuleSourceCodeGenerator
from transformer import ASTTransformer
@@ -150,7 +150,7 @@
])
class Suite(object):
- __slots__ = ['code', '_globals']
+ __slots__ = ['source', '_globals']
mode = 'exec'
@@ -167,17 +167,11 @@
tree = transform.visit(node)
filename = tree.filename = '<script>'
- # generate code
- gen = ModuleCodeGenerator(tree)
- gen.optimized = True
-
- from sourcecodegen import ModuleSourceCodeGenerator
- generated = ModuleSourceCodeGenerator(tree).getSourceCode()
-
- self.code = gen.getCode()
-
+ # generate source code
+ self.source = ModuleSourceCodeGenerator(tree).getSourceCode()
+
def __hash__(self):
- return hash(self.code)
+ return hash(self.source)
def __repr__(self):
- return '%s(%r)' % (self.__class__.__name__, self.code)
+ return '%s(%r)' % (self.__class__.__name__, self.source)
Modified: Sandbox/malthe/chameleon.core/src/chameleon/core/codegen.txt
===================================================================
--- Sandbox/malthe/chameleon.core/src/chameleon/core/codegen.txt 2008-11-27 00:49:59 UTC (rev 93376)
+++ Sandbox/malthe/chameleon.core/src/chameleon/core/codegen.txt 2008-11-27 01:58:44 UTC (rev 93377)
@@ -14,7 +14,7 @@
>>> suite = Suite("""\
... print 'Hello World!'
... """)
- >>> exec suite.code
+ >>> exec suite.source
Hello World!
Syntax extension: Dictionary lookup using dot operator
@@ -28,7 +28,7 @@
... a = {'b': 1}
... assert a['b'] == a.b
... """)
- >>> exec suite.code in lookup_globals.copy()
+ >>> exec suite.source in lookup_globals.copy()
Syntax extension: Dynamic scoping
---------------------------------
@@ -37,4 +37,4 @@
... econtext = {'a': 1}
... assert a == 1
... """)
- >>> exec suite.code in lookup_globals.copy()
+ >>> exec suite.source in lookup_globals.copy()
Modified: Sandbox/malthe/chameleon.core/src/chameleon/core/generation.py
===================================================================
--- Sandbox/malthe/chameleon.core/src/chameleon/core/generation.py 2008-11-27 00:49:59 UTC (rev 93376)
+++ Sandbox/malthe/chameleon.core/src/chameleon/core/generation.py 2008-11-27 01:58:44 UTC (rev 93377)
@@ -123,6 +123,17 @@
def annotate(self, annotation):
if annotation.label is not None:
annotation = annotation.label
+
+ # make sure the annotation is a base string type
+ if isinstance(annotation, unicode):
+ annotation = unicode(annotation)
+ else:
+ annotation = str(annotation)
+
+ # encode unicode string if required
+ if isinstance(annotation, unicode) and self.encoding:
+ annotation = annotation.encode(self.encoding)
+
self.annotation = self.annotations[self.l_counter] = annotation
def out(self, string):
@@ -147,13 +158,13 @@
indent = self.indentation_string * self.indentation
- # if a source code annotation is set, write it as a comment
- # prior to the source code line
+ # if a source code annotation is set, write it as a
+ # triple-quoted string prior to the source line
if self.annotation:
if isinstance(self.annotation, unicode) and self.encoding:
- self.annotation = self.annotation.encode(self.encoding)
+ self.annotation = self.annotation.encode(self.encoding)
BufferIO.write(
- self, "%s# %s\n" % (indent, self.annotation))
+ self, "%s%s\n" % (indent, repr(self.annotation)))
self.annotation = None
BufferIO.write(self, indent + string + '\n')
Modified: Sandbox/malthe/chameleon.core/src/chameleon/core/translation.py
===================================================================
--- Sandbox/malthe/chameleon.core/src/chameleon/core/translation.py 2008-11-27 00:49:59 UTC (rev 93376)
+++ Sandbox/malthe/chameleon.core/src/chameleon/core/translation.py 2008-11-27 01:58:44 UTC (rev 93377)
@@ -11,7 +11,10 @@
import config
import etree
import marshal
+import sys
+GLOBALS = globals()
+
class Node(object):
"""Element translation class.
@@ -787,7 +790,8 @@
args.append('%s = None' % selector)
defaults.append('%s = None' % selector)
- args.append('%(language)s = None' % __dict__)
+ if not stream.symbols.language in args:
+ args.append('%(language)s = None' % __dict__)
# prepare globals
_globals = ["from cPickle import loads as _loads"]
@@ -822,13 +826,13 @@
self.tree = tree
def compile(self):
- suite = codegen.Suite(self.source)
-
+ suite = codegen.Suite(self.source)
_locals = {}
- exec suite.code in _locals
+ exec suite.source in _locals
self.bind = _locals['bind']
self.func = self.bind()
-
+ self.source = suite.source
+
def __reduce__(self):
reconstructor, (cls, base, state), kwargs = \
GhostedByteCodeTemplate(self).__reduce__()
@@ -860,8 +864,6 @@
return selectors
class GhostedByteCodeTemplate(object):
- suite = codegen.Suite("def bind(): pass")
-
def __init__(self, template):
self.code = marshal.dumps(template.bind.func_code)
self.defaults = len(template.bind.func_defaults or ())
@@ -876,13 +878,9 @@
parser = state['parser']
tree, doctype = parser.parse(xmldoc)
- _locals = {}
- exec cls.suite.code in _locals
+ bind = sys.modules['types'].FunctionType(
+ marshal.loads(state['code']), GLOBALS, "bind")
- bind = _locals['bind']
- bind.func_defaults = ((None,)*state['defaults']) or None
- bind.func_code = marshal.loads(state['code'])
-
func = bind()
return dict(
Modified: Sandbox/malthe/chameleon.core/src/chameleon/core/utils.py
===================================================================
--- Sandbox/malthe/chameleon.core/src/chameleon/core/utils.py 2008-11-27 00:49:59 UTC (rev 93376)
+++ Sandbox/malthe/chameleon.core/src/chameleon/core/utils.py 2008-11-27 01:58:44 UTC (rev 93377)
@@ -34,6 +34,8 @@
'<!ENTITY %s "&#%s;">' % (name, text) for (name, text) in \
htmlentitydefs.name2codepoint.items()))
+re_annotation = re.compile(r'^\s*u?\'(.*)\'$')
+
s_counter = 0
marker = object()
@@ -331,8 +333,9 @@
# the template source as comments)
source = source.split('\n')
for i in reversed(range(lineno)):
- if source[i].lstrip().startswith('#'):
- annotation = source[i].split('#', 1)[-1].lstrip()
+ m = re_annotation.match(source[i])
+ if m is not None:
+ annotation = m.group(1)
break
else:
annotation = ""
More information about the Checkins
mailing list