[Checkins] SVN: zc.buildout/branches/gary-4/src/zc/buildout/ an interpreter variant that supports all interpreter flags (AFAIK). this commit contains core implementation (in easy_install module) and tests of all functionality except import_sitecustomize. Integration into zc.recipe.egg and tests for import_sitecustomize (which will be more involved, and in tests.py) will follow.
Gary Poster
gary.poster at canonical.com
Wed Dec 16 11:19:10 EST 2009
Log message for revision 106634:
an interpreter variant that supports all interpreter flags (AFAIK). this commit contains core implementation (in easy_install module) and tests of all functionality except import_sitecustomize. Integration into zc.recipe.egg and tests for import_sitecustomize (which will be more involved, and in tests.py) will follow.
Changed:
U zc.buildout/branches/gary-4/src/zc/buildout/easy_install.py
U zc.buildout/branches/gary-4/src/zc/buildout/easy_install.txt
U zc.buildout/branches/gary-4/src/zc/buildout/tests.py
-=-
Modified: zc.buildout/branches/gary-4/src/zc/buildout/easy_install.py
===================================================================
--- zc.buildout/branches/gary-4/src/zc/buildout/easy_install.py 2009-12-16 15:58:14 UTC (rev 106633)
+++ zc.buildout/branches/gary-4/src/zc/buildout/easy_install.py 2009-12-16 16:19:10 UTC (rev 106634)
@@ -968,18 +968,92 @@
return generated
-import_site_snippet = '''\
-# We have to import pkg_resources before namespace
-# package .pth files are processed or else the distribution's namespace
-# packages will mask all of the egg-based packages in the same namespace
-# package.
-try:
- import pkg_resources
-except ImportError:
- pass
-import site
-'''
+def interpreter(name, working_set, executable, dest, site_py_dest,
+ extra_paths=(),
+ initialization='',
+ relative_paths=False,
+ import_site=False,
+ import_sitecustomize=False
+ ):
+ generated = []
+ # Write sitecustomize.py.
+ sitecustomize_path = os.path.join(site_py_dest, 'sitecustomize.py')
+ sitecustomize = open(sitecustomize_path, 'w')
+ if initialization:
+ sitecustomize.write(initialization + '\n')
+ if import_sitecustomize:
+ real_sitecustomize_path = _get_module_file(
+ executable, 'sitecustomize')
+ if real_sitecustomize_path:
+ sitecustomize.write('execfile(%s)\n' % (real_sitecustomize_path,))
+ sitecustomize.close()
+ generated.append(sitecustomize_path)
+ # Write site.py.
+ path = [dist.location for dist in working_set]
+ path.extend(extra_paths)
+ path = map(realpath, path)
+ site_path = os.path.join(site_py_dest, 'site.py')
+ path_string, rpsetup = _relative_path_and_setup(
+ site_path, path, relative_paths)
+ site = open(site_path, 'w')
+ site.write(rpsetup)
+ site.write(sys_path_template % (path_string,))
+ if import_site:
+ real_site_path = _get_module_file(executable, 'site')
+ if real_site_path:
+ site.write(import_site_template % (real_site_path,))
+ else:
+ site.write('import sitecustomize\n')
+ generated.append(site_path)
+ # Write interpreter script.
+ script_name = full_name = os.path.join(dest, name)
+ site_py_dest_string, rpsetup = _relative_path_and_setup(
+ full_name, [site_py_dest], relative_paths)
+ if is_win32:
+ script_name += '-script.py'
+ contents = interpreter_template % dict(
+ python = _safe_arg(executable),
+ site_dest = site_py_dest_string,
+ relative_paths_setup = rpsetup,
+ )
+ changed = not (os.path.exists(script_name) and
+ open(script_name).read() == contents)
+ if is_win32:
+ # Generate exe file and give the script a magic name.
+ exe = full_name + '.exe'
+ open(exe, 'wb').write(
+ pkg_resources.resource_string('setuptools', 'cli.exe')
+ )
+ generated.append(exe)
+ if changed:
+ open(script_name, 'w').write(contents)
+ try:
+ os.chmod(script_name,0755)
+ except (AttributeError, os.error):
+ pass
+ logger.info("Generated interpreter %r.", name)
+ generated.append(script_name)
+ return generated
+def _get_module_file(executable, name):
+ cmd = [executable, "-c",
+ "import imp; fp, path, desc = imp.find_module(%r); fp.close; print path" % (name,)]
+ _proc = subprocess.Popen(
+ cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+ stdout, stderr = _proc.communicate();
+ if _proc.returncode:
+ logger.info(
+ 'Could not find file for module %s:\n%s', name, stderr)
+ return None
+ # else: ...
+ res = stdout.strip()
+ if res.endswith('.pyc') or res.endswith('.pyo'):
+ raise RuntimeError('Cannot find uncompiled version of %s' % (name,))
+ if not os.path.exists(res):
+ raise RuntimeError(
+ 'File does not exist for module %s:\n%s' % (name, res))
+ return res
+
def _relative_path_and_setup(sname, path, relative_paths):
if relative_paths:
relative_paths = os.path.normcase(relative_paths)
@@ -996,7 +1070,6 @@
rpsetup = ''
return spath, rpsetup
-
def _relative_depth(common, path):
n = 0
while 1:
@@ -1033,7 +1106,6 @@
else:
return repr(path)
-
relative_paths_setup = """
import os
@@ -1090,7 +1162,26 @@
else:
script_header = '#!%(python)s -S'
+_import_site_start = '''\
+# We have to import pkg_resources before namespace
+# package .pth files are processed or else the distribution's namespace
+# packages will mask all of the egg-based packages in the same namespace
+# package.
+try:
+ import pkg_resources
+except ImportError:
+ pass
+'''
+import_site_snippet = _import_site_start + 'import site\n'
+import_site_template = _import_site_start + 'execfile(%r)\n'
+sys_path_template = '''\
+import sys
+sys.path[0:0] = [
+ %s,
+ ]
+'''
+
script_template = script_header + '''\
%(relative_paths_setup)s
@@ -1106,7 +1197,21 @@
%(module_name)s.%(attrs)s(%(arguments)s)
'''
+interpreter_template = script_header + '''\
+%(relative_paths_setup)s
+import os
+import sys
+
+argv = [sys.executable] + sys.argv[1:]
+environ = os.environ.copy()
+path = %(site_dest)s
+if environ.get('PYTHONPATH'):
+ path = os.pathsep.join([path, environ['PYTHONPATH']])
+environ['PYTHONPATH'] = path
+os.execve(sys.executable, argv, environ)
+'''
+
def _pyscript(path, dest, executable, rsetup, import_site):
generated = []
script = dest
Modified: zc.buildout/branches/gary-4/src/zc/buildout/easy_install.txt
===================================================================
--- zc.buildout/branches/gary-4/src/zc/buildout/easy_install.txt 2009-12-16 15:58:14 UTC (rev 106633)
+++ zc.buildout/branches/gary-4/src/zc/buildout/easy_install.txt 2009-12-16 16:19:10 UTC (rev 106634)
@@ -999,6 +999,233 @@
__import__("code").interact(banner="", local=globals())
+Interpreter generation
+----------------------
+
+The interpreter created above is a script that emulates the Python
+interpreter. Sometimes it is useful to have a full-fledged Python interpreter
+available. The ``interpreter`` function supports this.
+
+It works by creating site.py and sitecustomize.py files that set up the
+desired paths and initialization. These must be placed within an otherwise
+empty directory. Typically this is in a recipe's parts directory.
+
+Here's the simplest example.
+
+ >>> interpreter_dir = tmpdir('interpreter')
+ >>> mkdir(interpreter_dir, 'bin')
+ >>> mkdir(interpreter_dir, 'eggs')
+ >>> mkdir(interpreter_dir, 'parts')
+ >>> mkdir(interpreter_dir, 'parts', 'interpreter')
+
+ >>> ws = zc.buildout.easy_install.install(
+ ... ['demo'], join(interpreter_dir, 'eggs'), links=[link_server],
+ ... index=link_server+'index/')
+ >>> generated = zc.buildout.easy_install.interpreter(
+ ... 'py', ws, sys.executable, join(interpreter_dir, 'bin'),
+ ... join(interpreter_dir, 'parts', 'interpreter'))
+
+On non-Windows systems, this creates three files.
+
+ >>> len(generated)
+ 3
+ >>> import pprint
+ >>> pprint.pprint(sorted(generated))
+ ['/interpreter/bin/py',
+ '/interpreter/parts/interpreter/site.py',
+ '/interpreter/parts/interpreter/sitecustomize.py']
+
+These are the three generated files.
+
+ >>> cat(interpreter_dir, 'bin', 'py')
+ #!/usr/bin/python2.4 -S
+ <BLANKLINE>
+ import os
+ import sys
+ <BLANKLINE>
+ argv = [sys.executable] + sys.argv[1:]
+ environ = os.environ.copy()
+ path = '/interpreter/parts/interpreter'
+ if environ.get('PYTHONPATH'):
+ path = os.pathsep.join([path, environ['PYTHONPATH']])
+ environ['PYTHONPATH'] = path
+ os.execve(sys.executable, argv, environ)
+ >>> ls(interpreter_dir, 'parts', 'interpreter')
+ - site.py
+ - sitecustomize.py
+ >>> cat(interpreter_dir, 'parts', 'interpreter', 'site.py')
+ import sys
+ sys.path[0:0] = [
+ '/interpreter/eggs/demo-0.3-py2.4.egg',
+ '/interpreter/eggs/demoneeded-1.1-py2.4.egg',
+ ]
+ import sitecustomize
+ >>> cat(interpreter_dir, 'parts', 'interpreter', 'sitecustomize.py')
+
+Here are some examples of the interpreter in use.
+
+ >>> print system(join(interpreter_dir, 'bin', 'py') + ' -c "print 16+26"')
+ 42
+ <BLANKLINE>
+ >>> res = system(join(interpreter_dir, 'bin', 'py') +
+ ... ' -c "import sys, pprint; pprint.pprint(sys.path)"')
+ >>> print res # doctest: +ELLIPSIS
+ ['',
+ '/interpreter/eggs/demo-0.3-py2.4.egg',
+ '/interpreter/eggs/demoneeded-1.1-py2.4.egg',
+ '/interpreter/parts/interpreter',
+ ...]
+ <BLANKLINE>
+ >>> clean_paths = eval(res.strip()) # This is used later for comparison.
+
+If you provide initialization, it goes in sitecustomize.py.
+
+ >>> def reset_interpreter():
+ ... # This is necessary because, in our tests, the timestamps of the
+ ... # .pyc files are not outdated when we want them to be.
+ ... rmdir(interpreter_dir, 'bin')
+ ... mkdir(interpreter_dir, 'bin')
+ ... rmdir(interpreter_dir, 'parts', 'interpreter')
+ ... mkdir(interpreter_dir, 'parts', 'interpreter')
+ ...
+ >>> reset_interpreter()
+
+ >>> initialization_string = """\
+ ... import os
+ ... os.environ['FOO'] = 'bar baz bing shazam'"""
+ >>> generated = zc.buildout.easy_install.interpreter(
+ ... 'py', ws, sys.executable, join(interpreter_dir, 'bin'),
+ ... join(interpreter_dir, 'parts', 'interpreter'),
+ ... initialization=initialization_string)
+ >>> cat(interpreter_dir, 'parts', 'interpreter', 'sitecustomize.py')
+ import os
+ os.environ['FOO'] = 'bar baz bing shazam'
+ >>> print system(join(interpreter_dir, 'bin', 'py') +
+ ... """ -c 'import os; print os.environ["FOO"]'""")
+ bar baz bing shazam
+ <BLANKLINE>
+
+If you use relative paths, this affects the interpreter and site.py.
+
+ >>> reset_interpreter()
+ >>> generated = zc.buildout.easy_install.interpreter(
+ ... 'py', ws, sys.executable, join(interpreter_dir, 'bin'),
+ ... join(interpreter_dir, 'parts', 'interpreter'),
+ ... relative_paths=interpreter_dir)
+ >>> cat(interpreter_dir, 'bin', 'py')
+ #!/usr/bin/python2.4 -S
+ <BLANKLINE>
+ import os
+ <BLANKLINE>
+ join = os.path.join
+ base = os.path.dirname(os.path.abspath(os.path.realpath(__file__)))
+ base = os.path.dirname(base)
+ <BLANKLINE>
+ import os
+ import sys
+ <BLANKLINE>
+ argv = [sys.executable] + sys.argv[1:]
+ environ = os.environ.copy()
+ path = join(base, 'parts/interpreter')
+ if environ.get('PYTHONPATH'):
+ path = os.pathsep.join([path, environ['PYTHONPATH']])
+ environ['PYTHONPATH'] = path
+ os.execve(sys.executable, argv, environ)
+ >>> cat(interpreter_dir, 'parts', 'interpreter', 'site.py')
+ <BLANKLINE>
+ import os
+ <BLANKLINE>
+ join = os.path.join
+ base = os.path.dirname(os.path.abspath(os.path.realpath(__file__)))
+ base = os.path.dirname(base)
+ base = os.path.dirname(base)
+ import sys
+ sys.path[0:0] = [
+ join(base, 'eggs/demo-0.3-pyN.N.egg'),
+ join(base, 'eggs/demoneeded-1.1-pyN.N.egg'),
+ ]
+ import sitecustomize
+ >>> print system(join(interpreter_dir, 'bin', 'py') +
+ ... ' -c "import sys, pprint; pprint.pprint(sys.path)"')
+ ... # doctest: +ELLIPSIS
+ ['',
+ '/interpreter/eggs/demo-0.3-py2.4.egg',
+ '/interpreter/eggs/demoneeded-1.1-py2.4.egg',
+ '/interpreter/parts/interpreter',
+ ...]
+ <BLANKLINE>
+
+The ``extra_paths`` argument affects the path in site.py.
+
+ >>> reset_interpreter()
+ >>> generated = zc.buildout.easy_install.interpreter(
+ ... 'py', ws, sys.executable, join(interpreter_dir, 'bin'),
+ ... join(interpreter_dir, 'parts', 'interpreter'),
+ ... extra_paths=[join(interpreter_dir, 'other')])
+ >>> cat(interpreter_dir, 'parts', 'interpreter', 'site.py')
+ import sys
+ sys.path[0:0] = [
+ '/interpreter/eggs/demo-0.3-py2.4.egg',
+ '/interpreter/eggs/demoneeded-1.1-py2.4.egg',
+ '/interpreter/other',
+ ]
+ import sitecustomize
+ >>> print system(join(interpreter_dir, 'bin', 'py') +
+ ... ' -c "import sys, pprint; pprint.pprint(sys.path)"')
+ ... # doctest: +ELLIPSIS
+ ['',
+ '/interpreter/eggs/demo-0.3-py2.4.egg',
+ '/interpreter/eggs/demoneeded-1.1-py2.4.egg',
+ '/interpreter/other',
+ '/interpreter/parts/interpreter',
+ ...]
+ <BLANKLINE>
+
+The ``import_site`` argument also affects site.py. It causes the executable's
+real site.py to be executed after our customizations have run.
+
+ >>> reset_interpreter()
+ >>> generated = zc.buildout.easy_install.interpreter(
+ ... 'py', ws, sys.executable, join(interpreter_dir, 'bin'),
+ ... join(interpreter_dir, 'parts', 'interpreter'),
+ ... import_site=True)
+ >>> cat(interpreter_dir, 'parts', 'interpreter', 'site.py')
+ import sys
+ sys.path[0:0] = [
+ '/interpreter/eggs/demo-0.3-py2.4.egg',
+ '/interpreter/eggs/demoneeded-1.1-py2.4.egg',
+ ]
+ # We have to import pkg_resources before namespace
+ # package .pth files are processed or else the distribution's namespace
+ # packages will mask all of the egg-based packages in the same namespace
+ # package.
+ try:
+ import pkg_resources
+ except ImportError:
+ pass
+ execfile('/usr/lib/python/site.py')
+ >>> res = system(join(interpreter_dir, 'bin', 'py') +
+ ... ' -c "import sys, pprint; pprint.pprint(sys.path)"')
+ >>> print res # doctest: +ELLIPSIS
+ ['',
+ '/interpreter/eggs/demo-0.3-py2.4.egg',
+ '/interpreter/eggs/demoneeded-1.1-py2.4.egg',
+ '/interpreter/parts/interpreter',
+ ...]
+ <BLANKLINE>
+
+The clean_paths gathered earlier is a subset of this full list of paths.
+
+ >>> full_paths = eval(res.strip())
+ >>> len(clean_paths) < len(full_paths)
+ True
+ >>> set(os.path.normpath(p) for p in clean_paths).difference(
+ ... os.path.normpath(p) for p in full_paths)
+ set([])
+
+The ``import_sitecustomize`` argument does the same thing for the
+sitecustomize module.
+
Handling custom build options for extensions provided in source distributions
-----------------------------------------------------------------------------
Modified: zc.buildout/branches/gary-4/src/zc/buildout/tests.py
===================================================================
--- zc.buildout/branches/gary-4/src/zc/buildout/tests.py 2009-12-16 15:58:14 UTC (rev 106633)
+++ zc.buildout/branches/gary-4/src/zc/buildout/tests.py 2009-12-16 16:19:10 UTC (rev 106634)
@@ -2933,6 +2933,8 @@
(re.compile('extdemo[.]pyd'), 'extdemo.so'),
(re.compile('[-d] setuptools-\S+[.]egg'), 'setuptools.egg'),
(re.compile(r'\\[\\]?'), '/'),
+ (re.compile('''execfile\(['"].+site.py['"]\)'''),
+ "execfile('/usr/lib/python/site.py')"),
]+(sys.version_info < (2, 5) and [
(re.compile('.*No module named runpy.*', re.S), ''),
(re.compile('.*usage: pdb.py scriptfile .*', re.S), ''),
More information about the checkins
mailing list