[Checkins] SVN: zc.buildout/branches/gary-support-system-python/ save an almost-certainly broken, but promising sketch. This attempts to do everything I wanted except a package whitelist. I initially implemented changes to bootstrap that removed site-packages, but decided that this was unnecessary, so ripped it out. Tests are in progress, already causing some rewrites, but this is also a sketch for Jim to consider, if he gets around to it.
Gary Poster
gary.poster at canonical.com
Fri Jun 26 17:09:25 EDT 2009
Log message for revision 101293:
save an almost-certainly broken, but promising sketch. This attempts to do everything I wanted except a package whitelist. I initially implemented changes to bootstrap that removed site-packages, but decided that this was unnecessary, so ripped it out. Tests are in progress, already causing some rewrites, but this is also a sketch for Jim to consider, if he gets around to it.
Changed:
U zc.buildout/branches/gary-support-system-python/bootstrap/bootstrap.py
U zc.buildout/branches/gary-support-system-python/src/zc/buildout/buildout.py
U zc.buildout/branches/gary-support-system-python/src/zc/buildout/easy_install.py
U zc.buildout/branches/gary-support-system-python/zc.recipe.egg_/src/zc/recipe/egg/egg.py
-=-
Modified: zc.buildout/branches/gary-support-system-python/bootstrap/bootstrap.py
===================================================================
--- zc.buildout/branches/gary-support-system-python/bootstrap/bootstrap.py 2009-06-26 21:06:12 UTC (rev 101292)
+++ zc.buildout/branches/gary-support-system-python/bootstrap/bootstrap.py 2009-06-26 21:09:24 UTC (rev 101293)
@@ -20,19 +20,87 @@
$Id$
"""
-import os, shutil, sys, tempfile, urllib2
+import os, re, shutil, sys, tempfile, textwrap, urllib, urllib2
-tmpeggs = tempfile.mkdtemp()
+# We have to manually parse our options rather than using one of the stdlib
+# tools because we want to pass the ones we don't recognize along to
+# zc.buildout.buildout.main.
-is_jython = sys.platform.startswith('java')
+configuration = {
+ '--ez_setup-source': 'http://peak.telecommunity.com/dist/ez_setup.py',
+ '--version': '',
+ '--download-base': None,
+ '--eggs': None}
+helpstring = __doc__ + textwrap.dedent('''
+ Options:
+ --version=ZC_BUILDOUT_VERSION
+ Specify a version number of the zc.buildout to use
+ --ez_setup-source=URL_OR_FILE
+ Specify a URL or file location for the ez_setup file.
+ Defaults to
+ %(--ez_setup-source)s
+ --download-base=URL_OR_DIRECTORY
+ Specify a URL or directory for downloading setuptools and
+ zc.buildout. Defaults to PyPI.
+ --eggs=DIRECTORY
+ Specify a directory for storing eggs. Defaults to a temporary
+ directory that is deleted when the bootstrap script completes.
+
+ By using --ez_setup-source and --download-base to point to local resources,
+ you can keep bootstrap from going over the network.
+ ''')
+match_equals = re.compile(r'(%s)=(\S*)' % ('|'.join(configuration),)).match
+args = sys.argv[1:]
+if args == ['--help']:
+ print helpstring
+ sys.exit(0)
+
+# defaults
+tmpeggs = None
+
+while args:
+ val = args[0]
+ elif val in configuration:
+ del args[0]
+ if not args or args[0].startswith('-'):
+ print "ERROR: %s requires an argument."
+ print helpstring
+ sys.exit(1)
+ configuration[val] = args[0]
+ else:
+ match = match_equals(val)
+ if match and match.group(1) in configuration:
+ configuration[match.group(1)] = match.group(2)
+ else:
+ break
+ del args[0]
+
+for name in ('--ez_setup-source', '--download-base'):
+ val = configuration[name]
+ if val is not None and '://' not in val: # we're being lazy.
+ configuration[name] = 'file://%s' % (
+ urllib.pathname2url(os.path.abspath(os.path.expanduser(val))),)
+
+if not configuration['--eggs']:
+ configuration['--eggs'] = tmpeggs = tempfile.mkdtemp()
+else:
+ configuration['--eggs'] = os.path.abspath(
+ os.path.expanduser(configuration['--eggs']))
+
+if configuration['--version']:
+ configuration['--version'] = '==' + configuration['--version']
+
try:
import pkg_resources
except ImportError:
ez = {}
- exec urllib2.urlopen('http://peak.telecommunity.com/dist/ez_setup.py'
- ).read() in ez
- ez['use_setuptools'](to_dir=tmpeggs, download_delay=0)
+ exec urllib2.urlopen(configuration['--ez_setup-source']).read() in ez
+ setuptools_args = dict(to_dir=configuration['--eggs'], download_delay=0)
+ if configuration['--download-base']:
+ setuptools_args['download_base'] = (
+ configuration['--download-base'] + '/')
+ ez['use_setuptools'](**setuptools_args)
import pkg_resources
@@ -45,40 +113,33 @@
else:
def quote (c):
return c
+cmd = [quote(sys.executable),
+ '-c',
+ quote('from setuptools.command.easy_install import main; main()'),
+ '-mqNxd',
+ quote(configuration['--eggs']),
+ 'zc.buildout' + configuration['--version']]
+ws = pkg_resources.working_set
+env = dict(
+ os.environ,
+ PYTHONPATH=ws.find(pkg_resources.Requirement.parse('setuptools')).location)
-cmd = 'from setuptools.command.easy_install import main; main()'
-ws = pkg_resources.working_set
-
-if len(sys.argv) > 2 and sys.argv[1] == '--version':
- VERSION = '==%s' % sys.argv[2]
- args = sys.argv[3:] + ['bootstrap']
-else:
- VERSION = ''
- args = sys.argv[1:] + ['bootstrap']
-
-if is_jython:
+try:
import subprocess
-
- assert subprocess.Popen([sys.executable] + ['-c', quote(cmd), '-mqNxd',
- quote(tmpeggs), 'zc.buildout' + VERSION],
- env=dict(os.environ,
- PYTHONPATH=
- ws.find(pkg_resources.Requirement.parse('setuptools')).location
- ),
- ).wait() == 0
-
+except ImportError:
+ exitcode = os.spawnle(*([os.P_WAIT, sys.executable] + cmd + [env]))
else:
- assert os.spawnle(
- os.P_WAIT, sys.executable, quote (sys.executable),
- '-c', quote (cmd), '-mqNxd', quote (tmpeggs), 'zc.buildout' + VERSION,
- dict(os.environ,
- PYTHONPATH=
- ws.find(pkg_resources.Requirement.parse('setuptools')).location
- ),
- ) == 0
+ # Jython can use subprocess but not spawn. We prefer it generally.
+ exitcode = subprocess.Popen(cmd, env=env).wait()
+if exitcode != 0:
+ # we shouldn't need an error message because a failure
+ # should have generated a visible traceback in the subprocess.
+ sys.exit(exitcode)
-ws.add_entry(tmpeggs)
-ws.require('zc.buildout' + VERSION)
+ws.add_entry(configuration['--eggs'])
+ws.require('zc.buildout' + configuration['--version'])
import zc.buildout.buildout
+args.append('bootstrap')
zc.buildout.buildout.main(args)
-shutil.rmtree(tmpeggs)
+if tmpeggs is not None:
+ shutil.rmtree(tmpeggs)
Modified: zc.buildout/branches/gary-support-system-python/src/zc/buildout/buildout.py
===================================================================
--- zc.buildout/branches/gary-support-system-python/src/zc/buildout/buildout.py 2009-06-26 21:06:12 UTC (rev 101292)
+++ zc.buildout/branches/gary-support-system-python/src/zc/buildout/buildout.py 2009-06-26 21:09:24 UTC (rev 101293)
@@ -217,6 +217,12 @@
prefer_final)
zc.buildout.easy_install.prefer_final(prefer_final=='true')
+ use_site_packages = options.get('use-site-packages', 'true')
+ if use_site_packages not in ('true', 'false'):
+ self._error('Invalid value for use-site-packages option: %s',
+ use_site_packages)
+ zc.buildout.easy_install.use_site_packages(use_site_packages=='true')
+
use_dependency_links = options.get('use-dependency-links', 'true')
if use_dependency_links not in ('true', 'false'):
self._error('Invalid value for use-dependency-links option: %s',
Modified: zc.buildout/branches/gary-support-system-python/src/zc/buildout/easy_install.py
===================================================================
--- zc.buildout/branches/gary-support-system-python/src/zc/buildout/easy_install.py 2009-06-26 21:06:12 UTC (rev 101292)
+++ zc.buildout/branches/gary-support-system-python/src/zc/buildout/easy_install.py 2009-06-26 21:09:24 UTC (rev 101293)
@@ -64,13 +64,49 @@
pkg_resources.Requirement.parse('setuptools')
).location
-# Include buildout and setuptools eggs in paths
-buildout_and_setuptools_path = [
- setuptools_loc,
- pkg_resources.working_set.find(
- pkg_resources.Requirement.parse('zc.buildout')).location,
- ]
+# Include buildout and setuptools eggs in paths. We prevent dupes just to
+# keep from duplicating any log messages about them.
+buildout_loc = pkg_resources.working_set.find(
+ pkg_resources.Requirement.parse('zc.buildout')).location
+buildout_and_setuptools_path = [setuptools_loc]
+if os.path.normpath(setuptools_loc) != os.path.normpath(buildout_loc):
+ buildout_and_setuptools_path.append(buildout_loc)
+def _get_system_packages(executable):
+ """return a pair of the standard lib and site packages for the executable.
+ """
+ # We want to get a list of the site packages, which is not easy. The
+ # canonical way to do this is to use distutils.sysconfig.get_python_lib(),
+ # but that only returns a single path, which does not reflect reality for
+ # many system Pythons, which have multiple additions. Instead, we start
+ # Python with -S, which does not import site.py and set up the extra paths
+ # like site-packages or (Ubuntu/Debian) dist-packages and python-support.
+ # We then compare that sys.path with ours. The set of ours minus the set
+ # of the ones in ``python -S`` is the set of packages that are effectively
+ # site-packages.
+ def get_sys_path(clean=False):
+ cmd = [executable, "-c",
+ "import sys, os;"
+ "print repr([os.path.normpath(p) for p in sys.path])"]
+ if clean:
+ cmd.insert(1, '-S')
+ _proc = subprocess.Popen(cmd, stdout=subprocess.PIPE)
+ _proc.wait();
+ res = eval(_proc.stdout.read())
+ try:
+ res.remove('.')
+ except ValueError:
+ pass
+ return res
+ # This code might be run in the context of code that has manipulated the
+ # sys.path--for instance, to add local zc.buildout or setuptools eggs.
+ # Therefore, even if the executable is the current executable, we want to
+ # get our Python's "standard" sys.path, we need to go out to a subprocess,
+ # just as we do for the "-S" variant.
+ stdlib = get_sys_path(clean=True)
+ site_packages = [p for p in get_sys_path() if p not in stdlib]
+ return (stdlib, site_packages)
+
class IncompatibleVersionError(zc.buildout.UserError):
"""A specified version is incompatible with a given requirement.
"""
@@ -152,6 +188,7 @@
_use_dependency_links = True
_allow_picked_versions = True
_always_unzip = False
+ _use_site_packages = True
def __init__(self,
dest=None,
@@ -163,6 +200,7 @@
newest=True,
versions=None,
use_dependency_links=None,
+ use_site_packages=None,
allow_hosts=('*',)
):
self._dest = dest
@@ -185,7 +223,24 @@
self._executable = executable
if always_unzip is not None:
self._always_unzip = always_unzip
- path = (path and path[:] or []) + buildout_and_setuptools_path
+ path = (path and path[:] or [])
+ if use_site_packages is not None:
+ self._use_site_packages = use_site_packages
+ stdlib, site_packages = _get_system_packages(executable)
+ if not self._use_site_packages:
+ for p in buildout_and_setuptools_path:
+ if os.path.normpath(p) not in site_packages:
+ logger.debug(
+ 'We normally include our own buildout and setuptools '
+ 'paths, but this one comes from site '
+ 'packages, which does not satisfy the requested '
+ 'installation constraints:'
+ '%s', p)
+ else:
+ path.append(p)
+ else:
+ path.extend(buildout_and_setuptools_path)
+ path.extend(site_packages)
if dest is not None and dest not in path:
path.insert(0, dest)
self._path = path
@@ -773,6 +828,12 @@
Installer._prefer_final = bool(setting)
return old
+def use_site_packages(setting=None):
+ old = Installer._use_site_packages
+ if setting is not None:
+ Installer._use_site_packages = bool(setting)
+ return old
+
def use_dependency_links(setting=None):
old = Installer._use_dependency_links
if setting is not None:
@@ -795,19 +856,21 @@
links=(), index=None,
executable=sys.executable, always_unzip=None,
path=None, working_set=None, newest=True, versions=None,
- use_dependency_links=None, allow_hosts=('*',)):
+ use_dependency_links=None, use_site_packages=None,
+ allow_hosts=('*',)):
installer = Installer(dest, links, index, executable, always_unzip, path,
newest, versions, use_dependency_links,
- allow_hosts=allow_hosts)
+ use_site_packages, allow_hosts=allow_hosts)
return installer.install(specs, working_set)
def build(spec, dest, build_ext,
links=(), index=None,
executable=sys.executable,
- path=None, newest=True, versions=None, allow_hosts=('*',)):
+ path=None, newest=True, versions=None, use_site_packages,
+ allow_hosts=('*',)):
installer = Installer(dest, links, index, executable, True, path, newest,
- versions, allow_hosts=allow_hosts)
+ versions, use_site_packages, allow_hosts=allow_hosts)
return installer.build(spec, build_ext)
@@ -903,22 +966,56 @@
[f() for f in undo]
-def working_set(specs, executable, path):
- return install(specs, None, executable=executable, path=path)
+def working_set(specs, executable, path, use_site_packages=None):
+ return install(
+ specs, None, executable=executable, path=path, use_site_packages)
+def get_path(working_set, executable, use_site_packages=True):
+ """Given working set and path to executable, return value for sys.path.
+
+ Distribution locations from the working set come first in the list. Within
+ that collection, this function pushes site-packages-based distribution
+ locations to the end of the list, so that they don't mask eggs.
+
+ This expects that the working_set has already been created to honor a
+ use_site_packages setting. That is, if use_site_packages is False, this
+ function does *not* verify that the working_set's distributions are not in
+ site packages.
+
+ However, it does explicitly include site packages if use_site_packages is
+ True.
+
+ The standard library (defined as what the given Python executable has on
+ the path before its site.py is run) is always included.
+ """
+ stdlib, site_packages = _get_system_packages(executable)
+ postponed = []
+ path = []
+ for dist in working_set:
+ location = os.path.normpath(dist.location)
+ if location in site_packages:
+ postponed.append(location)
+ else:
+ path.append(location)
+ path.extend(postponed)
+ path.extend(extra_paths)
+ path = map(realpath, path)
+ # now we add in all paths
+ if use_site_packages:
+ path.extend(site_packages)
+ path.extend(stdlib)
+ return path
+
def scripts(reqs, working_set, executable, dest,
scripts=None,
extra_paths=(),
arguments='',
interpreter=None,
initialization='',
- relative_paths=False,
+ use_site_packages=True,
+ relative_paths=False
):
-
- path = [dist.location for dist in working_set]
- path.extend(extra_paths)
- path = map(realpath, path)
-
+ path = get_path(working_set, executable, use_site_packages)
generated = []
if isinstance(reqs, str):
@@ -955,7 +1052,7 @@
generated.extend(
_script(module_name, attrs, spath, sname, executable, arguments,
- initialization, rpsetup)
+ initialization, rpsetup, exclude_site_packages)
)
if interpreter:
@@ -1075,7 +1172,7 @@
%(relative_paths_setup)s
import sys
-sys.path[0:0] = [
+sys.path[:] = [
%(path)s,
]
%(initialization)s
Modified: zc.buildout/branches/gary-support-system-python/zc.recipe.egg_/src/zc/recipe/egg/egg.py
===================================================================
--- zc.buildout/branches/gary-support-system-python/zc.recipe.egg_/src/zc/recipe/egg/egg.py 2009-06-26 21:06:12 UTC (rev 101292)
+++ zc.buildout/branches/gary-support-system-python/zc.recipe.egg_/src/zc/recipe/egg/egg.py 2009-06-26 21:09:24 UTC (rev 101293)
@@ -54,6 +54,13 @@
python = options.get('python', buildout['buildout']['python'])
options['executable'] = buildout[python]['executable']
+ use_site_packages = self.options.get(
+ 'use-site-packages',
+ self.buildout['buildout'].get('use-site-packages', 'true')
+ if use_site_packages not in ('true', 'false'):
+ self._error('Invalid value for use-site-packages option: %s',
+ use_site_packages)
+ self.use_site_packages = use_site_packages=='true'
def working_set(self, extra=()):
"""Separate method to just get the working set
@@ -72,7 +79,8 @@
if self.buildout['buildout'].get('offline') == 'true':
ws = zc.buildout.easy_install.working_set(
distributions, options['executable'],
- [options['develop-eggs-directory'], options['eggs-directory']]
+ [options['develop-eggs-directory'], options['eggs-directory']],
+ use_site_packages=self.use_site_packages
)
else:
kw = {}
@@ -81,12 +89,13 @@
ws = zc.buildout.easy_install.install(
distributions, options['eggs-directory'],
- links = self.links,
- index = self.index,
- executable = options['executable'],
+ links=self.links,
+ index=self.index,
+ executable=options['executable'],
path=[options['develop-eggs-directory']],
newest=self.buildout['buildout'].get('newest') == 'true',
allow_hosts=self.allow_hosts,
+ use_site_packages=self.use_site_packages,
**kw)
return orig_distributions, ws
@@ -166,7 +175,8 @@
interpreter=options.get('interpreter'),
initialization=options.get('initialization', ''),
arguments=options.get('arguments', ''),
- relative_paths=self._relative_paths,
+ use_site_packages=self.use_site_packages,
+ relative_paths=self._relative_paths
)
return ()
More information about the Checkins
mailing list