[Checkins] SVN: zc.buildout/branches/gary-5/ support limiting packages from site-packages
Gary Poster
gary.poster at canonical.com
Fri Feb 19 20:54:02 EST 2010
Log message for revision 109166:
support limiting packages from site-packages
Changed:
A zc.buildout/branches/gary-5/
A zc.buildout/branches/gary-5/.bzrignore
U zc.buildout/branches/gary-5/src/zc/buildout/easy_install.py
U zc.buildout/branches/gary-5/src/zc/buildout/easy_install.txt
U zc.buildout/branches/gary-5/src/zc/buildout/testing.py
U zc.buildout/branches/gary-5/src/zc/buildout/tests.py
-=-
Added: zc.buildout/branches/gary-5/.bzrignore
===================================================================
--- zc.buildout/branches/gary-5/.bzrignore (rev 0)
+++ zc.buildout/branches/gary-5/.bzrignore 2010-02-20 01:54:01 UTC (rev 109166)
@@ -0,0 +1,9 @@
+.installed.cfg
+bin
+build
+develop-eggs
+eggs
+parts
+src/zc.buildout.egg-info
+z3c.recipe.scripts_/src/z3c.recipe.scripts.egg-info
+zc.recipe.egg_/src/zc.recipe.egg.egg-info
Modified: zc.buildout/branches/gary-5/src/zc/buildout/easy_install.py
===================================================================
--- zc.buildout/branches/gary-4/src/zc/buildout/easy_install.py 2010-02-17 21:30:01 UTC (rev 109101)
+++ zc.buildout/branches/gary-5/src/zc/buildout/easy_install.py 2010-02-20 01:54:01 UTC (rev 109166)
@@ -19,6 +19,7 @@
"""
import distutils.errors
+import fnmatch
import glob
import logging
import os
@@ -67,7 +68,65 @@
pkg_resources.Requirement.parse('zc.buildout')).location,
]
+def _get_system_paths(executable):
+ """return lists of standard lib and site paths for executable.
+ """
+ # We want to get a list of the site packages, which is not easy.
+ # The canonical way to do this is to use
+ # distutils.sysconfig.get_python_lib(), but that only returns a
+ # single path, which does not reflect reality for many system
+ # Pythons, which have multiple additions. Instead, we start Python
+ # with -S, which does not import site.py and set up the extra paths
+ # like site-packages or (Ubuntu/Debian) dist-packages and
+ # python-support. We then compare that sys.path with the normal one
+ # (minus user packages if this is Python 2.6, because we don't
+ # support those (yet?). The set of the normal one minus the set of
+ # the ones in ``python -S`` is the set of packages that are
+ # effectively site-packages.
+ #
+ # The given executable might not be the current executable, so it is
+ # appropriate to do another subprocess to figure out what the
+ # additional site-package paths are. Moreover, even if this
+ # executable *is* the current executable, this code might be run in
+ # the context of code that has manipulated the sys.path--for
+ # instance, to add local zc.buildout or setuptools eggs.
+ def get_sys_path(*args, **kwargs):
+ cmd = [executable]
+ cmd.extend(args)
+ cmd.extend([
+ "-c", "import sys, os;"
+ "print repr([os.path.normpath(p) for p in sys.path if p])"])
+ # Windows needs some (as yet to be determined) part of the real env.
+ env = os.environ.copy()
+ env.update(kwargs)
+ _proc = subprocess.Popen(
+ cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env)
+ stdout, stderr = _proc.communicate();
+ if _proc.returncode:
+ raise RuntimeError(
+ 'error trying to get system packages:\n%s' % (stderr,))
+ res = eval(stdout.strip())
+ try:
+ res.remove('.')
+ except ValueError:
+ pass
+ return res
+ stdlib = get_sys_path('-S') # stdlib only
+ no_user_paths = get_sys_path(PYTHONNOUSERSITE='x')
+ site_paths = [p for p in no_user_paths if p not in stdlib]
+ return (stdlib, site_paths)
+def _get_version_info(executable):
+ cmd = [executable, '-Sc', 'import sys; print repr(sys.version_info)']
+ _proc = subprocess.Popen(
+ cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+ stdout, stderr = _proc.communicate();
+ if _proc.returncode:
+ raise RuntimeError(
+ 'error trying to get system packages:\n%s' % (stderr,))
+ return eval(stdout.strip())
+
+
class IncompatibleVersionError(zc.buildout.UserError):
"""A specified version is incompatible with a given requirement.
"""
@@ -109,7 +168,12 @@
_indexes = {}
-def _get_index(executable, index_url, find_links, allow_hosts=('*',)):
+def _get_index(executable, index_url, find_links, allow_hosts=('*',),
+ path=None):
+ # If path is None, the index will use sys.path. If you provide an empty
+ # path ([]), it will complain uselessly about missing index pages for
+ # packages found in the paths that you expect to use. Therefore, this path
+ # is always the same as the _env path in the Installer.
key = executable, index_url, tuple(find_links)
index = _indexes.get(key)
if index is not None:
@@ -118,7 +182,8 @@
if index_url is None:
index_url = default_index_url
index = AllowHostsPackageIndex(
- index_url, hosts=allow_hosts, python=_get_version(executable)
+ index_url, hosts=allow_hosts, search_path=path,
+ python=_get_version(executable)
)
if find_links:
@@ -192,6 +257,8 @@
_use_dependency_links = True
_allow_picked_versions = True
_always_unzip = False
+ _include_site_packages = True
+ _allowed_eggs_from_site_packages = ('*',)
def __init__(self,
dest=None,
@@ -203,6 +270,8 @@
newest=True,
versions=None,
use_dependency_links=None,
+ include_site_packages=None,
+ allowed_eggs_from_site_packages=None,
allow_hosts=('*',)
):
self._dest = dest
@@ -225,7 +294,28 @@
self._executable = executable
if always_unzip is not None:
self._always_unzip = always_unzip
- path = (path and path[:] or []) + buildout_and_setuptools_path
+ path = (path and path[:] or [])
+ if include_site_packages is not None:
+ self._include_site_packages = include_site_packages
+ if allowed_eggs_from_site_packages is not None:
+ self._allowed_eggs_from_site_packages = tuple(
+ allowed_eggs_from_site_packages)
+ stdlib, self._site_packages = _get_system_paths(executable)
+ version_info = _get_version_info(executable)
+ if version_info == sys.version_info:
+ # Maybe we can add the buildout and setuptools path. If we
+ # are including site_packages, we only have to include the extra
+ # bits here, so we don't duplicate. On the other hand, if we
+ # are not including site_packages, we only want to include the
+ # parts that are not in site_packages, so the code is the same.
+ path.extend(
+ set(buildout_and_setuptools_path).difference(
+ self._site_packages))
+ if self._include_site_packages:
+ path.extend(self._site_packages)
+ # else we could try to still include the buildout_and_setuptools_path
+ # if the elements are not in site_packages, but we're not bothering
+ # with this optimization for now, in the name of code simplicity.
if dest is not None and dest not in path:
path.insert(0, dest)
self._path = path
@@ -234,13 +324,42 @@
self._newest = newest
self._env = pkg_resources.Environment(path,
python=_get_version(executable))
- self._index = _get_index(executable, index, links, self._allow_hosts)
+ self._index = _get_index(executable, index, links, self._allow_hosts,
+ self._path)
if versions is not None:
self._versions = versions
+ _allowed_eggs_from_site_packages_regex = None
+ def allow_site_package_egg(self, name):
+ if (not self._include_site_packages or
+ not self._allowed_eggs_from_site_packages):
+ # If the answer is a blanket "no," perform a shortcut.
+ return False
+ if self._allowed_eggs_from_site_packages_regex is None:
+ pattern = '(%s)' % (
+ '|'.join(
+ fnmatch.translate(name)
+ for name in self._allowed_eggs_from_site_packages),
+ )
+ self._allowed_eggs_from_site_packages_regex = re.compile(pattern)
+ return bool(self._allowed_eggs_from_site_packages_regex.match(name))
+
def _satisfied(self, req, source=None):
- dists = [dist for dist in self._env[req.project_name] if dist in req]
+ # We get all distributions that match the given requirement. If we are
+ # not supposed to include site-packages for the given egg, we also
+ # filter those out. Even if include_site_packages is False and so we
+ # have excluded site packages from the _env's paths (see
+ # Installer.__init__), we need to do the filtering here because an
+ # .egg-link, such as one for setuptools or zc.buildout installed by
+ # zc.buildout.buildout.Buildout.bootstrap, can indirectly include a
+ # path in our _site_packages.
+ dists = [dist for dist in self._env[req.project_name] if (
+ dist in req and (
+ dist.location not in self._site_packages or
+ self.allow_site_package_egg(dist.project_name))
+ )
+ ]
if not dists:
logger.debug('We have no distributions for %s that satisfies %r.',
req.project_name, str(req))
@@ -441,14 +560,22 @@
# Nothing is available.
return None
- # Filter the available dists for the requirement and source flag
- dists = [dist for dist in index[requirement.project_name]
- if ((dist in requirement)
- and
- ((not source) or
- (dist.precedence == pkg_resources.SOURCE_DIST)
- )
- )
+ # Filter the available dists for the requirement and source flag. If
+ # we are not supposed to include site-packages for the given egg, we
+ # also filter those out. Even if include_site_packages is False and so
+ # we have excluded site packages from the _env's paths (see
+ # Installer.__init__), we need to do the filtering here because an
+ # .egg-link, such as one for setuptools or zc.buildout installed by
+ # zc.buildout.buildout.Buildout.bootstrap, can indirectly include a
+ # path in our _site_packages.
+ dists = [dist for dist in index[requirement.project_name] if (
+ dist in requirement and (
+ dist.location not in self._site_packages or
+ self.allow_site_package_egg(dist.project_name))
+ and (
+ (not source) or
+ (dist.precedence == pkg_resources.SOURCE_DIST))
+ )
]
# If we prefer final dists, filter for final and use the
@@ -608,7 +735,7 @@
self._links.append(link)
self._index = _get_index(self._executable,
self._index_url, self._links,
- self._allow_hosts)
+ self._allow_hosts, self._path)
for dist in dists:
# Check whether we picked a version and, if we did, report it:
@@ -689,35 +816,52 @@
self._maybe_add_setuptools(ws, dist)
# OK, we have the requested distributions and they're in the working
- # set, but they may have unmet requirements. We'll simply keep
- # trying to resolve requirements, adding missing requirements as they
- # are reported.
- #
- # Note that we don't pass in the environment, because we want
+ # set, but they may have unmet requirements. We'll resolve these
+ # requirements. This is code modified from
+ # pkg_resources.WorkingSet.resolve. We can't reuse that code directly
+ # because we have to constrain our requirements (see
+ # versions_section_ignored_for_dependency_in_favor_of_site_packages in
+ # zc.buildout.tests).
+ requirements.reverse() # Set up the stack.
+ processed = {} # This is a set of processed requirements.
+ best = {} # This is a mapping of key -> dist.
+ # Note that we don't use the existing environment, because we want
# to look for new eggs unless what we have is the best that
# matches the requirement.
- while 1:
- try:
- ws.resolve(requirements)
- except pkg_resources.DistributionNotFound, err:
- [requirement] = err
- requirement = self._constrain(requirement)
- if destination:
- logger.debug('Getting required %r', str(requirement))
- else:
- logger.debug('Adding required %r', str(requirement))
- _log_requirement(ws, requirement)
-
- for dist in self._get_dist(requirement, ws, self._always_unzip
- ):
-
- ws.add(dist)
- self._maybe_add_setuptools(ws, dist)
- except pkg_resources.VersionConflict, err:
- raise VersionConflict(err, ws)
- else:
- break
-
+ env = pkg_resources.Environment(ws.entries)
+ while requirements:
+ # Process dependencies breadth-first.
+ req = self._constrain(requirements.pop(0))
+ if req in processed:
+ # Ignore cyclic or redundant dependencies.
+ continue
+ dist = best.get(req.key)
+ if dist is None:
+ # Find the best distribution and add it to the map.
+ dist = ws.by_key.get(req.key)
+ if dist is None:
+ try:
+ dist = best[req.key] = env.best_match(req, ws)
+ except pkg_resources.VersionConflict, err:
+ raise VersionConflict(err, ws)
+ if dist is None:
+ if destination:
+ logger.debug('Getting required %r', str(req))
+ else:
+ logger.debug('Adding required %r', str(req))
+ _log_requirement(ws, req)
+ for dist in self._get_dist(req,
+ ws, self._always_unzip):
+ ws.add(dist)
+ self._maybe_add_setuptools(ws, dist)
+ if dist not in req:
+ # Oops, the "best" so far conflicts with a dependency.
+ raise VersionConflict(
+ pkg_resources.VersionConflict(dist, req), ws)
+ requirements.extend(dist.requires(req.extras)[::-1])
+ processed[req] = True
+ if dist.location in self._site_packages:
+ logger.debug('Egg from site-packages: %s', dist)
return ws
def build(self, spec, build_ext):
@@ -812,6 +956,18 @@
Installer._prefer_final = bool(setting)
return old
+def include_site_packages(setting=None):
+ old = Installer._include_site_packages
+ if setting is not None:
+ Installer._include_site_packages = bool(setting)
+ return old
+
+def allowed_eggs_from_site_packages(setting=None):
+ old = Installer._allowed_eggs_from_site_packages
+ if setting is not None:
+ Installer._allowed_eggs_from_site_packages = tuple(setting)
+ return old
+
def use_dependency_links(setting=None):
old = Installer._use_dependency_links
if setting is not None:
@@ -834,9 +990,13 @@
links=(), index=None,
executable=sys.executable, always_unzip=None,
path=None, working_set=None, newest=True, versions=None,
- use_dependency_links=None, allow_hosts=('*',)):
+ use_dependency_links=None, include_site_packages=None,
+ allowed_eggs_from_site_packages=None, allow_hosts=('*',)):
installer = Installer(dest, links, index, executable, always_unzip, path,
newest, versions, use_dependency_links,
+ include_site_packages=include_site_packages,
+ allowed_eggs_from_site_packages=
+ allowed_eggs_from_site_packages,
allow_hosts=allow_hosts)
return installer.install(specs, working_set)
@@ -844,9 +1004,14 @@
def build(spec, dest, build_ext,
links=(), index=None,
executable=sys.executable,
- path=None, newest=True, versions=None, allow_hosts=('*',)):
+ path=None, newest=True, versions=None, include_site_packages=None,
+ allowed_eggs_from_site_packages=None, allow_hosts=('*',)):
installer = Installer(dest, links, index, executable, True, path, newest,
- versions, allow_hosts=allow_hosts)
+ versions,
+ include_site_packages=include_site_packages,
+ allowed_eggs_from_site_packages=
+ allowed_eggs_from_site_packages,
+ allow_hosts=allow_hosts)
return installer.build(spec, build_ext)
@@ -941,10 +1106,13 @@
undo.reverse()
[f() for f in undo]
+def working_set(specs, executable, path, include_site_packages=None,
+ allowed_eggs_from_site_packages=None):
+ return install(
+ specs, None, executable=executable, path=path,
+ include_site_packages=include_site_packages,
+ allowed_eggs_from_site_packages=allowed_eggs_from_site_packages)
-def working_set(specs, executable, path):
- return install(specs, None, executable=executable, path=path)
-
############################################################################
# Script generation functions
@@ -1276,54 +1444,6 @@
# These are used only by the newer ``generate_scripts`` function.
-def _get_system_paths(executable):
- """return lists of standard lib and site paths for executable.
- """
- # We want to get a list of the site packages, which is not easy.
- # The canonical way to do this is to use
- # distutils.sysconfig.get_python_lib(), but that only returns a
- # single path, which does not reflect reality for many system
- # Pythons, which have multiple additions. Instead, we start Python
- # with -S, which does not import site.py and set up the extra paths
- # like site-packages or (Ubuntu/Debian) dist-packages and
- # python-support. We then compare that sys.path with the normal one
- # (minus user packages if this is Python 2.6, because we don't
- # support those (yet?). The set of the normal one minus the set of
- # the ones in ``python -S`` is the set of packages that are
- # effectively site-packages.
- #
- # The given executable might not be the current executable, so it is
- # appropriate to do another subprocess to figure out what the
- # additional site-package paths are. Moreover, even if this
- # executable *is* the current executable, this code might be run in
- # the context of code that has manipulated the sys.path--for
- # instance, to add local zc.buildout or setuptools eggs.
- def get_sys_path(*args, **kwargs):
- cmd = [executable]
- cmd.extend(args)
- cmd.extend([
- "-c", "import sys, os;"
- "print repr([os.path.normpath(p) for p in sys.path if p])"])
- # Windows needs some (as yet to be determined) part of the real env.
- env = os.environ.copy()
- env.update(kwargs)
- _proc = subprocess.Popen(
- cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env)
- stdout, stderr = _proc.communicate();
- if _proc.returncode:
- raise RuntimeError(
- 'error trying to get system packages:\n%s' % (stderr,))
- res = eval(stdout.strip())
- try:
- res.remove('.')
- except ValueError:
- pass
- return res
- stdlib = get_sys_path('-S') # stdlib only
- no_user_paths = get_sys_path(PYTHONNOUSERSITE='x')
- site_paths = [p for p in no_user_paths if p not in stdlib]
- return (stdlib, site_paths)
-
def _get_module_file(executable, name):
"""Return a module's file path.
Modified: zc.buildout/branches/gary-5/src/zc/buildout/easy_install.txt
===================================================================
--- zc.buildout/branches/gary-4/src/zc/buildout/easy_install.txt 2010-02-17 21:30:01 UTC (rev 109101)
+++ zc.buildout/branches/gary-5/src/zc/buildout/easy_install.txt 2010-02-20 01:54:01 UTC (rev 109166)
@@ -89,6 +89,14 @@
for using dependency_links in preference to other
locations. Defaults to true.
+include_site_packages
+ A flag indicating whether Python's non-standard-library packages should
+ be available for finding dependencies. Defaults to true.
+
+ Paths outside of Python's standard library--or more precisely, those that
+ are not included when Python is started with the -S argument--are loosely
+ referred to as "site-packages" here.
+
relative_paths
Adjust egg paths so they are relative to the script path. This
allows scripts to work when scripts and eggs are moved, as long as
@@ -399,6 +407,68 @@
>>> [d.version for d in ws]
['0.3', '1.1']
+Dependencies in Site Packages
+-----------------------------
+
+Paths outside of Python's standard library--or more precisely, those that are
+not included when Python is started with the -S argument--are loosely referred
+to as "site-packages" here. These site-packages are searched by default for
+distributions. This can be disabled, so that, for instance, a system Python
+can be used with buildout, cleaned of any packages installed by a user or
+system package manager.
+
+The default behavior can be controlled and introspected using
+zc.buildout.easy_install.include_site_packages.
+
+ >>> zc.buildout.easy_install.include_site_packages()
+ True
+
+Here's an example of using a Python executable that includes our dependencies.
+
+Our "py_path" will have the "demoneeded," and "demo" packages available.
+ We'll simply be asking for "demoneeded" here, but without any external
+ index or links.
+
+ >>> from zc.buildout.tests import create_sample_sys_install
+ >>> py_path, site_packages_path = make_py()
+ >>> create_sample_sys_install(site_packages_path)
+
+ >>> example_dest = tmpdir('site-packages-example-install')
+ >>> workingset = zc.buildout.easy_install.install(
+ ... ['demoneeded'], example_dest, links=[], executable=py_path,
+ ... index=None)
+ >>> [dist.project_name for dist in workingset]
+ ['demoneeded']
+
+That worked fine. Let's try again with site packages not allowed. We'll
+change the policy by changing the default. Notice that the function for
+changing the default value returns the previous value.
+
+ >>> zc.buildout.easy_install.include_site_packages(False)
+ True
+
+ >>> zc.buildout.easy_install.include_site_packages()
+ False
+
+ >>> zc.buildout.easy_install.clear_index_cache()
+ >>> rmdir(example_dest)
+ >>> example_dest = tmpdir('site-packages-example-install')
+ >>> workingset = zc.buildout.easy_install.install(
+ ... ['demoneeded'], example_dest, links=[], executable=py_path,
+ ... index=None)
+ Traceback (most recent call last):
+ ...
+ MissingDistribution: Couldn't find a distribution for 'demoneeded'.
+ >>> zc.buildout.easy_install.clear_index_cache()
+
+Now we'll reset the default.
+
+ >>> zc.buildout.easy_install.include_site_packages(True)
+ False
+
+ >>> zc.buildout.easy_install.include_site_packages()
+ True
+
Dependency links
----------------
@@ -1259,6 +1329,7 @@
>>> namespace_eggs = tmpdir('namespace_eggs')
>>> create_sample_namespace_eggs(namespace_eggs)
+ >>> reset_interpreter()
>>> ws = zc.buildout.easy_install.install(
... ['demo', 'tellmy.fortune'], join(interpreter_dir, 'eggs'),
... links=[link_server, namespace_eggs], index=link_server+'index/')
@@ -1319,6 +1390,7 @@
include site-packages, and use relative paths. For completeness, we'll look
at that result.
+ >>> reset_interpreter()
>>> generated = zc.buildout.easy_install.generate_scripts(
... interpreter_bin_dir, ws, sys.executable, interpreter_parts_dir,
... interpreter='py', add_site_packages=True,
Modified: zc.buildout/branches/gary-5/src/zc/buildout/testing.py
===================================================================
--- zc.buildout/branches/gary-4/src/zc/buildout/testing.py 2010-02-17 21:30:01 UTC (rev 109101)
+++ zc.buildout/branches/gary-5/src/zc/buildout/testing.py 2010-02-20 01:54:01 UTC (rev 109166)
@@ -222,11 +222,37 @@
time.sleep(0.01)
raise ValueError('Timed out waiting for: '+label)
+def get_installer_values():
+ """Get the current values for the easy_install module.
+
+ This is necessary because instantiating a Buildout will force the
+ Buildout's values on the installer.
+
+ Returns a dict of names-values suitable for set_installer_values."""
+ names = ('default_versions', 'download_cache', 'install_from_cache',
+ 'prefer_final', 'include_site_packages',
+ 'allowed_eggs_from_site_packages', 'use_dependency_links',
+ 'allow_picked_versions', 'always_unzip'
+ )
+ values = {}
+ for name in names:
+ values[name] = getattr(zc.buildout.easy_install, name)()
+ return values
+
+def set_installer_values(values):
+ """Set the given values on the installer."""
+ for name, value in values.items():
+ getattr(zc.buildout.easy_install, name)(value)
+
def make_buildout():
- # Create a basic buildout.cfg to avoid a warning from buildout:
+ """Make a buildout that uses this version of zc.buildout."""
+ # Create a basic buildout.cfg to avoid a warning from buildout.
open('buildout.cfg', 'w').write(
"[buildout]\nparts =\n"
)
+ # Get state of installer defaults so we can reinstate them (instantiating
+ # a Buildout will force the Buildout's defaults on the installer).
+ installer_values = get_installer_values()
# Use the buildout bootstrap command to create a buildout
zc.buildout.buildout.Buildout(
'buildout.cfg',
@@ -234,20 +260,23 @@
# trick bootstrap into putting the buildout develop egg
# in the eggs dir.
('buildout', 'develop-eggs-directory', 'eggs'),
- ]
+ ],
+ user_defaults=False,
).bootstrap([])
# Create the develop-eggs dir, which didn't get created the usual
# way due to the trick above:
os.mkdir('develop-eggs')
+ # Reinstate the default values of the installer.
+ set_installer_values(installer_values)
def buildoutSetUp(test):
test.globs['__tear_downs'] = __tear_downs = []
test.globs['register_teardown'] = register_teardown = __tear_downs.append
- prefer_final = zc.buildout.easy_install.prefer_final()
+ installer_values = get_installer_values()
register_teardown(
- lambda: zc.buildout.easy_install.prefer_final(prefer_final)
+ lambda: set_installer_values(installer_values)
)
here = os.getcwd()
@@ -367,8 +396,6 @@
make_py = make_py
))
- zc.buildout.easy_install.prefer_final(prefer_final)
-
def buildoutTearDown(test):
for f in test.globs['__tear_downs']:
f()
Modified: zc.buildout/branches/gary-5/src/zc/buildout/tests.py
===================================================================
--- zc.buildout/branches/gary-4/src/zc/buildout/tests.py 2010-02-17 21:30:01 UTC (rev 109101)
+++ zc.buildout/branches/gary-5/src/zc/buildout/tests.py 2010-02-20 01:54:01 UTC (rev 109166)
@@ -385,7 +385,65 @@
Error: Couldn't find a distribution for 'demoneeded'.
"""
+def show_eggs_from_site_packages():
+ """
+Sometimes you want to know what eggs are coming from site-packages. This
+might be for a diagnostic, or so that you can get a starting value for the
+allowed-eggs-from-site-packages option. The -v flag will also include this
+information.
+Our "py_path" has the "demoneeded," "demo"
+packages available. We'll ask for "bigdemo," which will get both of them.
+
+Here's our set up.
+
+ >>> py_path, site_packages_path = make_py()
+ >>> create_sample_sys_install(site_packages_path)
+
+ >>> write('buildout.cfg',
+ ... '''
+ ... [buildout]
+ ... parts = eggs
+ ... prefer-final = true
+ ... find-links = %(link_server)s
+ ...
+ ... [primed_python]
+ ... executable = %(py_path)s
+ ...
+ ... [eggs]
+ ... recipe = zc.recipe.egg:eggs
+ ... python = primed_python
+ ... eggs = bigdemo
+ ... ''' % globals())
+
+Now here is the output. The lines that begin with "Egg from site-packages:"
+indicate the eggs from site-packages that have been selected. You'll see
+we have two: demo 0.3 and demoneeded 1.1.
+
+ >>> print system(py_path+" "+buildout+" -v")
+ Installing 'zc.buildout', 'setuptools'.
+ We have a develop egg: zc.buildout V
+ We have the best distribution that satisfies 'setuptools'.
+ Picked: setuptools = V
+ Installing 'zc.recipe.egg'.
+ We have a develop egg: zc.recipe.egg V
+ Installing eggs.
+ Installing 'bigdemo'.
+ We have no distributions for bigdemo that satisfies 'bigdemo'.
+ Getting distribution for 'bigdemo'.
+ Got bigdemo 0.1.
+ Picked: bigdemo = 0.1
+ Getting required 'demo'
+ required by bigdemo 0.1.
+ We have a develop egg: demo V
+ Egg from site-packages: demo 0.3
+ Getting required 'demoneeded'
+ required by demo 0.3.
+ We have a develop egg: demoneeded V
+ Egg from site-packages: demoneeded 1.1
+ <BLANKLINE>
+ """
+
def test_comparing_saved_options_with_funny_characters():
"""
If an option has newlines, extra/odd spaces or a %, we need to make sure
@@ -2003,6 +2061,197 @@
"""
+def isolated_include_site_packages():
+ """
+
+This is an isolated test of the include_site_packages functionality, passing
+the argument directly to install, overriding a default.
+
+Our "py_path" has the "demoneeded" and "demo" packages available. We'll
+simply be asking for "demoneeded" here.
+
+ >>> py_path, site_packages_path = make_py()
+ >>> create_sample_sys_install(site_packages_path)
+ >>> zc.buildout.easy_install.include_site_packages(False)
+ True
+
+ >>> example_dest = tmpdir('site-packages-example-install')
+ >>> workingset = zc.buildout.easy_install.install(
+ ... ['demoneeded'], example_dest, links=[], executable=py_path,
+ ... index=None, include_site_packages=True)
+ >>> [dist.project_name for dist in workingset]
+ ['demoneeded']
+
+That worked fine. Let's try again with site packages not allowed (and
+reversing the default).
+
+ >>> zc.buildout.easy_install.include_site_packages(True)
+ False
+
+ >>> zc.buildout.easy_install.clear_index_cache()
+ >>> rmdir(example_dest)
+ >>> example_dest = tmpdir('site-packages-example-install')
+ >>> workingset = zc.buildout.easy_install.install(
+ ... ['demoneeded'], example_dest, links=[], executable=py_path,
+ ... index=None, include_site_packages=False)
+ Traceback (most recent call last):
+ ...
+ MissingDistribution: Couldn't find a distribution for 'demoneeded'.
+
+That's a failure, as expected.
+
+Now we explore an important edge case.
+
+Some system Pythons include setuptools (and other Python packages) in their
+site-packages (or equivalent) using a .egg-info directory. The pkg_resources
+module (from setuptools) considers a package installed using .egg-info to be a
+develop egg.
+
+zc.buildout.buildout.Buildout.bootstrap will make setuptools and zc.buildout
+available to the buildout via the eggs directory, for normal eggs; or the
+develop-eggs directory, for develop-eggs.
+
+If setuptools or zc.buildout is found in site-packages and considered by
+pkg_resources to be a develop egg, then the bootstrap code will use a .egg-link
+in the local develop-eggs, pointing to site-packages, in its entirety. Because
+develop-eggs must always be available for searching for distributions, this
+indirectly brings site-packages back into the search path for distributions.
+
+Because of this, we have to take special care that we still exclude
+site-packages even in this case. See the comments about site packages in the
+Installer._satisfied and Installer._obtain methods for the implementation
+(as of this writing).
+
+In this demonstration, we insert a link to the "demoneeded" distribution
+in our develop-eggs, which would bring the package back in, except for
+the special care we have taken to exclude it.
+
+ >>> zc.buildout.easy_install.clear_index_cache()
+ >>> rmdir(example_dest)
+ >>> example_dest = tmpdir('site-packages-example-install')
+ >>> mkdir(example_dest, 'develop-eggs')
+ >>> write(example_dest, 'develop-eggs', 'demoneeded.egg-link',
+ ... site_packages_path)
+ >>> workingset = zc.buildout.easy_install.install(
+ ... ['demoneeded'], example_dest, links=[],
+ ... path=[join(example_dest, 'develop-eggs')],
+ ... executable=py_path,
+ ... index=None, include_site_packages=False)
+ Traceback (most recent call last):
+ ...
+ MissingDistribution: Couldn't find a distribution for 'demoneeded'.
+
+The MissingDistribution error shows that buildout correctly excluded the
+"site-packages" source even though it was indirectly included in the path
+via a .egg-link file.
+
+ """
+
+def allowed_eggs_from_site_packages():
+ """
+Sometimes you need or want to control what eggs from site-packages are used.
+The allowed-eggs-from-site-packages option allows you to specify a whitelist of
+project names that may be included from site-packages. You can use globs to
+specify the value. It defaults to a single value of '*', indicating that any
+package may come from site-packages.
+
+This option interacts with include-site-packages in the following ways.
+
+If include-site-packages is true, then allowed-eggs-from-site-packages filters
+what eggs from site-packages may be chosen. If allowed-eggs-from-site-packages
+is an empty list, then no eggs from site-packages are chosen, but site-packages
+will still be included at the end of path lists.
+
+If include-site-packages is false, allowed-eggs-from-site-packages is
+irrelevant.
+
+This test shows the interaction with the zc.buildout.easy_install API. Another
+test below (allow_site_package_eggs_option) shows using it with a buildout.cfg.
+
+Our "py_path" has the "demoneeded" and "demo" packages available. We'll
+simply be asking for "demoneeded" here.
+
+ >>> py_path, site_packages_path = make_py()
+ >>> create_sample_sys_install(site_packages_path)
+
+ >>> example_dest = tmpdir('site-packages-example-install')
+ >>> workingset = zc.buildout.easy_install.install(
+ ... ['demoneeded'], example_dest, links=[], executable=py_path,
+ ... index=None,
+ ... allowed_eggs_from_site_packages=['demoneeded', 'other'])
+ >>> [dist.project_name for dist in workingset]
+ ['demoneeded']
+
+That worked fine. It would work fine for a glob too.
+
+ >>> zc.buildout.easy_install.clear_index_cache()
+ >>> rmdir(example_dest)
+ >>> example_dest = tmpdir('site-packages-example-install')
+ >>> workingset = zc.buildout.easy_install.install(
+ ... ['demoneeded'], example_dest, links=[], executable=py_path,
+ ... index=None,
+ ... allowed_eggs_from_site_packages=['?emon*', 'other'])
+ >>> [dist.project_name for dist in workingset]
+ ['demoneeded']
+
+But now let's try again with 'demoneeded' not allowed.
+
+ >>> zc.buildout.easy_install.clear_index_cache()
+ >>> rmdir(example_dest)
+ >>> example_dest = tmpdir('site-packages-example-install')
+ >>> workingset = zc.buildout.easy_install.install(
+ ... ['demoneeded'], example_dest, links=[], executable=py_path,
+ ... index=None,
+ ... allowed_eggs_from_site_packages=['demo'])
+ Traceback (most recent call last):
+ ...
+ MissingDistribution: Couldn't find a distribution for 'demoneeded'.
+
+Here's the same, but with an empty list.
+
+ >>> zc.buildout.easy_install.clear_index_cache()
+ >>> rmdir(example_dest)
+ >>> example_dest = tmpdir('site-packages-example-install')
+ >>> workingset = zc.buildout.easy_install.install(
+ ... ['demoneeded'], example_dest, links=[], executable=py_path,
+ ... index=None,
+ ... allowed_eggs_from_site_packages=[])
+ Traceback (most recent call last):
+ ...
+ MissingDistribution: Couldn't find a distribution for 'demoneeded'.
+
+Of course, this doesn't stop us from getting a package from elsewhere. Here,
+we add a link server.
+
+ >>> zc.buildout.easy_install.clear_index_cache()
+ >>> rmdir(example_dest)
+ >>> example_dest = tmpdir('site-packages-example-install')
+ >>> workingset = zc.buildout.easy_install.install(
+ ... ['demoneeded'], example_dest, executable=py_path,
+ ... links=[link_server], index=link_server+'index/',
+ ... allowed_eggs_from_site_packages=['other'])
+ >>> [dist.project_name for dist in workingset]
+ ['demoneeded']
+ >>> [dist.location for dist in workingset]
+ ['/site-packages-example-install/demoneeded-1.1-py2.6.egg']
+
+Finally, here's an example of an interaction: we say that it is OK to
+allow the "demoneeded" egg to come from site-packages, but we don't
+include-site-packages.
+
+ >>> zc.buildout.easy_install.clear_index_cache()
+ >>> rmdir(example_dest)
+ >>> example_dest = tmpdir('site-packages-example-install')
+ >>> workingset = zc.buildout.easy_install.install(
+ ... ['demoneeded'], example_dest, links=[], executable=py_path,
+ ... index=None, include_site_packages=False,
+ ... allowed_eggs_from_site_packages=['demoneeded'])
+ Traceback (most recent call last):
+ ...
+ MissingDistribution: Couldn't find a distribution for 'demoneeded'.
+
+ """
+
if sys.version_info > (2, 4):
def test_exit_codes():
"""
@@ -2930,24 +3179,59 @@
finally:
shutil.rmtree(tmp)
+def _write_eggrecipedemoneeded(tmp, minor_version, suffix=''):
+ from zc.buildout.testing import write
+ write(tmp, 'README.txt', '')
+ write(tmp, 'eggrecipedemoneeded.py',
+ 'y=%s\ndef f():\n pass' % minor_version)
+ write(
+ tmp, 'setup.py',
+ "from setuptools import setup\n"
+ "setup(name='demoneeded', py_modules=['eggrecipedemoneeded'],"
+ " zip_safe=True, version='1.%s%s', author='bob', url='bob', "
+ "author_email='bob')\n"
+ % (minor_version, suffix)
+ )
+
+def _write_eggrecipedemo(tmp, minor_version, suffix=''):
+ from zc.buildout.testing import write
+ write(tmp, 'README.txt', '')
+ write(
+ tmp, 'eggrecipedemo.py',
+ 'import eggrecipedemoneeded\n'
+ 'x=%s\n'
+ 'def main(): print x, eggrecipedemoneeded.y\n'
+ % minor_version)
+ write(
+ tmp, 'setup.py',
+ "from setuptools import setup\n"
+ "setup(name='demo', py_modules=['eggrecipedemo'],"
+ " install_requires = 'demoneeded',"
+ " entry_points={'console_scripts': "
+ "['demo = eggrecipedemo:main']},"
+ " zip_safe=True, version='0.%s%s')\n" % (minor_version, suffix)
+ )
+
+def create_sample_sys_install(site_packages_path):
+ for creator, minor_version in (
+ (_write_eggrecipedemoneeded, 1),
+ (_write_eggrecipedemo, 3)):
+ # Write the files and install in site_packages_path.
+ tmp = tempfile.mkdtemp()
+ try:
+ creator(tmp, minor_version)
+ zc.buildout.testing.sys_install(tmp, site_packages_path)
+ finally:
+ shutil.rmtree(tmp)
+
def create_sample_eggs(test, executable=sys.executable):
- write = test.globs['write']
+ from zc.buildout.testing import write
dest = test.globs['sample_eggs']
tmp = tempfile.mkdtemp()
try:
- write(tmp, 'README.txt', '')
-
for i in (0, 1, 2):
- write(tmp, 'eggrecipedemoneeded.py', 'y=%s\ndef f():\n pass' % i)
- c1 = i==2 and 'c1' or ''
- write(
- tmp, 'setup.py',
- "from setuptools import setup\n"
- "setup(name='demoneeded', py_modules=['eggrecipedemoneeded'],"
- " zip_safe=True, version='1.%s%s', author='bob', url='bob', "
- "author_email='bob')\n"
- % (i, c1)
- )
+ suffix = i==2 and 'c1' or ''
+ _write_eggrecipedemoneeded(tmp, i, suffix)
zc.buildout.testing.sdist(tmp, dest)
write(
@@ -2961,22 +3245,8 @@
os.remove(os.path.join(tmp, 'eggrecipedemoneeded.py'))
for i in (1, 2, 3, 4):
- write(
- tmp, 'eggrecipedemo.py',
- 'import eggrecipedemoneeded\n'
- 'x=%s\n'
- 'def main(): print x, eggrecipedemoneeded.y\n'
- % i)
- c1 = i==4 and 'c1' or ''
- write(
- tmp, 'setup.py',
- "from setuptools import setup\n"
- "setup(name='demo', py_modules=['eggrecipedemo'],"
- " install_requires = 'demoneeded',"
- " entry_points={'console_scripts': "
- "['demo = eggrecipedemo:main']},"
- " zip_safe=True, version='0.%s%s')\n" % (i, c1)
- )
+ suffix = i==4 and 'c1' or ''
+ _write_eggrecipedemo(tmp, i, suffix)
zc.buildout.testing.bdist_egg(tmp, executable, dest)
write(tmp, 'eggrecipebigdemo.py', 'import eggrecipedemo')
More information about the checkins
mailing list