[Checkins] SVN: zc.buildout/branches/gary-4-include-site-packages/ add include-site-packages and include-site-packages-for-buildout
Gary Poster
gary.poster at canonical.com
Fri Sep 25 17:15:10 EDT 2009
Log message for revision 104551:
add include-site-packages and include-site-packages-for-buildout
Changed:
U zc.buildout/branches/gary-4-include-site-packages/CHANGES.txt
U zc.buildout/branches/gary-4-include-site-packages/src/zc/buildout/buildout.py
U zc.buildout/branches/gary-4-include-site-packages/src/zc/buildout/buildout.txt
U zc.buildout/branches/gary-4-include-site-packages/src/zc/buildout/easy_install.py
U zc.buildout/branches/gary-4-include-site-packages/src/zc/buildout/easy_install.txt
U zc.buildout/branches/gary-4-include-site-packages/src/zc/buildout/testing.py
U zc.buildout/branches/gary-4-include-site-packages/src/zc/buildout/tests.py
U zc.buildout/branches/gary-4-include-site-packages/src/zc/buildout/testselectingpython.py
U zc.buildout/branches/gary-4-include-site-packages/src/zc/buildout/update.txt
U zc.buildout/branches/gary-4-include-site-packages/zc.recipe.egg_/src/zc/recipe/egg/README.txt
U zc.buildout/branches/gary-4-include-site-packages/zc.recipe.egg_/src/zc/recipe/egg/api.txt
U zc.buildout/branches/gary-4-include-site-packages/zc.recipe.egg_/src/zc/recipe/egg/egg.py
-=-
Modified: zc.buildout/branches/gary-4-include-site-packages/CHANGES.txt
===================================================================
--- zc.buildout/branches/gary-4-include-site-packages/CHANGES.txt 2009-09-25 21:13:39 UTC (rev 104550)
+++ zc.buildout/branches/gary-4-include-site-packages/CHANGES.txt 2009-09-25 21:15:09 UTC (rev 104551)
@@ -6,6 +6,42 @@
New Features:
+- Support and bugfixes for using a system Python with buildout.
+
+ In all of these descriptions, "site-packages" is an imprecise term for a
+ precise definition: the packages that are added by the Python's site.py.
+ Practically, this is the difference of the set of paths normally used by
+ Python minus those used when Python is started with the -S flag.
+
+ * A new boolean option, 'include-site-packages', includes or excludes site
+ packages from finding requirements, and from generated scripts.
+ zc.buildout's own buildout.cfg dogfoods this option. This defaults
+ to 'true', which is very similar to buildout's previous behavior.
+
+ * A new boolean option, 'include-site-packages-for-buildout', does the
+ same thing but only for the bin/buildout script. This can be important
+ for getting recipes and their dependencies without conflicts. This
+ defaults to 'false', which is different from buildout's previous behavior.
+
+ * Script generation pushes dependency paths that are in site-packages to
+ the end of the dependency paths in sys.path (but, as before, these
+ are still before extra paths and the rest of the site-package paths).
+
+ * Fix an error when at least two dependencies were in a shared location like
+ site-packages, and the first one met the "versions" setting. The first
+ dependency would be added, but subsequent dependencies from the same
+ location (e.g., site-packages) would use the version of the package found
+ in the shared location, ignoring the version setting.
+
+- The generated interpreter scripts now have a few more similarities to a
+ real Python interpreter.
+
+ * -m and -c incorporates all subsequent arguments as part of the command.
+
+ * -V returns the version.
+
+ * -S causes the script to not modify the standard path (for tests)
+
- Improve bootstrap.
* New options let you specify where to find ez_setup.py and where to find
Modified: zc.buildout/branches/gary-4-include-site-packages/src/zc/buildout/buildout.py
===================================================================
--- zc.buildout/branches/gary-4-include-site-packages/src/zc/buildout/buildout.py 2009-09-25 21:13:39 UTC (rev 104550)
+++ zc.buildout/branches/gary-4-include-site-packages/src/zc/buildout/buildout.py 2009-09-25 21:15:09 UTC (rev 104551)
@@ -119,6 +119,8 @@
'eggs-directory': 'eggs',
'executable': sys.executable,
'find-links': '',
+ 'include-site-packages-for-buildout': 'false',
+ 'include-site-packages': 'true',
'install-from-cache': 'false',
'installed': '.installed.cfg',
'log-format': '',
@@ -284,6 +286,23 @@
prefer_final)
zc.buildout.easy_install.prefer_final(prefer_final=='true')
+ include_site_packages = options['include-site-packages']
+ if include_site_packages not in ('true', 'false'):
+ self._error('Invalid value for include-site-packages option: %s',
+ include_site_packages)
+ zc.buildout.easy_install.include_site_packages(
+ include_site_packages=='true')
+
+ include_site_packages_for_buildout = options[
+ 'include-site-packages-for-buildout']
+ if include_site_packages_for_buildout not in ('true', 'false'):
+ self._error(
+ 'Invalid value for include-site-packages-for-buildout option: '
+ '%s',
+ include_site_packages_for_buildout)
+ self.include_site_packages_for_buildout = (
+ include_site_packages_for_buildout=='true')
+
use_dependency_links = options['use-dependency-links']
if use_dependency_links not in ('true', 'false'):
self._error('Invalid value for use-dependency-links option: %s',
@@ -355,7 +374,8 @@
if options.get('offline') == 'true':
ws = zc.buildout.easy_install.working_set(
distributions, options['executable'],
- [options['develop-eggs-directory'], options['eggs-directory']]
+ [options['develop-eggs-directory'], options['eggs-directory']],
+ include_site_packages=self.include_site_packages_for_buildout,
)
else:
ws = zc.buildout.easy_install.install(
@@ -365,7 +385,8 @@
executable=options['executable'],
path=[options['develop-eggs-directory']],
newest=self.newest,
- allow_hosts=self._allow_hosts
+ allow_hosts=self._allow_hosts,
+ include_site_packages=self.include_site_packages_for_buildout,
)
# Now copy buildout and setuptools eggs, and record destination eggs:
@@ -393,7 +414,8 @@
ws.require('zc.buildout')
zc.buildout.easy_install.scripts(
['zc.buildout'], ws, options['executable'],
- options['bin-directory'])
+ options['bin-directory'],
+ include_site_packages=self.include_site_packages_for_buildout)
init = bootstrap
@@ -946,7 +968,7 @@
fd, tsetup = tempfile.mkstemp()
try:
os.write(fd, zc.buildout.easy_install.runsetup_template % dict(
- setuptools=pkg_resources_loc,
+ sys_path=',\n '.join(repr(p) for p in sys.path),
setupdir=os.path.dirname(setup),
setup=setup,
__file__ = setup,
Modified: zc.buildout/branches/gary-4-include-site-packages/src/zc/buildout/buildout.txt
===================================================================
--- zc.buildout/branches/gary-4-include-site-packages/src/zc/buildout/buildout.txt 2009-09-25 21:13:39 UTC (rev 104550)
+++ zc.buildout/branches/gary-4-include-site-packages/src/zc/buildout/buildout.txt 2009-09-25 21:15:09 UTC (rev 104551)
@@ -742,6 +742,10 @@
DEFAULT_VALUE
find-links=
DEFAULT_VALUE
+ include-site-packages= true
+ DEFAULT_VALUE
+ include-site-packages-for-buildout= false
+ DEFAULT_VALUE
install-from-cache= false
DEFAULT_VALUE
installed= .installed.cfg
@@ -2226,6 +2230,8 @@
eggs-directory = /sample-buildout/eggs
executable = python
find-links =
+ include-site-packages = true
+ include-site-packages-for-buildout = false
install-from-cache = false
installed = /sample-buildout/.installed.cfg
log-format =
@@ -2332,6 +2338,36 @@
/some/otherpath
/some/path/someegg-1.0.0-py2.3.egg
+include-site-packages
+ By default, buildout will look for dependencies in the system's
+ site-packages. For this purpose, paths outside of Python's standard
+ library--or more precisely, those that are not included when Python is
+ started with the -S argument--are loosely referred to as "site-packages"
+ here. The include-site-packages buildout option can be used to override
+ the default behavior of using site packages
+ ("include-site-packages = false").
+
+include-site-packages-for-buildout
+ When buildout gets a recipe egg (as opposed to runs a recipe), it starts
+ with the current Python working set--the one that the bin/buildout script
+ uses itself. If this working set includes site-packages, and site-packages
+ includes an egg for a package that the recipe needs, *and* the recipe
+ specifies a newer version of that package, this can generate a version
+ conflict.
+
+ One solution to this is to not use site-packages
+ (``include-site-packages = false``). However, if you want your scripts to
+ use site-packages, then you have to specify 'include-site-packages = true'
+ for all of them.
+
+ To make this use case easier to handle, you can instead specify
+ ``include-site-packages-with-buildout = false``, which indicates that the
+ bin/buildout should *not* use site-packages; and
+ ``include-site-packages = true``, which indicates that the rest of the
+ scripts should use site-packages.
+
+ This is the default configuration.
+
install-from-cache
A download cache can be used as the basis of application source releases.
In an application source release, we want to distribute an application that
@@ -2445,6 +2481,7 @@
>>> print system(buildout
... +' -c'+os.path.join(sample_bootstrapped, 'setup.cfg')
+ ... +' buildout:include-site-packages-for-buildout=true'
... +' init'),
Creating '/sample-bootstrapped/setup.cfg'.
Creating directory '/sample-bootstrapped/bin'.
@@ -2486,6 +2523,7 @@
>>> print system(buildout
... +' -c'+os.path.join(sample_bootstrapped2, 'setup.cfg')
+ ... +' buildout:include-site-packages-for-buildout=true'
... +' bootstrap'),
While:
Initializing.
@@ -2499,6 +2537,7 @@
>>> print system(buildout
... +' -c'+os.path.join(sample_bootstrapped2, 'setup.cfg')
+ ... +' buildout:include-site-packages-for-buildout=true'
... +' bootstrap'),
Creating directory '/sample-bootstrapped2/bin'.
Creating directory '/sample-bootstrapped2/parts'.
Modified: zc.buildout/branches/gary-4-include-site-packages/src/zc/buildout/easy_install.py
===================================================================
--- zc.buildout/branches/gary-4-include-site-packages/src/zc/buildout/easy_install.py 2009-09-25 21:13:39 UTC (rev 104550)
+++ zc.buildout/branches/gary-4-include-site-packages/src/zc/buildout/easy_install.py 2009-09-25 21:15:09 UTC (rev 104551)
@@ -60,14 +60,66 @@
pkg_resources.Requirement.parse('setuptools')
).location
-# Include buildout and setuptools eggs in paths
-buildout_and_setuptools_path = [
- setuptools_loc,
- pkg_resources.working_set.find(
- pkg_resources.Requirement.parse('zc.buildout')).location,
- ]
+# Include buildout and setuptools eggs in paths. We prevent dupes just to
+# keep from duplicating any log messages about them.
+buildout_loc = pkg_resources.working_set.find(
+ pkg_resources.Requirement.parse('zc.buildout')).location
+buildout_and_setuptools_path = [setuptools_loc]
+if os.path.normpath(setuptools_loc) != os.path.normpath(buildout_loc):
+ buildout_and_setuptools_path.append(buildout_loc)
+def _get_system_packages(executable):
+ """return a pair of the standard lib and site packages for the executable.
+ """
+ # We want to get a list of the site packages, which is not easy. The
+ # canonical way to do this is to use distutils.sysconfig.get_python_lib(),
+ # but that only returns a single path, which does not reflect reality for
+ # many system Pythons, which have multiple additions. Instead, we start
+ # Python with -S, which does not import site.py and set up the extra paths
+ # like site-packages or (Ubuntu/Debian) dist-packages and python-support.
+ # We then compare that sys.path with the normal one. The set of the normal
+ # one minus the set of the ones in ``python -S`` is the set of packages
+ # that are effectively site-packages.
+ def get_sys_path(clean=False):
+ cmd = [executable, "-c",
+ "import sys, os;"
+ "print repr([os.path.normpath(p) for p in sys.path])"]
+ if clean:
+ cmd.insert(1, '-S')
+ _proc = subprocess.Popen(
+ cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+ stdout, stderr = _proc.communicate();
+ if _proc.returncode:
+ raise RuntimeError(
+ 'error trying to get system packages:\n%s' % (stderr,))
+ res = eval(stdout)
+ try:
+ res.remove('.')
+ except ValueError:
+ pass
+ return res
+ stdlib = get_sys_path(clean=True)
+ # The given executable might not be the current executable, so it is
+ # appropriate to do another subprocess to figure out what the additional
+ # site-package paths are. Moreover, even if this executable *is* the
+ # current executable, this code might be run in the context of code that
+ # has manipulated the sys.path--for instance, to add local zc.buildout or
+ # setuptools eggs.
+ site_packages = [p for p in get_sys_path() if p not in stdlib]
+ return (stdlib, site_packages)
+def _get_clean_sys_modules(executable):
+ cmd = [executable, '-S', '-c',
+ 'import sys; print repr(sys.modules.keys())']
+ _proc = subprocess.Popen(
+ cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+ stdout, stderr = _proc.communicate();
+ if _proc.returncode:
+ raise RuntimeError(
+ 'error trying to get system packages:\n%s' % (stderr,))
+ return eval(stdout)
+
+
class IncompatibleVersionError(zc.buildout.UserError):
"""A specified version is incompatible with a given requirement.
"""
@@ -109,7 +161,12 @@
_indexes = {}
-def _get_index(executable, index_url, find_links, allow_hosts=('*',)):
+def _get_index(executable, index_url, find_links, allow_hosts=('*',),
+ path=None):
+ # If path is None, the index will use sys.path. If you provide an empty
+ # path ([]), it will complain uselessly about missing index pages for
+ # packages found in the paths that you expect to use. Therefore, this path
+ # is always the same as the _env path in the Installer.
key = executable, index_url, tuple(find_links)
index = _indexes.get(key)
if index is not None:
@@ -118,7 +175,8 @@
if index_url is None:
index_url = default_index_url
index = AllowHostsPackageIndex(
- index_url, hosts=allow_hosts, python=_get_version(executable)
+ index_url, hosts=allow_hosts, search_path=path,
+ python=_get_version(executable)
)
if find_links:
@@ -151,6 +209,7 @@
_use_dependency_links = True
_allow_picked_versions = True
_always_unzip = False
+ _include_site_packages = True
def __init__(self,
dest=None,
@@ -162,6 +221,7 @@
newest=True,
versions=None,
use_dependency_links=None,
+ include_site_packages=None,
allow_hosts=('*',)
):
self._dest = dest
@@ -184,7 +244,16 @@
self._executable = executable
if always_unzip is not None:
self._always_unzip = always_unzip
- path = (path and path[:] or []) + buildout_and_setuptools_path
+ path = (path and path[:] or [])
+ if include_site_packages is not None:
+ self._include_site_packages = include_site_packages
+ stdlib, self._site_packages = _get_system_packages(executable)
+ if self._include_site_packages:
+ path.extend(buildout_and_setuptools_path)
+ path.extend(self._site_packages)
+ # else we could try to still include the buildout_and_setuptools_path
+ # if the elements are not in site_packages, but we're not bothering
+ # with this optimization for now, in the name of code simplicity.
if dest is not None and dest not in path:
path.insert(0, dest)
self._path = path
@@ -193,13 +262,27 @@
self._newest = newest
self._env = pkg_resources.Environment(path,
python=_get_version(executable))
- self._index = _get_index(executable, index, links, self._allow_hosts)
+ self._index = _get_index(executable, index, links, self._allow_hosts,
+ self._path)
if versions is not None:
self._versions = versions
def _satisfied(self, req, source=None):
- dists = [dist for dist in self._env[req.project_name] if dist in req]
+ # We get all distributions that match the given requirement. If we are
+ # not supposed to include site-packages for the given egg, we also
+ # filter those out. Even if include_site_packages is False and so we
+ # have excluded site packages from the _env's paths (see
+ # Installer.__init__), we need to do the filtering here because an
+ # .egg-link, such as one for setuptools or zc.buildout installed by
+ # zc.buildout.buildout.Buildout.bootstrap, can indirectly include a
+ # path in our _site_packages.
+ dists = [dist for dist in self._env[req.project_name] if (
+ dist in req and (
+ self._include_site_packages or
+ dist.location not in self._site_packages)
+ )
+ ]
if not dists:
logger.debug('We have no distributions for %s that satisfies %r.',
req.project_name, str(req))
@@ -304,7 +387,7 @@
ws, False,
)[0].location
- args = ('-c', _easy_install_cmd, '-mUNxd', _safe_arg(tmp))
+ args = ('-Sc', _easy_install_cmd, '-mUNxd', _safe_arg(tmp))
if self._always_unzip:
args += ('-Z', )
level = logger.getEffectiveLevel()
@@ -403,10 +486,20 @@
# Nothing is available.
return None
- # Filter the available dists for the requirement and source flag
+ # Filter the available dists for the requirement and source flag. If
+ # we are not supposed to include site-packages for the given egg, we
+ # also filter those out. Even if include_site_packages is False and so
+ # we have excluded site packages from the _env's paths (see
+ # Installer.__init__), we need to do the filtering here because an
+ # .egg-link, such as one for setuptools or zc.buildout installed by
+ # zc.buildout.buildout.Buildout.bootstrap, can indirectly include a
+ # path in our _site_packages.
dists = [dist for dist in index[requirement.project_name]
if ((dist in requirement)
and
+ (self._include_site_packages or
+ dist.location not in self._site_packages)
+ and
((not source) or
(dist.precedence == pkg_resources.SOURCE_DIST)
)
@@ -570,7 +663,7 @@
self._links.append(link)
self._index = _get_index(self._executable,
self._index_url, self._links,
- self._allow_hosts)
+ self._allow_hosts, self._path)
for dist in dists:
# Check whether we picked a version and, if we did, report it:
@@ -631,9 +724,9 @@
logger.debug('Installing %s.', repr(specs)[1:-1])
path = self._path
- dest = self._dest
- if dest is not None and dest not in path:
- path.insert(0, dest)
+ destination = self._dest
+ if destination is not None and destination not in path:
+ path.insert(0, destination)
requirements = [self._constrain(pkg_resources.Requirement.parse(spec))
for spec in specs]
@@ -651,35 +744,50 @@
self._maybe_add_setuptools(ws, dist)
# OK, we have the requested distributions and they're in the working
- # set, but they may have unmet requirements. We'll simply keep
- # trying to resolve requirements, adding missing requirements as they
- # are reported.
- #
- # Note that we don't pass in the environment, because we want
+ # set, but they may have unmet requirements. We'll resolve these
+ # requirements. This is code modified from
+ # pkg_resources.WorkingSet.resolve. We can't reuse that code directly
+ # because we have to constrain our requirements (see
+ # versions_section_ignored_for_dependency_in_favor_of_site_packages in
+ # zc.buildout.tests).
+ requirements.reverse() # Set up the stack.
+ processed = {} # This is a set of processed requirements.
+ best = {} # This is a mapping of key -> dist.
+ # Note that we don't use the environment, because we want
# to look for new eggs unless what we have is the best that
# matches the requirement.
- while 1:
- try:
- ws.resolve(requirements)
- except pkg_resources.DistributionNotFound, err:
- [requirement] = err
- requirement = self._constrain(requirement)
- if dest:
- logger.debug('Getting required %r', str(requirement))
- else:
- logger.debug('Adding required %r', str(requirement))
- _log_requirement(ws, requirement)
-
- for dist in self._get_dist(requirement, ws, self._always_unzip
- ):
-
- ws.add(dist)
- self._maybe_add_setuptools(ws, dist)
- except pkg_resources.VersionConflict, err:
- raise VersionConflict(err, ws)
- else:
- break
-
+ env = pkg_resources.Environment(ws.entries)
+ while requirements:
+ # Process dependencies breadth-first.
+ req = self._constrain(requirements.pop(0))
+ if req in processed:
+ # Ignore cyclic or redundant dependencies.
+ continue
+ dist = best.get(req.key)
+ if dist is None:
+ # Find the best distribution and add it to the map.
+ dist = ws.by_key.get(req.key)
+ if dist is None:
+ try:
+ dist = best[req.key] = env.best_match(req, ws)
+ except pkg_resources.VersionConflict, err:
+ raise VersionConflict(err, ws)
+ if dist is None:
+ if destination:
+ logger.debug('Getting required %r', str(req))
+ else:
+ logger.debug('Adding required %r', str(req))
+ _log_requirement(ws, req)
+ for dist in self._get_dist(req,
+ ws, self._always_unzip):
+ ws.add(dist)
+ self._maybe_add_setuptools(ws, dist)
+ if dist not in req:
+ # Oops, the "best" so far conflicts with a dependency.
+ raise VersionConflict(
+ pkg_resources.VersionConflict(dist, req), ws)
+ requirements.extend(dist.requires(req.extras)[::-1])
+ processed[req] = True
return ws
def build(self, spec, build_ext):
@@ -774,6 +882,12 @@
Installer._prefer_final = bool(setting)
return old
+def include_site_packages(setting=None):
+ old = Installer._include_site_packages
+ if setting is not None:
+ Installer._include_site_packages = bool(setting)
+ return old
+
def use_dependency_links(setting=None):
old = Installer._use_dependency_links
if setting is not None:
@@ -796,9 +910,11 @@
links=(), index=None,
executable=sys.executable, always_unzip=None,
path=None, working_set=None, newest=True, versions=None,
- use_dependency_links=None, allow_hosts=('*',)):
+ use_dependency_links=None, include_site_packages=None,
+ allow_hosts=('*',)):
installer = Installer(dest, links, index, executable, always_unzip, path,
newest, versions, use_dependency_links,
+ include_site_packages=include_site_packages,
allow_hosts=allow_hosts)
return installer.install(specs, working_set)
@@ -806,9 +922,12 @@
def build(spec, dest, build_ext,
links=(), index=None,
executable=sys.executable,
- path=None, newest=True, versions=None, allow_hosts=('*',)):
+ path=None, newest=True, versions=None, include_site_packages=None,
+ allow_hosts=('*',)):
installer = Installer(dest, links, index, executable, True, path, newest,
- versions, allow_hosts=allow_hosts)
+ versions,
+ include_site_packages=include_site_packages,
+ allow_hosts=allow_hosts)
return installer.build(spec, build_ext)
@@ -867,7 +986,7 @@
undo.append(lambda: os.close(fd))
os.write(fd, runsetup_template % dict(
- setuptools=setuptools_loc,
+ sys_path=',\n '.join(repr(p) for p in sys.path),
setupdir=directory,
setup=setup,
__file__ = setup,
@@ -904,22 +1023,73 @@
[f() for f in undo]
-def working_set(specs, executable, path):
- return install(specs, None, executable=executable, path=path)
+def working_set(specs, executable, path, include_site_packages=None):
+ return install(
+ specs, None, executable=executable, path=path,
+ include_site_packages=include_site_packages)
+def get_path(working_set, executable, extra_paths=(),
+ include_site_packages=True):
+ """Given working set and path to executable, return values for sys.path.
+
+ Return values are three lists: the standard library, the pure eggs, and
+ the paths like site-packages (which might contain eggs) that need to be
+ processed with site.addsitedir. It is expected that scripts will want
+ to assemble sys.path in that order.
+
+ Distribution locations from the working set come first in the list. Within
+ that collection, this function pushes site-packages-based distribution
+ locations to the end of the list, so that they don't mask eggs.
+
+ This expects that the working_set has already been created to honor a
+ include_site_packages setting. That is, if include_site_packages is False,
+ this function does *not* verify that the working_set's distributions are
+ not in site packages.
+
+ However, it does explicitly include site packages if include_site_packages
+ is True.
+
+ The standard library (defined as what the given Python executable has on
+ the path before its site.py is run) is always included.
+ """
+ stdlib, site_packages = _get_system_packages(executable)
+ site_directories = []
+ path = []
+ for dist in working_set:
+ location = os.path.normpath(dist.location)
+ if location in path:
+ path.remove(location)
+ site_directories.append(location)
+ elif location in site_packages:
+ site_directories.append(location)
+ site_packages.remove(location)
+ elif location not in site_directories:
+ path.append(location)
+ site_directories.extend(extra_paths)
+ # Now we add in all paths.
+ if include_site_packages:
+ # These are the remaining site_packages not already found in
+ # site_directories.
+ site_directories.extend(site_packages)
+ path = map(realpath, path)
+ site_directories = map(realpath, site_directories)
+ return stdlib, path, site_directories
+
def scripts(reqs, working_set, executable, dest,
scripts=None,
extra_paths=(),
arguments='',
interpreter=None,
initialization='',
+ include_site_packages=True,
relative_paths=False,
):
+ stdlib, eggs, site_dirs = get_path(
+ working_set, executable, extra_paths, include_site_packages)
+ clean_modules = set(_get_clean_sys_modules(executable))
+ clean_modules.add('site')
+ clean_modules.add('sys')
- path = [dist.location for dist in working_set]
- path.extend(extra_paths)
- path = map(realpath, path)
-
generated = []
if isinstance(reqs, str):
@@ -943,46 +1113,58 @@
else:
entry_points.append(req)
+ stdlib = repr(stdlib)[1:-1].replace(', ', ',\n ')
+ clean_modules = repr(sorted(clean_modules))[1:-1].replace(', ', ',\n ')
+
for name, module_name, attrs in entry_points:
if scripts is not None:
- sname = scripts.get(name)
- if sname is None:
+ script_name = scripts.get(name)
+ if script_name is None:
continue
else:
- sname = name
+ script_name = name
- sname = os.path.join(dest, sname)
- spath, rpsetup = _relative_path_and_setup(sname, path, relative_paths)
+ script_name = os.path.join(dest, script_name)
+ script_eggs, script_site_dirs, rpsetup = _relative_path_and_setup(
+ script_name, eggs, site_dirs, relative_paths)
generated.extend(
- _script(module_name, attrs, spath, sname, executable, arguments,
- initialization, rpsetup)
+ _script(module_name, attrs, clean_modules, stdlib, script_eggs,
+ script_site_dirs, script_name, executable,
+ arguments, initialization, rpsetup)
)
if interpreter:
- sname = os.path.join(dest, interpreter)
- spath, rpsetup = _relative_path_and_setup(sname, path, relative_paths)
- generated.extend(_pyscript(spath, sname, executable, rpsetup))
+ script_name = os.path.join(dest, interpreter)
+ script_eggs, script_site_dirs, rpsetup = _relative_path_and_setup(
+ script_name, eggs, site_dirs, relative_paths)
+ generated.extend(
+ _pyscript(clean_modules, stdlib, script_eggs, script_site_dirs,
+ script_name, executable, rpsetup))
return generated
-def _relative_path_and_setup(sname, path, relative_paths):
+def _relative_path_and_setup(script_name, eggs, site_dirs, relative_paths):
+ result = []
if relative_paths:
relative_paths = os.path.normcase(relative_paths)
- sname = os.path.normcase(os.path.abspath(sname))
- spath = ',\n '.join(
- [_relativitize(os.path.normcase(path_item), sname, relative_paths)
- for path_item in path]
- )
+ script_name = os.path.normcase(os.path.abspath(script_name))
+ for paths in (eggs, site_dirs):
+ result.append(',\n '.join(
+ [_relativitize(os.path.normcase(path_item), script_name,
+ relative_paths)
+ for path_item in paths]
+ ))
rpsetup = relative_paths_setup
- for i in range(_relative_depth(relative_paths, sname)):
+ for i in range(_relative_depth(relative_paths, script_name)):
rpsetup += "base = os.path.dirname(base)\n"
+ result.append(rpsetup)
else:
- spath = repr(path)[1:-1].replace(', ', ',\n ')
- rpsetup = ''
- return spath, rpsetup
+ for paths in (eggs, site_dirs):
+ result.append(repr(paths)[1:-1].replace(', ', ',\n '))
+ result.append('')
+ return result
-
def _relative_depth(common, path):
n = 0
while 1:
@@ -1027,8 +1209,8 @@
base = os.path.dirname(os.path.abspath(os.path.realpath(__file__)))
"""
-def _script(module_name, attrs, path, dest, executable, arguments,
- initialization, rsetup):
+def _script(module_name, attrs, clean_modules, stdlib, eggs, site_dirs, dest,
+ executable, arguments, initialization, rsetup):
generated = []
script = dest
if is_win32:
@@ -1036,7 +1218,10 @@
contents = script_template % dict(
python = _safe_arg(executable),
- path = path,
+ clean_modules = clean_modules,
+ stdlib = stdlib,
+ eggs = eggs,
+ site_dirs = site_dirs,
module_name = module_name,
attrs = attrs,
arguments = arguments,
@@ -1071,14 +1256,41 @@
else:
script_header = '#!%(python)s'
-
script_template = script_header + '''\
%(relative_paths_setup)s
+import site
import sys
-sys.path[0:0] = [
- %(path)s,
- ]
+
+# Clean out sys.modules from site's processing of .pth files.
+clean_modules = [
+ %(clean_modules)s
+ ]
+for k in sys.modules.keys():
+ if k not in clean_modules:
+ del sys.modules[k]
+
+sys.path[:] = [
+ %(stdlib)s
+ ]
+sys.path.extend([
+ %(eggs)s
+ ])
+site_dirs = [
+ %(site_dirs)s
+ ]
+# Add the site_dirs before `addsitedir` in case it has setuptools.
+sys.path.extend(site_dirs)
+# Process all buildout-controlled eggs before site-packages by importing
+# pkg_resources. This is only important for namespace packages, so it may
+# not have been added, so ignore import errors.
+try:
+ import pkg_resources
+except ImportError:
+ pass
+# Process .pth files.
+for p in site_dirs:
+ site.addsitedir(p)
%(initialization)s
import %(module_name)s
@@ -1086,8 +1298,8 @@
%(module_name)s.%(attrs)s(%(arguments)s)
'''
-
-def _pyscript(path, dest, executable, rsetup):
+def _pyscript(clean_modules, stdlib, eggs, site_dirs, dest, executable,
+ rsetup):
generated = []
script = dest
if is_win32:
@@ -1095,7 +1307,10 @@
contents = py_script_template % dict(
python = _safe_arg(executable),
- path = path,
+ clean_modules = clean_modules,
+ stdlib = stdlib,
+ eggs = eggs,
+ site_dirs = site_dirs,
relative_paths_setup = rsetup,
)
changed = not (os.path.exists(dest) and open(dest).read() == contents)
@@ -1121,43 +1336,120 @@
py_script_template = script_header + '''\
+# Get a clean copy of globals before import or variable definition.
+globs = globals().copy()
+
%(relative_paths_setup)s
+
+import site
import sys
-sys.path[0:0] = [
- %(path)s,
- ]
+# Clean out sys.modules from site's processing of .pth files.
+clean_modules = [
+ %(clean_modules)s
+ ]
+for k in sys.modules.keys():
+ if k not in clean_modules:
+ del sys.modules[k]
-_interactive = True
-if len(sys.argv) > 1:
- _options, _args = __import__("getopt").getopt(sys.argv[1:], 'ic:m:')
- _interactive = False
- for (_opt, _val) in _options:
- if _opt == '-i':
- _interactive = True
- elif _opt == '-c':
- exec _val
- elif _opt == '-m':
- sys.argv[1:] = _args
- _args = []
- __import__("runpy").run_module(
- _val, {}, "__main__", alter_sys=True)
+import code
+import optparse
+import os
- if _args:
- sys.argv[:] = _args
- __file__ = _args[0]
- del _options, _args
- execfile(__file__)
-if _interactive:
- del _interactive
- __import__("code").interact(banner="", local=globals())
+def _version_callback(*args, **kwargs):
+ print 'Python ' + sys.version.split()[0]
+ sys.exit(0)
+
+def _runner_callback(option, opt_str, value, parser, *args, **kwargs):
+ args = parser.rargs[:]
+ args.insert(0, opt_str)
+ del parser.rargs[:]
+ parser.values.runnable = value
+ parser.values.sys_argv = args
+ if opt_str == '-c':
+ parser.values.command = True
+ else:
+ assert opt_str == '-m'
+ parser.values.module = True
+
+parser = optparse.OptionParser(
+ usage='usage: %%prog [option] ... [-c cmd | -m mod | file ] [arg] ...')
+parser.add_option(
+ '-i', action='store_true', dest='inspect', default=False,
+ help='inspect interactively after running script')
+parser.add_option(
+ '-V', '--version', action='callback', callback=_version_callback,
+ help='print the Python version number and exit')
+parser.add_option(
+ '-S', action='store_false', dest='import_site', default=True,
+ help="Only use stdlib (mimics not initializing via 'import site')")
+parser.add_option(
+ '-c', action='callback', type='string', callback=_runner_callback,
+ help='program passed in as string (terminates option list)')
+parser.add_option(
+ '-m', action='callback', type='string', callback=_runner_callback,
+ help='run library module as a script (terminates option list)')
+parser.disable_interspersed_args()
+(options, args) = parser.parse_args()
+
+stdlib = [
+ %(stdlib)s,
+ ]
+egg_paths = [
+ %(eggs)s
+ ]
+site_dirs = [
+ %(site_dirs)s
+ ]
+
+sys.path[:] = stdlib
+
+if options.import_site:
+ sys.path.extend(egg_paths)
+ # Add the site_dirs before `addsitedir` in case it has setuptools.
+ sys.path.extend(site_dirs)
+ # Process all buildout-controlled eggs before site-packages by importing
+ # pkg_resources. This is only important for namespace packages, so it may
+ # not have been added, so ignore import errors.
+ try:
+ import pkg_resources
+ except ImportError:
+ pass
+ # Process .pth files.
+ for p in site_dirs:
+ site.addsitedir(p)
+sys.path.insert(0, '.')
+pythonpath = os.environ.get('PYTHONPATH', '')
+sys.path[0:0] = filter(None, (p.strip() for p in pythonpath.split(':')))
+
+interactive = options.inspect
+if getattr(options, 'command', False):
+ sys.argv[:] = options.sys_argv
+ exec options.runnable in globs
+elif getattr(options, 'module', False):
+ # runpy is only available in Python >= 2.5, so only try if we're asked
+ import runpy
+ sys.argv[:] = options.sys_argv
+ runpy.run_module(options.runnable, {}, "__main__", alter_sys=True)
+elif args:
+ sys.argv[:] = args
+ globs['__file__'] = args[0]
+ execfile(args[0], globs)
+else:
+ interactive = True
+
+if interactive:
+ del globs['__file__']
+ code.interact(banner="", local=globs)
'''
runsetup_template = """
import sys
-sys.path.insert(0, %(setupdir)r)
-sys.path.insert(0, %(setuptools)r)
+sys.path[:] = [
+ %(setupdir)r,
+ %(sys_path)s
+ ]
import os, setuptools
__file__ = %(__file__)r
Modified: zc.buildout/branches/gary-4-include-site-packages/src/zc/buildout/easy_install.txt
===================================================================
--- zc.buildout/branches/gary-4-include-site-packages/src/zc/buildout/easy_install.txt 2009-09-25 21:13:39 UTC (rev 104550)
+++ zc.buildout/branches/gary-4-include-site-packages/src/zc/buildout/easy_install.txt 2009-09-25 21:15:09 UTC (rev 104551)
@@ -89,6 +89,14 @@
for using dependency_links in preference to other
locations. Defaults to true.
+include_site_packages
+ A flag indicating whether Python's non-standard-library packages should
+ be available for finding dependencies. Defaults to true.
+
+ Paths outside of Python's standard library--or more precisely, those that
+ are not included when Python is started with the -S argument--are loosely
+ referred to as "site-packages" here.
+
relative_paths
Adjust egg paths so they are relative to the script path. This
allows scripts to work when scripts and eggs are moved, as long as
@@ -399,6 +407,65 @@
>>> [d.version for d in ws]
['0.3', '1.1']
+Dependencies in Site Packages
+-----------------------------
+
+Paths outside of Python's standard library--or more precisely, those that are
+not included when Python is started with the -S argument--are loosely referred
+to as "site-packages" here. These site-packages are searched by default for
+distributions. This can be disabled, so that, for instance, a system Python
+can be used with buildout, cleaned of any packages installed by a user or
+system package manager.
+
+The default behavior can be controlled and introspected using
+zc.buildout.easy_install.include_site_packages.
+
+ >>> zc.buildout.easy_install.include_site_packages()
+ True
+
+Here's an example of using a Python executable that includes our dependencies.
+
+Our "primed_executable" has the "demoneeded," "other," and "setuptools"
+packages available. We'll simply be asking for "other" here.
+
+ >>> primed_executable = get_executable_with_site_packages()
+
+ >>> example_dest = tmpdir('site-packages-example-install')
+ >>> workingset = zc.buildout.easy_install.install(
+ ... ['other'], example_dest, links=[], executable=primed_executable,
+ ... index=None)
+ >>> [dist.project_name for dist in workingset]
+ ['other']
+
+That worked fine. Let's try again with site packages not allowed. We'll
+change the policy by changing the default. Notice that the function for
+changing the default value returns the previous value.
+
+ >>> zc.buildout.easy_install.include_site_packages(False)
+ True
+
+ >>> zc.buildout.easy_install.include_site_packages()
+ False
+
+ >>> zc.buildout.easy_install.clear_index_cache()
+ >>> rmdir(example_dest)
+ >>> example_dest = tmpdir('site-packages-example-install')
+ >>> workingset = zc.buildout.easy_install.install(
+ ... ['other'], example_dest, links=[], executable=primed_executable,
+ ... index=None)
+ Traceback (most recent call last):
+ ...
+ MissingDistribution: Couldn't find a distribution for 'other'.
+ >>> zc.buildout.easy_install.clear_index_cache()
+
+Now we'll reset the default.
+
+ >>> zc.buildout.easy_install.include_site_packages(True)
+ False
+
+ >>> zc.buildout.easy_install.include_site_packages()
+ True
+
Dependency links
----------------
@@ -580,15 +647,43 @@
The demo script run the entry point defined in the demo egg:
- >>> cat(bin, 'demo') # doctest: +NORMALIZE_WHITESPACE
+ >>> cat(bin, 'demo') # doctest: +NORMALIZE_WHITESPACE +ELLIPSIS
#!/usr/local/bin/python2.4
<BLANKLINE>
+ import site
import sys
- sys.path[0:0] = [
- '/sample-install/demo-0.3-py2.4.egg',
- '/sample-install/demoneeded-1.1-py2.4.egg',
- ]
<BLANKLINE>
+ # Clean out sys.modules from site's processing of .pth files.
+ clean_modules = [
+ ...
+ ]
+ for k in sys.modules.keys():
+ if k not in clean_modules:
+ del sys.modules[k]
+ <BLANKLINE>
+ sys.path[:] = [
+ ...
+ ]
+ sys.path.extend([
+ '/sample-install/demo-0.3-pyN.N.egg',
+ '/sample-install/demoneeded-1.1-pyN.N.egg'
+ ])
+ site_dirs = [
+ ...
+ ]
+ # Add the site_dirs before `addsitedir` in case it has setuptools.
+ sys.path.extend(site_dirs)
+ # Process all buildout-controlled eggs before site-packages by importing
+ # pkg_resources. This is only important for namespace packages, so it may
+ # not have been added, so ignore import errors.
+ try:
+ import pkg_resources
+ except ImportError:
+ pass
+ # Process .pth files.
+ for p in site_dirs:
+ site.addsitedir(p)
+ <BLANKLINE>
import eggrecipedemo
<BLANKLINE>
if __name__ == '__main__':
@@ -596,8 +691,12 @@
Some things to note:
-- The demo and demoneeded eggs are added to the beginning of sys.path.
+- The script controls the entire path. The standard library goes in first
+ (the first ellipsis).
+- All non-standard-library paths are processed with site.addsitedir, so .pth
+ files work.
+
- The module for the script entry point is imported and the entry
point, in this case, 'main', is run.
@@ -617,15 +716,43 @@
... [('demo', 'eggrecipedemo', 'main')],
... ws, sys.executable, bin)
- >>> cat(bin, 'demo') # doctest: +NORMALIZE_WHITESPACE
+ >>> cat(bin, 'demo') # doctest: +NORMALIZE_WHITESPACE +ELLIPSIS
#!/usr/local/bin/python2.4
<BLANKLINE>
+ import site
import sys
- sys.path[0:0] = [
- '/sample-install/demo-0.3-py2.4.egg',
- '/sample-install/demoneeded-1.1-py2.4.egg',
- ]
<BLANKLINE>
+ # Clean out sys.modules from site's processing of .pth files.
+ clean_modules = [
+ ...
+ ]
+ for k in sys.modules.keys():
+ if k not in clean_modules:
+ del sys.modules[k]
+ <BLANKLINE>
+ sys.path[:] = [
+ ...
+ ]
+ sys.path.extend([
+ '/sample-install/demo-0.3-pyN.N.egg',
+ '/sample-install/demoneeded-1.1-pyN.N.egg'
+ ])
+ site_dirs = [
+ ...
+ ]
+ # Add the site_dirs before `addsitedir` in case it has setuptools.
+ sys.path.extend(site_dirs)
+ # Process all buildout-controlled eggs before site-packages by importing
+ # pkg_resources. This is only important for namespace packages, so it may
+ # not have been added, so ignore import errors.
+ try:
+ import pkg_resources
+ except ImportError:
+ pass
+ # Process .pth files.
+ for p in site_dirs:
+ site.addsitedir(p)
+ <BLANKLINE>
import eggrecipedemo
<BLANKLINE>
if __name__ == '__main__':
@@ -661,45 +788,124 @@
The py script simply runs the Python interactive interpreter with
the path set:
- >>> cat(bin, 'py') # doctest: +NORMALIZE_WHITESPACE
+The py script can run the Python interactive interpreter, supporting some of
+the interpreter's usual options, with the path set:
+
+ >>> cat(bin, 'py') # doctest: +NORMALIZE_WHITESPACE +ELLIPSIS
#!/usr/local/bin/python2.4
+ # Get a clean copy of globals before import or variable definition.
+ globs = globals().copy()
<BLANKLINE>
+ <BLANKLINE>
+ <BLANKLINE>
+ import site
import sys
<BLANKLINE>
- sys.path[0:0] = [
- '/sample-install/demo-0.3-pyN.N.egg',
- '/sample-install/demoneeded-1.1-pyN.N.egg',
- ]
+ # Clean out sys.modules from site's processing of .pth files.
+ clean_modules = [
+ ...
+ ]
+ for k in sys.modules.keys():
+ if k not in clean_modules:
+ del sys.modules[k]
<BLANKLINE>
- _interactive = True
- if len(sys.argv) > 1:
- _options, _args = __import__("getopt").getopt(sys.argv[1:], 'ic:m:')
- _interactive = False
- for (_opt, _val) in _options:
- if _opt == '-i':
- _interactive = True
- elif _opt == '-c':
- exec _val
- elif _opt == '-m':
- sys.argv[1:] = _args
- _args = []
- __import__("runpy").run_module(
- _val, {}, "__main__", alter_sys=True)
+ import code
+ import optparse
+ import os
<BLANKLINE>
- if _args:
- sys.argv[:] = _args
- __file__ = _args[0]
- del _options, _args
- execfile(__file__)
<BLANKLINE>
- if _interactive:
- del _interactive
- __import__("code").interact(banner="", local=globals())
+ def _version_callback(*args, **kwargs):
+ print 'Python ' + sys.version.split()[0]
+ sys.exit(0)
+ <BLANKLINE>
+ def _runner_callback(option, opt_str, value, parser, *args, **kwargs):
+ args = parser.rargs[:]
+ args.insert(0, opt_str)
+ del parser.rargs[:]
+ parser.values.runnable = value
+ parser.values.sys_argv = args
+ if opt_str == '-c':
+ parser.values.command = True
+ else:
+ assert opt_str == '-m'
+ parser.values.module = True
+ <BLANKLINE>
+ parser = optparse.OptionParser(
+ usage='usage: %prog [option] ... [-c cmd | -m mod | file ] [arg] ...')
+ parser.add_option(
+ '-i', action='store_true', dest='inspect', default=False,
+ help='inspect interactively after running script')
+ parser.add_option(
+ '-V', '--version', action='callback', callback=_version_callback,
+ help='print the Python version number and exit')
+ parser.add_option(
+ '-S', action='store_false', dest='import_site', default=True,
+ help="Only use stdlib (mimics not initializing via 'import site')")
+ parser.add_option(
+ '-c', action='callback', type='string', callback=_runner_callback,
+ help='program passed in as string (terminates option list)')
+ parser.add_option(
+ '-m', action='callback', type='string', callback=_runner_callback,
+ help='run library module as a script (terminates option list)')
+ parser.disable_interspersed_args()
+ (options, args) = parser.parse_args()
+ <BLANKLINE>
+ stdlib = [
+ ...
+ ]
+ egg_paths = [
+ '/sample-install/demo-0.3-pyN.N.egg',
+ '/sample-install/demoneeded-1.1-pyN.N.egg'
+ ]
+ site_dirs = [
+ ...
+ ]
+ <BLANKLINE>
+ sys.path[:] = stdlib
+ <BLANKLINE>
+ if options.import_site:
+ sys.path.extend(egg_paths)
+ # Add the site_dirs before `addsitedir` in case it has setuptools.
+ sys.path.extend(site_dirs)
+ # Process all buildout-controlled eggs before site-packages by importing
+ # pkg_resources. This is only important for namespace packages, so it may
+ # not have been added, so ignore import errors.
+ try:
+ import pkg_resources
+ except ImportError:
+ pass
+ # Process .pth files.
+ for p in site_dirs:
+ site.addsitedir(p)
+ sys.path.insert(0, '.')
+ pythonpath = os.environ.get('PYTHONPATH', '')
+ sys.path[0:0] = filter(None, (p.strip() for p in pythonpath.split(':')))
+ <BLANKLINE>
+ interactive = options.inspect
+ if getattr(options, 'command', False):
+ sys.argv[:] = options.sys_argv
+ exec options.runnable in globs
+ elif getattr(options, 'module', False):
+ # runpy is only available in Python >= 2.5, so only try if we're asked
+ import runpy
+ sys.argv[:] = options.sys_argv
+ runpy.run_module(options.runnable, {}, "__main__", alter_sys=True)
+ elif args:
+ sys.argv[:] = args
+ globs['__file__'] = args[0]
+ execfile(args[0], globs)
+ else:
+ interactive = True
+ <BLANKLINE>
+ if interactive:
+ del globs['__file__']
+ code.interact(banner="", local=globs)
If invoked with a script name and arguments, it will run that script, instead.
>>> write('ascript', '''
... "demo doc"
+ ... import sys
... print sys.argv
... print (__name__, __file__, __doc__)
... ''')
@@ -747,21 +953,120 @@
... ['demo'], ws, sys.executable, bin, dict(demo='run'),
... extra_paths=[foo])
- >>> cat(bin, 'run') # doctest: +NORMALIZE_WHITESPACE
+ >>> cat(bin, 'run') # doctest: +NORMALIZE_WHITESPACE +ELLIPSIS
#!/usr/local/bin/python2.4
<BLANKLINE>
+ import site
import sys
- sys.path[0:0] = [
- '/sample-install/demo-0.3-py2.4.egg',
- '/sample-install/demoneeded-1.1-py2.4.egg',
- '/foo',
- ]
<BLANKLINE>
+ # Clean out sys.modules from site's processing of .pth files.
+ clean_modules = [
+ ...
+ ]
+ for k in sys.modules.keys():
+ if k not in clean_modules:
+ del sys.modules[k]
+ <BLANKLINE>
+ sys.path[:] = [
+ ...
+ ]
+ sys.path.extend([
+ '/sample-install/demo-0.3-pyN.N.egg',
+ '/sample-install/demoneeded-1.1-pyN.N.egg'
+ ])
+ site_dirs = [
+ '/foo',
+ ...
+ ]
+ # Add the site_dirs before `addsitedir` in case it has setuptools.
+ sys.path.extend(site_dirs)
+ # Process all buildout-controlled eggs before site-packages by importing
+ # pkg_resources. This is only important for namespace packages, so it may
+ # not have been added, so ignore import errors.
+ try:
+ import pkg_resources
+ except ImportError:
+ pass
+ # Process .pth files.
+ for p in site_dirs:
+ site.addsitedir(p)
+ <BLANKLINE>
import eggrecipedemo
<BLANKLINE>
if __name__ == '__main__':
eggrecipedemo.main()
+Ordering paths
+--------------
+
+We have already seen that we have a precise definition for a loose term:
+"site-packages". Paths outside of Python's standard library--or more
+precisely, those that are not included when Python is started with the -S
+argument--are loosely referred to as "site-packages" here.
+
+When generating scripts, paths that come from the site-packages are ordered
+after the other specific dependencies generated from the working set. This
+is so that directories such as "site-packages" that can contain multiple
+dependencies come after the more specific distributions found by setuptools,
+reducing the chance of the distributions being masked by the system folders.
+
+This is controlled by the ``get_path`` function, which is available for
+other script recipes to use. It returns three lists: a list of the
+paths from the standard library, a list of the stand-alone eggs, and a
+list of directories that might contain .pth files. We typically replace
+the sys.path with the standard library paths, add the other two lists,
+and then also iterate over the list of directories that might contain
+.pth files, calling site.addsitedir with them so that any .pth files are
+honored.
+
+As a demonstration, we will create a working set that has dependencies
+on "bigdemo" and "other". In our first case, we will use a clean Python
+without any of these dependencies installed.
+
+ >>> dest1 = tmpdir('path-install-1')
+ >>> ws1 = zc.buildout.easy_install.install(
+ ... ['other', 'bigdemo'], dest1,
+ ... links=[link_server], index=link_server+'index/')
+ >>> stdlib, egg_paths, dir_paths = zc.buildout.easy_install.get_path(
+ ... ws1, sys.executable)
+
+ >>> import pprint
+ >>> pprint.pprint(egg_paths) # doctest: +ELLIPSIS
+ ['/path-install-1/other-1.0-py...egg',
+ '/path-install-1/bigdemo-0.1-py...egg',
+ '/path-install-1/demo-0.3-py...egg',
+ '/path-install-1/demoneeded-1.1-py...egg']
+ >>> len(dir_paths) # site-packages
+ 1
+
+We will now compare the results using a Python that has bigdemo's indirect
+dependency available, "demoneeded," and "other," but not "demo" or "bigdemo".
+
+ >>> dest2 = tmpdir('path-install-2')
+ >>> ws2 = zc.buildout.easy_install.install(
+ ... ['other', 'bigdemo'], dest2,
+ ... links=[link_server], index=link_server+'index/',
+ ... executable=primed_executable)
+ >>> stdlib, egg_paths, dir_paths = zc.buildout.easy_install.get_path(
+ ... ws2, primed_executable)
+ >>> pprint.pprint(egg_paths) # doctest: +ELLIPSIS
+ ['/path-install-2/bigdemo-0.1-py...egg',
+ '/path-install-2/demo-0.3-py...egg']
+ >>> pprint.pprint(dir_paths) # doctest: +ELLIPSIS
+ ['/executable/eggs/other-1.0-py...egg',
+ '/executable/eggs/demoneeded-1.1-py...egg',
+ '/executable/eggs/setuptools-0.6c9-py...egg',
+ ...]
+ >>> zc.buildout.easy_install.clear_index_cache() # clean up
+
+Notice that the paths from the executable come after the ones for this
+buildout. This is most evident in the change of order for the "other" egg.
+
+In fact, this ordering is not important in this example, because the
+executable's paths all are individual packages; but if a path were a directory
+that shared many packages, like a classic "site-packages" directory, its shared
+packages would not mask those selected by the buildout.
+
Providing script arguments
--------------------------
@@ -773,14 +1078,43 @@
... ['demo'], ws, sys.executable, bin, dict(demo='run'),
... arguments='1, 2')
- >>> cat(bin, 'run') # doctest: +NORMALIZE_WHITESPACE
+ >>> cat(bin, 'run') # doctest: +NORMALIZE_WHITESPACE +ELLIPSIS
#!/usr/local/bin/python2.4
+ <BLANKLINE>
+ import site
import sys
- sys.path[0:0] = [
- '/sample-install/demo-0.3-py2.4.egg',
- '/sample-install/demoneeded-1.1-py2.4.egg',
- ]
<BLANKLINE>
+ # Clean out sys.modules from site's processing of .pth files.
+ clean_modules = [
+ ...
+ ]
+ for k in sys.modules.keys():
+ if k not in clean_modules:
+ del sys.modules[k]
+ <BLANKLINE>
+ sys.path[:] = [
+ ...
+ ]
+ sys.path.extend([
+ '/sample-install/demo-0.3-pyN.N.egg',
+ '/sample-install/demoneeded-1.1-pyN.N.egg'
+ ])
+ site_dirs = [
+ ...
+ ]
+ # Add the site_dirs before `addsitedir` in case it has setuptools.
+ sys.path.extend(site_dirs)
+ # Process all buildout-controlled eggs before site-packages by importing
+ # pkg_resources. This is only important for namespace packages, so it may
+ # not have been added, so ignore import errors.
+ try:
+ import pkg_resources
+ except ImportError:
+ pass
+ # Process .pth files.
+ for p in site_dirs:
+ site.addsitedir(p)
+ <BLANKLINE>
import eggrecipedemo
<BLANKLINE>
if __name__ == '__main__':
@@ -796,14 +1130,43 @@
... arguments='1, 2',
... initialization='import os\nos.chdir("foo")')
- >>> cat(bin, 'run') # doctest: +NORMALIZE_WHITESPACE
+ >>> cat(bin, 'run') # doctest: +NORMALIZE_WHITESPACE +ELLIPSIS
#!/usr/local/bin/python2.4
+ <BLANKLINE>
+ import site
import sys
- sys.path[0:0] = [
- '/sample-install/demo-0.3-py2.4.egg',
- '/sample-install/demoneeded-1.1-py2.4.egg',
- ]
<BLANKLINE>
+ # Clean out sys.modules from site's processing of .pth files.
+ clean_modules = [
+ ...
+ ]
+ for k in sys.modules.keys():
+ if k not in clean_modules:
+ del sys.modules[k]
+ <BLANKLINE>
+ sys.path[:] = [
+ ...
+ ]
+ sys.path.extend([
+ '/sample-install/demo-0.3-pyN.N.egg',
+ '/sample-install/demoneeded-1.1-pyN.N.egg'
+ ])
+ site_dirs = [
+ ...
+ ]
+ # Add the site_dirs before `addsitedir` in case it has setuptools.
+ sys.path.extend(site_dirs)
+ # Process all buildout-controlled eggs before site-packages by importing
+ # pkg_resources. This is only important for namespace packages, so it may
+ # not have been added, so ignore import errors.
+ try:
+ import pkg_resources
+ except ImportError:
+ pass
+ # Process .pth files.
+ for p in site_dirs:
+ site.addsitedir(p)
+ <BLANKLINE>
import os
os.chdir("foo")
<BLANKLINE>
@@ -836,7 +1199,7 @@
... interpreter='py',
... relative_paths=bo)
- >>> cat(bo, 'bin', 'run')
+ >>> cat(bo, 'bin', 'run') # doctest: +ELLIPSIS
#!/usr/local/bin/python2.4
<BLANKLINE>
import os
@@ -845,14 +1208,42 @@
base = os.path.dirname(os.path.abspath(os.path.realpath(__file__)))
base = os.path.dirname(base)
<BLANKLINE>
+ import site
import sys
- sys.path[0:0] = [
- join(base, 'eggs/demo-0.3-pyN.N.egg'),
- join(base, 'eggs/demoneeded-1.1-pyN.N.egg'),
- '/ba',
- join(base, 'bar'),
- ]
<BLANKLINE>
+ # Clean out sys.modules from site's processing of .pth files.
+ clean_modules = [
+ ...
+ ]
+ for k in sys.modules.keys():
+ if k not in clean_modules:
+ del sys.modules[k]
+ <BLANKLINE>
+ sys.path[:] = [
+ ...
+ ]
+ sys.path.extend([
+ join(base, 'eggs/demo-0.3-pyN.N.egg'),
+ join(base, 'eggs/demoneeded-1.1-pyN.N.egg')
+ ])
+ site_dirs = [
+ '/ba',
+ join(base, 'bar'),
+ ...
+ ]
+ # Add the site_dirs before `addsitedir` in case it has setuptools.
+ sys.path.extend(site_dirs)
+ # Process all buildout-controlled eggs before site-packages by importing
+ # pkg_resources. This is only important for namespace packages, so it may
+ # not have been added, so ignore import errors.
+ try:
+ import pkg_resources
+ except ImportError:
+ pass
+ # Process .pth files.
+ for p in site_dirs:
+ site.addsitedir(p)
+ <BLANKLINE>
import eggrecipedemo
<BLANKLINE>
if __name__ == '__main__':
@@ -868,50 +1259,124 @@
We specified an interpreter and its paths are adjusted too:
- >>> cat(bo, 'bin', 'py')
+ >>> cat(bo, 'bin', 'py') # doctest: +ELLIPSIS
#!/usr/local/bin/python2.4
+ # Get a clean copy of globals before import or variable definition.
+ globs = globals().copy()
<BLANKLINE>
+ <BLANKLINE>
import os
<BLANKLINE>
join = os.path.join
base = os.path.dirname(os.path.abspath(os.path.realpath(__file__)))
base = os.path.dirname(base)
<BLANKLINE>
+ <BLANKLINE>
+ import site
import sys
<BLANKLINE>
- sys.path[0:0] = [
- join(base, 'eggs/demo-0.3-pyN.N.egg'),
- join(base, 'eggs/demoneeded-1.1-pyN.N.egg'),
- '/ba',
- join(base, 'bar'),
- ]
+ # Clean out sys.modules from site's processing of .pth files.
+ clean_modules = [
+ ...
+ ]
+ for k in sys.modules.keys():
+ if k not in clean_modules:
+ del sys.modules[k]
<BLANKLINE>
- _interactive = True
- if len(sys.argv) > 1:
- _options, _args = __import__("getopt").getopt(sys.argv[1:], 'ic:m:')
- _interactive = False
- for (_opt, _val) in _options:
- if _opt == '-i':
- _interactive = True
- elif _opt == '-c':
- exec _val
- elif _opt == '-m':
- sys.argv[1:] = _args
- _args = []
- __import__("runpy").run_module(
- _val, {}, "__main__", alter_sys=True)
+ import code
+ import optparse
+ import os
<BLANKLINE>
- if _args:
- sys.argv[:] = _args
- __file__ = _args[0]
- del _options, _args
- execfile(__file__)
<BLANKLINE>
- if _interactive:
- del _interactive
- __import__("code").interact(banner="", local=globals())
+ def _version_callback(*args, **kwargs):
+ print 'Python ' + sys.version.split()[0]
+ sys.exit(0)
+ <BLANKLINE>
+ def _runner_callback(option, opt_str, value, parser, *args, **kwargs):
+ args = parser.rargs[:]
+ args.insert(0, opt_str)
+ del parser.rargs[:]
+ parser.values.runnable = value
+ parser.values.sys_argv = args
+ if opt_str == '-c':
+ parser.values.command = True
+ else:
+ assert opt_str == '-m'
+ parser.values.module = True
+ <BLANKLINE>
+ parser = optparse.OptionParser(
+ usage='usage: %prog [option] ... [-c cmd | -m mod | file ] [arg] ...')
+ parser.add_option(
+ '-i', action='store_true', dest='inspect', default=False,
+ help='inspect interactively after running script')
+ parser.add_option(
+ '-V', '--version', action='callback', callback=_version_callback,
+ help='print the Python version number and exit')
+ parser.add_option(
+ '-S', action='store_false', dest='import_site', default=True,
+ help="Only use stdlib (mimics not initializing via 'import site')")
+ parser.add_option(
+ '-c', action='callback', type='string', callback=_runner_callback,
+ help='program passed in as string (terminates option list)')
+ parser.add_option(
+ '-m', action='callback', type='string', callback=_runner_callback,
+ help='run library module as a script (terminates option list)')
+ parser.disable_interspersed_args()
+ (options, args) = parser.parse_args()
+ <BLANKLINE>
+ stdlib = [
+ ...
+ ]
+ egg_paths = [
+ join(base, 'eggs/demo-0.3-pyN.N.egg'),
+ join(base, 'eggs/demoneeded-1.1-pyN.N.egg')
+ ]
+ site_dirs = [
+ '/ba',
+ join(base, 'bar'),
+ ...
+ ]
+ <BLANKLINE>
+ sys.path[:] = stdlib
+ <BLANKLINE>
+ if options.import_site:
+ sys.path.extend(egg_paths)
+ # Add the site_dirs before `addsitedir` in case it has setuptools.
+ sys.path.extend(site_dirs)
+ # Process all buildout-controlled eggs before site-packages by importing
+ # pkg_resources. This is only important for namespace packages, so it may
+ # not have been added, so ignore import errors.
+ try:
+ import pkg_resources
+ except ImportError:
+ pass
+ # Process .pth files.
+ for p in site_dirs:
+ site.addsitedir(p)
+ sys.path.insert(0, '.')
+ pythonpath = os.environ.get('PYTHONPATH', '')
+ sys.path[0:0] = filter(None, (p.strip() for p in pythonpath.split(':')))
+ <BLANKLINE>
+ interactive = options.inspect
+ if getattr(options, 'command', False):
+ sys.argv[:] = options.sys_argv
+ exec options.runnable in globs
+ elif getattr(options, 'module', False):
+ # runpy is only available in Python >= 2.5, so only try if we're asked
+ import runpy
+ sys.argv[:] = options.sys_argv
+ runpy.run_module(options.runnable, {}, "__main__", alter_sys=True)
+ elif args:
+ sys.argv[:] = args
+ globs['__file__'] = args[0]
+ execfile(args[0], globs)
+ else:
+ interactive = True
+ <BLANKLINE>
+ if interactive:
+ del globs['__file__']
+ code.interact(banner="", local=globs)
-
Handling custom build options for extensions provided in source distributions
-----------------------------------------------------------------------------
Modified: zc.buildout/branches/gary-4-include-site-packages/src/zc/buildout/testing.py
===================================================================
--- zc.buildout/branches/gary-4-include-site-packages/src/zc/buildout/testing.py 2009-09-25 21:13:39 UTC (rev 104550)
+++ zc.buildout/branches/gary-4-include-site-packages/src/zc/buildout/testing.py 2009-09-25 21:15:09 UTC (rev 104551)
@@ -116,8 +116,13 @@
args = [zc.buildout.easy_install._safe_arg(arg)
for arg in args]
args.insert(0, '-q')
- args.append(dict(os.environ, PYTHONPATH=setuptools_location))
+ env = dict(os.environ)
+ if executable == sys.executable:
+ env['PYTHONPATH'] = setuptools_location
+ # else pass an executable that has setuptools! See testselectingpython.py.
+ args.append(env)
+
here = os.getcwd()
try:
os.chdir(d)
@@ -135,6 +140,11 @@
def bdist_egg(setup, executable, dest):
_runsetup(setup, executable, 'bdist_egg', '-d', dest)
+def sys_install(setup, dest):
+ _runsetup(setup, sys.executable, 'install', '--home', dest,
+ '--single-version-externally-managed',
+ '--record', os.path.join(dest, 'added'))
+
def find_python(version):
e = os.environ.get('PYTHON%s' % version)
if e is not None:
@@ -268,6 +278,8 @@
# trick bootstrap into putting the buildout develop egg
# in the eggs dir.
('buildout', 'develop-eggs-directory', 'eggs'),
+ # we need to have setuptools around.
+ ('buildout', 'include-site-packages-for-buildout', 'true'),
]
).bootstrap([])
Modified: zc.buildout/branches/gary-4-include-site-packages/src/zc/buildout/tests.py
===================================================================
--- zc.buildout/branches/gary-4-include-site-packages/src/zc/buildout/tests.py 2009-09-25 21:13:39 UTC (rev 104550)
+++ zc.buildout/branches/gary-4-include-site-packages/src/zc/buildout/tests.py 2009-09-25 21:15:09 UTC (rev 104551)
@@ -19,6 +19,7 @@
import shutil
import sys
import tempfile
+import textwrap
import unittest
import zc.buildout.easy_install
import zc.buildout.testing
@@ -605,6 +606,7 @@
>>> os.chdir(d)
>>> print system(os.path.join(sample_buildout, 'bin', 'buildout')
+ ... + ' buildout:include-site-packages-for-buildout=true'
... + ' bootstrap'),
Creating directory '/sample-bootstrap/bin'.
Creating directory '/sample-bootstrap/parts'.
@@ -632,6 +634,7 @@
>>> os.chdir(d)
>>> print system(os.path.join(sample_buildout, 'bin', 'buildout')
+ ... + ' buildout:include-site-packages-for-buildout=true'
... + ' bootstrap'),
Creating directory '/sample-bootstrap/bin'.
Creating directory '/sample-bootstrap/parts'.
@@ -1769,6 +1772,54 @@
1 2
"""
+def versions_section_ignored_for_dependency_in_favor_of_site_packages():
+ r"""
+This is a test for a bugfix.
+
+The error showed itself when at least two dependencies were in a shared
+location like site-packages, and the first one met the "versions" setting. The
+first dependency would be added, but subsequent dependencies from the same
+location (e.g., site-packages) would use the version of the package found in
+the shared location, ignoring the version setting.
+
+We begin with a Python that has demoneeded version 1.1 installed and a
+demo version 0.3, all in a site-packages-like shared directory. We need
+to create this. ``eggrecipedemo.main()`` shows the number after the dot
+(that is, ``X`` in ``1.X``), for the demo package and the demoneeded
+package, so this demonstrates that our Python does in fact have demo
+version 0.3 and demoneeded version 1.1.
+
+ >>> primed_executable = get_executable_with_system_installed_packages()
+ >>> print system(primed_executable+" -c "+
+ ... "'import eggrecipedemo; eggrecipedemo.main()'")
+ 3 1
+ <BLANKLINE>
+
+Now we will install bigdemo, specifying different versions of demo
+and demoneeded in a versions section. Before the bugfix, the demo version
+would be honored, but not the demoneeded.
+
+Now here's a setup that would expose the bug, using the
+zc.buildout.easy_install API.
+
+ >>> example_dest = tmpdir('example_dest')
+ >>> workingset = zc.buildout.easy_install.install(
+ ... ['bigdemo'], example_dest, links=[sample_eggs],
+ ... executable=primed_executable,
+ ... index=None, include_site_packages=True,
+ ... versions={'demoneeded': '1.2c1', 'demo': '0.3'})
+ >>> for dist in workingset:
+ ... print dist
+ bigdemo 0.1
+ demo 0.3
+ demoneeded 1.2c1
+
+Before the bugfix, the demoneeded distribution was not included in the working
+set, and the demoneeded in site-packages (of the wrong number) would have been
+used.
+
+ """
+
if sys.version_info > (2, 4):
def test_exit_codes():
"""
@@ -2339,6 +2390,582 @@
"""
+def handle_sys_path_version_hack():
+ r"""
+If you use a Python that has a different version of one of your
+dependencies, and the new package tries to do sys.path tricks in the
+setup.py to get a __version__, and it uses namespace packages, the older
+package will be loaded first, making the setup version the wrong number.
+While very arguably packages simply shouldn't do this, some do, and we
+don't want buildout to fall over when they do.
+
+To demonstrate this, we will need to create a distribution that has one of
+these unpleasant tricks, and a Python that has an older version installed.
+
+ >>> for version in ('1.0', '1.1'):
+ ... tmp = tempfile.mkdtemp()
+ ... try:
+ ... write(tmp, 'README.txt', '')
+ ... mkdir(tmp, 'src')
+ ... mkdir(tmp, 'src', 'tellmy')
+ ... write(tmp, 'src', 'tellmy', '__init__.py',
+ ... "__import__("
+ ... "'pkg_resources').declare_namespace(__name__)\n")
+ ... mkdir(tmp, 'src', 'tellmy', 'version')
+ ... write(tmp, 'src', 'tellmy', 'version',
+ ... '__init__.py', '__version__=%r\n' % version)
+ ... write(
+ ... tmp, 'setup.py',
+ ... "from setuptools import setup\n"
+ ... "import sys\n"
+ ... "sys.path.insert(0, 'src')\n"
+ ... "from tellmy.version import __version__\n"
+ ... "setup(\n"
+ ... " name='tellmy.version',\n"
+ ... " package_dir = {'': 'src'},\n"
+ ... " packages = ['tellmy', 'tellmy.version'],\n"
+ ... " install_requires = ['setuptools'],\n"
+ ... " namespace_packages=['tellmy'],\n"
+ ... " zip_safe=True, version=__version__,\n"
+ ... " author='bob', url='bob', author_email='bob')\n"
+ ... )
+ ... zc.buildout.testing.sdist(tmp, sample_eggs)
+ ... if version == '1.0':
+ ... # We install the 1.0 version in site packages the way a
+ ... # system packaging system (debs, rpms) would do it.
+ ... zc.buildout.testing.sys_install(tmp, site_packages)
+ ... finally:
+ ... shutil.rmtree(tmp)
+ >>> primed_python = get_executable_with_system_installed_packages()
+ >>> print system(
+ ... primed_python + " -c '" +
+ ... "import tellmy.version\n" +
+ ... "print tellmy.version.__version__\n" +
+ ... "'")
+ 1.0
+ <BLANKLINE>
+ >>> write('buildout.cfg',
+ ... '''
+ ... [buildout]
+ ... parts = eggs
+ ... find-links = %(sample_eggs)s
+ ...
+ ... [primed_python]
+ ... executable = %(primed_python)s
+ ...
+ ... [eggs]
+ ... recipe = zc.recipe.egg:eggs
+ ... python = primed_python
+ ... eggs = tellmy.version == 1.1
+ ... ''' % globals())
+
+Before the bugfix, running this buildout would generate this error:
+
+ Installing eggs.
+ Getting distribution for 'tellmy.version==1.1'.
+ Installing tellmy.version 1.1
+ Caused installation of a distribution:
+ tellmy.version 1.0
+ with a different version.
+ Got None.
+ While:
+ Installing eggs.
+ Error: There is a version conflict.
+ We already have: tellmy.version 1.0
+ <BLANKLINE>
+
+The bugfix was simply to add Python's "-S" option when calling
+easyinstall (see zc.buildout.easy_install.Installer._call_easy_install).
+Now the install works correctly, as seen here.
+
+ >>> print system(primed_python+" "+buildout)
+ Installing eggs.
+ Getting distribution for 'tellmy.version==1.1'.
+ Got tellmy.version 1.1.
+ <BLANKLINE>
+
+ """
+
+def handle_namespace_package_in_both_site_packages_and_buildout_eggs():
+ r"""
+If you have the same namespace package in both site-packages and in
+buildout, we need to be very careful that faux-Python-executables and
+scripts correctly combine the two.
+
+To demonstrate this, we will create three packages: tellmy.version 1.0,
+tellmy.version 1.1, and tellmy.fortune 1.0.
+
+ >>> for pkg, version in (('version', '1.0'), ('version', '1.1'),
+ ... ('fortune', '1.0')):
+ ... tmp = tempfile.mkdtemp()
+ ... try:
+ ... write(tmp, 'README.txt', '')
+ ... mkdir(tmp, 'src')
+ ... mkdir(tmp, 'src', 'tellmy')
+ ... write(tmp, 'src', 'tellmy', '__init__.py',
+ ... "__import__("
+ ... "'pkg_resources').declare_namespace(__name__)\n")
+ ... mkdir(tmp, 'src', 'tellmy', pkg)
+ ... write(tmp, 'src', 'tellmy', pkg,
+ ... '__init__.py', '__version__=%r\n' % version)
+ ... write(
+ ... tmp, 'setup.py',
+ ... "from setuptools import setup\n"
+ ... "setup(\n"
+ ... " name='tellmy.%(pkg)s',\n"
+ ... " package_dir = {'': 'src'},\n"
+ ... " packages = ['tellmy', 'tellmy.%(pkg)s'],\n"
+ ... " install_requires = ['setuptools'],\n"
+ ... " namespace_packages=['tellmy'],\n"
+ ... " zip_safe=True, version=%(version)r,\n"
+ ... " author='bob', url='bob', author_email='bob')\n"
+ ... % globals()
+ ... )
+ ... zc.buildout.testing.sdist(tmp, sample_eggs)
+ ... if pkg == 'version' and version == '1.1':
+ ... # We install the 1.1 version in site packages the way a
+ ... # system packaging system (debs, rpms) would do it.
+ ... zc.buildout.testing.sys_install(tmp, site_packages)
+ ... finally:
+ ... shutil.rmtree(tmp)
+ >>> primed_python = get_executable_with_system_installed_packages()
+ >>> print system(
+ ... primed_python + " -c '" +
+ ... "import tellmy.version\n" +
+ ... "print tellmy.version.__version__\n" +
+ ... "'")
+ 1.1
+ <BLANKLINE>
+
+Now we will create a buildout that creates a script and a faux-Python script.
+We want to see that both can successfully import the specified versions of
+tellmy.version and tellmy.fortune.
+
+ >>> write('buildout.cfg',
+ ... '''
+ ... [buildout]
+ ... parts = eggs
+ ... find-links = %(link_server)s
+ ...
+ ... [primed_python]
+ ... executable = %(primed_python)s
+ ...
+ ... [eggs]
+ ... recipe = zc.recipe.egg
+ ... python = primed_python
+ ... interpreter = py
+ ... eggs = tellmy.version == 1.0
+ ... tellmy.fortune == 1.0
+ ... demo
+ ... initialization =
+ ... import tellmy.version
+ ... print tellmy.version.__version__
+ ... import tellmy.fortune
+ ... print tellmy.fortune.__version__
+ ... ''' % globals())
+
+ >>> print system(primed_python+" "+buildout)
+ Installing eggs.
+ Getting distribution for 'tellmy.version==1.0'.
+ Got tellmy.version 1.0.
+ Getting distribution for 'tellmy.fortune==1.0'.
+ Got tellmy.fortune 1.0.
+ Generated script '/sample-buildout/bin/demo'.
+ Generated interpreter '/sample-buildout/bin/py'.
+ <BLANKLINE>
+
+Finally, we are ready for the actual test. Prior to the bug fix that
+this tests, the results of both calls below was the following::
+
+ 1.1
+ Traceback (most recent call last):
+ ...
+ ImportError: No module named fortune
+ <BLANKLINE>
+
+In other words, we got the site-packages version of tellmy.version, and
+we could not import tellmy.fortune at all. The following are the correct
+results.
+
+ >>> print system(
+ ... join('bin', 'py') + " -c '" +
+ ... "import tellmy.version\n" +
+ ... "print tellmy.version.__version__\n" +
+ ... "import tellmy.fortune\n" +
+ ... "print tellmy.fortune.__version__\n" +
+ ... "'")
+ 1.0
+ 1.0
+ <BLANKLINE>
+ >>> print system(join('bin', 'demo'))
+ 1.0
+ 1.0
+ 3 1
+ <BLANKLINE>
+ """
+
+def isolated_include_site_packages():
+ """
+
+This is an isolated test of the include_site_packages functionality, passing
+the argument directly to install, overriding a default.
+
+Our "primed_executable" has the "demoneeded," "other," and "setuptools"
+packages available. We'll simply be asking for "other" here.
+
+ >>> primed_executable = get_executable_with_site_packages()
+ >>> zc.buildout.easy_install.include_site_packages(False)
+ True
+
+ >>> example_dest = tmpdir('site-packages-example-install')
+ >>> workingset = zc.buildout.easy_install.install(
+ ... ['other'], example_dest, links=[], executable=primed_executable,
+ ... index=None, include_site_packages=True)
+ >>> [dist.project_name for dist in workingset]
+ ['other']
+
+That worked fine. Let's try again with site packages not allowed (and
+reversing the default).
+
+ >>> zc.buildout.easy_install.include_site_packages(True)
+ False
+
+ >>> zc.buildout.easy_install.clear_index_cache()
+ >>> rmdir(example_dest)
+ >>> example_dest = tmpdir('site-packages-example-install')
+ >>> workingset = zc.buildout.easy_install.install(
+ ... ['other'], example_dest, links=[], executable=primed_executable,
+ ... index=None, include_site_packages=False)
+ Traceback (most recent call last):
+ ...
+ MissingDistribution: Couldn't find a distribution for 'other'.
+
+That's a failure, as expected.
+
+Now we explore an important edge case.
+
+Some system Pythons include setuptools (and other Python packages) in their
+site-packages (or equivalent) using a .egg-info directory. The pkg_resources
+module (from setuptools) considers a package installed using .egg-info to be a
+develop egg.
+
+zc.buildout.buildout.Buildout.bootstrap will make setuptools and zc.buildout
+available to the buildout via the eggs directory, for normal eggs; or the
+develop-eggs directory, for develop-eggs.
+
+If setuptools or zc.buildout is found in site-packages and considered by
+pkg_resources to be a develop egg, then the bootstrap code will use a .egg-link
+in the local develop-eggs, pointing to site-packages, in its entirety. Because
+develop-eggs must always be available for searching for distributions, this
+indirectly brings site-packages back into the search path for distributions.
+
+Because of this, we have to take special care that we still exclude
+site-packages even in this case. See the comments about site packages in the
+Installer._satisfied and Installer._obtain methods for the implementation
+(as of this writing).
+
+In this demonstration, we insert a link to the "other" distribution in our
+develop-eggs, which would bring the package back in, except for the special
+care we have taken to exclude it.
+
+ >>> zc.buildout.easy_install.clear_index_cache()
+ >>> rmdir(example_dest)
+ >>> example_dest = tmpdir('site-packages-example-install')
+ >>> mkdir(example_dest, 'develop-eggs')
+ >>> stdlib, site_packages = (
+ ... zc.buildout.easy_install._get_system_packages(primed_executable))
+ >>> path_to_other = [p for p in site_packages if 'other' in p][0]
+ >>> write(example_dest, 'develop-eggs', 'other.egg-link', path_to_other)
+ >>> workingset = zc.buildout.easy_install.install(
+ ... ['other'], example_dest, links=[],
+ ... path=[join(example_dest, 'develop-eggs')],
+ ... executable=primed_executable,
+ ... index=None, include_site_packages=False)
+ Traceback (most recent call last):
+ ...
+ MissingDistribution: Couldn't find a distribution for 'other'.
+
+The MissingDistribution error shows that buildout correctly excluded the
+"site-packages" source even though it was indirectly included in the path
+via a .egg-link file.
+
+ """
+
+def buildout_include_site_packages_option():
+ """
+The include-site-packages buildout option can be used to override the default
+behavior of using site packages.
+
+The default is include-site-packages = true. As a demonstration, notice we do
+not set find-links, but the eggs are still found because they are in the
+executable's path.
+
+Our "primed_executable" has the "demoneeded," "other," and "setuptools"
+packages available. We'll simply be asking for "other" here.
+
+ >>> primed_executable = get_executable_with_site_packages()
+ >>> write('buildout.cfg',
+ ... '''
+ ... [buildout]
+ ... parts = eggs
+ ... find-links =
+ ...
+ ... [primed_python]
+ ... executable = %(primed_executable)s
+ ...
+ ... [eggs]
+ ... recipe = zc.recipe.egg:eggs
+ ... python = primed_python
+ ... eggs = other
+ ... ''' % globals())
+
+ >>> print system(primed_executable+" "+buildout)
+ Installing eggs.
+ <BLANKLINE>
+
+However, if we set include-site-packages to false, we get an error, because
+the packages are not available in any links, and they are not allowed to be
+obtained from the executable's site packages.
+
+ >>> zc.buildout.easy_install.clear_index_cache()
+ >>> write('buildout.cfg',
+ ... '''
+ ... [buildout]
+ ... parts = eggs
+ ... find-links =
+ ... include-site-packages = false
+ ...
+ ... [primed_python]
+ ... executable = %(primed_executable)s
+ ...
+ ... [eggs]
+ ... recipe = zc.recipe.egg:eggs
+ ... eggs = other
+ ... ''' % globals())
+ >>> print system(primed_executable+" "+buildout)
+ Uninstalling eggs.
+ Installing eggs.
+ Couldn't find index page for 'other' (maybe misspelled?)
+ Getting distribution for 'other'.
+ While:
+ Installing eggs.
+ Getting distribution for 'other'.
+ Error: Couldn't find a distribution for 'other'.
+ <BLANKLINE>
+
+We get an error if we specify anything but true or false:
+
+ >>> write('buildout.cfg',
+ ... '''
+ ... [buildout]
+ ... parts = eggs
+ ... find-links = %(link_server)s
+ ... include-site-packages = no
+ ...
+ ... [eggs]
+ ... recipe = zc.recipe.egg:eggs
+ ... eggs = other
+ ... ''' % globals())
+
+ >>> print system(primed_executable+" "+buildout)
+ While:
+ Initializing.
+ Error: Invalid value for include-site-packages option: no
+ <BLANKLINE>
+
+ """
+
+def scripts_clean_sys_modules_when_site_packages_excluded():
+ r"""
+If you are using a Python with site-packages, the faux-Python-executables
+and scripts must be sure to not let any .pth files in site-packages affect
+sys.modules, which they can do normally because of importing site.py.
+
+Of course, .pth files that only contain paths simply affect sys.path, not
+sys.modules. They can also contain arbitrary Python, however: if a line
+begins with "import" (followed by a space or a tab) then the line will be
+interpreted as Python (see http://docs.python.org/library/site.html ).
+
+The setuptools package uses this mechanism to create namespace packages, for
+instance. The line ends up modifying sys.modules.
+
+This test verifies that, if ``include-site-packages = false``, sys.modules is
+not affected by the namespace package .pth files in the Python's
+site-packages.
+
+We begin by creating a namespace package installed as a distribution might
+install it.
+
+ >>> tmp = tempfile.mkdtemp()
+ >>> try:
+ ... write(tmp, 'README.txt', '')
+ ... mkdir(tmp, 'src')
+ ... mkdir(tmp, 'src', 'tellmy')
+ ... write(tmp, 'src', 'tellmy', '__init__.py',
+ ... "__import__("
+ ... "'pkg_resources').declare_namespace(__name__)\n")
+ ... mkdir(tmp, 'src', 'tellmy', 'fortune')
+ ... write(tmp, 'src', 'tellmy', 'fortune',
+ ... '__init__.py', '__version__="1.0"\n')
+ ... write(
+ ... tmp, 'setup.py',
+ ... "from setuptools import setup\n"
+ ... "setup(\n"
+ ... " name='tellmy.fortune',\n"
+ ... " package_dir = {'': 'src'},\n"
+ ... " packages = ['tellmy', 'tellmy.fortune'],\n"
+ ... " install_requires = ['setuptools'],\n"
+ ... " namespace_packages=['tellmy'],\n"
+ ... " zip_safe=True, version='1.0',\n"
+ ... " author='bob', url='bob', author_email='bob')\n"
+ ... )
+ ... zc.buildout.testing.sdist(tmp, sample_eggs)
+ ... zc.buildout.testing.sys_install(tmp, site_packages)
+ ... finally:
+ ... shutil.rmtree(tmp)
+ >>> primed_python = get_executable_with_system_installed_packages()
+
+Now we create a buildout that uses this Python but does not want to
+include site-packages.
+
+ >>> write('buildout.cfg',
+ ... '''
+ ... [buildout]
+ ... parts = eggs
+ ... find-links = %(link_server)s
+ ... include-site-packages = false
+ ...
+ ... [primed_python]
+ ... executable = %(primed_python)s
+ ...
+ ... [eggs]
+ ... recipe = zc.recipe.egg
+ ... python = primed_python
+ ... interpreter = py
+ ... eggs = demo
+ ... initialization =
+ ... import tellmy
+ ... ''' % globals())
+
+ >>> print system(primed_python+" "+buildout)
+ Installing eggs.
+ Getting distribution for 'demo'.
+ Got demo 0.4c1.
+ Getting distribution for 'demoneeded'.
+ Got demoneeded 1.2c1.
+ Generated script '/sample-buildout/bin/demo'.
+ Generated interpreter '/sample-buildout/bin/py'.
+ <BLANKLINE>
+
+Finally, we are ready for the actual test. Prior to the bug fix that
+this tests, the results of both calls was no errors. In other words, you
+could still import tellmy because it was stashed in sys.modules, even
+though sys.path had been cleaned out.
+
+ >>> print system(
+ ... join('bin', 'py') + " -c '" +
+ ... "import eggrecipedemo\n" +
+ ... "import tellmy\n" +
+ ... "'") # doctest: +ELLIPSIS
+ Traceback (most recent call last):
+ ...
+ ImportError: No module named tellmy
+ <BLANKLINE>
+ >>> print system(join('bin', 'demo')) # doctest: +ELLIPSIS
+ Traceback (most recent call last):
+ ...
+ ImportError: No module named tellmy
+ <BLANKLINE>
+ """
+
+def include_site_packages_with_buildout():
+ """
+When buildout gets a recipe egg (as opposed to runs a recipe), it starts with
+the current Python working set--the one that the bin/buildout script uses
+itself. If this working set includes site-packages, and site-packages includes
+an egg for package that the recipe needs, and the recipe specifies a newer
+version of that package, this can generate a version conflict.
+
+One solution to this is to not use site-packages
+('include-site-packages = false'). However, if you want your scripts to use
+site-packages, then you have to specify 'include-site-packages = true' for
+all of them.
+
+To make this use case easier to handle, you can instead specify
+``include-site-packages-with-buildout = false``, which indicates that
+the bin/buildout should *not* use site-packages; and
+``include-site-packages = true``, which indicates that the rest of the scripts
+should use site-packages.
+
+This is the default configuration.
+
+ >>> from zc.buildout.buildout import Buildout
+ >>> write('buildout.cfg',
+ ... '''
+ ... [buildout]
+ ... find-links = %(link_server)s
+ ... ''' % globals())
+ >>> buildout = Buildout('buildout.cfg', ())
+ >>> buildout['buildout']['include-site-packages-for-buildout']
+ 'false'
+ >>> buildout['buildout']['include-site-packages']
+ 'true'
+ >>> buildout.include_site_packages_for_buildout
+ False
+ >>> zc.buildout.easy_install.include_site_packages()
+ True
+
+This means that, when the buildout script is created by buildout, it explicitly
+specifies that site-packages should not be used. We'll monkeypatch the
+zc.buildout.easy_install install function so we can see this happens. (We test
+that this argument actually does what we want in other tests.)
+
+ >>> original_install = zc.buildout.easy_install.install
+ >>> def install(*args, **kwargs):
+ ... print 'include_site_packages =', kwargs['include_site_packages']
+ ... return original_install(*args, **kwargs)
+ ...
+ >>> zc.buildout.easy_install.install = install
+ >>> buildout.bootstrap(()) # doctest: +ELLIPSIS
+ include_site_packages = False...
+
+Now we'll do the reverse settings to show that the value will be honored in
+that case.
+
+ >>> write('buildout.cfg',
+ ... '''
+ ... [buildout]
+ ... include-site-packages-for-buildout = true
+ ... include-site-packages = false
+ ... find-links = %(link_server)s
+ ... ''' % globals())
+ >>> buildout = Buildout('buildout.cfg', ())
+ >>> buildout['buildout']['include-site-packages-for-buildout']
+ 'true'
+ >>> buildout['buildout']['include-site-packages']
+ 'false'
+ >>> buildout.include_site_packages_for_buildout
+ True
+ >>> zc.buildout.easy_install.include_site_packages()
+ False
+ >>> buildout.bootstrap(()) # doctest: +ELLIPSIS
+ include_site_packages = True...
+ >>> zc.buildout.easy_install.install = original_install
+
+Now we'll show that a value that is not 'true' or 'false' generates an error.
+
+ >>> write('buildout.cfg',
+ ... '''
+ ... [buildout]
+ ... include-site-packages-for-buildout = shazbot
+ ... ''')
+ >>> buildout = Buildout('buildout.cfg', ())
+ Traceback (most recent call last):
+ ...
+ UserError: Invalid value for include-site-packages-for-buildout option: shazbot
+
+ """
+
def develop_with_modules():
"""
Distribution setup scripts can import modules in the distribution directory:
@@ -2655,8 +3282,17 @@
######################################################################
def create_sample_eggs(test, executable=sys.executable):
+ """Creates sample eggs, source distributions, and a faux site-packages."
+
+ Unlike the faux site-packages created by
+ ``get_executable_with_site_packages``, this one has packages installed the
+ way distributions often install eggs in system Pythons (via
+ zc.buildout.testing.sys_install).
+ """
write = test.globs['write']
dest = test.globs['sample_eggs']
+ site_packages = test.globs['tmpdir']('site_packages')
+ test.globs['site_packages'] = site_packages
tmp = tempfile.mkdtemp()
try:
write(tmp, 'README.txt', '')
@@ -2673,6 +3309,8 @@
% (i, c1)
)
zc.buildout.testing.sdist(tmp, dest)
+ if i==1:
+ zc.buildout.testing.sys_install(tmp, site_packages)
write(
tmp, 'setup.py',
@@ -2702,6 +3340,8 @@
" zip_safe=True, version='0.%s%s')\n" % (i, c1)
)
zc.buildout.testing.bdist_egg(tmp, executable, dest)
+ if i==3:
+ zc.buildout.testing.sys_install(tmp, site_packages)
write(tmp, 'eggrecipebigdemo.py', 'import eggrecipedemo')
write(
@@ -2776,7 +3416,94 @@
test.globs['sample_eggs'])
test.globs['update_extdemo'] = lambda : add_source_dist(test, 1.5)
zc.buildout.testing.install_develop('zc.recipe.egg', test)
+ # Most tests don't need this set up, and it takes some time, so we just
+ # make it available as a convenience.
+ def get_executable_with_site_packages(requirements=None):
+ executable_buildout = test.globs['tmpdir']('executable')
+ old_wd = os.getcwd()
+ os.chdir(executable_buildout)
+ if requirements is None:
+ requirements = ['demoneeded', 'setuptools', 'other']
+ elif len([req for req in requirements
+ if req.startswith('setuptools')]) == 0:
+ requirements.append('setuptools') # you always need that.
+ requirements = '\n '.join(requirements)
+ test.globs['write']('buildout.cfg', textwrap.dedent(
+ '''
+ [buildout]
+ parts = interpreter
+ find-links = %(link_server)s
+ prefer-final = true
+ [interpreter]
+ recipe = zc.recipe.egg
+ interpreter = py
+ eggs = %(requirements)s
+ ''') % {'requirements': requirements,
+ 'link_server': test.globs['link_server']})
+ zc.buildout.buildout.Buildout(
+ 'buildout.cfg',
+ [('buildout', 'log-level', 'WARNING'),
+ # trick bootstrap into putting the buildout develop egg
+ # in the eggs dir.
+ ('buildout', 'develop-eggs-directory', 'eggs'),
+ # we need to have setuptools around.
+ ('buildout', 'include-site-packages-for-buildout', 'true'),
+ ]
+ ).bootstrap([])
+ os.mkdir('develop-eggs')
+ zc.buildout.testing.install_develop(
+ 'zc.recipe.egg',
+ os.path.join(executable_buildout, 'develop-eggs'))
+ test.globs['system'](
+ os.path.join(executable_buildout, 'bin', 'buildout'))
+ os.chdir(old_wd)
+ return os.path.join(executable_buildout, 'bin', 'py')
+ test.globs['get_executable_with_site_packages'] = (
+ get_executable_with_site_packages)
+ # Most tests also don't need this one. This creates an executable with
+ # eggs installed as a system might.
+ def get_executable_with_system_installed_packages():
+ executable_buildout = test.globs['tmpdir']('executable_buildout')
+ old_wd = os.getcwd()
+ os.chdir(executable_buildout)
+ import textwrap
+ test.globs['write']('buildout.cfg', textwrap.dedent(
+ '''
+ [buildout]
+ parts = interpreter
+
+ [interpreter]
+ recipe = zc.recipe.egg
+ scripts = py
+ interpreter = py
+ extra-paths = %(site-packages)s
+ include-site-packages = false
+ eggs = setuptools
+ ''') % {
+ 'site-packages': os.path.join(
+ test.globs['site_packages'], 'lib', 'python')})
+ zc.buildout.buildout.Buildout(
+ 'buildout.cfg',
+ [('buildout', 'log-level', 'WARNING'),
+ # trick bootstrap into putting the buildout develop egg
+ # in the eggs dir.
+ ('buildout', 'develop-eggs-directory', 'eggs'),
+ # we need to have setuptools around.
+ ('buildout', 'include-site-packages-for-buildout', 'true'),
+ ]
+ ).bootstrap([])
+ os.mkdir('develop-eggs')
+ zc.buildout.testing.install_develop(
+ 'zc.recipe.egg',
+ os.path.join(executable_buildout, 'develop-eggs'))
+ test.globs['system'](
+ os.path.join(executable_buildout, 'bin', 'buildout'))
+ os.chdir(old_wd)
+ return os.path.join(executable_buildout, 'bin', 'py')
+ test.globs['get_executable_with_system_installed_packages'] = (
+ get_executable_with_system_installed_packages)
+
egg_parse = re.compile('([0-9a-zA-Z_.]+)-([0-9a-zA-Z_.]+)-py(\d[.]\d).egg$'
).match
def makeNewRelease(project, ws, dest):
Modified: zc.buildout/branches/gary-4-include-site-packages/src/zc/buildout/testselectingpython.py
===================================================================
--- zc.buildout/branches/gary-4-include-site-packages/src/zc/buildout/testselectingpython.py 2009-09-25 21:13:39 UTC (rev 104550)
+++ zc.buildout/branches/gary-4-include-site-packages/src/zc/buildout/testselectingpython.py 2009-09-25 21:15:09 UTC (rev 104551)
@@ -11,7 +11,7 @@
# FOR A PARTICULAR PURPOSE.
#
##############################################################################
-import os, re, sys, unittest
+import os, re, subprocess, sys, textwrap, unittest
from zope.testing import doctest, renormalizing
import zc.buildout.tests
import zc.buildout.testing
@@ -31,7 +31,8 @@
>>> ws = zc.buildout.easy_install.install(
... ['demo'], dest, links=[link_server],
... index='http://www.python.org/pypi/',
- ... always_unzip=True, executable=other_executable)
+ ... always_unzip=True, executable=other_executable,
+ ... include_site_packages=False)
>>> ls(dest)
d demo-0.3-py%(other_version)s.egg
@@ -43,6 +44,31 @@
def multi_python(test):
other_executable = zc.buildout.testing.find_python(other_version)
+ command = textwrap.dedent('''\
+ try:
+ import setuptools
+ except ImportError:
+ import sys
+ sys.exit(1)
+ ''')
+ if subprocess.call([other_executable, '-c', command],
+ env=os.environ):
+ # the other executable does not have setuptools. Get setuptools.
+ # We will do this using the same tools we are testing, for better or
+ # worse. Alternatively, we could try using bootstrap.
+ executable_dir = test.globs['tmpdir']('executable_dir')
+ ws = zc.buildout.easy_install.install(
+ ['setuptools'], executable_dir,
+ index='http://www.python.org/pypi/',
+ always_unzip=True, executable=other_executable)
+ zc.buildout.easy_install.scripts(
+ ['setuptools'], ws, other_executable, executable_dir,
+ interpreter='py')
+ original_executable = other_executable
+ other_executable = os.path.join(executable_dir, 'py')
+ assert not subprocess.call(
+ [other_executable, '-c', command], env=os.environ), (
+ 'test set up failed')
sample_eggs = test.globs['tmpdir']('sample_eggs')
os.mkdir(os.path.join(sample_eggs, 'index'))
test.globs['sample_eggs'] = sample_eggs
Modified: zc.buildout/branches/gary-4-include-site-packages/src/zc/buildout/update.txt
===================================================================
--- zc.buildout/branches/gary-4-include-site-packages/src/zc/buildout/update.txt 2009-09-25 21:13:39 UTC (rev 104550)
+++ zc.buildout/branches/gary-4-include-site-packages/src/zc/buildout/update.txt 2009-09-25 21:15:09 UTC (rev 104551)
@@ -80,15 +80,43 @@
Our buildout script has been updated to use the new eggs:
- >>> cat(sample_buildout, 'bin', 'buildout')
+ >>> cat(sample_buildout, 'bin', 'buildout') # doctest: +ELLIPSIS
#!/usr/local/bin/python2.4
<BLANKLINE>
+ import site
import sys
- sys.path[0:0] = [
- '/sample-buildout/eggs/zc.buildout-99.99-py2.4.egg',
- '/sample-buildout/eggs/setuptools-99.99-py2.4.egg',
- ]
<BLANKLINE>
+ # Clean out sys.modules from site's processing of .pth files.
+ clean_modules = [
+ ...
+ ]
+ for k in sys.modules.keys():
+ if k not in clean_modules:
+ del sys.modules[k]
+ <BLANKLINE>
+ sys.path[:] = [
+ ...
+ ]
+ sys.path.extend([
+ '/sample-buildout/eggs/zc.buildout-NINETYNINE.NINETYNINE-pyN.N.egg',
+ '/sample-buildout/eggs/setuptools-NINETYNINE.NINETYNINE-pyN.N.egg'
+ ])
+ site_dirs = [
+ ...
+ ]
+ # Add the site_dirs before `addsitedir` in case it has setuptools.
+ sys.path.extend(site_dirs)
+ # Process all buildout-controlled eggs before site-packages by importing
+ # pkg_resources. This is only important for namespace packages, so it may
+ # not have been added, so ignore import errors.
+ try:
+ import pkg_resources
+ except ImportError:
+ pass
+ # Process .pth files.
+ for p in site_dirs:
+ site.addsitedir(p)
+ <BLANKLINE>
import zc.buildout.buildout
<BLANKLINE>
if __name__ == '__main__':
Modified: zc.buildout/branches/gary-4-include-site-packages/zc.recipe.egg_/src/zc/recipe/egg/README.txt
===================================================================
--- zc.buildout/branches/gary-4-include-site-packages/zc.recipe.egg_/src/zc/recipe/egg/README.txt 2009-09-25 21:13:39 UTC (rev 104550)
+++ zc.buildout/branches/gary-4-include-site-packages/zc.recipe.egg_/src/zc/recipe/egg/README.txt 2009-09-25 21:15:09 UTC (rev 104551)
@@ -372,17 +372,46 @@
Let's look at the script that was generated:
- >>> cat(sample_buildout, 'bin', 'foo') # doctest: +NORMALIZE_WHITESPACE
+ >>> cat(sample_buildout, 'bin', 'foo')
+ ... # doctest: +NORMALIZE_WHITESPACE +ELLIPSIS
#!/usr/local/bin/python2.4
<BLANKLINE>
+ import site
import sys
- sys.path[0:0] = [
- '/sample-buildout/eggs/demo-0.4c1-py2.4.egg',
- '/sample-buildout/eggs/demoneeded-1.2c1-py2.4.egg',
- '/foo/bar',
- '/sample-buildout/spam',
- ]
<BLANKLINE>
+ # Clean out sys.modules from site's processing of .pth files.
+ clean_modules = [
+ ...
+ ]
+ for k in sys.modules.keys():
+ if k not in clean_modules:
+ del sys.modules[k]
+ <BLANKLINE>
+ sys.path[:] = [
+ ...
+ ]
+ sys.path.extend([
+ '/sample-buildout/eggs/demo-0.4c1-pyN.N.egg',
+ '/sample-buildout/eggs/demoneeded-1.2c1-pyN.N.egg'
+ ])
+ site_dirs = [
+ '/foo/bar',
+ '/sample-buildout/spam',
+ ...
+ ]
+ # Add the site_dirs before `addsitedir` in case it has setuptools.
+ sys.path.extend(site_dirs)
+ # Process all buildout-controlled eggs before site-packages by importing
+ # pkg_resources. This is only important for namespace packages, so it may
+ # not have been added, so ignore import errors.
+ try:
+ import pkg_resources
+ except ImportError:
+ pass
+ # Process .pth files.
+ for p in site_dirs:
+ site.addsitedir(p)
+ <BLANKLINE>
import eggrecipedemo
<BLANKLINE>
if __name__ == '__main__':
@@ -419,7 +448,8 @@
Let's look at the script that was generated:
- >>> cat(sample_buildout, 'bin', 'foo') # doctest: +NORMALIZE_WHITESPACE
+ >>> cat(sample_buildout, 'bin', 'foo')
+ ... # doctest: +NORMALIZE_WHITESPACE +ELLIPSIS
#!/usr/local/bin/python2.4
<BLANKLINE>
import os
@@ -428,14 +458,42 @@
base = os.path.dirname(os.path.abspath(os.path.realpath(__file__)))
base = os.path.dirname(base)
<BLANKLINE>
+ import site
import sys
- sys.path[0:0] = [
- join(base, 'eggs/demo-0.4c1-pyN.N.egg'),
- join(base, 'eggs/demoneeded-1.2c1-pyN.N.egg'),
- '/foo/bar',
- join(base, 'spam'),
- ]
<BLANKLINE>
+ # Clean out sys.modules from site's processing of .pth files.
+ clean_modules = [
+ ...
+ ]
+ for k in sys.modules.keys():
+ if k not in clean_modules:
+ del sys.modules[k]
+ <BLANKLINE>
+ sys.path[:] = [
+ ...
+ ]
+ sys.path.extend([
+ join(base, 'eggs/demo-0.4c1-pyN.N.egg'),
+ join(base, 'eggs/demoneeded-1.2c1-pyN.N.egg')
+ ])
+ site_dirs = [
+ '/foo/bar',
+ join(base, 'spam'),
+ ...
+ ]
+ # Add the site_dirs before `addsitedir` in case it has setuptools.
+ sys.path.extend(site_dirs)
+ # Process all buildout-controlled eggs before site-packages by importing
+ # pkg_resources. This is only important for namespace packages, so it may
+ # not have been added, so ignore import errors.
+ try:
+ import pkg_resources
+ except ImportError:
+ pass
+ # Process .pth files.
+ for p in site_dirs:
+ site.addsitedir(p)
+ <BLANKLINE>
import eggrecipedemo
<BLANKLINE>
if __name__ == '__main__':
@@ -444,7 +502,6 @@
You can specify relative paths in the buildout section, rather than in
each individual script section:
-
>>> write(sample_buildout, 'buildout.cfg',
... """
... [buildout]
@@ -466,7 +523,8 @@
Installing demo.
Generated script '/sample-buildout/bin/foo'.
- >>> cat(sample_buildout, 'bin', 'foo') # doctest: +NORMALIZE_WHITESPACE
+ >>> cat(sample_buildout, 'bin', 'foo')
+ ... # doctest: +NORMALIZE_WHITESPACE +ELLIPSIS
#!/usr/local/bin/python2.4
<BLANKLINE>
import os
@@ -475,14 +533,42 @@
base = os.path.dirname(os.path.abspath(os.path.realpath(__file__)))
base = os.path.dirname(base)
<BLANKLINE>
+ import site
import sys
- sys.path[0:0] = [
- join(base, 'eggs/demo-0.4c1-pyN.N.egg'),
- join(base, 'eggs/demoneeded-1.2c1-pyN.N.egg'),
- '/foo/bar',
- join(base, 'spam'),
- ]
<BLANKLINE>
+ # Clean out sys.modules from site's processing of .pth files.
+ clean_modules = [
+ ...
+ ]
+ for k in sys.modules.keys():
+ if k not in clean_modules:
+ del sys.modules[k]
+ <BLANKLINE>
+ sys.path[:] = [
+ ...
+ ]
+ sys.path.extend([
+ join(base, 'eggs/demo-0.4c1-pyN.N.egg'),
+ join(base, 'eggs/demoneeded-1.2c1-pyN.N.egg')
+ ])
+ site_dirs = [
+ '/foo/bar',
+ join(base, 'spam'),
+ ...
+ ]
+ # Add the site_dirs before `addsitedir` in case it has setuptools.
+ sys.path.extend(site_dirs)
+ # Process all buildout-controlled eggs before site-packages by importing
+ # pkg_resources. This is only important for namespace packages, so it may
+ # not have been added, so ignore import errors.
+ try:
+ import pkg_resources
+ except ImportError:
+ pass
+ # Process .pth files.
+ for p in site_dirs:
+ site.addsitedir(p)
+ <BLANKLINE>
import eggrecipedemo
<BLANKLINE>
if __name__ == '__main__':
@@ -519,17 +605,46 @@
Installing demo.
Generated script '/sample-buildout/bin/foo'.
- >>> cat(sample_buildout, 'bin', 'foo') # doctest: +NORMALIZE_WHITESPACE
+ >>> cat(sample_buildout, 'bin', 'foo')
+ ... # doctest: +NORMALIZE_WHITESPACE +ELLIPSIS
#!/usr/local/bin/python2.4
<BLANKLINE>
+ import site
import sys
- sys.path[0:0] = [
- '/sample-buildout/eggs/demo-0.4c1-py2.4.egg',
- '/sample-buildout/eggs/demoneeded-1.2c1-py2.4.egg',
- '/foo/bar',
- '/sample-buildout/spam',
- ]
<BLANKLINE>
+ # Clean out sys.modules from site's processing of .pth files.
+ clean_modules = [
+ ...
+ ]
+ for k in sys.modules.keys():
+ if k not in clean_modules:
+ del sys.modules[k]
+ <BLANKLINE>
+ sys.path[:] = [
+ ...
+ ]
+ sys.path.extend([
+ '/sample-buildout/eggs/demo-0.4c1-pyN.N.egg',
+ '/sample-buildout/eggs/demoneeded-1.2c1-pyN.N.egg'
+ ])
+ site_dirs = [
+ '/foo/bar',
+ '/sample-buildout/spam',
+ ...
+ ]
+ # Add the site_dirs before `addsitedir` in case it has setuptools.
+ sys.path.extend(site_dirs)
+ # Process all buildout-controlled eggs before site-packages by importing
+ # pkg_resources. This is only important for namespace packages, so it may
+ # not have been added, so ignore import errors.
+ try:
+ import pkg_resources
+ except ImportError:
+ pass
+ # Process .pth files.
+ for p in site_dirs:
+ site.addsitedir(p)
+ <BLANKLINE>
a = (1, 2
3, 4)
<BLANKLINE>
@@ -578,16 +693,45 @@
- other
>>> cat(sample_buildout, 'bin', 'other')
+ ... # doctest: +NORMALIZE_WHITESPACE +ELLIPSIS
#!/usr/local/bin/python2.4
<BLANKLINE>
+ import site
import sys
- sys.path[0:0] = [
- '/sample-buildout/eggs/demo-0.4c1-py2.4.egg',
- '/sample-buildout/eggs/demoneeded-1.2c1-py2.4.egg',
- '/foo/bar',
- '/sample-buildout/spam',
- ]
<BLANKLINE>
+ # Clean out sys.modules from site's processing of .pth files.
+ clean_modules = [
+ ...
+ ]
+ for k in sys.modules.keys():
+ if k not in clean_modules:
+ del sys.modules[k]
+ <BLANKLINE>
+ sys.path[:] = [
+ ...
+ ]
+ sys.path.extend([
+ '/sample-buildout/eggs/demo-0.4c1-pyN.N.egg',
+ '/sample-buildout/eggs/demoneeded-1.2c1-pyN.N.egg'
+ ])
+ site_dirs = [
+ '/foo/bar',
+ '/sample-buildout/spam',
+ ...
+ ]
+ # Add the site_dirs before `addsitedir` in case it has setuptools.
+ sys.path.extend(site_dirs)
+ # Process all buildout-controlled eggs before site-packages by importing
+ # pkg_resources. This is only important for namespace packages, so it may
+ # not have been added, so ignore import errors.
+ try:
+ import pkg_resources
+ except ImportError:
+ pass
+ # Process .pth files.
+ for p in site_dirs:
+ site.addsitedir(p)
+ <BLANKLINE>
import foo.bar
<BLANKLINE>
if __name__ == '__main__':
Modified: zc.buildout/branches/gary-4-include-site-packages/zc.recipe.egg_/src/zc/recipe/egg/api.txt
===================================================================
--- zc.buildout/branches/gary-4-include-site-packages/zc.recipe.egg_/src/zc/recipe/egg/api.txt 2009-09-25 21:13:39 UTC (rev 104550)
+++ zc.buildout/branches/gary-4-include-site-packages/zc.recipe.egg_/src/zc/recipe/egg/api.txt 2009-09-25 21:15:09 UTC (rev 104551)
@@ -15,7 +15,7 @@
around the egg recipe:
>>> mkdir(sample_buildout, 'sample')
- >>> write(sample_buildout, 'sample', 'sample.py',
+ >>> write(sample_buildout, 'sample', 'sample.py',
... """
... import logging, os
... import zc.recipe.egg
@@ -53,7 +53,7 @@
>>> write(sample_buildout, 'sample', 'setup.py',
... """
... from setuptools import setup
- ...
+ ...
... setup(
... name = "sample",
... entry_points = {'zc.buildout': ['default = sample:Sample']},
@@ -95,12 +95,13 @@
computed by the egg recipe by looking at .installed.cfg:
>>> cat(sample_buildout, '.installed.cfg')
+ ... # doctest: +NORMALIZE_WHITESPACE
[buildout]
installed_develop_eggs = /sample-buildout/develop-eggs/sample.egg-link
parts = sample-part
<BLANKLINE>
[sample-part]
- __buildout_installed__ =
+ __buildout_installed__ =
__buildout_signature__ = sample-6aWMvV2EJ9Ijq+bR8ugArQ==
zc.recipe.egg-cAsnudgkduAa/Fd+WJIM6Q==
setuptools-0.6-py2.4.egg
@@ -112,10 +113,12 @@
develop-eggs-directory = /sample-buildout/develop-eggs
eggs = demo<0.3
eggs-directory = /sample-buildout/eggs
- executable = /usr/local/bin/python2.3
+ executable = python
extras = other
- find-links = http://localhost:27071/
- index = http://localhost:27071/index
+ find-links = http://localhost:8080/
+ include-site-packages = true
+ index = http://localhost:8080/index
+ python = buildout
recipe = sample
If we use the extra-paths option:
Modified: zc.buildout/branches/gary-4-include-site-packages/zc.recipe.egg_/src/zc/recipe/egg/egg.py
===================================================================
--- zc.buildout/branches/gary-4-include-site-packages/zc.recipe.egg_/src/zc/recipe/egg/egg.py 2009-09-25 21:13:39 UTC (rev 104550)
+++ zc.buildout/branches/gary-4-include-site-packages/zc.recipe.egg_/src/zc/recipe/egg/egg.py 2009-09-25 21:15:09 UTC (rev 104551)
@@ -48,11 +48,16 @@
options['_e'] = options['eggs-directory'] # backward compat.
options['develop-eggs-directory'] = b_options['develop-eggs-directory']
options['_d'] = options['develop-eggs-directory'] # backward compat.
-
# verify that this is None, 'true' or 'false'
get_bool(options, 'unzip')
-
- python = options.get('python', b_options['python'])
+ value = options.setdefault('include-site-packages',
+ b_options['include-site-packages'])
+ if value not in ('true', 'false'):
+ raise zc.buildout.UserError(
+ "Invalid value for include-site-packages option: %s" %
+ (value,))
+ self.include_site_packages = (value == 'true')
+ python = options.setdefault('python', b_options['python'])
options['executable'] = buildout[python]['executable']
def working_set(self, extra=()):
@@ -73,7 +78,9 @@
if self.buildout['buildout'].get('offline') == 'true':
ws = zc.buildout.easy_install.working_set(
distributions, options['executable'],
- [options['develop-eggs-directory'], options['eggs-directory']]
+ [options['develop-eggs-directory'],
+ options['eggs-directory']],
+ include_site_packages = self.include_site_packages,
)
else:
kw = {}
@@ -87,6 +94,7 @@
path=[options['develop-eggs-directory']],
newest=self.buildout['buildout'].get('newest') == 'true',
allow_hosts=self.allow_hosts,
+ include_site_packages = self.include_site_packages,
**kw)
return orig_distributions, ws
@@ -167,6 +175,7 @@
initialization=options.get('initialization', ''),
arguments=options.get('arguments', ''),
relative_paths=self._relative_paths,
+ include_site_packages = self.include_site_packages,
)
return ()
More information about the checkins
mailing list