[Checkins] SVN: zc.buildout/branches/gary-1.5.2/ make include-site-packages, allowed-eggs-from-site-packages, and exec-sitecustomize parts of the core buildout configuration. Have them control the environment for recipes and extensions. Set the defaults to values to closer match the behavior of previous versions of buildout.
Gary Poster
gary.poster at canonical.com
Wed Sep 1 18:58:35 EDT 2010
Log message for revision 116116:
make include-site-packages, allowed-eggs-from-site-packages, and exec-sitecustomize parts of the core buildout configuration. Have them control the environment for recipes and extensions. Set the defaults to values to closer match the behavior of previous versions of buildout.
Changed:
A zc.buildout/branches/gary-1.5.2/
U zc.buildout/branches/gary-1.5.2/src/zc/buildout/buildout.py
U zc.buildout/branches/gary-1.5.2/src/zc/buildout/buildout.txt
U zc.buildout/branches/gary-1.5.2/src/zc/buildout/easy_install.py
U zc.buildout/branches/gary-1.5.2/src/zc/buildout/testing.py
U zc.buildout/branches/gary-1.5.2/src/zc/buildout/update.txt
U zc.buildout/branches/gary-1.5.2/src/zc/buildout/virtualenv.txt
U zc.buildout/branches/gary-1.5.2/z3c.recipe.scripts_/src/z3c/recipe/scripts/README.txt
U zc.buildout/branches/gary-1.5.2/z3c.recipe.scripts_/src/z3c/recipe/scripts/tests.py
-=-
Modified: zc.buildout/branches/gary-1.5.2/src/zc/buildout/buildout.py
===================================================================
--- zc.buildout/trunk/src/zc/buildout/buildout.py 2010-08-29 20:58:59 UTC (rev 116013)
+++ zc.buildout/branches/gary-1.5.2/src/zc/buildout/buildout.py 2010-09-01 22:58:35 UTC (rev 116116)
@@ -119,11 +119,14 @@
'accept-buildout-test-releases': 'false',
'allow-hosts': '*',
'allow-picked-versions': 'true',
+ 'allowed-eggs-from-site-packages': '*',
'bin-directory': 'bin',
'develop-eggs-directory': 'develop-eggs',
'eggs-directory': 'eggs',
'executable': sys.executable,
+ 'exec-sitecustomize': 'true',
'find-links': '',
+ 'include-site-packages': 'true',
'install-from-cache': 'false',
'installed': '.installed.cfg',
'log-format': '',
@@ -290,6 +293,18 @@
zc.buildout.easy_install.install_from_cache(
options.get_bool('install-from-cache'))
zc.buildout.easy_install.always_unzip(options.get_bool('unzip'))
+ allowed_eggs = tuple(name.strip() for name in options[
+ 'allowed-eggs-from-site-packages'].split('\n'))
+ self.include_site_packages = options.get_bool('include-site-packages')
+ self.exec_sitecustomize = options.get_bool('exec-sitecustomize')
+ if (_sys_executable_has_broken_dash_S and
+ (not self.include_site_packages or allowed_eggs != ('*',))):
+ # We can't do this if the executable has a broken -S.
+ warnings.warn(zc.buildout.easy_install.BROKEN_DASH_S_WARNING)
+ self.include_site_packages = True
+ zc.buildout.easy_install.allowed_eggs_from_site_packages(allowed_eggs)
+ zc.buildout.easy_install.include_site_packages(
+ self.include_site_packages)
download_cache = options.get('download-cache')
if download_cache:
@@ -337,7 +352,6 @@
distributions, options['executable'],
[options['develop-eggs-directory'],
options['eggs-directory']],
- include_site_packages=_sys_executable_has_broken_dash_S,
prefer_final=not self.accept_buildout_test_releases,
)
else:
@@ -349,7 +363,6 @@
path=[options['develop-eggs-directory']],
newest=self.newest,
allow_hosts=self._allow_hosts,
- include_site_packages=_sys_executable_has_broken_dash_S,
prefer_final=not self.accept_buildout_test_releases,
)
@@ -399,8 +412,10 @@
zc.buildout.easy_install.sitepackage_safe_scripts(
options['bin-directory'], ws, options['executable'], partsdir,
reqs=['zc.buildout'], relative_paths=relative_paths,
- include_site_packages=_sys_executable_has_broken_dash_S,
- script_initialization=script_initialization,)
+ include_site_packages=self.include_site_packages,
+ script_initialization=script_initialization,
+ exec_sitecustomize=self.exec_sitecustomize,
+ )
init = bootstrap
@@ -846,7 +861,6 @@
index = options.get('index'),
path = [options['develop-eggs-directory']],
allow_hosts = self._allow_hosts,
- include_site_packages=_sys_executable_has_broken_dash_S,
prefer_final=not self.accept_buildout_test_releases,
)
@@ -911,11 +925,20 @@
script_initialization = _early_release_initialization_code
else:
script_initialization = ''
+ # (Honor the relative-paths option.)
+ relative_paths = options.get('relative-paths', 'false')
+ if relative_paths == 'true':
+ relative_paths = options['directory']
+ else:
+ assert relative_paths == 'false'
+ relative_paths = ''
zc.buildout.easy_install.sitepackage_safe_scripts(
- options['bin-directory'], ws, sys.executable, partsdir,
- reqs=['zc.buildout'],
- include_site_packages=_sys_executable_has_broken_dash_S,
- script_initialization=script_initialization)
+ options['bin-directory'], ws, options['executable'], partsdir,
+ reqs=['zc.buildout'], relative_paths=relative_paths,
+ include_site_packages=self.include_site_packages,
+ script_initialization=script_initialization,
+ exec_sitecustomize=self.exec_sitecustomize,
+ )
# Restart
args = map(zc.buildout.easy_install._safe_arg, sys.argv)
@@ -956,7 +979,6 @@
links = self['buildout'].get('find-links', '').split(),
index = self['buildout'].get('index'),
newest=self.newest, allow_hosts=self._allow_hosts,
- include_site_packages=_sys_executable_has_broken_dash_S,
prefer_final=not self.accept_buildout_test_releases)
# Clear cache because extensions might now let us read pages we
@@ -1072,7 +1094,6 @@
working_set=pkg_resources.working_set,
newest=buildout.newest,
allow_hosts=buildout._allow_hosts,
- include_site_packages=_sys_executable_has_broken_dash_S,
prefer_final=not buildout.accept_buildout_test_releases)
__doing__ = 'Loading %s recipe entry %s:%s.', group, spec, entry
Modified: zc.buildout/branches/gary-1.5.2/src/zc/buildout/buildout.txt
===================================================================
--- zc.buildout/trunk/src/zc/buildout/buildout.txt 2010-08-29 20:58:59 UTC (rev 116013)
+++ zc.buildout/branches/gary-1.5.2/src/zc/buildout/buildout.txt 2010-09-01 22:58:35 UTC (rev 116116)
@@ -735,6 +735,8 @@
DEFAULT_VALUE
allow-picked-versions= true
DEFAULT_VALUE
+ allowed-eggs-from-site-packages= *
+ DEFAULT_VALUE
bin-directory= bin
DEFAULT_VALUE
develop= recipes
@@ -745,10 +747,14 @@
COMPUTED_VALUE
eggs-directory= eggs
DEFAULT_VALUE
+ exec-sitecustomize= true
+ DEFAULT_VALUE
executable= ...
DEFAULT_VALUE
find-links=
DEFAULT_VALUE
+ include-site-packages= true
+ DEFAULT_VALUE
install-from-cache= false
DEFAULT_VALUE
installed= .installed.cfg
@@ -2232,12 +2238,15 @@
accept-buildout-test-releases = false
allow-hosts = *
allow-picked-versions = true
+ allowed-eggs-from-site-packages = *
bin-directory = /sample-buildout/bin
develop-eggs-directory = /sample-buildout/develop-eggs
directory = /sample-buildout
eggs-directory = /sample-buildout/eggs
+ exec-sitecustomize = true
executable = python
find-links =
+ include-site-packages = true
install-from-cache = false
installed = /sample-buildout/.installed.cfg
log-format =
@@ -2259,6 +2268,37 @@
command-line assignments. We've discussed most of these options
already, but let's review them and touch on some we haven't discussed:
+allowed-eggs-from-site-packages
+ Sometimes you need or want to control what eggs from site-packages are
+ used. The allowed-eggs-from-site-packages option allows you to specify a
+ whitelist of project names that may be included from site-packages. You
+ can use globs to specify the value. It defaults to a single value of '*',
+ indicating that any package may come from site-packages.
+
+ Here's a usage example::
+
+ [buildout]
+ ...
+
+ allowed-eggs-from-site-packages =
+ demo
+ bigdemo
+ zope.*
+
+ This option interacts with the ``include-site-packages`` option in the
+ following ways.
+
+ If ``include-site-packages`` is true, then
+ ``allowed-eggs-from-site-packages`` filters what eggs from site-packages
+ may be chosen. Therefore, if ``allowed-eggs-from-site-packages`` is an
+ empty list, then no eggs from site-packages are chosen, but site-packages
+ will still be included at the end of path lists.
+
+ If ``include-site-packages`` is false, the value of
+ ``allowed-eggs-from-site-packages`` is irrelevant.
+
+ See the ``include-site-packages`` description for more information.
+
bin-directory
The directory path where scripts are written. This can be a
relative path, which is interpreted relative to the directory
@@ -2279,10 +2319,27 @@
*never* be modified. This can be a relative path, which is
interpreted relative to the directory option.
+exec-sitecustomize
+ Normally the Python's real sitecustomize module is processed.
+ If you do not want it to be processed in order to increase the
+ repeatability of your buildout, set this value to 'false'. This will
+ be honored irrespective of the setting for include-site-packages.
+ This option will be honored by some recipes and not others.
+ z3c.recipe.scripts honors this and zc.recipe.egg does not, for
+ instance.
+
executable
The Python executable used to run the buildout. See the python
option below.
+include-site-packages
+ You can choose not to have the site-packages of the underlying Python
+ available to your script or interpreter, in addition to the packages
+ from your eggs. This can increase repeatability for your buildout.
+ This option will be better used by some recipes than others.
+ z3c.recipe.scripts honors this fully and zc.recipe.egg only
+ partially, for instance.
+
installed
The file path where information about the results of the previous
buildout run is written. This can be a relative path, which is
Modified: zc.buildout/branches/gary-1.5.2/src/zc/buildout/easy_install.py
===================================================================
--- zc.buildout/trunk/src/zc/buildout/easy_install.py 2010-08-29 20:58:59 UTC (rev 116013)
+++ zc.buildout/branches/gary-1.5.2/src/zc/buildout/easy_install.py 2010-09-01 22:58:35 UTC (rev 116116)
@@ -1564,7 +1564,7 @@
# These are used only by the newer ``sitepackage_safe_scripts`` function.
-def _get_module_file(executable, name):
+def _get_module_file(executable, name, silent=False):
"""Return a module's file path.
- executable is a path to the desired Python executable.
@@ -1585,8 +1585,9 @@
cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env)
stdout, stderr = _proc.communicate();
if _proc.returncode:
- logger.info(
- 'Could not find file for module %s:\n%s', name, stderr)
+ if not silent:
+ logger.info(
+ 'Could not find file for module %s:\n%s', name, stderr)
return None
# else: ...
res = stdout.strip()
@@ -1610,7 +1611,7 @@
sitecustomize.write(initialization + '\n')
if exec_sitecustomize:
real_sitecustomize_path = _get_module_file(
- executable, 'sitecustomize')
+ executable, 'sitecustomize', silent=True)
if real_sitecustomize_path:
real_sitecustomize = open(real_sitecustomize_path, 'r')
sitecustomize.write(
Modified: zc.buildout/branches/gary-1.5.2/src/zc/buildout/testing.py
===================================================================
--- zc.buildout/trunk/src/zc/buildout/testing.py 2010-08-29 20:58:59 UTC (rev 116013)
+++ zc.buildout/branches/gary-1.5.2/src/zc/buildout/testing.py 2010-09-01 22:58:35 UTC (rev 116116)
@@ -365,6 +365,8 @@
write('buildout.cfg', textwrap.dedent('''\
[buildout]
parts = py
+ include-site-packages = false
+ exec-sitecustomize = false
[py]
recipe = z3c.recipe.scripts
Modified: zc.buildout/branches/gary-1.5.2/src/zc/buildout/update.txt
===================================================================
--- zc.buildout/trunk/src/zc/buildout/update.txt 2010-08-29 20:58:59 UTC (rev 116013)
+++ zc.buildout/branches/gary-1.5.2/src/zc/buildout/update.txt 2010-09-01 22:58:35 UTC (rev 116116)
@@ -95,6 +95,10 @@
"""Add site packages, as determined by zc.buildout.
<BLANKLINE>
See original_addsitepackages, below, for the original version."""
+ setuptools_path = '/sample-buildout/eggs/setuptools-99.99-pyN.N.egg'
+ sys.path.append(setuptools_path)
+ known_paths.add(os.path.normcase(setuptools_path))
+ import pkg_resources
buildout_paths = [
'/sample-buildout/eggs/zc.buildout-99.99-pyN.N.egg',
'/sample-buildout/eggs/setuptools-99.99-pyN.N.egg'
@@ -104,6 +108,14 @@
if not sitedircase in known_paths and os.path.exists(sitedir):
sys.path.append(sitedir)
known_paths.add(sitedircase)
+ pkg_resources.working_set.add_entry(sitedir)
+ sys.__egginsert = len(buildout_paths) # Support setuptools.
+ original_paths = [
+ ...
+ ]
+ for path in original_paths:
+ if path == setuptools_path or path not in known_paths:
+ addsitedir(path, known_paths)
return known_paths
...
Modified: zc.buildout/branches/gary-1.5.2/src/zc/buildout/virtualenv.txt
===================================================================
--- zc.buildout/trunk/src/zc/buildout/virtualenv.txt 2010-08-29 20:58:59 UTC (rev 116013)
+++ zc.buildout/branches/gary-1.5.2/src/zc/buildout/virtualenv.txt 2010-09-01 22:58:35 UTC (rev 116116)
@@ -196,6 +196,7 @@
... [buildout]
... parts = eggs
... find-links = %(link_server)s
+ ... include-site-packages = false
...
... [primed_python]
... executable = %(py_path)s
@@ -207,7 +208,7 @@
... eggs = demo
... ''' % globals())
- >>> print system(buildout) # doctest: +NORMALIZE_WHITESPACE +ELLIPSIS
+ >>> print system(buildout), # doctest: +NORMALIZE_WHITESPACE +ELLIPSIS
Installing eggs.
Getting distribution for 'demo'.
Got demo 0.4c1.
@@ -226,20 +227,17 @@
to the buildout script. Alternatively, use a Python executable with a
working -S (such as a standard Python binary).
warnings.warn(BROKEN_DASH_S_WARNING)
- <BLANKLINE>
So, it did what we asked as best it could, but gave a big warning. If
you don't want those warnings for those particular recipes that use the
new features, you can use the "-s" option to squelch the warnings.
- >>> print system(buildout + ' -s')
+ >>> print system(buildout + ' -s'),
Updating eggs.
- <BLANKLINE>
A lower verbosity (one or more -q options) also quiets the warning.
- >>> print system(buildout + ' -q')
- <BLANKLINE>
+ >>> print system(buildout + ' -q'),
Notice that, as we saw before with bin/buildout, the generated scripts
are old-style, because the new-style feature gracefully degrades to the
Modified: zc.buildout/branches/gary-1.5.2/z3c.recipe.scripts_/src/z3c/recipe/scripts/README.txt
===================================================================
--- zc.buildout/trunk/z3c.recipe.scripts_/src/z3c/recipe/scripts/README.txt 2010-08-29 20:58:59 UTC (rev 116013)
+++ zc.buildout/branches/gary-1.5.2/z3c.recipe.scripts_/src/z3c/recipe/scripts/README.txt 2010-09-01 22:58:35 UTC (rev 116116)
@@ -37,7 +37,8 @@
You can choose to have the site-packages of the underlying Python
available to your script or interpreter, in addition to the packages
from your eggs. See the section on this option for motivations and
- warnings.
+ warnings. As of zc.buildout 1.5.2, this defaults to true, for increased
+ backwards compatibility with pre 1.5 recipes and buildouts.
allowed-eggs-from-site-packages
Sometimes you need or want to control what eggs from site-packages are
@@ -153,6 +154,7 @@
... """
... [buildout]
... parts = py
+ ... include-site-packages = false
...
... [py]
... recipe = z3c.recipe.scripts:interpreter
@@ -244,18 +246,19 @@
>>> print site_packages_path
/executable_buildout/site-packages
-Now let's take a look at include-site-packages. The default is false,
-so we will set it to true.
+Now let's take a look at include-site-packages.
+The default is value true (as of zc.buildout 1.5.2).
+
>>> write(sample_buildout, 'buildout.cfg',
... """
... [buildout]
... parts = py
... executable = %(py_path)s
+ ... exec-sitecustomize = false
...
... [py]
... recipe = z3c.recipe.scripts:interpreter
- ... include-site-packages = true
... eggs = demo<0.3
... find-links = %(server)s
... index = %(server)s/index
@@ -280,6 +283,38 @@
'/executable_buildout/site-packages']
<BLANKLINE>
+If you set it to false, they are excluded.
+
+ >>> write(sample_buildout, 'buildout.cfg',
+ ... """
+ ... [buildout]
+ ... parts = py
+ ... executable = %(py_path)s
+ ... exec-sitecustomize = false
+ ...
+ ... [py]
+ ... recipe = z3c.recipe.scripts:interpreter
+ ... include-site-packages = false
+ ... eggs = demo<0.3
+ ... find-links = %(server)s
+ ... index = %(server)s/index
+ ... """ % dict(server=link_server, py_path=py_path))
+
+ >>> print system(buildout),
+ Uninstalling py.
+ Installing py.
+ Generated interpreter '/sample-buildout/bin/py'.
+
+ >>> print system(join(sample_buildout, 'bin', 'py') +
+ ... ''' -c "import sys, pprint; pprint.pprint(sys.path)"''')
+ ... # doctest: +ELLIPSIS
+ ['',
+ '/sample-buildout/parts/py',
+ ...,
+ '/sample-buildout/eggs/demo-0.2-pyN.N.egg',
+ '/sample-buildout/eggs/demoneeded-1.2c1-pyN.N.egg']
+ <BLANKLINE>
+
As described above, the allowed-eggs-from-site-packages option lets us
control what site-packages eggs zc.buildout will allow to fulfill
dependencies. The behavior was described above with an example (and the
@@ -320,7 +355,6 @@
...
... [eggs]
... recipe = z3c.recipe.scripts
- ... include-site-packages = true
... python = primed_python
... eggs = demoneeded
... ''' % globals())
@@ -349,7 +383,6 @@
...
... [eggs]
... recipe = z3c.recipe.scripts
- ... include-site-packages = true
... python = primed_python
... allowed-eggs-from-site-packages =
... eggs = demoneeded
@@ -382,7 +415,6 @@
... [buildout]
... parts = eggs
... eggs-directory = tmpeggs
- ... include-site-packages = true
... find-links =
...
... [primed_python]
@@ -406,7 +438,6 @@
... [buildout]
... parts = eggs
... eggs-directory = tmpeggs
- ... include-site-packages = true
... allowed-eggs-from-site-packages =
... find-links =
...
@@ -522,6 +553,7 @@
... [demo]
... recipe = z3c.recipe.scripts
... eggs = demo<0.3
+ ... include-site-packages = false
... find-links = %(server)s
... index = %(server)s/index
... initialization =
Modified: zc.buildout/branches/gary-1.5.2/z3c.recipe.scripts_/src/z3c/recipe/scripts/tests.py
===================================================================
--- zc.buildout/trunk/z3c.recipe.scripts_/src/z3c/recipe/scripts/tests.py 2010-08-29 20:58:59 UTC (rev 116013)
+++ zc.buildout/branches/gary-1.5.2/z3c.recipe.scripts_/src/z3c/recipe/scripts/tests.py 2010-09-01 22:58:35 UTC (rev 116116)
@@ -209,6 +209,7 @@
... recipe = z3c.recipe.scripts:interpreter
... find-links = %(server)s
... index = %(server)s/index
+ ... include-site-packages = false
... relative-paths = true
... extra-paths =
... /foo/bar
@@ -245,11 +246,15 @@
it did not, it might fail to exclude site-packages because one of the
dependencies actually was supposed to be fulfilled with it.
-The default is ``include-site-packages = false``. This makes it possible to
-easily use a system Python. As a demonstration, we will start with a
-Python executable that has the "demoneeded" and "demo" eggs installed.
-The eggs are not found.
+The default is ``include-site-packages = true``. This is backwards
+compatible with previous versions of zc.buildout.
+As a demonstration, we will start with a Python executable that has the
+"demoneeded" and "demo" eggs installed. With the value of
+include-site-packages to true in the default, the package will be found.
+Notice we do not set find-links, but the eggs are still found because
+they are in the executable's path.
+
>>> from zc.buildout.tests import create_sample_sys_install
>>> py_path, site_packages_path = make_py()
>>> create_sample_sys_install(site_packages_path)
@@ -265,21 +270,16 @@
... [eggs]
... recipe = z3c.recipe.scripts
... python = primed_python
+ ... include-site-packages = true
... eggs = demoneeded
... ''' % globals())
- >>> print system(buildout)
+
+ >>> print system(buildout),
Installing eggs.
- Couldn't find index page for 'demoneeded' (maybe misspelled?)
- Getting distribution for 'demoneeded'.
- While:
- Installing eggs.
- Getting distribution for 'demoneeded'.
- Error: Couldn't find a distribution for 'demoneeded'.
- <BLANKLINE>
-However, if we set include-site-packages to true, the package will be found.
-Notice we do not set find-links, but the eggs are still found because
-they are in the executable's path.
+You can set the value false explicitly. This makes it possible to
+get a more repeatable build from a system Python. In our example, the
+eggs are not found, even though the system Python provides them.
>>> write('buildout.cfg',
... '''
@@ -292,13 +292,19 @@
...
... [eggs]
... recipe = z3c.recipe.scripts
+ ... include-site-packages = false
... python = primed_python
- ... include-site-packages = true
... eggs = demoneeded
... ''' % globals())
-
>>> print system(buildout)
+ Uninstalling eggs.
Installing eggs.
+ Couldn't find index page for 'demoneeded' (maybe misspelled?)
+ Getting distribution for 'demoneeded'.
+ While:
+ Installing eggs.
+ Getting distribution for 'demoneeded'.
+ Error: Couldn't find a distribution for 'demoneeded'.
<BLANKLINE>
We get an error if we specify anything but true or false:
More information about the checkins
mailing list