[Checkins] SVN: zc.buildout/branches/help-api/ merged the allow-hosts option
Godefroid Chapelle
gotcha at bubblenet.be
Sun Mar 29 17:19:17 EDT 2009
Log message for revision 98487:
merged the allow-hosts option
Changed:
U zc.buildout/branches/help-api/CHANGES.txt
A zc.buildout/branches/help-api/src/zc/buildout/allowhosts.txt
U zc.buildout/branches/help-api/src/zc/buildout/buildout.py
U zc.buildout/branches/help-api/src/zc/buildout/buildout.txt
U zc.buildout/branches/help-api/src/zc/buildout/easy_install.py
U zc.buildout/branches/help-api/src/zc/buildout/tests.py
U zc.buildout/branches/help-api/zc.recipe.egg_/src/zc/recipe/egg/egg.py
-=-
Modified: zc.buildout/branches/help-api/CHANGES.txt
===================================================================
--- zc.buildout/branches/help-api/CHANGES.txt 2009-03-29 21:19:09 UTC (rev 98486)
+++ zc.buildout/branches/help-api/CHANGES.txt 2009-03-29 21:19:16 UTC (rev 98487)
@@ -7,6 +7,8 @@
1.0.4 (unreleased)
==================
+- Added the `allow-hosts` option (tarek)
+
- Quote the 'executable' argument when trying to detect the python
version using popen4. (sidnei)
Added: zc.buildout/branches/help-api/src/zc/buildout/allowhosts.txt
===================================================================
--- zc.buildout/branches/help-api/src/zc/buildout/allowhosts.txt (rev 0)
+++ zc.buildout/branches/help-api/src/zc/buildout/allowhosts.txt 2009-03-29 21:19:16 UTC (rev 98487)
@@ -0,0 +1,109 @@
+Allow hosts
+-----------
+
+On some environments the links visited by `zc.buildout` can be forbidden
+by paranoiac firewalls. These URL might be on the chain of links
+visited by `zc.buildout` wheter they are defined in the `find-links` option,
+wheter they are defined by various eggs in their `url`, `download_url`,
+`dependency_links` metadata.
+
+It is even harder to track that package_index works like a spider and
+might visit links and go to other location.
+
+The `allow-hosts` option provides a way to prevent this, and
+works exactly like the one provided in `easy_install`
+(see `easy_install allow-hosts option`_).
+
+You can provide a list of allowed host, together with wildcards::
+
+ [buildout]
+ ...
+
+ allow-hosts =
+ *.python.org
+ example.com
+
+Let's create a develop egg in our buildout that specifies
+`dependency_links` which points to a server in the outside world::
+
+ >>> mkdir(sample_buildout, 'allowdemo')
+ >>> write(sample_buildout, 'allowdemo', 'dependencydemo.py',
+ ... 'import eggrecipekss.core')
+ >>> write(sample_buildout, 'allowdemo', 'setup.py',
+ ... '''from setuptools import setup; setup(
+ ... name='allowdemo', py_modules=['dependencydemo'],
+ ... install_requires = 'kss.core',
+ ... dependency_links = ['http://dist.plone.org'],
+ ... zip_safe=True, version='1')
+ ... ''')
+
+Now let's configure the buildout to use the develop egg,
+together with some rules that disallow any website but PyPI and
+local files::
+
+ >>> write(sample_buildout, 'buildout.cfg',
+ ... '''
+ ... [buildout]
+ ... develop = allowdemo
+ ... parts = eggs
+ ... allow-hosts =
+ ... pypi.python.org
+ ...
+ ... [eggs]
+ ... recipe = zc.recipe.egg:eggs
+ ... eggs = allowdemo
+ ... ''')
+
+Now we can run the buildout and make sure all attempts to dist.plone.org fails::
+
+ >>> print system(buildout)
+ Develop: '/sample-buildout/allowdemo'
+ Installing eggs.
+ <BLANKLINE>
+ Link to http://dist.plone.org ***BLOCKED*** by --allow-hosts
+ <BLANKLINE>
+ Couldn't find index page for 'kss.core' (maybe misspelled?)
+ Getting distribution for 'kss.core'.
+ While:
+ Installing eggs.
+ Getting distribution for 'kss.core'.
+ Error: Couldn't find a distribution for 'kss.core'.
+ <BLANKLINE>
+
+That's what we wanted : this will prevent any attempt to access
+unwanted domains. For instance, some packages are listing in their
+links `svn://` links. These can lead to error in some cases, and
+can therefore be protected like this::
+
+XXX (showcase with a svn:// file)
+
+ >>> write(sample_buildout, 'buildout.cfg',
+ ... '''
+ ... [buildout]
+ ... develop = allowdemo
+ ... parts = eggs
+ ... allow-hosts =
+ ... ^(!svn://).*
+ ...
+ ... [eggs]
+ ... recipe = zc.recipe.egg:eggs
+ ... eggs = allowdemo
+ ... ''')
+
+Now we can run the buildout and make sure all attempts to dist.plone.org fails::
+
+ >>> print system(buildout)
+ Develop: '/sample-buildout/allowdemo'
+ Installing eggs.
+ <BLANKLINE>
+ Link to http://dist.plone.org ***BLOCKED*** by --allow-hosts
+ <BLANKLINE>
+ Couldn't find index page for 'kss.core' (maybe misspelled?)
+ Getting distribution for 'kss.core'.
+ While:
+ Installing eggs.
+ Getting distribution for 'kss.core'.
+ Error: Couldn't find a distribution for 'kss.core'.
+ <BLANKLINE>
+
+
Modified: zc.buildout/branches/help-api/src/zc/buildout/buildout.py
===================================================================
--- zc.buildout/branches/help-api/src/zc/buildout/buildout.py 2009-03-29 21:19:09 UTC (rev 98486)
+++ zc.buildout/branches/help-api/src/zc/buildout/buildout.py 2009-03-29 21:19:16 UTC (rev 98487)
@@ -134,6 +134,10 @@
links = options.get('find-links', '')
self._links = links and links.split() or ()
+
+ allow_hosts = options.get('allow-hosts', '*').split('\n')
+ self._allow_hosts = tuple([host.strip() for host in allow_hosts
+ if host.strip() != ''])
self._buildout_dir = options['directory']
for name in ('bin', 'parts', 'eggs', 'develop-eggs'):
@@ -664,7 +668,8 @@
self['buildout']['eggs-directory'],
links = self['buildout'].get('find-links', '').split(),
index = self['buildout'].get('index'),
- path = [self['buildout']['develop-eggs-directory']]
+ path = [self['buildout']['develop-eggs-directory']],
+ allow_hosts = self._allow_hosts
)
upgraded = []
@@ -742,7 +747,7 @@
working_set=pkg_resources.working_set,
links = self['buildout'].get('find-links', '').split(),
index = self['buildout'].get('index'),
- newest=self.newest)
+ newest=self.newest, allow_hosts=self._allow_hosts)
# Clear cache because extensions might now let us read pages we
# couldn't read before.
@@ -923,6 +928,7 @@
path=path,
working_set=pkg_resources.working_set,
newest=buildout.newest,
+ allow_hosts=buildout._allow_hosts
)
__doing__ = 'Loading %s recipe entry %s:%s.', group, spec, entry
Modified: zc.buildout/branches/help-api/src/zc/buildout/buildout.txt
===================================================================
--- zc.buildout/branches/help-api/src/zc/buildout/buildout.txt 2009-03-29 21:19:09 UTC (rev 98486)
+++ zc.buildout/branches/help-api/src/zc/buildout/buildout.txt 2009-03-29 21:19:16 UTC (rev 98487)
@@ -2385,8 +2385,32 @@
ext ['buildout']
Develop: '/sample-bootstrapped/demo'
+Allow hosts
+-----------
+On some environments the links visited by `zc.buildout` can be forbidden
+by paranoiac firewalls. These URL might be on the chain of links
+visited by `zc.buildout` wheter they are defined in the `find-links` option,
+wheter they are defined by various eggs in their `url`, `download_url`,
+`dependency_links` metadata.
+It is even harder to track that package_index works like a spider and
+might visit links and go to other location.
+
+The `allow-hosts` option provides a way to prevent this, and
+works exactly like the one provided in `easy_install`.
+
+You can provide a list of allowed host, together with wildcards::
+
+ [buildout]
+ ...
+
+ allow-hosts =
+ *.python.org
+ example.com
+
+All urls that does not match these hosts will not be visited.
+
.. [#future_recipe_methods] In the future, additional methods may be
added. Older recipes with fewer methods will still be
supported.
Modified: zc.buildout/branches/help-api/src/zc/buildout/easy_install.py
===================================================================
--- zc.buildout/branches/help-api/src/zc/buildout/easy_install.py 2009-03-29 21:19:09 UTC (rev 98486)
+++ zc.buildout/branches/help-api/src/zc/buildout/easy_install.py 2009-03-29 21:19:16 UTC (rev 98487)
@@ -71,8 +71,21 @@
_versions[executable] = version
return version
+FILE_SCHEME = re.compile('file://', re.I).match
+
+class AllowHostsPackageIndex(setuptools.package_index.PackageIndex):
+ """Will allow urls that are local to the system.
+
+ No matter what is allow_hosts.
+ """
+ def url_ok(self, url, fatal=False):
+ if FILE_SCHEME(url):
+ return True
+ return setuptools.package_index.PackageIndex.url_ok(self, url, False)
+
+
_indexes = {}
-def _get_index(executable, index_url, find_links):
+def _get_index(executable, index_url, find_links, allow_hosts=('*',)):
key = executable, index_url, tuple(find_links)
index = _indexes.get(key)
if index is not None:
@@ -80,9 +93,8 @@
if index_url is None:
index_url = default_index_url
-
- index = setuptools.package_index.PackageIndex(
- index_url, python=_get_version(executable)
+ index = AllowHostsPackageIndex(
+ index_url, hosts=allow_hosts, python=_get_version(executable)
)
if find_links:
@@ -124,8 +136,10 @@
newest=True,
versions=None,
use_dependency_links=None,
+ allow_hosts=('*',)
):
self._dest = dest
+ self._allow_hosts = allow_hosts
if self._install_from_cache:
if not self._download_cache:
@@ -152,7 +166,7 @@
self._newest = newest
self._env = pkg_resources.Environment(path,
python=_get_version(executable))
- self._index = _get_index(executable, index, links)
+ self._index = _get_index(executable, index, links, self._allow_hosts)
if versions is not None:
self._versions = versions
@@ -162,9 +176,9 @@
if not dists:
logger.debug('We have no distributions for %s that satisfies %r.',
req.project_name, str(req))
+
return None, self._obtain(req, source)
-
# Note that dists are sorted from best to worst, as promised by
# env.__getitem__
@@ -345,9 +359,9 @@
shutil.rmtree(tmp)
def _obtain(self, requirement, source=None):
-
# initialize out index for this project:
index = self._index
+
if index.obtain(requirement) is None:
# Nothing is available.
return None
@@ -425,7 +439,7 @@
# Maybe an existing dist is already the best dist that satisfies the
# requirement
dist, avail = self._satisfied(requirement)
-
+
if dist is None:
if self._dest is not None:
logger.info(*__doing__)
@@ -514,7 +528,8 @@
logger.debug('Adding find link %r from %s', link, dist)
self._links.append(link)
self._index = _get_index(self._executable,
- self._index_url, self._links)
+ self._index_url, self._links,
+ self._allow_hosts)
for dist in dists:
# Check whether we picked a version and, if we did, report it:
@@ -729,18 +744,19 @@
links=(), index=None,
executable=sys.executable, always_unzip=False,
path=None, working_set=None, newest=True, versions=None,
- use_dependency_links=None):
+ use_dependency_links=None, allow_hosts=('*',)):
installer = Installer(dest, links, index, executable, always_unzip, path,
- newest, versions, use_dependency_links)
+ newest, versions, use_dependency_links,
+ allow_hosts=allow_hosts)
return installer.install(specs, working_set)
def build(spec, dest, build_ext,
links=(), index=None,
executable=sys.executable,
- path=None, newest=True, versions=None):
+ path=None, newest=True, versions=None, allow_hosts=('*',)):
installer = Installer(dest, links, index, executable, True, path, newest,
- versions)
+ versions, allow_hosts=allow_hosts)
return installer.build(spec, build_ext)
Modified: zc.buildout/branches/help-api/src/zc/buildout/tests.py
===================================================================
--- zc.buildout/branches/help-api/src/zc/buildout/tests.py 2009-03-29 21:19:09 UTC (rev 98486)
+++ zc.buildout/branches/help-api/src/zc/buildout/tests.py 2009-03-29 21:19:16 UTC (rev 98487)
@@ -3067,6 +3067,7 @@
doctest.DocFileSuite(
'easy_install.txt', 'downloadcache.txt', 'dependencylinks.txt',
+ 'allowhosts.txt',
setUp=easy_install_SetUp,
tearDown=zc.buildout.testing.buildoutTearDown,
Modified: zc.buildout/branches/help-api/zc.recipe.egg_/src/zc/recipe/egg/egg.py
===================================================================
--- zc.buildout/branches/help-api/zc.recipe.egg_/src/zc/recipe/egg/egg.py 2009-03-29 21:19:09 UTC (rev 98486)
+++ zc.buildout/branches/help-api/zc.recipe.egg_/src/zc/recipe/egg/egg.py 2009-03-29 21:19:16 UTC (rev 98487)
@@ -39,6 +39,11 @@
options['index'] = index
self.index = index
+ allow_hosts = buildout['buildout'].get('allow-hosts', '*')
+ allow_hosts = tuple([host.strip() for host in allow_hosts.split('\n')
+ if host.strip()!=''])
+ self.allow_hosts = allow_hosts
+
options['eggs-directory'] = buildout['buildout']['eggs-directory']
options['_e'] = options['eggs-directory'] # backward compat.
options['develop-eggs-directory'
@@ -78,6 +83,7 @@
always_unzip=options.get('unzip') == 'true',
path=[options['develop-eggs-directory']],
newest=self.buildout['buildout'].get('newest') == 'true',
+ allow_hosts=self.allow_hosts
)
return orig_distributions, ws
More information about the Checkins
mailing list