[Checkins] SVN: zc.buildout/branches/tlotze-download-api/src/zc/buildout/ - use the download API to download extended configuration using a cache
Thomas Lotze
tl at gocept.com
Wed Jun 3 03:24:29 EDT 2009
Log message for revision 100607:
- use the download API to download extended configuration using a cache
- added basic tests for the new behaviour
Changed:
U zc.buildout/branches/tlotze-download-api/src/zc/buildout/buildout.py
A zc.buildout/branches/tlotze-download-api/src/zc/buildout/extends-cache.txt
U zc.buildout/branches/tlotze-download-api/src/zc/buildout/tests.py
-=-
Modified: zc.buildout/branches/tlotze-download-api/src/zc/buildout/buildout.py
===================================================================
--- zc.buildout/branches/tlotze-download-api/src/zc/buildout/buildout.py 2009-06-03 06:43:14 UTC (rev 100606)
+++ zc.buildout/branches/tlotze-download-api/src/zc/buildout/buildout.py 2009-06-03 07:24:29 UTC (rev 100607)
@@ -1,6 +1,6 @@
##############################################################################
#
-# Copyright (c) 2005 Zope Corporation and Contributors.
+# Copyright (c) 2005-2009 Zope Corporation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
@@ -25,13 +25,13 @@
import cStringIO
import sys
import tempfile
-import urllib2
import ConfigParser
import UserDict
import glob
import pkg_resources
import zc.buildout
+import zc.buildout.download
import zc.buildout.easy_install
from rmtree import rmtree
@@ -111,17 +111,28 @@
else:
base = None
+ for section, option, value in cloptions:
+ if (section, option) == ('buildout', 'offline'):
+ offline = value
+ break
+ else:
+ offline = False
+
# load user defaults, which override defaults
if user_defaults:
user_config = os.path.join(os.path.expanduser('~'),
'.buildout', 'default.cfg')
if os.path.exists(user_config):
_update(data, _open(os.path.dirname(user_config), user_config,
- []))
+ [], None, offline))
+ extends_cache = data['buildout'].get('extends-cache')
+ offline = offline or data['buildout'].get('offline', False)
+
# load configuration files
if config_file:
- _update(data, _open(os.path.dirname(config_file), config_file, []))
+ _update(data, _open(os.path.dirname(config_file), config_file, [],
+ extends_cache, offline))
# apply command-line options
for (section, option, value) in cloptions:
@@ -129,7 +140,6 @@
if options is None:
options = data[section] = {}
options[option] = value
- # The egg dire
self._raw = data
self._data = {}
@@ -267,6 +277,11 @@
for name in _buildout_default_options:
options[name]
+ # Do the same for extends-cache which is not among the defaults but
+ # wasn't recognized as having been used since it was used before
+ # tracking was turned on.
+ options.get('extends-cache')
+
os.chdir(options['directory'])
def _buildout_path(self, name):
@@ -1155,14 +1170,17 @@
for option, value in items:
_save_option(option, value, f)
-def _open(base, filename, seen):
+def _open(base, filename, seen, cache, offline):
"""Open a configuration file and return the result as a dictionary,
Recursively open other files based on buildout options found.
"""
+ download = zc.buildout.download.Download(
+ {'download-cache': cache, 'offline': offline},
+ use_cache=zc.buildout.download.FALLBACK, hash_name=True)
if _isurl(filename):
- fp = urllib2.urlopen(filename)
+ fp = open(download(filename))
base = filename[:filename.rfind('/')]
elif _isurl(base):
if os.path.isabs(filename):
@@ -1170,7 +1188,7 @@
base = os.path.dirname(filename)
else:
filename = base + '/' + filename
- fp = urllib2.urlopen(filename)
+ fp = open(download(filename))
base = filename[:filename.rfind('/')]
else:
filename = os.path.join(base, filename)
@@ -1180,6 +1198,7 @@
if filename in seen:
raise zc.buildout.UserError("Recursive file include", seen, filename)
+ root_config_file = not seen
seen.append(filename)
result = {}
@@ -1195,18 +1214,22 @@
extended_by = options.pop('extended-by', extended_by)
result[section] = options
+ if root_config_file and 'buildout' in result:
+ cache = result['buildout'].get('extends-cache', cache)
+ offline = offline or result['buildout'].get('offline', False)
+
if extends:
extends = extends.split()
extends.reverse()
for fname in extends:
- result = _update(_open(base, fname, seen), result)
+ result = _update(_open(base, fname, seen, cache, offline), result)
if extended_by:
self._logger.warn(
"The extendedBy option is deprecated. Stop using it."
)
for fname in extended_by.split():
- result = _update(result, _open(base, fname, seen))
+ result = _update(result, _open(base, fname, seen, cache, offline))
seen.pop()
return result
Added: zc.buildout/branches/tlotze-download-api/src/zc/buildout/extends-cache.txt
===================================================================
--- zc.buildout/branches/tlotze-download-api/src/zc/buildout/extends-cache.txt (rev 0)
+++ zc.buildout/branches/tlotze-download-api/src/zc/buildout/extends-cache.txt 2009-06-03 07:24:29 UTC (rev 100607)
@@ -0,0 +1,115 @@
+Caching extended configuration
+==============================
+
+As mentioned in the general buildout documentation, configuration files can
+extend each other, including the ability to download configuration being
+extended from a URL. If desired, zc.buildout caches downloaded configuration
+in order to be able to use it when run offline.
+
+As we're going to talk about downloading things, let's start an HTTP server.
+Also, all of the following will take place inside the sample buildout.
+
+>>> server_data = tmpdir('server_data')
+>>> server_url = start_server(server_data)
+>>> cd(sample_buildout)
+
+
+Basic use of the extends cache
+------------------------------
+
+We put some base configuration on a server and reference it from a sample
+buildout:
+
+>>> write(server_data, 'base.cfg', """\
+... [buildout]
+... parts =
+... foo = bar
+... """)
+
+>>> write('buildout.cfg', """\
+... [buildout]
+... extends = %sbase.cfg
+... """ % server_url)
+
+When trying to run this buildout offline, we'll find that we cannot read all
+of the required configuration:
+
+>>> print system(buildout + ' -o')
+While:
+ Initializing.
+Error: Couldn't download 'http://localhost/base.cfg' in offline mode.
+
+Trying the same online, we can:
+
+>>> print system(buildout)
+Unused options for buildout: 'foo'.
+
+As long as we haven't said anything about caching downloaded configuration,
+nothing gets cached. Offline mode will still cause the buildout to fail:
+
+>>> print system(buildout + ' -o')
+While:
+ Initializing.
+Error: Couldn't download 'http://localhost/base.cfg' in offline mode.
+
+Let's now specify a cache for base configuration files. This cache is
+different from the download cache used by recipes for caching distributions
+and other files; one might, however, use a namespace subdirectory of the
+download cache for it. The configuration cache we specify will be created when
+running buildout and the base.cfg file will be put in it (with the file name
+being a hash of the complete URL):
+
+>>> write('buildout.cfg', """\
+... [buildout]
+... extends = %sbase.cfg
+... extends-cache = cache
+... """ % server_url)
+
+>>> print system(buildout)
+Unused options for buildout: 'foo'.
+
+>>> cache = join(sample_buildout, 'cache')
+>>> ls(cache)
+- 5aedc98d7e769290a29d654a591a3a45
+
+>>> import os
+>>> cat(cache, os.listdir(cache)[0])
+[buildout]
+parts =
+foo = bar
+
+We can now run buildout offline as it will read base.cfg from the cache:
+
+>>> print system(buildout + ' -o')
+Unused options for buildout: 'foo'.
+
+The cache is being used purely as a fall-back in case we are offline or don't
+have access to a configuration file to be downloaded. As long as we are
+online, buildout attempts to download a fresh copy of each file even if a
+cached copy of the file exists. To see this, we put different configuration in
+the same place on the server and run buildout in offline mode so it takes
+base.cfg from the cache:
+
+>>> write(server_data, 'base.cfg', """\
+... [buildout]
+... parts =
+... bar = baz
+... """)
+
+>>> print system(buildout + ' -o')
+Unused options for buildout: 'foo'.
+
+In online mode, buildout will download and use the modified version:
+
+>>> print system(buildout)
+Unused options for buildout: 'bar'.
+
+Trying offline mode again, the new version will be used as it has been put in
+the cache now:
+
+>>> print system(buildout + ' -o')
+Unused options for buildout: 'bar'.
+
+
+Specifying extends cache and offline mode
+-----------------------------------------
Property changes on: zc.buildout/branches/tlotze-download-api/src/zc/buildout/extends-cache.txt
___________________________________________________________________
Added: svn:keywords
+ Id Rev Date
Added: svn:eol-style
+ native
Modified: zc.buildout/branches/tlotze-download-api/src/zc/buildout/tests.py
===================================================================
--- zc.buildout/branches/tlotze-download-api/src/zc/buildout/tests.py 2009-06-03 06:43:14 UTC (rev 100606)
+++ zc.buildout/branches/tlotze-download-api/src/zc/buildout/tests.py 2009-06-03 07:24:29 UTC (rev 100607)
@@ -2807,12 +2807,11 @@
]),
),
-
doctest.DocFileSuite(
- 'download.txt',
+ 'download.txt', 'extends-cache.txt',
setUp=easy_install_SetUp,
tearDown=zc.buildout.testing.buildoutTearDown,
- optionflags=doctest.NORMALIZE_WHITESPACE,
+ optionflags=doctest.NORMALIZE_WHITESPACE | doctest.ELLIPSIS,
checker=renormalizing.RENormalizing([
(re.compile('0x[0-9a-f]+'), '<MEM ADDRESS>'),
(re.compile('http://localhost:[0-9]{4,5}/'),
More information about the Checkins
mailing list