[Checkins] SVN: zc.buildout/trunk/ made sure to download extended configuration files only once per buildout run (based on patch by Rafael Monnerat)
Thomas Lotze
tl at gocept.com
Wed Feb 16 10:47:25 EST 2011
Log message for revision 120386:
made sure to download extended configuration files only once per buildout run (based on patch by Rafael Monnerat)
Changed:
U zc.buildout/trunk/CHANGES.txt
U zc.buildout/trunk/src/zc/buildout/buildout.py
U zc.buildout/trunk/src/zc/buildout/extends-cache.txt
-=-
Modified: zc.buildout/trunk/CHANGES.txt
===================================================================
--- zc.buildout/trunk/CHANGES.txt 2011-02-16 15:39:27 UTC (rev 120385)
+++ zc.buildout/trunk/CHANGES.txt 2011-02-16 15:47:25 UTC (rev 120386)
@@ -12,6 +12,9 @@
due to missing support for the ``-E`` option, which only got added
afterwards.
+- Made sure to download extended configuration files only once per buildout
+ run even if they are referenced multiple times (patch by Rafael Monnerat).
+
Bugs fixed:
- In the download module, fixed the handling of directories that are pointed
Modified: zc.buildout/trunk/src/zc/buildout/buildout.py
===================================================================
--- zc.buildout/trunk/src/zc/buildout/buildout.py 2011-02-16 15:39:27 UTC (rev 120385)
+++ zc.buildout/trunk/src/zc/buildout/buildout.py 2011-02-16 15:47:25 UTC (rev 120386)
@@ -193,12 +193,12 @@
'.buildout', 'default.cfg')
if os.path.exists(user_config):
_update(data, _open(os.path.dirname(user_config), user_config,
- [], data['buildout'].copy(), override))
+ [], data['buildout'].copy(), override, set()))
# load configuration files
if config_file:
_update(data, _open(os.path.dirname(config_file), config_file, [],
- data['buildout'].copy(), override))
+ data['buildout'].copy(), override, set()))
# apply command-line options
_update(data, cloptions)
@@ -1387,17 +1387,19 @@
for option, value in items:
_save_option(option, value, f)
-def _open(base, filename, seen, dl_options, override):
+def _open(base, filename, seen, dl_options, override, downloaded):
"""Open a configuration file and return the result as a dictionary,
Recursively open other files based on buildout options found.
"""
_update_section(dl_options, override)
_dl_options = _unannotate_section(dl_options.copy())
+ newest = _convert_bool('newest', _dl_options.get('newest', 'false'))
+ fallback = newest and not (filename in downloaded)
+ download = zc.buildout.download.Download(
+ _dl_options, cache=_dl_options.get('extends-cache'),
+ fallback=fallback, hash_name=True)
is_temp = False
- download = zc.buildout.download.Download(
- _dl_options, cache=_dl_options.get('extends-cache'), fallback=True,
- hash_name=True)
if _isurl(filename):
path, is_temp = download(filename)
fp = open(path)
@@ -1415,6 +1417,7 @@
filename = os.path.join(base, filename)
fp = open(filename)
base = os.path.dirname(filename)
+ downloaded.add(filename)
if filename in seen:
if is_temp:
@@ -1449,9 +1452,11 @@
if extends:
extends = extends.split()
- eresult = _open(base, extends.pop(0), seen, dl_options, override)
+ eresult = _open(base, extends.pop(0), seen, dl_options, override,
+ downloaded)
for fname in extends:
- _update(eresult, _open(base, fname, seen, dl_options, override))
+ _update(eresult, _open(base, fname, seen, dl_options, override,
+ downloaded))
result = _update(eresult, result)
if extended_by:
@@ -1459,9 +1464,9 @@
"The extendedBy option is deprecated. Stop using it."
)
for fname in extended_by.split():
- result = _update(result,
- _open(base, fname, seen, dl_options, override))
-
+ result = _update(
+ result,
+ _open(base, fname, seen, dl_options, override, downloaded))
seen.pop()
return result
Modified: zc.buildout/trunk/src/zc/buildout/extends-cache.txt
===================================================================
--- zc.buildout/trunk/src/zc/buildout/extends-cache.txt 2011-02-16 15:39:27 UTC (rev 120385)
+++ zc.buildout/trunk/src/zc/buildout/extends-cache.txt 2011-02-16 15:47:25 UTC (rev 120386)
@@ -386,7 +386,105 @@
An internal error occurred ...
ValueError: install_from_cache set to true with no download cache
+>>> rmdir('home', '.buildout')
+Newest and non-newest behaviour for extends cache
+-------------------------------------------------
+
+While offline mode forbids network access completely, 'newest' mode determines
+whether to look for updated versions of a resource even if some version of it
+is already present locally. If we run buildout in newest mode
+(``newest = true``), the configuration files are updated with each run:
+
+>>> mkdir("cache")
+>>> write(server_data, 'base.cfg', """\
+... [buildout]
+... parts =
+... """)
+>>> write('buildout.cfg', """\
+... [buildout]
+... extends-cache = cache
+... extends = %sbase.cfg
+... """ % server_url)
+>>> print system(buildout)
+>>> ls('cache')
+- 5aedc98d7e769290a29d654a591a3a45
+>>> cat('cache', os.listdir(cache)[0])
+[buildout]
+parts =
+
+A change to ``base.cfg`` is picked up on the next buildout run:
+
+>>> write(server_data, 'base.cfg', """\
+... [buildout]
+... parts =
+... foo = bar
+... """)
+>>> print system(buildout + " -n")
+Unused options for buildout: 'foo'.
+>>> cat('cache', os.listdir(cache)[0])
+[buildout]
+parts =
+foo = bar
+
+In contrast, when not using ``newest`` mode (``newest = false``), the files
+already present in the extends cache will not be updated:
+
+>>> write(server_data, 'base.cfg', """\
+... [buildout]
+... parts =
+... """)
+>>> print system(buildout + " -N")
+Unused options for buildout: 'foo'.
+>>> cat('cache', os.listdir(cache)[0])
+[buildout]
+parts =
+foo = bar
+
+Even when updating base configuration files with a buildout run, any given
+configuration file will be downloaded only once during that particular run. If
+some base configuration file is extended more than once, its cached copy is
+used:
+
+>>> write(server_data, 'baseA.cfg', """\
+... [buildout]
+... extends = %sbase.cfg
+... foo = bar
+... """ % server_url)
+>>> write(server_data, 'baseB.cfg', """\
+... [buildout]
+... extends-cache = cache
+... extends = %sbase.cfg
+... bar = foo
+... """ % server_url)
+>>> write('buildout.cfg', """\
+... [buildout]
+... extends-cache = cache
+... newest = true
+... extends = %sbaseA.cfg %sbaseB.cfg
+... """ % (server_url, server_url))
+>>> print system(buildout + " -n")
+Unused options for buildout: 'bar' 'foo'.
+
+(XXX We patch download utility's API to produce readable output for the test;
+a better solution would utilise the logging already done by the utility.)
+
+>>> import zc.buildout
+>>> old_download = zc.buildout.download.Download.download
+>>> def wrapper_download(self, url, md5sum=None, path=None):
+... print "The URL %s was downloaded." % url
+... return old_download(url, md5sum, path)
+>>> zc.buildout.download.Download.download = wrapper_download
+
+>>> zc.buildout.buildout.main([])
+The URL http://localhost/baseA.cfg was downloaded.
+The URL http://localhost/base.cfg was downloaded.
+The URL http://localhost/baseB.cfg was downloaded.
+Unused options for buildout: 'bar' 'foo'.
+
+>>> zc.buildout.download.Download.download = old_download
+
+
Clean up
--------
More information about the checkins
mailing list