[Checkins] SVN: zc.buildout/branches/tlotze-download-api/src/zc/buildout/ refactored download API so it can be used in a more readable way
Thomas Lotze
tl at gocept.com
Tue Jun 23 02:52:28 EDT 2009
Log message for revision 101241:
refactored download API so it can be used in a more readable way
Changed:
U zc.buildout/branches/tlotze-download-api/src/zc/buildout/buildout.py
U zc.buildout/branches/tlotze-download-api/src/zc/buildout/download.py
U zc.buildout/branches/tlotze-download-api/src/zc/buildout/download.txt
-=-
Modified: zc.buildout/branches/tlotze-download-api/src/zc/buildout/buildout.py
===================================================================
--- zc.buildout/branches/tlotze-download-api/src/zc/buildout/buildout.py 2009-06-23 00:04:17 UTC (rev 101240)
+++ zc.buildout/branches/tlotze-download-api/src/zc/buildout/buildout.py 2009-06-23 06:52:27 UTC (rev 101241)
@@ -1186,13 +1186,8 @@
Recursively open other files based on buildout options found.
"""
- if offline:
- offline_option = 'true'
- else:
- offline_option = 'false'
download = zc.buildout.download.Download(
- {'download-cache': cache, 'offline': offline_option},
- use_cache=zc.buildout.download.FALLBACK, hash_name=True)
+ cache=cache, offline=offline, fallback=True, hash_name=True)
if _isurl(filename):
fp = open(download(filename))
base = filename[:filename.rfind('/')]
Modified: zc.buildout/branches/tlotze-download-api/src/zc/buildout/download.py
===================================================================
--- zc.buildout/branches/tlotze-download-api/src/zc/buildout/download.py 2009-06-23 00:04:17 UTC (rev 101240)
+++ zc.buildout/branches/tlotze-download-api/src/zc/buildout/download.py 2009-06-23 06:52:27 UTC (rev 101241)
@@ -40,46 +40,40 @@
url_opener = URLOpener()
-FALLBACK = object()
-
-
class Download(object):
"""Configurable download utility.
Handles the download cache and offline mode.
- Download(buildout, use_cache=True, namespace=None, hash_name=False)
+ Download(options=None, cache=None, namespace=None, hash_name=False)
- buildout: mapping of buildout options (the ``buildout`` config section)
- use_cache: whether to use the cache at all
+ options: mapping of buildout options (e.g. a ``buildout`` config section)
+ cache: path to the download cache (excluding namespaces)
namespace: namespace directory to use inside the cache
hash_name: whether to use a hash of the URL as cache file name
logger: an optional logger to receive download-related log messages
"""
- def __init__(self, buildout,
- use_cache=True, namespace=None, hash_name=False,
- logger=None):
- self.buildout = buildout
- self.set_cache(use_cache, namespace)
+ def __init__(self, options={}, cache=-1, namespace=None,
+ offline=-1, fallback=False, hash_name=False, logger=None):
+ self.cache = cache
+ if cache == -1:
+ self.cache = options.get('download-cache')
+ self.namespace = namespace
+ self.offline = offline
+ if offline == -1:
+ self.offline = (options.get('offline') == 'true'
+ or options.get('install-from-cache') == 'true')
+ self.fallback = fallback
self.hash_name = hash_name
self.logger = logger or logging.getLogger('zc.buildout')
- def set_cache(self, use_cache=True, namespace=None):
- """Configure the caching properties.
+ @property
+ def cache_dir(self):
+ if self.cache is not None:
+ return os.path.join(realpath(self.cache), self.namespace or '')
- See __init__.
-
- """
- self.use_cache = use_cache
- self.namespace = namespace
- if use_cache and self.buildout.get('download-cache'):
- self.cache = os.path.join(
- realpath(self.buildout['download-cache']), namespace or '')
- else:
- self.cache = None
-
def __call__(self, url, md5sum=None, path=None):
"""Download a file according to the utility's configuration.
@@ -105,14 +99,15 @@
but will not remove the copy in that case.
"""
- if not os.path.exists(self.cache):
- os.makedirs(self.cache)
+ cache_dir = self.cache_dir
+ if not os.path.exists(cache_dir):
+ os.makedirs(cache_dir)
cache_key = self.filename(url)
- cached_path = os.path.join(self.cache, cache_key)
+ cached_path = os.path.join(cache_dir, cache_key)
- self.logger.debug('Searching cache at %s' % self.cache)
+ self.logger.debug('Searching cache at %s' % cache_dir)
if os.path.isfile(cached_path):
- if self.use_cache is FALLBACK:
+ if self.fallback:
try:
self.download(url, md5sum, cached_path)
except ChecksumError:
@@ -150,8 +145,7 @@
parsed_url.path)
return locate_at(parsed_url.path, path)
- if (self.buildout.get('offline') == 'true'
- or self.buildout.get('install-from-cache') == 'true'):
+ if self.offline:
raise zc.buildout.UserError(
"Couldn't download %r in offline mode." % url)
Modified: zc.buildout/branches/tlotze-download-api/src/zc/buildout/download.txt
===================================================================
--- zc.buildout/branches/tlotze-download-api/src/zc/buildout/download.txt 2009-06-23 00:04:17 UTC (rev 101240)
+++ zc.buildout/branches/tlotze-download-api/src/zc/buildout/download.txt 2009-06-23 06:52:27 UTC (rev 101241)
@@ -17,11 +17,11 @@
-----------------------------------
If no download cache should be used, the download utility is instantiated
-given buildout's options and switching off the cache:
+without any arguments:
>>> from zc.buildout.download import Download
->>> download = Download({}, use_cache=False)
->>> print download.cache
+>>> download = Download()
+>>> print download.cache_dir
None
Downloading a file is achieved by calling the utility with the URL as an
@@ -83,7 +83,7 @@
Trying to download a file in offline mode will result in an error:
->>> download = Download({'offline': 'true'}, use_cache=False)
+>>> download = Download(cache=None, offline=True)
>>> download(server_url+'foo.txt')
Traceback (most recent call last):
UserError: Couldn't download 'http://localhost/foo.txt' in offline mode.
@@ -102,40 +102,14 @@
------------------------------------
In order to make use of the download cache, we need to configure the download
-utility differently. In the simplest case, we don't turn off using the cache
-and provide a ``download-cache`` buildout option:
+utility differently. To do this, we pass a directory path as the ``cache``
+attribute upon instantiation:
>>> cache = tmpdir('download-cache')
->>> download = Download({'download-cache': cache})
->>> print download.cache
+>>> download = Download(cache=cache)
+>>> print download.cache_dir
/download-cache/
-If either the ``use_cache`` parameter is set to False or no download cache is
-specified for the buildout, the utility will not have a cache associated:
-
->>> download = Download({})
->>> print download.cache
-None
-
->>> download = Download({'download-cache': None})
->>> print download.cache
-None
-
->>> download = Download({'download-cache': ''})
->>> print download.cache
-None
-
->>> download = Download({'download-cache': cache}, use_cache=False)
->>> print download.cache
-None
-
-We can turn on the download cache of an existing download utility using the
-``set_cache`` method:
-
->>> download.set_cache(use_cache=True)
->>> print download.cache
-/download-cache/
-
Simple usage
~~~~~~~~~~~~
@@ -209,7 +183,7 @@
In offline mode, downloads from any URL will be successful if the file is
found in the cache:
->>> download = Download({'download-cache': cache, 'offline': 'true'})
+>>> download = Download(cache=cache, offline=True)
>>> cat(download(server_url+'foo.txt'))
This is a foo text.
@@ -220,7 +194,7 @@
>>> ls(cache)
>>> write(server_data, 'foo.txt', 'This is a foo text.')
->>> download = Download({'download-cache': cache})
+>>> download = Download(cache=cache)
>>> cat(download('file://' + join(server_data, 'foo.txt'), path=path))
This is a foo text.
@@ -255,16 +229,10 @@
namespaces to use, so the download utility stored files directly inside the
download cache. Let's use a namespace "test" instead:
->>> download.set_cache(namespace='test')
->>> print download.cache
+>>> download = Download(cache=cache, namespace='test')
+>>> print download.cache_dir
/download-cache/test
-The namespace parameter can also be passed to the utility's constructor:
-
->>> download = Download({'download-cache': cache}, namespace='test')
->>> print download.cache
-/download-cache/test
-
The namespace sub-directory hasn't been created yet:
>>> ls(cache)
@@ -309,10 +277,7 @@
depends on URL parameters. In such cases, an MD5 hash of the complete URL may
be used as the filename in the cache:
->>> download = Download({'download-cache': cache}, hash_name=True)
->>> download.hash_name
-True
-
+>>> download = Download(cache=cache, hash_name=True)
>>> path = download(server_url+'foo.txt')
>>> print path
/download-cache/09f5793fcdc1716727f72d49519c688d
@@ -363,21 +328,16 @@
>>> write(server_data, 'foo.txt', 'This is a foo text.')
-Downloading with the cache being used when offline
---------------------------------------------------
+Using the cache purely as a fall-back
+-------------------------------------
-Another way the download cache can be used is purely as a fall-back option in
-the case that a file cannot be accessed on the net, such as when a server is
+Sometimes it is desirable to try downloading a file from the net if at all
+possible, and use the cache purely as a fall-back option when a server is
down or if we are in offline mode. This mode is only in effect if a download
cache is configured in the first place:
->>> from zc.buildout.download import FALLBACK
->>> download = Download({}, use_cache=FALLBACK)
->>> print download.cache
-None
-
->>> download = Download({'download-cache': cache}, use_cache=FALLBACK)
->>> print download.cache
+>>> download = Download(cache=cache, fallback=True)
+>>> print download.cache_dir
/download-cache/
A downloaded file will be cached:
@@ -392,7 +352,7 @@
If the file cannot be served, the cached copy will be used:
>>> remove(server_data, 'foo.txt')
->>> Download({})(server_url+'foo.txt')
+>>> Download()(server_url+'foo.txt')
Traceback (most recent call last):
IOError: ('http error', 404, 'Not Found',
<httplib.HTTPMessage instance at 0xa35d36c>)
@@ -404,10 +364,9 @@
using the cache:
>>> write(server_data, 'foo.txt', 'The wrong text.')
->>> cat(Download({})(server_url+'foo.txt'))
+>>> cat(Download()(server_url+'foo.txt'))
The wrong text.
->>> offline_download = Download({'download-cache': cache, 'offline': 'true'},
-... use_cache=FALLBACK)
+>>> offline_download = Download(cache=cache, offline=True, fallback=True)
>>> path = offline_download(server_url+'foo.txt')
>>> cat(path)
This is a foo text.
@@ -433,37 +392,65 @@
The wrong text.
-Specifying offline mode and installation from cache
----------------------------------------------------
+Configuring the download utility from buildout options
+------------------------------------------------------
-As usual with zc.buildout, the offline option must assume one of the values
-'true' and 'false'. Turning offline mode on with 'true' has been shown above;
-setting the option to 'false' turns offline mode off explicitly:
+Some aspects of how a download utility is configured are determined by the use
+cache that the calling code covers while others depend on options read from
+the buildout configuration. The latter can be evaluated in a simple and
+uniform way by configuring the download utility with a dictionary of options.
->>> download = Download({'offline': 'false'})
->>> cat(download(server_url+'foo.txt'))
-This is a foo text.
+The location of the download cache is specified by the ``download-cache``
+option:
-Another way to determine offline mode is the ``install-from-cache`` option. It
-accepts the same values as the offline option:
+>>> download = Download({'download-cache': '/var/cache/buildout'},
+... namespace='cmmi')
+>>> print download.cache_dir
+/var/cache/buildout/cmmi
->>> download = Download({'install-from-cache': 'false'})
->>> cat(download(server_url+'foo.txt'))
-This is a foo text.
+Keyword parameters take precedence over the corresponding options:
+>>> download = Download({'download-cache': '/var/cache/buildout'}, cache=None)
+>>> print download.cache_dir
+None
+
+Whether to assume offline mode can be inferred from either the ``offline`` or
+the ``install-from-cache`` option. As usual with zc.buildout, these options
+must assume one of the values 'true' and 'false':
+
+>>> download = Download({'offline': 'true'})
+>>> download.offline
+True
+
+>>> download = Download({'offline': 'false'})
+>>> download.offline
+False
+
>>> download = Download({'install-from-cache': 'true'})
->>> cat(download(server_url+'foo.txt'))
-Traceback (most recent call last):
-UserError: Couldn't download 'http://localhost/foo.txt' in offline mode.
+>>> download.offline
+True
+>>> download = Download({'install-from-cache': 'false'})
+>>> download.offline
+False
+
These two options are combined using logical 'or':
->>> download = Download({'install-from-cache': 'true', 'offline': 'false'})
->>> cat(download(server_url+'foo.txt'))
-Traceback (most recent call last):
-UserError: Couldn't download 'http://localhost/foo.txt' in offline mode.
+>>> download = Download({'offline': 'true', 'install-from-cache': 'false'})
+>>> download.offline
+True
->>> download = Download({'install-from-cache': 'false', 'offline': 'true'})
->>> cat(download(server_url+'foo.txt'))
-Traceback (most recent call last):
-UserError: Couldn't download 'http://localhost/foo.txt' in offline mode.
+>>> download = Download({'offline': 'false', 'install-from-cache': 'true'})
+>>> download.offline
+True
+
+The ``offline`` keyword parameter takes precedence over both the ``offline``
+and ``install-from-cache`` options:
+
+>>> download = Download({'offline': 'true'}, offline=False)
+>>> download.offline
+False
+
+>>> download = Download({'install-from-cache': 'false'}, offline=True)
+>>> download.offline
+True
More information about the Checkins
mailing list