[Checkins] SVN: zc.buildout/branches/tlotze-download-api/src/zc/buildout/ refactored download API so it can be used in a more readable way

Thomas Lotze tl at gocept.com
Tue Jun 23 02:52:28 EDT 2009


Log message for revision 101241:
  refactored download API so it can be used in a more readable way

Changed:
  U   zc.buildout/branches/tlotze-download-api/src/zc/buildout/buildout.py
  U   zc.buildout/branches/tlotze-download-api/src/zc/buildout/download.py
  U   zc.buildout/branches/tlotze-download-api/src/zc/buildout/download.txt

-=-
Modified: zc.buildout/branches/tlotze-download-api/src/zc/buildout/buildout.py
===================================================================
--- zc.buildout/branches/tlotze-download-api/src/zc/buildout/buildout.py	2009-06-23 00:04:17 UTC (rev 101240)
+++ zc.buildout/branches/tlotze-download-api/src/zc/buildout/buildout.py	2009-06-23 06:52:27 UTC (rev 101241)
@@ -1186,13 +1186,8 @@
 
     Recursively open other files based on buildout options found.
     """
-    if offline:
-        offline_option = 'true'
-    else:
-        offline_option = 'false'
     download = zc.buildout.download.Download(
-        {'download-cache': cache, 'offline': offline_option},
-        use_cache=zc.buildout.download.FALLBACK, hash_name=True)
+        cache=cache, offline=offline, fallback=True, hash_name=True)
     if _isurl(filename):
         fp = open(download(filename))
         base = filename[:filename.rfind('/')]

Modified: zc.buildout/branches/tlotze-download-api/src/zc/buildout/download.py
===================================================================
--- zc.buildout/branches/tlotze-download-api/src/zc/buildout/download.py	2009-06-23 00:04:17 UTC (rev 101240)
+++ zc.buildout/branches/tlotze-download-api/src/zc/buildout/download.py	2009-06-23 06:52:27 UTC (rev 101241)
@@ -40,46 +40,40 @@
 url_opener = URLOpener()
 
 
-FALLBACK = object()
-
-
 class Download(object):
     """Configurable download utility.
 
     Handles the download cache and offline mode.
 
-    Download(buildout, use_cache=True, namespace=None, hash_name=False)
+    Download(options=None, cache=None, namespace=None, hash_name=False)
 
-    buildout: mapping of buildout options (the ``buildout`` config section)
-    use_cache: whether to use the cache at all
+    options: mapping of buildout options (e.g. a ``buildout`` config section)
+    cache: path to the download cache (excluding namespaces)
     namespace: namespace directory to use inside the cache
     hash_name: whether to use a hash of the URL as cache file name
     logger: an optional logger to receive download-related log messages
 
     """
 
-    def __init__(self, buildout,
-                 use_cache=True, namespace=None, hash_name=False,
-                 logger=None):
-        self.buildout = buildout
-        self.set_cache(use_cache, namespace)
+    def __init__(self, options={}, cache=-1, namespace=None,
+                 offline=-1, fallback=False, hash_name=False, logger=None):
+        self.cache = cache
+        if cache == -1:
+            self.cache = options.get('download-cache')
+        self.namespace = namespace
+        self.offline = offline
+        if offline == -1:
+            self.offline = (options.get('offline') == 'true'
+                            or options.get('install-from-cache') == 'true')
+        self.fallback = fallback
         self.hash_name = hash_name
         self.logger = logger or logging.getLogger('zc.buildout')
 
-    def set_cache(self, use_cache=True, namespace=None):
-        """Configure the caching properties.
+    @property
+    def cache_dir(self):
+        if self.cache is not None:
+            return os.path.join(realpath(self.cache), self.namespace or '')
 
-        See __init__.
-
-        """
-        self.use_cache = use_cache
-        self.namespace = namespace
-        if use_cache and self.buildout.get('download-cache'):
-            self.cache = os.path.join(
-                realpath(self.buildout['download-cache']), namespace or '')
-        else:
-            self.cache = None
-
     def __call__(self, url, md5sum=None, path=None):
         """Download a file according to the utility's configuration.
 
@@ -105,14 +99,15 @@
         but will not remove the copy in that case.
 
         """
-        if not os.path.exists(self.cache):
-            os.makedirs(self.cache)
+        cache_dir = self.cache_dir
+        if not os.path.exists(cache_dir):
+            os.makedirs(cache_dir)
         cache_key = self.filename(url)
-        cached_path = os.path.join(self.cache, cache_key)
+        cached_path = os.path.join(cache_dir, cache_key)
 
-        self.logger.debug('Searching cache at %s' % self.cache)
+        self.logger.debug('Searching cache at %s' % cache_dir)
         if os.path.isfile(cached_path):
-            if self.use_cache is FALLBACK:
+            if self.fallback:
                 try:
                     self.download(url, md5sum, cached_path)
                 except ChecksumError:
@@ -150,8 +145,7 @@
                     parsed_url.path)
             return locate_at(parsed_url.path, path)
 
-        if (self.buildout.get('offline') == 'true'
-            or self.buildout.get('install-from-cache') == 'true'):
+        if self.offline:
             raise zc.buildout.UserError(
                 "Couldn't download %r in offline mode." % url)
 

Modified: zc.buildout/branches/tlotze-download-api/src/zc/buildout/download.txt
===================================================================
--- zc.buildout/branches/tlotze-download-api/src/zc/buildout/download.txt	2009-06-23 00:04:17 UTC (rev 101240)
+++ zc.buildout/branches/tlotze-download-api/src/zc/buildout/download.txt	2009-06-23 06:52:27 UTC (rev 101241)
@@ -17,11 +17,11 @@
 -----------------------------------
 
 If no download cache should be used, the download utility is instantiated
-given buildout's options and switching off the cache:
+without any arguments:
 
 >>> from zc.buildout.download import Download
->>> download = Download({}, use_cache=False)
->>> print download.cache
+>>> download = Download()
+>>> print download.cache_dir
 None
 
 Downloading a file is achieved by calling the utility with the URL as an
@@ -83,7 +83,7 @@
 
 Trying to download a file in offline mode will result in an error:
 
->>> download = Download({'offline': 'true'}, use_cache=False)
+>>> download = Download(cache=None, offline=True)
 >>> download(server_url+'foo.txt')
 Traceback (most recent call last):
 UserError: Couldn't download 'http://localhost/foo.txt' in offline mode.
@@ -102,40 +102,14 @@
 ------------------------------------
 
 In order to make use of the download cache, we need to configure the download
-utility differently. In the simplest case, we don't turn off using the cache
-and provide a ``download-cache`` buildout option:
+utility differently. To do this, we pass a directory path as the ``cache``
+attribute upon instantiation:
 
 >>> cache = tmpdir('download-cache')
->>> download = Download({'download-cache': cache})
->>> print download.cache
+>>> download = Download(cache=cache)
+>>> print download.cache_dir
 /download-cache/
 
-If either the ``use_cache`` parameter is set to False or no download cache is
-specified for the buildout, the utility will not have a cache associated:
-
->>> download = Download({})
->>> print download.cache
-None
-
->>> download = Download({'download-cache': None})
->>> print download.cache
-None
-
->>> download = Download({'download-cache': ''})
->>> print download.cache
-None
-
->>> download = Download({'download-cache': cache}, use_cache=False)
->>> print download.cache
-None
-
-We can turn on the download cache of an existing download utility using the
-``set_cache`` method:
-
->>> download.set_cache(use_cache=True)
->>> print download.cache
-/download-cache/
-
 Simple usage
 ~~~~~~~~~~~~
 
@@ -209,7 +183,7 @@
 In offline mode, downloads from any URL will be successful if the file is
 found in the cache:
 
->>> download = Download({'download-cache': cache, 'offline': 'true'})
+>>> download = Download(cache=cache, offline=True)
 >>> cat(download(server_url+'foo.txt'))
 This is a foo text.
 
@@ -220,7 +194,7 @@
 >>> ls(cache)
 
 >>> write(server_data, 'foo.txt', 'This is a foo text.')
->>> download = Download({'download-cache': cache})
+>>> download = Download(cache=cache)
 
 >>> cat(download('file://' + join(server_data, 'foo.txt'), path=path))
 This is a foo text.
@@ -255,16 +229,10 @@
 namespaces to use, so the download utility stored files directly inside the
 download cache. Let's use a namespace "test" instead:
 
->>> download.set_cache(namespace='test')
->>> print download.cache
+>>> download = Download(cache=cache, namespace='test')
+>>> print download.cache_dir
 /download-cache/test
 
-The namespace parameter can also be passed to the utility's constructor:
-
->>> download = Download({'download-cache': cache}, namespace='test')
->>> print download.cache
-/download-cache/test
-
 The namespace sub-directory hasn't been created yet:
 
 >>> ls(cache)
@@ -309,10 +277,7 @@
 depends on URL parameters. In such cases, an MD5 hash of the complete URL may
 be used as the filename in the cache:
 
->>> download = Download({'download-cache': cache}, hash_name=True)
->>> download.hash_name
-True
-
+>>> download = Download(cache=cache, hash_name=True)
 >>> path = download(server_url+'foo.txt')
 >>> print path
 /download-cache/09f5793fcdc1716727f72d49519c688d
@@ -363,21 +328,16 @@
 >>> write(server_data, 'foo.txt', 'This is a foo text.')
 
 
-Downloading with the cache being used when offline
---------------------------------------------------
+Using the cache purely as a fall-back
+-------------------------------------
 
-Another way the download cache can be used is purely as a fall-back option in
-the case that a file cannot be accessed on the net, such as when a server is
+Sometimes it is desirable to try downloading a file from the net if at all
+possible, and use the cache purely as a fall-back option when a server is
 down or if we are in offline mode. This mode is only in effect if a download
 cache is configured in the first place:
 
->>> from zc.buildout.download import FALLBACK
->>> download = Download({}, use_cache=FALLBACK)
->>> print download.cache
-None
-
->>> download = Download({'download-cache': cache}, use_cache=FALLBACK)
->>> print download.cache
+>>> download = Download(cache=cache, fallback=True)
+>>> print download.cache_dir
 /download-cache/
 
 A downloaded file will be cached:
@@ -392,7 +352,7 @@
 If the file cannot be served, the cached copy will be used:
 
 >>> remove(server_data, 'foo.txt')
->>> Download({})(server_url+'foo.txt')
+>>> Download()(server_url+'foo.txt')
 Traceback (most recent call last):
 IOError: ('http error', 404, 'Not Found',
           <httplib.HTTPMessage instance at 0xa35d36c>)
@@ -404,10 +364,9 @@
 using the cache:
 
 >>> write(server_data, 'foo.txt', 'The wrong text.')
->>> cat(Download({})(server_url+'foo.txt'))
+>>> cat(Download()(server_url+'foo.txt'))
 The wrong text.
->>> offline_download = Download({'download-cache': cache, 'offline': 'true'},
-...                             use_cache=FALLBACK)
+>>> offline_download = Download(cache=cache, offline=True, fallback=True)
 >>> path = offline_download(server_url+'foo.txt')
 >>> cat(path)
 This is a foo text.
@@ -433,37 +392,65 @@
 The wrong text.
 
 
-Specifying offline mode and installation from cache
----------------------------------------------------
+Configuring the download utility from buildout options
+------------------------------------------------------
 
-As usual with zc.buildout, the offline option must assume one of the values
-'true' and 'false'. Turning offline mode on with 'true' has been shown above;
-setting the option to 'false' turns offline mode off explicitly:
+Some aspects of how a download utility is configured are determined by the use
+cache that the calling code covers while others depend on options read from
+the buildout configuration. The latter can be evaluated in a simple and
+uniform way by configuring the download utility with a dictionary of options.
 
->>> download = Download({'offline': 'false'})
->>> cat(download(server_url+'foo.txt'))
-This is a foo text.
+The location of the download cache is specified by the ``download-cache``
+option:
 
-Another way to determine offline mode is the ``install-from-cache`` option. It
-accepts the same values as the offline option:
+>>> download = Download({'download-cache': '/var/cache/buildout'},
+...                     namespace='cmmi')
+>>> print download.cache_dir
+/var/cache/buildout/cmmi
 
->>> download = Download({'install-from-cache': 'false'})
->>> cat(download(server_url+'foo.txt'))
-This is a foo text.
+Keyword parameters take precedence over the corresponding options:
 
+>>> download = Download({'download-cache': '/var/cache/buildout'}, cache=None)
+>>> print download.cache_dir
+None
+
+Whether to assume offline mode can be inferred from either the ``offline`` or
+the ``install-from-cache`` option. As usual with zc.buildout, these options
+must assume one of the values 'true' and 'false':
+
+>>> download = Download({'offline': 'true'})
+>>> download.offline
+True
+
+>>> download = Download({'offline': 'false'})
+>>> download.offline
+False
+
 >>> download = Download({'install-from-cache': 'true'})
->>> cat(download(server_url+'foo.txt'))
-Traceback (most recent call last):
-UserError: Couldn't download 'http://localhost/foo.txt' in offline mode.
+>>> download.offline
+True
 
+>>> download = Download({'install-from-cache': 'false'})
+>>> download.offline
+False
+
 These two options are combined using logical 'or':
 
->>> download = Download({'install-from-cache': 'true', 'offline': 'false'})
->>> cat(download(server_url+'foo.txt'))
-Traceback (most recent call last):
-UserError: Couldn't download 'http://localhost/foo.txt' in offline mode.
+>>> download = Download({'offline': 'true', 'install-from-cache': 'false'})
+>>> download.offline
+True
 
->>> download = Download({'install-from-cache': 'false', 'offline': 'true'})
->>> cat(download(server_url+'foo.txt'))
-Traceback (most recent call last):
-UserError: Couldn't download 'http://localhost/foo.txt' in offline mode.
+>>> download = Download({'offline': 'false', 'install-from-cache': 'true'})
+>>> download.offline
+True
+
+The ``offline`` keyword parameter takes precedence over both the ``offline``
+and ``install-from-cache`` options:
+
+>>> download = Download({'offline': 'true'}, offline=False)
+>>> download.offline
+False
+
+>>> download = Download({'install-from-cache': 'false'}, offline=True)
+>>> download.offline
+True



More information about the Checkins mailing list