[Checkins] SVN: zc.recipe.cmmi/branches/miles-download-cache/zc/recipe/cmmi/__init__.py Changes to support the following:

[Miles Waller]

Log message for revision 78078:
  Changes to support the following:
  
   - download-cache: downloaded files are cached in the 'cmmi' subdirectory of the cache
     cache keys are hashes of the url that the file was downloaded from
     cache information recorded in the cache.ini file within each directory
  
   - offline mode: cmmi will not go online if the package is not in the cache
  
   - variable location: build files other than in the parts directory if required
  
   - some additional logging/output so you can see what's going on
  
  

Changed:
  U   zc.recipe.cmmi/branches/miles-download-cache/zc/recipe/cmmi/__init__.py

-=-
Modified: zc.recipe.cmmi/branches/miles-download-cache/zc/recipe/cmmi/__init__.py
===================================================================
--- zc.recipe.cmmi/branches/miles-download-cache/zc/recipe/cmmi/__init__.py	2007-07-17 15:44:08 UTC (rev 78077)
+++ zc.recipe.cmmi/branches/miles-download-cache/zc/recipe/cmmi/__init__.py	2007-07-17 15:53:01 UTC (rev 78078)
@@ -14,6 +14,9 @@
 
 import logging, os, shutil, tempfile, urllib2, urlparse
 import setuptools.archive_util
+import datetime
+import sha
+import zc.buildout
 
 def system(c):
     if os.system(c):
@@ -23,12 +26,35 @@
 
     def __init__(self, buildout, name, options):
         self.name, self.options = name, options
-        options['location'] = options['prefix'] = os.path.join(
-            buildout['buildout']['parts-directory'],
-            name)
+        directory = buildout['buildout']['directory']
+        self.download_cache = buildout['buildout'].get('download-cache')
+        self.install_from_cache = buildout['buildout'].get('install-from-cache')
 
+        url = self.options['url']
+        _, _, urlpath, _, _, _ = urlparse.urlparse(url)
+        self.filename = urlpath.split('/')[-1]
+        if self.download_cache:
+            # cache keys are hashes of url, to ensure repeatability if the
+            # downloads do not have a version number in the filename
+            # cache key is a directory which contains the downloaded file
+            # download details stored with each key as cache.ini
+            cache_fname = sha.new(url).hexdigest()
+            self.download_cache = os.path.join(directory, self.download_cache, 'cmmi')
+            self.cache_name = os.path.join(self.download_cache, cache_fname)
+
+        # we assume that install_from_cache and download_cache values
+        # are correctly set, and that the download_cache directory has
+        # been created: this is done by the main zc.buildout anyway
+          
+        if not options.has_key('location'):
+            options['location'] = os.path.join(
+                buildout['buildout']['parts-directory'],
+                name)
+        options['prefix'] = options['location']
+
     def install(self):
         dest = self.options['location']
+        url = self.options['url']
         extra_options = self.options.get('extra_options', '')
         # get rid of any newlines that may be in the options so they
         # do not get passed through to the commandline
@@ -36,13 +62,69 @@
         patch = self.options.get('patch', '')
         patch_options = self.options.get('patch_options', '-p0')
 
-        url = self.options['url']
-        _, _, urlpath, _, _, _ = urlparse.urlparse(url)
-        tmp = tempfile.mkdtemp('buildout-'+self.name)
-        tmp2 = tempfile.mkdtemp('buildout-'+self.name)
+        if self.download_cache:
+            if not os.path.isdir(self.download_cache):
+                os.mkdir(self.download_cache)
+
+        # get the file from the right place
+        fname, tmp2 = None, None
+        if self.download_cache:
+            # if we have a cache, try and use it
+            logging.getLogger(self.name).debug(
+                'Searching cache at %s' % self.download_cache )
+            if os.path.isdir(self.cache_name):
+                # just cache files for now
+                fname = os.path.join(self.cache_name, self.filename)
+                logging.getLogger(self.name).debug(
+                    'Using cache file %s' % self.cache_name )
+
+            else:
+                logging.getLogger(self.name).debug(
+                    'Did not find %s under cache key %s' % (self.filename, self.cache_name) )
+
+        if not fname:
+            if self.install_from_cache:
+                # no file in the cache, but we are staying offline
+                raise zc.buildout.UserError(
+                    "Offline mode: file from %s not found in the cache at %s" % 
+                    (url, self.download_cache) )
+
+            try:
+                # okay, we've got to download now
+                # XXX: do we need to do something about permissions
+                # XXX: in case the cache is shared across users?
+                tmp2 = None
+                if self.download_cache:
+                    # set up the cache and download into it
+                    os.mkdir(self.cache_name)
+                    fname = os.path.join(self.cache_name, self.filename)
+                    if self.filename != "cache.ini":
+                        now = datetime.datetime.utcnow()
+                        cache_ini = open(os.path.join(self.cache_name, "cache.ini"), "w")
+                        print >>cache_ini, "[cache]"
+                        print >>cache_ini, "download_url =", url
+                        print >>cache_ini, "retrieved =", now.isoformat() + "Z"
+                        cache_ini.close()
+                    logging.getLogger(self.name).debug(
+                        'Cache download %s as %s' % (url, self.cache_name) )
+                else:
+                    # use tempfile
+                    tmp2 = tempfile.mkdtemp('buildout-'+self.name)
+                    fname = os.path.join(tmp2, self.filename)
+                    logging.getLogger(self.name).info(
+                        'Downloading %s' % url )
+                open(fname, 'w').write(urllib2.urlopen(url).read())
+            except:
+                if tmp2 is not None:
+                   shutil.rmtree(tmp2)
+                if self.download_cache:
+                   shutil.rmtree(self.cache_name)
+                raise
+
         try:
-            fname = os.path.join(tmp2, urlpath.split('/')[-1])
-            open(fname, 'w').write(urllib2.urlopen(url).read())
+            # now unpack and work as normal
+            tmp = tempfile.mkdtemp('buildout-'+self.name)
+            logging.getLogger(self.name).info( 'Unpacking and configuring %s' % self.filename)
             setuptools.archive_util.unpack_archive(fname, tmp)
             
             os.mkdir(dest)
@@ -70,11 +152,11 @@
 
         finally:
             shutil.rmtree(tmp)
-            shutil.rmtree(tmp2)
+            if tmp2 is not None:
+               shutil.rmtree(tmp2)
 
         return dest
 
     def update(self):
         pass
-
-                
+