[Checkins] SVN: zc.recipe.cmmi/trunk/ merge of download cache work
Miles Waller
miles at jamkit.com
Thu Aug 9 05:47:06 EDT 2007
Log message for revision 78723:
merge of download cache work
Changed:
U zc.recipe.cmmi/trunk/CHANGES.txt
U zc.recipe.cmmi/trunk/setup.py
U zc.recipe.cmmi/trunk/zc/recipe/cmmi/README.txt
U zc.recipe.cmmi/trunk/zc/recipe/cmmi/__init__.py
A zc.recipe.cmmi/trunk/zc/recipe/cmmi/downloadcache.txt
U zc.recipe.cmmi/trunk/zc/recipe/cmmi/tests.py
-=-
Modified: zc.recipe.cmmi/trunk/CHANGES.txt
===================================================================
--- zc.recipe.cmmi/trunk/CHANGES.txt 2007-08-09 03:26:51 UTC (rev 78722)
+++ zc.recipe.cmmi/trunk/CHANGES.txt 2007-08-09 09:47:05 UTC (rev 78723)
@@ -1,6 +1,21 @@
Release History
***************
+1.1.0
+=====
+
+Added support for:
+
+ - download-cache: downloaded files are cached in the 'cmmi' subdirectory of
+ the cache cache keys are hashes of the url that the file was downloaded from
+ cache information recorded in the cache.ini file within each directory
+
+ - offline mode: cmmi will not go online if the package is not in the cache
+
+ - variable location: build files other than in the parts directory if required
+
+ - additional logging/output
+
1.0.2 (2007-06-03)
==================
Modified: zc.recipe.cmmi/trunk/setup.py
===================================================================
--- zc.recipe.cmmi/trunk/setup.py 2007-08-09 03:26:51 UTC (rev 78722)
+++ zc.recipe.cmmi/trunk/setup.py 2007-08-09 09:47:05 UTC (rev 78723)
@@ -21,7 +21,7 @@
name = "zc.recipe.cmmi"
setup(
name = name,
- version = "1.0.2",
+ version = "1.1.0",
author = "Jim Fulton",
author_email = "jim at zope.com",
description = "ZC Buildout recipe for configure/make/make install",
@@ -38,6 +38,11 @@
+ '\n' +
read('zc', 'recipe', 'cmmi', 'README.txt')
+ '\n' +
+ 'Download Cache\n'
+ '**************\n'
+ 'The recipe supports use of a download cache in the same way\n'
+ 'as zc.buildout. See downloadcache.txt for details\n'
+ + '\n' +
'Download\n'
'**********************\n'
),
Modified: zc.recipe.cmmi/trunk/zc/recipe/cmmi/README.txt
===================================================================
--- zc.recipe.cmmi/trunk/zc/recipe/cmmi/README.txt 2007-08-09 03:26:51 UTC (rev 78722)
+++ zc.recipe.cmmi/trunk/zc/recipe/cmmi/README.txt 2007-08-09 09:47:05 UTC (rev 78723)
@@ -22,6 +22,8 @@
>>> print system('bin/buildout'),
Installing foo.
+ foo: Downloading .../distros/foo.tgz
+ foo: Unpacking and configuring
configuring foo --prefix=/sample-buildout/parts/foo
echo building foo
building foo
@@ -55,6 +57,8 @@
>>> print system('bin/buildout'),
Uninstalling foo.
Installing foo.
+ foo: Downloading .../distros/foo.tgz
+ foo: Unpacking and configuring
configuring foo --prefix=/sample-buildout/parts/foo -a -b c
echo building foo
building foo
@@ -125,10 +129,12 @@
>>> print system('bin/buildout'),
Uninstalling foo.
Installing foo.
+ foo: Downloading .../distros/foo.tgz
+ foo: Unpacking and configuring
patching file configure
configuring foo patched --prefix=/sample_buildout/parts/foo
echo building foo patched
building foo patched
echo installing foo patched
installing foo patched
-
\ No newline at end of file
+
Modified: zc.recipe.cmmi/trunk/zc/recipe/cmmi/__init__.py
===================================================================
--- zc.recipe.cmmi/trunk/zc/recipe/cmmi/__init__.py 2007-08-09 03:26:51 UTC (rev 78722)
+++ zc.recipe.cmmi/trunk/zc/recipe/cmmi/__init__.py 2007-08-09 09:47:05 UTC (rev 78723)
@@ -14,6 +14,9 @@
import logging, os, shutil, tempfile, urllib2, urlparse
import setuptools.archive_util
+import datetime
+import sha
+import zc.buildout
def system(c):
if os.system(c):
@@ -23,12 +26,30 @@
def __init__(self, buildout, name, options):
self.name, self.options = name, options
- options['location'] = options['prefix'] = os.path.join(
- buildout['buildout']['parts-directory'],
- name)
+ directory = buildout['buildout']['directory']
+ self.download_cache = buildout['buildout'].get('download-cache')
+ self.install_from_cache = buildout['buildout'].get('install-from-cache')
+ if self.download_cache:
+ # cache keys are hashes of url, to ensure repeatability if the
+ # downloads do not have a version number in the filename
+ # cache key is a directory which contains the downloaded file
+ # download details stored with each key as cache.ini
+ self.download_cache = os.path.join(
+ directory, self.download_cache, 'cmmi')
+
+ # we assume that install_from_cache and download_cache values
+ # are correctly set, and that the download_cache directory has
+ # been created: this is done by the main zc.buildout anyway
+
+ location = options.get(
+ 'location', buildout['buildout']['parts-directory'])
+ options['location'] = os.path.join(location, name)
+ options['prefix'] = options['location']
+
def install(self):
dest = self.options['location']
+ url = self.options['url']
extra_options = self.options.get('extra_options', '')
# get rid of any newlines that may be in the options so they
# do not get passed through to the commandline
@@ -36,45 +57,104 @@
patch = self.options.get('patch', '')
patch_options = self.options.get('patch_options', '-p0')
- url = self.options['url']
- _, _, urlpath, _, _, _ = urlparse.urlparse(url)
+ fname = getFromCache(
+ url, self.name, self.download_cache, self.install_from_cache)
+
+ # now unpack and work as normal
tmp = tempfile.mkdtemp('buildout-'+self.name)
- tmp2 = tempfile.mkdtemp('buildout-'+self.name)
+ logging.getLogger(self.name).info('Unpacking and configuring')
+ setuptools.archive_util.unpack_archive(fname, tmp)
+
+ here = os.getcwd()
+ os.mkdir(dest)
+
try:
- fname = os.path.join(tmp2, urlpath.split('/')[-1])
- open(fname, 'w').write(urllib2.urlopen(url).read())
- setuptools.archive_util.unpack_archive(fname, tmp)
-
- os.mkdir(dest)
- here = os.getcwd()
+ os.chdir(tmp)
try:
- os.chdir(tmp)
- try:
- if not os.path.exists('configure'):
- entries = os.listdir(tmp)
- if len(entries) == 1:
- os.chdir(entries[0])
- else:
- raise ValueError("Couldn't find configure")
- if patch is not '':
- system("patch %s < %s" % (patch_options, patch))
- system("./configure --prefix=%s %s" %
- (dest, extra_options))
- system("make")
- system("make install")
- finally:
- os.chdir(here)
- except:
- os.rmdir(dest)
- raise
+ if not os.path.exists('configure'):
+ entries = os.listdir(tmp)
+ if len(entries) == 1:
+ os.chdir(entries[0])
+ else:
+ raise ValueError("Couldn't find configure")
+ if patch is not '':
+ system("patch %s < %s" % (patch_options, patch))
+ system("./configure --prefix=%s %s" %
+ (dest, extra_options))
+ system("make")
+ system("make install")
+ finally:
+ os.chdir(here)
+ except:
+ os.rmdir(dest)
+ raise
- finally:
- shutil.rmtree(tmp)
- shutil.rmtree(tmp2)
-
return dest
def update(self):
pass
-
+def getFromCache(url, name, download_cache=None, install_from_cache=False):
+ if download_cache:
+ cache_fname = sha.new(url).hexdigest()
+ cache_name = os.path.join(download_cache, cache_fname)
+ if not os.path.isdir(download_cache):
+ os.mkdir(download_cache)
+
+ _, _, urlpath, _, _ = urlparse.urlsplit(url)
+ filename = urlpath.split('/')[-1]
+
+ # get the file from the right place
+ fname = tmp2 = None
+ if download_cache:
+ # if we have a cache, try and use it
+ logging.getLogger(name).debug(
+ 'Searching cache at %s' % download_cache)
+ if os.path.isdir(cache_name):
+ # just cache files for now
+ fname = os.path.join(cache_name, filename)
+ logging.getLogger(name).debug(
+ 'Using cache file %s' % cache_name)
+
+ else:
+ logging.getLogger(name).debug(
+ 'Did not find %s under cache key %s' % (filename, cache_name))
+
+ if not fname:
+ if install_from_cache:
+ # no file in the cache, but we are staying offline
+ raise zc.buildout.UserError(
+ "Offline mode: file from %s not found in the cache at %s" %
+ (url, download_cache))
+ try:
+ # okay, we've got to download now
+ # XXX: do we need to do something about permissions
+ # XXX: in case the cache is shared across users?
+ tmp2 = None
+ if download_cache:
+ # set up the cache and download into it
+ os.mkdir(cache_name)
+ fname = os.path.join(cache_name, filename)
+ if filename != "cache.ini":
+ now = datetime.datetime.utcnow()
+ cache_ini = open(os.path.join(cache_name, "cache.ini"), "w")
+ print >>cache_ini, "[cache]"
+ print >>cache_ini, "download_url =", url
+ print >>cache_ini, "retrieved =", now.isoformat() + "Z"
+ cache_ini.close()
+ logging.getLogger(name).debug(
+ 'Cache download %s as %s' % (url, cache_name))
+ else:
+ # use tempfile
+ tmp2 = tempfile.mkdtemp('buildout-' + name)
+ fname = os.path.join(tmp2, filename)
+ logging.getLogger(name).info('Downloading %s' % url)
+ open(fname, 'w').write(urllib2.urlopen(url).read())
+ except:
+ if tmp2 is not None:
+ shutil.rmtree(tmp2)
+ if download_cache:
+ shutil.rmtree(cache_name)
+ raise
+
+ return fname
Copied: zc.recipe.cmmi/trunk/zc/recipe/cmmi/downloadcache.txt (from rev 78722, zc.recipe.cmmi/branches/miles-download-cache/zc/recipe/cmmi/downloadcache.txt)
===================================================================
--- zc.recipe.cmmi/trunk/zc/recipe/cmmi/downloadcache.txt (rev 0)
+++ zc.recipe.cmmi/trunk/zc/recipe/cmmi/downloadcache.txt 2007-08-09 09:47:05 UTC (rev 78723)
@@ -0,0 +1,231 @@
+Using a download cache
+======================
+
+Normally, when distributions are installed, if any processing is
+needed, they are downloaded from the internet to a temporary directory
+and then installed from there. A download cache can be used to avoid
+the download step. This can be useful to reduce network access and to
+create source distributions of an entire buildout.
+
+The buildout download-cache option can be used to specify a directory
+to be used as a download cache.
+
+In this example, we'll create a directory to hold the cache:
+
+ >>> cache = tmpdir('cache')
+
+We have an archive with a demo foo tar ball:
+
+ >>> ls(distros)
+ - foo.tgz
+
+Let's update a sample buildout to install it:
+
+ >>> write('buildout.cfg',
+ ... """
+ ... [buildout]
+ ... parts = foo
+ ... download-cache = %s
+ ... log-level = DEBUG
+ ...
+ ... [foo]
+ ... recipe = zc.recipe.cmmi
+ ... url = file://%s/foo.tgz
+ ... """ % (cache, distros))
+
+We used the url option to specify the location of the archive.
+
+It creates a make file which is also run:
+
+ >>> print system('bin/buildout')
+ In...
+ ...
+ Installing foo.
+ foo: Searching cache at /cache/cmmi
+ foo: Did not find foo.tgz under cache key /cache/cmmi/...
+ foo: Cache download /distros/foo.tgz as /cache/cmmi/...
+ foo: Unpacking and configuring
+ configuring foo /sample-buildout/parts/foo
+ echo building foo
+ building foo
+ echo installing foo
+ installing foo
+ <BLANKLINE>
+
+We'll also get the download cache populated. The buildout doesn't put
+files in the cache directly. It creates an intermediate directory,
+cmmi:
+
+ >>> ls(cache)
+ d cmmi
+ d dist
+
+The cmmi directory contains the cache keys - these are hashes of the
+download url:
+
+ >>> import os
+ >>> cache_path = os.path.join(cache, 'cmmi')
+ >>> cache_key = os.listdir(cache_path)[0]
+
+Each directory contains two files, the downloaded file and a record
+describing the download:
+
+ >>> cache_entry = os.path.join(cache_path, cache_key)
+ >>> ls(cache_entry)
+ - cache.ini
+ - foo.tgz
+
+If we remove the installed parts and then re-run, we'll see that the
+files are not downloaded afresh:
+
+ >>> import os
+ >>> for f in os.listdir('parts'):
+ ... remove('parts', f)
+
+ >>> print system(buildout)
+ In...
+ ...
+ Uninstalling foo.
+ Installing foo.
+ foo: Searching cache at /cache/cmmi
+ foo: Using cache file /cache/cmmi/...
+ foo: Unpacking and configuring
+ configuring foo /sample-buildout/parts/foo
+ echo building foo
+ building foo
+ echo installing foo
+ installing foo
+ <BLANKLINE>
+
+This is because the ones in the download cache are used.
+
+If a file directory is removed from the cache, and the installed parts
+are also removed, then it is downloaded afresh:
+
+ >>> for f in os.listdir( cache_path ):
+ ... remove (cache_path, f)
+
+ >>> for f in os.listdir('parts'):
+ ... remove('parts', f)
+
+ >>> print system('bin/buildout')
+ In...
+ ...
+ Installing foo.
+ foo: Searching cache at /cache/cmmi
+ foo: Did not find foo.tgz under cache key /cache/cmmi/...
+ foo: Cache download /distros/foo.tgz as /cache/cmmi/...
+ foo: Unpacking and configuring
+ configuring foo /sample-buildout/parts/foo
+ echo building foo
+ building foo
+ echo installing foo
+ installing foo
+ <BLANKLINE>
+
+If the cache location is changed, and the installed parts are removed,
+the new cache is created and repopulated:
+
+ >>> for f in os.listdir('parts'):
+ ... remove('parts', f)
+
+ >>> cache2 = tmpdir('cache2')
+ >>> write('buildout.cfg',
+ ... """
+ ... [buildout]
+ ... parts = foo
+ ... download-cache = %s
+ ... log-level = DEBUG
+ ...
+ ... [foo]
+ ... recipe = zc.recipe.cmmi
+ ... url = file://%s/foo.tgz
+ ... """ % (cache2, distros))
+
+ >>> print system('bin/buildout')
+ In...
+ ...
+ Installing foo.
+ foo: Searching cache at /cache2/cmmi
+ foo: Did not find foo.tgz under cache key /cache2/cmmi/...
+ foo: Cache download /distros/foo.tgz as /cache2/cmmi/...
+ foo: Unpacking and configuring
+ configuring foo /sample-buildout/parts/foo
+ echo building foo
+ building foo
+ echo installing foo
+ installing foo
+ <BLANKLINE>
+
+The old cache is left in place:
+
+ >>> ls(cache_path)
+ d ...
+
+Installing solely from a download cache
+---------------------------------------
+
+A download cache can be used as the basis of application source
+releases. In an application source release, we want to distribute an
+application that can be built without making any network accesses. In
+this case, we distribute a buildout with download cache and tell the
+buildout to install from the download cache only, without making
+network accesses. The buildout install-from-cache option can be used
+to signal that packages should be installed only from the download
+cache.
+
+If the buildout is run in offline mode, once the installed parts have
+been removed, the files from the cache are used:
+
+ >>> write('buildout.cfg',
+ ... """
+ ... [buildout]
+ ... parts = foo
+ ... download-cache = %s
+ ... log-level = DEBUG
+ ... install-from-cache = true
+ ...
+ ... [foo]
+ ... recipe = zc.recipe.cmmi
+ ... url = file://%s/foo.tgz
+ ... """ % (cache, distros))
+
+ >>> for f in os.listdir('parts'):
+ ... remove('parts', f)
+
+ >>> print system(buildout)
+ In...
+ ...
+ Uninstalling foo.
+ Installing foo.
+ foo: Searching cache at /cache/cmmi
+ foo: Using cache file /cache/cmmi/...
+ foo: Unpacking and configuring
+ configuring foo /sample-buildout/parts/foo
+ echo building foo
+ building foo
+ echo installing foo
+ installing foo
+ <BLANKLINE>
+
+However, in offline mode, if we remove the installed parts and clear
+the cache, an error is raised because the file is not in the cache:
+
+ >>> for f in os.listdir( cache_path ):
+ ... remove (cache_path, f)
+
+ >>> for f in os.listdir('parts'):
+ ... remove('parts', f)
+
+ >>> print system(buildout)
+ In...
+ ...
+ Uninstalling foo.
+ Installing foo.
+ foo: Searching cache at /cache/cmmi
+ foo: Did not find foo.tgz under cache key /cache/cmmi/...
+ While:
+ Installing foo.
+ Error: Offline mode: file from /distros/foo.tgz not found in the cache at /cache/cmmi
+ <BLANKLINE>
+
Modified: zc.recipe.cmmi/trunk/zc/recipe/cmmi/tests.py
===================================================================
--- zc.recipe.cmmi/trunk/zc/recipe/cmmi/tests.py 2007-08-09 03:26:51 UTC (rev 78722)
+++ zc.recipe.cmmi/trunk/zc/recipe/cmmi/tests.py 2007-08-09 09:47:05 UTC (rev 78723)
@@ -19,6 +19,8 @@
import doctest
import zope.testing
from zope.testing import renormalizing
+from zc.buildout.tests import easy_install_SetUp
+from zc.buildout.tests import normalize_bang
def setUp(test):
zc.buildout.testing.buildoutSetUp(test)
@@ -65,7 +67,22 @@
'--prefix=/sample_buildout'),
(re.compile(' = \S+sample-buildout'),
' = /sample_buildout'),
- ])
+ ]),
+ optionflags = doctest.ELLIPSIS
),
+ doctest.DocFileSuite(
+ 'downloadcache.txt',
+ setUp=setUp,
+ tearDown=zc.buildout.testing.buildoutTearDown,
+
+ checker=renormalizing.RENormalizing([
+ zc.buildout.testing.normalize_path,
+ zc.buildout.testing.normalize_script,
+ zc.buildout.testing.normalize_egg_py,
+ normalize_bang,
+ (re.compile('extdemo[.]pyd'), 'extdemo.so')
+ ]),
+ optionflags = doctest.ELLIPSIS
+ ),
))
More information about the Checkins
mailing list