[Checkins] SVN: zc.recipe.cmmi/trunk/ merged the tlotze-download-api branch
Thomas Lotze
tl at gocept.com
Tue Jul 21 07:38:44 EDT 2009
Log message for revision 102052:
merged the tlotze-download-api branch
Changed:
U zc.recipe.cmmi/trunk/CHANGES.txt
U zc.recipe.cmmi/trunk/setup.py
U zc.recipe.cmmi/trunk/src/zc/recipe/cmmi/README.txt
U zc.recipe.cmmi/trunk/src/zc/recipe/cmmi/__init__.py
U zc.recipe.cmmi/trunk/src/zc/recipe/cmmi/downloadcache.txt
U zc.recipe.cmmi/trunk/src/zc/recipe/cmmi/patching.txt
U zc.recipe.cmmi/trunk/src/zc/recipe/cmmi/shared.txt
U zc.recipe.cmmi/trunk/src/zc/recipe/cmmi/tests.py
-=-
Modified: zc.recipe.cmmi/trunk/CHANGES.txt
===================================================================
--- zc.recipe.cmmi/trunk/CHANGES.txt 2009-07-21 11:37:27 UTC (rev 102051)
+++ zc.recipe.cmmi/trunk/CHANGES.txt 2009-07-21 11:38:43 UTC (rev 102052)
@@ -1,10 +1,11 @@
Release History
***************
-1.2.1 (unreleased)
+1.3.0 (unreleased)
==================
-- Nothing changed yet.
+- Use zc.buildout's download API. As this allows MD5 checking, added the
+ md5sum and patch-md5sum options.
1.2.0 (2009-05-18)
Modified: zc.recipe.cmmi/trunk/setup.py
===================================================================
--- zc.recipe.cmmi/trunk/setup.py 2009-07-21 11:37:27 UTC (rev 102051)
+++ zc.recipe.cmmi/trunk/setup.py 2009-07-21 11:38:43 UTC (rev 102052)
@@ -1,6 +1,6 @@
##############################################################################
#
-# Copyright (c) 2006 Zope Corporation and Contributors.
+# Copyright (c) 2006-2009 Zope Corporation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
@@ -21,14 +21,21 @@
name = "zc.recipe.cmmi"
setup(
name = name,
- version = '1.2.1 dev',
+ version = '1.3dev',
author = "Jim Fulton",
author_email = "jim at zope.com",
description = "ZC Buildout recipe for configure/make/make install",
license = "ZPL 2.1",
- keywords = "zope3",
- classifiers = ["Framework :: Buildout"],
- url='http://www.python.org/pypi/'+name,
+ keywords = "zc.buildout buildout recipe cmmi configure make install",
+ classifiers = [
+ "Environment :: Plugins",
+ "Framework :: Buildout",
+ "Intended Audience :: Developers",
+ "License :: OSI Approved :: Zope Public License",
+ "Topic :: Software Development :: Build Tools",
+ "Topic :: System :: Software Distribution",
+ ],
+ url='http://pypi.python.org/pypi/'+name,
long_description=(
read('README.txt')
+ '\n' +
Modified: zc.recipe.cmmi/trunk/src/zc/recipe/cmmi/README.txt
===================================================================
--- zc.recipe.cmmi/trunk/src/zc/recipe/cmmi/README.txt 2009-07-21 11:37:27 UTC (rev 102051)
+++ zc.recipe.cmmi/trunk/src/zc/recipe/cmmi/README.txt 2009-07-21 11:38:43 UTC (rev 102052)
@@ -1,9 +1,12 @@
-We have an archive with a demo foo tar ball:
+We have an archive with a demo foo tar ball and publish it by http in order
+to see offline effects:
>>> ls(distros)
- bar.tgz
- foo.tgz
+ >>> distros_url = start_server(distros)
+
Let's update a sample buildout to installs it:
>>> write('buildout.cfg',
@@ -13,8 +16,8 @@
...
... [foo]
... recipe = zc.recipe.cmmi
- ... url = file://%s/foo.tgz
- ... """ % distros)
+ ... url = %sfoo.tgz
+ ... """ % distros_url)
We used the url option to specify the location of the archive.
@@ -23,7 +26,7 @@
>>> print system('bin/buildout'),
Installing foo.
- foo: Downloading .../distros/foo.tgz
+ foo: Downloading http://localhost/foo.tgz
foo: Unpacking and configuring
configuring foo --prefix=/sample-buildout/parts/foo
echo building foo
@@ -51,14 +54,14 @@
...
... [foo]
... recipe = zc.recipe.cmmi
- ... url = file://%s/foo.tgz
+ ... url = %sfoo.tgz
... extra_options = -a -b c
- ... """ % distros)
+ ... """ % distros_url)
>>> print system('bin/buildout'),
Uninstalling foo.
Installing foo.
- foo: Downloading .../distros/foo.tgz
+ foo: Downloading http://localhost/foo.tgz
foo: Unpacking and configuring
configuring foo --prefix=/sample-buildout/parts/foo -a -b c
echo building foo
@@ -92,16 +95,16 @@
...
... [foo]
... recipe = zc.recipe.cmmi
- ... url = file://%s/foo.tgz
+ ... url = %sfoo.tgz
... environment =
... CFLAGS=-I/usr/lib/postgresql7.4/include
- ... """ % distros)
+ ... """ % distros_url)
>>> print system('bin/buildout'),
Uninstalling foo.
Installing foo.
- foo: Downloading .../distros/foo.tgz
+ foo: Downloading http://localhost/foo.tgz
foo: Unpacking and configuring
foo: Updating environment: CFLAGS=-I/usr/lib/postgresql7.4/include
configuring foo --prefix=/sample_buildout/parts/foo
@@ -150,15 +153,15 @@
...
... [foo]
... recipe = zc.recipe.cmmi
- ... url = file://%s/foo.tgz
+ ... url = %sfoo.tgz
... patch = ${buildout:directory}/patches/config.patch
... patch_options = -p0
- ... """ % distros)
+ ... """ % distros_url)
>>> print system('bin/buildout'),
Uninstalling foo.
Installing foo.
- foo: Downloading .../distros/foo.tgz
+ foo: Downloading http://localhost/foo.tgz
foo: Unpacking and configuring
patching file configure
configuring foo patched --prefix=/sample_buildout/parts/foo
@@ -176,14 +179,14 @@
...
... [foo]
... recipe = zc.recipe.cmmi
- ... url = file://%s/bar.tgz
+ ... url = %s/bar.tgz
... autogen = autogen.sh
- ... """ % distros)
+ ... """ % distros_url)
>>> print system('bin/buildout'),
Uninstalling foo.
Installing foo.
- foo: Downloading .../distros/bar.tgz
+ foo: Downloading http://localhost//bar.tgz
foo: Unpacking and configuring
foo: auto generating configure files
configuring foo --prefix=/sample_buildout/parts/foo
@@ -191,3 +194,78 @@
building foo
echo installing foo
installing foo
+
+When downloading a source archive or a patch, we can optionally make sure of
+its authenticity by supplying an MD5 checksum that must be matched. If it
+matches, we'll not be bothered with the check by buildout's output:
+
+ >>> try: from hashlib import md5
+ ... except ImportError: from md5 import new as md5
+ >>> foo_md5sum = md5(open(join(distros, 'foo.tgz')).read()).hexdigest()
+
+ >>> write('buildout.cfg',
+ ... """
+ ... [buildout]
+ ... parts = foo
+ ...
+ ... [foo]
+ ... recipe = zc.recipe.cmmi
+ ... url = %sfoo.tgz
+ ... md5sum = %s
+ ... """ % (distros_url, foo_md5sum))
+
+ >>> print system('bin/buildout'),
+ Uninstalling foo.
+ Installing foo.
+ foo: Downloading http://localhost/foo.tgz
+ foo: Unpacking and configuring
+ configuring foo --prefix=/sample_buildout/parts/foo
+ echo building foo
+ building foo
+ echo installing foo
+ installing foo
+
+But if the archive doesn't match the checksum, the recipe refuses to install:
+
+ >>> write('buildout.cfg',
+ ... """
+ ... [buildout]
+ ... parts = foo
+ ...
+ ... [foo]
+ ... recipe = zc.recipe.cmmi
+ ... url = %sbar.tgz
+ ... md5sum = %s
+ ... patch = ${buildout:directory}/patches/config.patch
+ ... """ % (distros_url, foo_md5sum))
+
+ >>> print system('bin/buildout'),
+ Uninstalling foo.
+ Installing foo.
+ foo: Downloading http://localhost:20617/bar.tgz
+ While:
+ Installing foo.
+ Error: MD5 checksum mismatch downloading 'http://localhost/bar.tgz'
+
+Similarly, a checksum mismatch for the patch will cause the buildout run to be
+aborted:
+
+ >>> write('buildout.cfg',
+ ... """
+ ... [buildout]
+ ... parts = foo
+ ...
+ ... [foo]
+ ... recipe = zc.recipe.cmmi
+ ... url = %sfoo.tgz
+ ... patch = ${buildout:directory}/patches/config.patch
+ ... patch-md5sum = %s
+ ... """ % (distros_url, foo_md5sum))
+
+ >>> print system('bin/buildout'),
+ Installing foo.
+ foo: Downloading http://localhost:21669/foo.tgz
+ foo: Unpacking and configuring
+ While:
+ Installing foo.
+ Error: MD5 checksum mismatch for local resource at '/.../sample-buildout/patches/config.patch'.
Modified: zc.recipe.cmmi/trunk/src/zc/recipe/cmmi/__init__.py
===================================================================
--- zc.recipe.cmmi/trunk/src/zc/recipe/cmmi/__init__.py 2009-07-21 11:37:27 UTC (rev 102051)
+++ zc.recipe.cmmi/trunk/src/zc/recipe/cmmi/__init__.py 2009-07-21 11:38:43 UTC (rev 102052)
@@ -1,6 +1,6 @@
##############################################################################
#
-# Copyright (c) 2006 Zope Corporation and Contributors.
+# Copyright (c) 2006-2009 Zope Corporation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
@@ -12,38 +12,33 @@
#
##############################################################################
-import logging, os, shutil, tempfile, urllib2, urlparse
-import setuptools.archive_util
-import datetime
try:
from hashlib import sha1
except ImportError: # Python < 2.5
from sha import new as sha1
+import datetime
+import logging
+import os
+import os.path
+import setuptools.archive_util
import shutil
+import tempfile
import zc.buildout
+import zc.buildout.download
+
def system(c):
if os.system(c):
raise SystemError("Failed", c)
-class Recipe:
+class Recipe(object):
+
def __init__(self, buildout, name, options):
- self.name, self.options = name, options
+ self.buildout, self.name, self.options = buildout, name, options
directory = buildout['buildout']['directory']
- self.download_cache = buildout['buildout'].get('download-cache')
- self.install_from_cache = buildout['buildout'].get('install-from-cache')
+ download_cache = buildout['buildout'].get('download-cache')
- if self.download_cache:
- # cache keys are hashes of url, to ensure repeatability if the
- # downloads do not have a version number in the filename
- # cache key is a directory which contains the downloaded file
- # download details stored with each key as cache.ini
- self.download_cache = os.path.join(
- directory, self.download_cache, 'cmmi')
- if not os.path.isdir(self.download_cache):
- os.mkdir(self.download_cache)
-
location = options.get(
'location', buildout['buildout']['parts-directory'])
options['location'] = os.path.join(location, name)
@@ -72,15 +67,13 @@
# since we remove it in case of build errors
self.shared = os.path.join(self.shared, 'cmmi')
else:
- if not self.download_cache:
+ if not download_cache:
raise ValueError(
"Set the 'shared' option of zc.recipe.cmmi to an existing"
" directory, or set ${buildout:download-cache}")
self.shared = os.path.join(
- directory, self.download_cache, 'build')
- if not os.path.isdir(self.shared):
- os.mkdir(self.shared)
+ directory, download_cache, 'cmmi', 'build')
self.shared = os.path.join(self.shared, self._state_hash())
options['location'] = self.shared
@@ -96,21 +89,23 @@
def install(self):
logger = logging.getLogger(self.name)
+ download = zc.buildout.download.Download(
+ self.buildout['buildout'], namespace='cmmi', hash_name=True,
+ logger=logger)
if self.shared:
if os.path.isdir(self.shared):
logger.info('using existing shared build')
return ()
else:
- os.mkdir(self.shared)
+ os.makedirs(self.shared)
dest = self.options['location']
here = os.getcwd()
if not os.path.exists(dest):
os.mkdir(dest)
- fname = getFromCache(
- self.url, self.name, self.download_cache, self.install_from_cache)
+ fname = download(self.url, md5sum=self.options.get('md5sum'))
# now unpack and work as normal
tmp = tempfile.mkdtemp('buildout-'+self.name)
@@ -132,12 +127,8 @@
if self.patch is not '':
# patch may be a filesystem path or url
# url patches can go through the cache
- if urlparse.urlparse(self.patch, None)[0] is not None:
- self.patch = getFromCache( self.patch
- , self.name
- , self.download_cache
- , self.install_from_cache
- )
+ self.patch = download(
+ self.patch, md5sum=self.options.get('patch-md5sum'))
system("patch %s < %s" % (self.patch_options, self.patch))
if self.autogen is not '':
logger.info('auto generating configure files')
@@ -162,67 +153,3 @@
def update(self):
pass
-
-
-def getFromCache(url, name, download_cache=None, install_from_cache=False):
- if download_cache:
- cache_fname = sha1(url).hexdigest()
- cache_name = os.path.join(download_cache, cache_fname)
-
- _, _, urlpath, _, _ = urlparse.urlsplit(url)
- filename = urlpath.split('/')[-1]
-
- # get the file from the right place
- fname = tmp2 = None
- if download_cache:
- # if we have a cache, try and use it
- logging.getLogger(name).debug(
- 'Searching cache at %s' % download_cache)
- if os.path.isdir(cache_name):
- # just cache files for now
- fname = os.path.join(cache_name, filename)
- logging.getLogger(name).debug(
- 'Using cache file %s' % cache_name)
-
- else:
- logging.getLogger(name).debug(
- 'Did not find %s under cache key %s' % (filename, cache_name))
-
- if not fname:
- if install_from_cache:
- # no file in the cache, but we are staying offline
- raise zc.buildout.UserError(
- "Offline mode: file from %s not found in the cache at %s" %
- (url, download_cache))
- try:
- # okay, we've got to download now
- # XXX: do we need to do something about permissions
- # XXX: in case the cache is shared across users?
- tmp2 = None
- if download_cache:
- # set up the cache and download into it
- os.mkdir(cache_name)
- fname = os.path.join(cache_name, filename)
- if filename != "cache.ini":
- now = datetime.datetime.utcnow()
- cache_ini = open(os.path.join(cache_name, "cache.ini"), "w")
- print >>cache_ini, "[cache]"
- print >>cache_ini, "download_url =", url
- print >>cache_ini, "retrieved =", now.isoformat() + "Z"
- cache_ini.close()
- logging.getLogger(name).debug(
- 'Cache download %s as %s' % (url, cache_name))
- else:
- # use tempfile
- tmp2 = tempfile.mkdtemp('buildout-' + name)
- fname = os.path.join(tmp2, filename)
- logging.getLogger(name).info('Downloading %s' % url)
- open(fname, 'wb').write(urllib2.urlopen(url).read())
- except:
- if tmp2 is not None:
- shutil.rmtree(tmp2)
- if download_cache:
- shutil.rmtree(cache_name)
- raise
-
- return fname
Modified: zc.recipe.cmmi/trunk/src/zc/recipe/cmmi/downloadcache.txt
===================================================================
--- zc.recipe.cmmi/trunk/src/zc/recipe/cmmi/downloadcache.txt 2009-07-21 11:37:27 UTC (rev 102051)
+++ zc.recipe.cmmi/trunk/src/zc/recipe/cmmi/downloadcache.txt 2009-07-21 11:38:43 UTC (rev 102052)
@@ -14,12 +14,15 @@
>>> cache = tmpdir('cache')
-We have an archive with a demo foo tar ball:
+We have an archive with a demo foo tar ball and publish it by http in order
+to see offline effects:
>>> ls(distros)
- bar.tgz
- foo.tgz
+ >>> distros = start_server(distros)
+
Let's update a sample buildout to install it:
>>> write('buildout.cfg',
@@ -31,7 +34,7 @@
...
... [foo]
... recipe = zc.recipe.cmmi
- ... url = file://%s/foo.tgz
+ ... url = %s/foo.tgz
... """ % (cache, distros))
We used the url option to specify the location of the archive.
@@ -43,8 +46,8 @@
...
Installing foo.
foo: Searching cache at /cache/cmmi
- foo: Did not find foo.tgz under cache key /cache/cmmi/...
- foo: Cache download /distros/foo.tgz as /cache/cmmi/...
+ foo: Cache miss; will cache http://localhost//foo.tgz as /cache/cmmi/...
+ foo: Downloading http://localhost//foo.tgz
foo: Unpacking and configuring
configuring foo /sample-buildout/parts/foo
echo building foo
@@ -66,16 +69,9 @@
>>> import os
>>> cache_path = os.path.join(cache, 'cmmi')
- >>> cache_key = os.listdir(cache_path)[0]
+ >>> len(os.listdir(cache_path))
+ 1
-Each directory contains two files, the downloaded file and a record
-describing the download:
-
- >>> cache_entry = os.path.join(cache_path, cache_key)
- >>> ls(cache_entry)
- - cache.ini
- - foo.tgz
-
If we remove the installed parts and then re-run, we'll see that the
files are not downloaded afresh:
@@ -114,8 +110,7 @@
...
Installing foo.
foo: Searching cache at /cache/cmmi
- foo: Did not find foo.tgz under cache key /cache/cmmi/...
- foo: Cache download /distros/foo.tgz as /cache/cmmi/...
+ foo: Cache miss; will cache http://localhost//foo.tgz as /cache/cmmi/...
foo: Unpacking and configuring
configuring foo /sample-buildout/parts/foo
echo building foo
@@ -140,7 +135,7 @@
...
... [foo]
... recipe = zc.recipe.cmmi
- ... url = file://%s/foo.tgz
+ ... url = %s/foo.tgz
... """ % (cache2, distros))
>>> print system('bin/buildout')
@@ -148,8 +143,7 @@
...
Installing foo.
foo: Searching cache at /cache2/cmmi
- foo: Did not find foo.tgz under cache key /cache2/cmmi/...
- foo: Cache download /distros/foo.tgz as /cache2/cmmi/...
+ foo: Cache miss; will cache http://localhost//foo.tgz as /cache2/cmmi/...
foo: Unpacking and configuring
configuring foo /sample-buildout/parts/foo
echo building foo
@@ -161,7 +155,7 @@
The old cache is left in place:
>>> ls(cache_path)
- d ...
+ - ...
Installing solely from a download cache
---------------------------------------
@@ -188,7 +182,7 @@
...
... [foo]
... recipe = zc.recipe.cmmi
- ... url = file://%s/foo.tgz
+ ... url = %s/foo.tgz
... """ % (cache, distros))
>>> for f in os.listdir('parts'):
@@ -224,9 +218,9 @@
Uninstalling foo.
Installing foo.
foo: Searching cache at /cache/cmmi
- foo: Did not find foo.tgz under cache key /cache/cmmi/...
+ foo: Cache miss; will cache http://localhost//foo.tgz as /cache/cmmi/...
While:
Installing foo.
- Error: Offline mode: file from /distros/foo.tgz not found in the cache at /cache/cmmi
+ Error: Couldn't download 'http://localhost//foo.tgz' in offline mode.
<BLANKLINE>
Modified: zc.recipe.cmmi/trunk/src/zc/recipe/cmmi/patching.txt
===================================================================
--- zc.recipe.cmmi/trunk/src/zc/recipe/cmmi/patching.txt 2009-07-21 11:37:27 UTC (rev 102051)
+++ zc.recipe.cmmi/trunk/src/zc/recipe/cmmi/patching.txt 2009-07-21 11:38:43 UTC (rev 102052)
@@ -59,12 +59,12 @@
In...
Installing foo.
foo: Searching cache at /cache/cmmi
- foo: Did not find foo.tgz under cache key /cache/cmmi/...
- foo: Cache download /distros/foo.tgz as /cache/cmmi/...
+ foo: Cache miss; will cache /distros/foo.tgz as /cache/cmmi/...
+ foo: Using local resource /distros/foo.tgz
foo: Unpacking and configuring
foo: Searching cache at /cache/cmmi
- foo: Did not find config.patch under cache key /cache/cmmi/...
- foo: Cache download .../config.patch as /cache/cmmi/...
+ foo: Cache miss; will cache http://localhost//config.patch as /cache/cmmi/...
+ foo: Downloading http://localhost//config.patch
patching file configure
configuring foo patched /sample-buildout/parts/foo
echo building foo patched
@@ -81,6 +81,6 @@
>>> import os
>>> cache_path = os.path.join(cache, 'cmmi')
>>> ls(cache_path)
- d ...
- d ...
+ - ...
+ - ...
Modified: zc.recipe.cmmi/trunk/src/zc/recipe/cmmi/shared.txt
===================================================================
--- zc.recipe.cmmi/trunk/src/zc/recipe/cmmi/shared.txt 2009-07-21 11:37:27 UTC (rev 102051)
+++ zc.recipe.cmmi/trunk/src/zc/recipe/cmmi/shared.txt 2009-07-21 11:38:43 UTC (rev 102052)
@@ -109,7 +109,6 @@
>>> remove('.installed.cfg')
>>> print system('bin/buildout')
Installing foo.
- foo: Downloading /distros/foo.tgz
foo: Unpacking and configuring
configuring foo /cache/existing/cmmi
echo building foo
Modified: zc.recipe.cmmi/trunk/src/zc/recipe/cmmi/tests.py
===================================================================
--- zc.recipe.cmmi/trunk/src/zc/recipe/cmmi/tests.py 2009-07-21 11:37:27 UTC (rev 102051)
+++ zc.recipe.cmmi/trunk/src/zc/recipe/cmmi/tests.py 2009-07-21 11:38:43 UTC (rev 102052)
@@ -84,6 +84,8 @@
'--prefix=/sample_buildout'),
(re.compile(' = \S+sample-buildout'),
' = /sample_buildout'),
+ (re.compile('http://localhost:[0-9]{4,5}/'),
+ 'http://localhost/'),
]),
optionflags = doctest.ELLIPSIS
),
@@ -101,6 +103,8 @@
zc.buildout.testing.normalize_script,
zc.buildout.testing.normalize_egg_py,
normalize_bang,
+ (re.compile('http://localhost:[0-9]{4,5}/'),
+ 'http://localhost/'),
(re.compile('extdemo[.]pyd'), 'extdemo.so')
]),
optionflags = doctest.ELLIPSIS|doctest.NORMALIZE_WHITESPACE
More information about the Checkins
mailing list