[Checkins] SVN: zc.recipe.cmmi/trunk/ Implemented using a shared directory for completed builds.
Wolfgang Schnerring
wosc at wosc.de
Tue Mar 24 10:58:56 EDT 2009
Log message for revision 98334:
Implemented using a shared directory for completed builds.
Changed:
U zc.recipe.cmmi/trunk/CHANGES.txt
U zc.recipe.cmmi/trunk/src/zc/recipe/cmmi/__init__.py
A zc.recipe.cmmi/trunk/src/zc/recipe/cmmi/shared.txt
U zc.recipe.cmmi/trunk/src/zc/recipe/cmmi/tests.py
-=-
Modified: zc.recipe.cmmi/trunk/CHANGES.txt
===================================================================
--- zc.recipe.cmmi/trunk/CHANGES.txt 2009-03-24 13:08:07 UTC (rev 98333)
+++ zc.recipe.cmmi/trunk/CHANGES.txt 2009-03-24 14:58:56 UTC (rev 98334)
@@ -1,6 +1,11 @@
Release History
***************
+1.1.7 (unreleased)
+==================
+
+Enabled using a shared directory for completed builds.
+
1.1.6 (2009-03-17)
==================
Modified: zc.recipe.cmmi/trunk/src/zc/recipe/cmmi/__init__.py
===================================================================
--- zc.recipe.cmmi/trunk/src/zc/recipe/cmmi/__init__.py 2009-03-24 13:08:07 UTC (rev 98333)
+++ zc.recipe.cmmi/trunk/src/zc/recipe/cmmi/__init__.py 2009-03-24 14:58:56 UTC (rev 98334)
@@ -41,70 +41,107 @@
# download details stored with each key as cache.ini
self.download_cache = os.path.join(
directory, self.download_cache, 'cmmi')
+ if not os.path.isdir(self.download_cache):
+ os.mkdir(self.download_cache)
- # we assume that install_from_cache and download_cache values
- # are correctly set, and that the download_cache directory has
- # been created: this is done by the main zc.buildout anyway
-
location = options.get(
'location', buildout['buildout']['parts-directory'])
options['location'] = os.path.join(location, name)
- options['prefix'] = options['location']
- def install(self):
- logger = logging.getLogger(self.name)
- dest = self.options['location']
- url = self.options['url']
+ self.url = self.options['url']
extra_options = self.options.get('extra_options', '')
# get rid of any newlines that may be in the options so they
# do not get passed through to the commandline
- extra_options = ' '.join(extra_options.split())
+ self.extra_options = ' '.join(extra_options.split())
- autogen = self.options.get('autogen', '')
+ self.autogen = self.options.get('autogen', '')
- patch = self.options.get('patch', '')
- patch_options = self.options.get('patch_options', '-p0')
+ self.patch = self.options.get('patch', '')
+ self.patch_options = self.options.get('patch_options', '-p0')
+ self.environ = self.options.get('environment', '').split()
+ if self.environ:
+ self.environ = dict([x.split('=', 1) for x in self.environ])
+ else:
+ self.environ = {}
+
+ self.shared = options.get('shared', None)
+ if self.shared:
+ if os.path.isdir(self.shared):
+ # to prevent nasty surprises, don't use the directory directly
+ # since we remove it in case of build errors
+ self.shared = os.path.join(self.shared, 'cmmi')
+ else:
+ if not self.download_cache:
+ raise ValueError(
+ "Set the 'shared' option of zc.recipe.cmmi to an existing"
+ " directory, or set ${buildout:download-cache}")
+
+ self.shared = os.path.join(
+ directory, self.download_cache, 'build')
+ if not os.path.isdir(self.shared):
+ os.mkdir(self.shared)
+ self.shared = os.path.join(self.shared, self._state_hash())
+
+ options['location'] = self.shared
+
+ def _state_hash(self):
+ # hash of our configuration state, so that e.g. different
+ # ./configure options will get a different build directory
+ env = ''.join(['%s%s' % (key, value) for key, value
+ in self.environ.items()])
+ state = [self.url, self.extra_options, self.autogen,
+ self.patch, self.patch_options, env]
+ return sha1(''.join(state)).hexdigest()
+
+ def install(self):
+ logger = logging.getLogger(self.name)
+
+ if self.shared:
+ if os.path.isdir(self.shared):
+ logger.info('using existing shared build')
+ return ()
+ else:
+ os.mkdir(self.shared)
+
+ dest = self.options['location']
+ here = os.getcwd()
+ if not os.path.exists(dest):
+ os.mkdir(dest)
+
fname = getFromCache(
- url, self.name, self.download_cache, self.install_from_cache)
+ self.url, self.name, self.download_cache, self.install_from_cache)
# now unpack and work as normal
tmp = tempfile.mkdtemp('buildout-'+self.name)
logger.info('Unpacking and configuring')
setuptools.archive_util.unpack_archive(fname, tmp)
- here = os.getcwd()
- if not os.path.exists(dest):
- os.mkdir(dest)
+ for key, value in self.environ.items():
+ logger.info('Updating environment: %s=%s' % (key, value))
+ os.environ.update(self.environ)
- environ = self.options.get('environment', '').split()
- if environ:
- for entry in environ:
- logger.info('Updating environment: %s' % entry)
- environ = dict([x.split('=', 1) for x in environ])
- os.environ.update(environ)
-
try:
os.chdir(tmp)
try:
if not (os.path.exists('configure') or
- os.path.exists(autogen)):
+ os.path.exists(self.autogen)):
entries = os.listdir(tmp)
if len(entries) == 1:
os.chdir(entries[0])
- if patch is not '':
+ if self.patch is not '':
# patch may be a filesystem path or url
# url patches can go through the cache
- if urlparse.urlparse( patch, None)[0] is not None:
- patch = getFromCache( patch
+ if urlparse.urlparse(self.patch, None)[0] is not None:
+ self.patch = getFromCache( self.patch
, self.name
, self.download_cache
, self.install_from_cache
)
- system("patch %s < %s" % (patch_options, patch))
- if autogen is not '':
+ system("patch %s < %s" % (self.patch_options, self.patch))
+ if self.autogen is not '':
logger.info('auto generating configure files')
- system("./%s" % autogen)
+ system("./%s" % self.autogen)
if not os.path.exists('configure'):
entries = os.listdir(tmp)
if len(entries) == 1:
@@ -112,7 +149,7 @@
else:
raise ValueError("Couldn't find configure")
system("./configure --prefix=%s %s" %
- (dest, extra_options))
+ (dest, self.extra_options))
system("make")
system("make install")
finally:
@@ -126,12 +163,11 @@
def update(self):
pass
+
def getFromCache(url, name, download_cache=None, install_from_cache=False):
if download_cache:
cache_fname = sha1(url).hexdigest()
cache_name = os.path.join(download_cache, cache_fname)
- if not os.path.isdir(download_cache):
- os.mkdir(download_cache)
_, _, urlpath, _, _ = urlparse.urlsplit(url)
filename = urlpath.split('/')[-1]
Added: zc.recipe.cmmi/trunk/src/zc/recipe/cmmi/shared.txt
===================================================================
--- zc.recipe.cmmi/trunk/src/zc/recipe/cmmi/shared.txt (rev 0)
+++ zc.recipe.cmmi/trunk/src/zc/recipe/cmmi/shared.txt 2009-03-24 14:58:56 UTC (rev 98334)
@@ -0,0 +1,218 @@
+==============================
+Using a shared build directory
+==============================
+
+For builds that take a long time, it can be convenient to reuse them across
+several buildouts. To do this, use the `shared` option:
+
+ >>> cache = tmpdir('cache')
+ >>> write('buildout.cfg',
+ ... """
+ ... [buildout]
+ ... parts = foo
+ ... download-cache = %s
+ ...
+ ... [foo]
+ ... recipe = zc.recipe.cmmi
+ ... url = file://%s/foo.tgz
+ ... shared = True
+ ... """ % (cache, distros))
+
+When run the first time, the build is executed as usual:
+
+ >>> print system('bin/buildout')
+ Installing foo.
+ foo: Unpacking and configuring
+ configuring foo /cache/cmmi/build/...
+ echo building foo
+ building foo
+ echo installing foo
+ installing foo
+ <BLANKLINE>
+
+But after that, the existing shared build directory is used instead of running
+the build again:
+
+ >>> remove('.installed.cfg')
+ >>> print system('bin/buildout')
+ Installing foo.
+ foo: using existing shared build
+ <BLANKLINE>
+
+
+The shared directory
+====================
+
+By default, the shared build directory is named with a hash of the recipe's
+configuration options (but it can also be configured manually, see below):
+
+ >>> ls(cache, 'cmmi', 'build')
+ d ...
+
+For example, if the download url changes, the build is executed again:
+
+ >>> import os
+ >>> import shutil
+ >>> shutil.copy(os.path.join(distros, 'foo.tgz'),
+ ... os.path.join(distros, 'qux.tgz'))
+
+ >>> remove('.installed.cfg')
+ >>> write('buildout.cfg',
+ ... """
+ ... [buildout]
+ ... parts = qux
+ ... download-cache = %s
+ ...
+ ... [qux]
+ ... recipe = zc.recipe.cmmi
+ ... url = file://%s/qux.tgz
+ ... shared = True
+ ... """ % (cache, distros))
+ >>> print system('bin/buildout')
+ Installing qux.
+ qux: Unpacking and configuring
+ configuring foo /cache/cmmi/build/...
+ echo building foo
+ building foo
+ echo installing foo
+ installing foo
+
+and another shared directory is created:
+
+ >>> ls(cache, 'cmmi', 'build')
+ d ...
+ d ...
+
+(Other recipes can retrieve the shared build directory from our part's
+`location` as usual, so the SHA-names shouldn't be a problem.)
+
+
+Configuring the shared directory
+================================
+
+If you set `shared` to an existing directory, that will be used as the build
+directory directly (instead of a name computed from to the recipe options):
+
+ >>> shared = os.path.join(cache, 'existing')
+ >>> os.mkdir(shared)
+ >>> write('buildout.cfg',
+ ... """
+ ... [buildout]
+ ... parts = foo
+ ...
+ ... [foo]
+ ... recipe = zc.recipe.cmmi
+ ... url = file://%s/foo.tgz
+ ... shared = %s
+ ... """ % (distros, shared))
+
+ >>> remove('.installed.cfg')
+ >>> print system('bin/buildout')
+ Installing foo.
+ foo: Downloading /distros/foo.tgz
+ foo: Unpacking and configuring
+ configuring foo /cache/existing/cmmi
+ echo building foo
+ building foo
+ echo installing foo
+ installing foo
+ <BLANKLINE>
+
+If no download-cache is set, and `shared` is not a directory, an error is raised:
+
+ >>> write('buildout.cfg',
+ ... """
+ ... [buildout]
+ ... parts = foo
+ ...
+ ... [foo]
+ ... recipe = zc.recipe.cmmi
+ ... url = file://%s/foo.tgz
+ ... shared = True
+ ... """ % distros)
+
+ >>> print system('bin/buildout')
+ While:
+ Installing.
+ Getting section foo.
+ Initializing part foo.
+ ...
+ ValueError: Set the 'shared' option of zc.recipe.cmmi to an existing
+ directory, or set ${buildout:download-cache}
+
+
+Build errors
+============
+
+If an error occurs during the build (or it is aborted by the user),
+the build directory is removed, so there is no risk of accidentally
+mistaking some half-baked build directory as a good cached shared build.
+
+Let's simulate a build error. First, we backup a working build.
+
+ >>> shutil.copy(os.path.join(distros, 'foo.tgz'),
+ ... os.path.join(distros, 'foo.tgz.bak'))
+
+Then we create a broken tarball:
+
+ >>> import tarfile
+ >>> import StringIO
+ >>> import sys
+ >>> tarpath = os.path.join(distros, 'foo.tgz')
+ >>> tar = tarfile.open(tarpath, 'w:gz')
+ >>> configure = 'invalid'
+ >>> info = tarfile.TarInfo('configure')
+ >>> info.size = len(configure)
+ >>> info.mode = 0755
+ >>> tar.addfile(info, StringIO.StringIO(configure))
+
+Now we reset the cache to force our broken tarball to be used:
+
+ >>> shutil.rmtree(cache)
+ >>> cache = tmpdir('cache')
+ >>> write('buildout.cfg',
+ ... """
+ ... [buildout]
+ ... parts = foo
+ ... download-cache = %s
+ ...
+ ... [foo]
+ ... recipe = zc.recipe.cmmi
+ ... url = file://%s/foo.tgz
+ ... shared = True
+ ... """ % (cache, distros))
+
+ >>> remove('.installed.cfg')
+ >>> print system('bin/buildout')
+ Installing foo.
+ ...
+ ValueError: Couldn't find configure
+
+When we now fix the error (by copying back the working version and resetting the
+cache), the build will be run again, and we don't use a half-baked shared
+directory:
+
+ >>> shutil.copy(os.path.join(distros, 'foo.tgz.bak'),
+ ... os.path.join(distros, 'foo.tgz'))
+ >>> shutil.rmtree(cache)
+ >>> cache = tmpdir('cache')
+ >>> write('buildout.cfg',
+ ... """
+ ... [buildout]
+ ... parts = foo
+ ... download-cache = %s
+ ...
+ ... [foo]
+ ... recipe = zc.recipe.cmmi
+ ... url = file://%s/foo.tgz
+ ... shared = True
+ ... """ % (cache, distros))
+ >>> print system('bin/buildout')
+ Installing foo.
+ foo: Unpacking and configuring
+ configuring foo /cache/cmmi/build/...
+ echo building foo
+ building foo
+ echo installing foo
+ installing foo
+ <BLANKLINE>
Modified: zc.recipe.cmmi/trunk/src/zc/recipe/cmmi/tests.py
===================================================================
--- zc.recipe.cmmi/trunk/src/zc/recipe/cmmi/tests.py 2009-03-24 13:08:07 UTC (rev 98333)
+++ zc.recipe.cmmi/trunk/src/zc/recipe/cmmi/tests.py 2009-03-24 14:58:56 UTC (rev 98334)
@@ -89,9 +89,10 @@
),
doctest.DocFileSuite(
- 'patching.txt',
'downloadcache.txt',
'misc.txt',
+ 'patching.txt',
+ 'shared.txt',
setUp=setUp,
tearDown=zc.buildout.testing.buildoutTearDown,
@@ -102,6 +103,6 @@
normalize_bang,
(re.compile('extdemo[.]pyd'), 'extdemo.so')
]),
- optionflags = doctest.ELLIPSIS
+ optionflags = doctest.ELLIPSIS|doctest.NORMALIZE_WHITESPACE
),
))
More information about the Checkins
mailing list