[Checkins] SVN: zc.buildout/trunk/ Refactored the way recipes are
run and how they should be written.
Jim Fulton
cvs-admin at zope.org
Wed Jun 14 14:34:23 EDT 2006
Log message for revision 68634:
Refactored the way recipes are run and how they should be written.
If a recipe uses any data from other sections, the recipe needs to
update it's data when the recipe is constructed.
Need more discussion of this in the docs.
Changed:
U zc.buildout/trunk/eggrecipe/src/zc/recipe/egg/egg.py
U zc.buildout/trunk/src/zc/buildout/buildout.py
U zc.buildout/trunk/src/zc/buildout/buildout.txt
U zc.buildout/trunk/src/zc/buildout/testing.py
U zc.buildout/trunk/src/zc/buildout/tests.py
U zc.buildout/trunk/testrunnerrecipe/src/zc/recipe/testrunner/__init__.py
-=-
Modified: zc.buildout/trunk/eggrecipe/src/zc/recipe/egg/egg.py
===================================================================
--- zc.buildout/trunk/eggrecipe/src/zc/recipe/egg/egg.py 2006-06-14 16:30:52 UTC (rev 68633)
+++ zc.buildout/trunk/eggrecipe/src/zc/recipe/egg/egg.py 2006-06-14 18:34:19 UTC (rev 68634)
@@ -16,6 +16,7 @@
$Id$
"""
+import os
import zc.buildout.egglinker
import zc.buildout.easy_install
@@ -25,26 +26,27 @@
self.buildout = buildout
self.name = name
self.options = options
-
- def install(self):
- distribution = self.options.get('distribution', self.name)
- links = self.options.get(
- 'find-links',
- self.buildout['buildout'].get('find-links'),
- )
+ links = options.get('find-links',
+ buildout['buildout'].get('find-links'))
if links:
- links = links.split()
+ buildout_directory = buildout['buildout']['directory']
+ links = [os.path.join(buildout_directory, link)
+ for link in links.split()]
+ options['find-links'] = '\n'.join(links)
else:
links = ()
+ self.links = links
- buildout = self.buildout
+ options['_b'] = buildout['buildout']['bin-directory']
+ options['_e'] = buildout['buildout']['eggs-directory']
+
+ def install(self):
+ options = self.options
+ distribution = options.get('distribution', self.name)
zc.buildout.easy_install.install(
- distribution,
- buildout.eggs,
- [buildout.buildout_path(link) for link in links],
- )
+ distribution, options['_e'], self.links)
- scripts = self.options.get('scripts')
+ scripts = options.get('scripts')
if scripts or scripts is None:
if scripts is not None:
scripts = scripts.split()
@@ -53,6 +55,6 @@
for s in scripts
])
return zc.buildout.egglinker.scripts(
- [distribution], buildout.bin, [buildout.eggs],
- scripts=scripts)
+ [distribution],
+ options['_b'], [options['_e']], scripts=scripts)
Modified: zc.buildout/trunk/src/zc/buildout/buildout.py
===================================================================
--- zc.buildout/trunk/src/zc/buildout/buildout.py 2006-06-14 16:30:52 UTC (rev 68633)
+++ zc.buildout/trunk/src/zc/buildout/buildout.py 2006-06-14 18:34:19 UTC (rev 68634)
@@ -46,6 +46,12 @@
except KeyError:
raise MissingOption("Missing option", self.section, option)
+ # XXX need test
+ def __setitem__(self, option, value):
+ if not isinstance(value, str):
+ raise TypeError('Option values must be strings', value)
+ super(Options, self).__setitem__(option, value)
+
def copy(self):
return Options(self.buildout, self.section, self)
@@ -63,6 +69,8 @@
'bin-directory': 'bin',
'parts-directory': 'parts',
'installed': '.installed.cfg',
+ 'python': 'buildout',
+ 'executable': sys.executable,
},
)
@@ -106,8 +114,8 @@
self._buildout_dir = options['directory']
for name in ('bin', 'parts', 'eggs'):
- d = self.buildout_path(options[name+'-directory'])
- setattr(self, name, d)
+ d = self._buildout_path(options[name+'-directory'])
+ options[name+'-directory'] = d
if not os.path.exists(d):
os.mkdir(d)
@@ -148,45 +156,75 @@
return ''.join([''.join(v) for v in zip(value[::2], subs)])
- def buildout_path(self, *names):
+ def _buildout_path(self, *names):
return os.path.join(self._buildout_dir, *names)
def install(self, install_parts):
self._develop()
- new_part_options = self._gather_part_info()
+
+ # load installed data
installed_part_options = self._read_installed_part_options()
- old_parts = installed_part_options['buildout']['parts'].split()
- old_parts.reverse()
- new_old_parts = []
- for part in old_parts:
- if install_parts and (part not in install_parts):
- # We were asked to install specific parts and this
- # wasn't one of them. Leave it alone.
- new_old_parts.append(part)
- continue
-
- installed_options = installed_part_options[part].copy()
- installed = installed_options.pop('__buildout_installed__')
- if installed_options != new_part_options.get(part):
- self._uninstall(installed)
- del installed_part_options[part]
- else:
- new_old_parts.append(part)
- new_old_parts.reverse()
+ # get configured and installed part lists
+ conf_parts = self['buildout']['parts']
+ conf_parts = conf_parts and conf_parts.split() or []
+ installed_parts = installed_part_options['buildout']['parts']
+ installed_parts = installed_parts and installed_parts.split() or []
- new_parts = []
+
+ # If install_parts is given, then they must be listed in parts
+ # and we don't uninstall anything. Otherwise, we install
+ # the configured parts and uninstall anything else.
+ if install_parts:
+ extra = [p for p in install_parts if p not in conf_parts]
+ if extra:
+ error('Invalid install parts:', *extra)
+ uninstall_missing = False
+ else:
+ install_parts = conf_parts
+ uninstall_missing = True
+
+ # load recipes
+ recipes = self._load_recipes(install_parts)
+
+ # compute new part recipe signatures
+ self._compute_part_signatures(install_parts)
+
try:
- for part in new_part_options['buildout']['parts'].split():
- if (not install_parts) or (part in install_parts):
- installed = self._install(part)
- new_part_options[part]['__buildout_installed__'] = installed
- installed_part_options[part] = new_part_options[part]
- new_parts.append(part)
- new_old_parts = [p for p in new_old_parts if p != part]
+ # uninstall parts that are no-longer used or who's configs
+ # have changed
+ for part in reversed(installed_parts):
+ if part in install_parts:
+ old_options = installed_part_options[part].copy()
+ old_options.pop('__buildout_installed__')
+ if old_options == self.get(part):
+ continue
+ elif not uninstall_missing:
+ continue
+
+ # ununstall part
+ self._uninstall(
+ installed_part_options[part]['__buildout_installed__'])
+ installed_parts = [p for p in installed_parts if p != part]
+
+ # install new parts
+ for part in install_parts:
+ installed_part_options[part] = self[part].copy()
+ del self[part]['__buildout_signature__']
+ installed_files = recipes[part].install() or ()
+ if isinstance(installed_files, str):
+ installed_files = [installed_files]
+ installed_part_options[part]['__buildout_installed__'] = (
+ '\n'.join(installed_files)
+ )
+ if part not in installed_parts:
+ installed_parts.append(part)
finally:
- new_parts.extend(new_old_parts)
- installed_part_options['buildout']['parts'] = ' '.join(new_parts)
+ installed_part_options['buildout']['parts'] = ' '.join(
+ [p for p in conf_parts if p in installed_parts]
+ +
+ [p for p in installed_parts if p not in conf_parts]
+ )
self._save_installed_options(installed_part_options)
def _develop(self):
@@ -197,7 +235,7 @@
here = os.getcwd()
try:
for setup in develop.split():
- setup = self.buildout_path(setup)
+ setup = self._buildout_path(setup)
if os.path.isdir(setup):
setup = os.path.join(setup, 'setup.py')
@@ -206,46 +244,53 @@
os.P_WAIT, sys.executable, sys.executable,
setup, '-q', 'develop', '-m', '-x',
'-f', ' '.join(self._links),
- '-d', self.eggs,
+ '-d', self['buildout']['eggs-directory'],
{'PYTHONPATH':
os.path.dirname(pkg_resources.__file__)},
)
finally:
os.chdir(os.path.dirname(here))
- def _gather_part_info(self):
- """Get current part info, including part options and recipe info
- """
- parts = self['buildout']['parts']
- part_info = {'buildout': {'parts': parts}}
+ def _load_recipes(self, parts):
+ recipes = {}
recipes_requirements = []
- pkg_resources.working_set.add_entry(self.eggs)
+ pkg_resources.working_set.add_entry(self['buildout']['eggs-directory'])
- parts = parts and parts.split() or []
+ # Install the recipe distros
for part in parts:
options = self.get(part)
if options is None:
options = self[part] = {}
- options = options.copy()
recipe, entry = self._recipe(part, options)
zc.buildout.easy_install.install(
- recipe, self.eggs, self._links)
+ recipe, self['buildout']['eggs-directory'], self._links)
recipes_requirements.append(recipe)
- part_info[part] = options
- # Load up the recipe distros
+ # Add the distros to the working set
pkg_resources.require(recipes_requirements)
- base = self.eggs + os.path.sep
+ # instantiate the recipes
for part in parts:
- options = part_info[part]
+ options = self[part]
recipe, entry = self._recipe(part, options)
+ recipe_class = pkg_resources.load_entry_point(
+ recipe, 'zc.buildout', entry)
+ recipes[part] = recipe_class(self, part, options)
+
+ return recipes
+
+ def _compute_part_signatures(self, parts):
+ # Compute recipe signature and add to options
+ base = self['buildout']['eggs-directory'] + os.path.sep
+ for part in parts:
+ options = self.get(part)
+ if options is None:
+ options = self[part] = {}
+ recipe, entry = self._recipe(part, options)
req = pkg_resources.Requirement.parse(recipe)
sig = _dists_sig(pkg_resources.working_set.resolve([req]), base)
options['__buildout_signature__'] = ' '.join(sig)
- return part_info
-
def _recipe(self, part, options):
recipe = options.get('recipe', part)
if ':' in recipe:
@@ -260,17 +305,18 @@
if os.path.isfile(old):
parser = ConfigParser.SafeConfigParser()
parser.read(old)
- return dict([(section, dict(parser.items(section)))
- for section in parser.sections()])
+ return dict([
+ (section, Options(self, section, parser.items(section)))
+ for section in parser.sections()])
else:
- return {'buildout': {'parts': ''}}
+ return {'buildout': Options(self, 'buildout', {'parts': ''})}
def _installed_path(self):
- return self.buildout_path(self['buildout']['installed'])
+ return self._buildout_path(self['buildout']['installed'])
def _uninstall(self, installed):
for f in installed.split():
- f = self.buildout_path(f)
+ f = self._buildout_path(f)
if os.path.isdir(f):
shutil.rmtree(f)
elif os.path.isfile(f):
@@ -286,7 +332,7 @@
installed = []
elif isinstance(installed, basestring):
installed = [installed]
- base = self.buildout_path('')
+ base = self._buildout_path('')
installed = [d.startswith(base) and d[len(base):] or d
for d in installed]
return ' '.join(installed)
@@ -412,3 +458,9 @@
command = 'install'
getattr(buildout, command)(args)
+
+if sys.version_info[:2] < (2, 4):
+ def reversed(iterable):
+ result = list(iterable);
+ result.reverse()
+ return result
Modified: zc.buildout/trunk/src/zc/buildout/buildout.txt
===================================================================
--- zc.buildout/trunk/src/zc/buildout/buildout.txt 2006-06-14 16:30:52 UTC (rev 68633)
+++ zc.buildout/trunk/src/zc/buildout/buildout.txt 2006-06-14 18:34:19 UTC (rev 68634)
@@ -92,9 +92,13 @@
... self.buildout = buildout
... self.name = name
... self.options = options
+ ... options['path'] = os.path.join(
+ ... buildout['buildout']['directory'],
+ ... options['path'],
+ ... )
...
... def install(self):
- ... path = self.buildout.buildout_path(self.options['path'])
+ ... path = self.options['path']
... if not os.path.isdir(path):
... print 'Creating directory', os.path.basename(path)
... os.mkdir(path)
@@ -104,14 +108,21 @@
The recipe defines a constructor that takes a buildout object, a part
name, and an options dictionary. It saves them in instance attributes.
+If the path is relative, we'll interpret it as relative to the
+buildout directory. The buildout object passed in is a mapping from
+section name to a mapping of options for that section. The buildout
+directory is available as the directory option of the buildout
+section. We normalize the path and save it back into the options
+directory.
+
+**IMPORTANT**: Any time we use data from another section, it is important
+to reflect that data in the recipe options, as this data is used to
+decide if a part configuration has changed and a part needs to be
+reinstalled.
+
The install method is responsible for creating the part. In this
case, we need the path of the directory to create. We'll use a
-buildout option from our options dictionary. If the path is relative,
-we'll interpret it relative to the buildout directory. The buildout
-buildout_path method gives us a path relative to the buildout. It
-uses os.path.join, so if we pass it an absolute path, we'll get the
-absolute path back. (If no arguments are passed to base_path, then the
-buildout directory is returned.)
+path option from our options dictionary.
We made the method chatty so that we can observe what it's doing.
@@ -134,14 +145,13 @@
... )
... """)
-Here we've defined a package containing just our module. We've also
-defined an entry point. Entry points provide a way for an egg to
-define the services it provides. Here we've said that we define a
-zc.buildout entry point named default. Recipe classes must be exposed
-as entry points in the zc.buildout group. we give entry points names
-within the group. The name "default" is somewhat special because it
-allows a recipe to be referenced using a package name without naming
-an entry point.
+Here we've defined a package with an entry_point. Entry points provide
+a way for an egg to define the services it provides. Here we've said
+that we define a zc.buildout entry point named default. Recipe
+classes must be exposed as entry points in the zc.buildout group. we
+give entry points names within the group. The name "default" is
+somewhat special because it allows a recipe to be referenced using a
+package name without naming an entry point.
We also need a README.txt for our recipes to avoid a warning:
@@ -219,12 +229,14 @@
parts = data_dir
<BLANKLINE>
[data_dir]
- __buildout_installed__ = mystuff
- __buildout_signature__ = recipes-O3ypTgKOkHMqMwKvMfvHnA==
- path = mystuff
+ __buildout_installed__ = /tmp/sample-buildout/mystuff
+ __buildout_signature__ = recipes-c7vHV6ekIDUPy/7fjAaYjg==
+ path = /tmp/sample-buildout/mystuff
recipe = recipes:mkdir
Note that the directory we installed is included in .installed.cfg.
+In addition, the path option includes the actual destination
+directory.
If we change the name of the directory in the configuration file,
we'll see that the directory gets removed and recreated:
@@ -639,27 +651,27 @@
<BLANKLINE>
[debug]
__buildout_installed__ =
- __buildout_signature__ = recipes-IX/o5hMSw90MtZVxRpjz0Q==
+ __buildout_signature__ = recipes-PiIFiO8ny5yNZ1S3JfT0xg==
op1 = 1
op7 = 7
recipe = recipes:debug
<BLANKLINE>
[d1]
- __buildout_installed__ = d1
- __buildout_signature__ = recipes-IX/o5hMSw90MtZVxRpjz0Q==
- path = d1
+ __buildout_installed__ = /tmp/sample-buildout/d1
+ __buildout_signature__ = recipes-PiIFiO8ny5yNZ1S3JfT0xg==
+ path = /tmp/sample-buildout/d1
recipe = recipes:mkdir
<BLANKLINE>
[d2]
- __buildout_installed__ = d2
- __buildout_signature__ = recipes-IX/o5hMSw90MtZVxRpjz0Q==
- path = d2
+ __buildout_installed__ = /tmp/sample-buildout/d2
+ __buildout_signature__ = recipes-PiIFiO8ny5yNZ1S3JfT0xg==
+ path = /tmp/sample-buildout/d2
recipe = recipes:mkdir
<BLANKLINE>
[d3]
- __buildout_installed__ = d3
- __buildout_signature__ = recipes-IX/o5hMSw90MtZVxRpjz0Q==
- path = d3
+ __buildout_installed__ = /tmp/sample-buildout/d3
+ __buildout_signature__ = recipes-PiIFiO8ny5yNZ1S3JfT0xg==
+ path = /tmp/sample-buildout/d3
recipe = recipes:mkdir
Now we'll update our configuration file:
@@ -721,33 +733,33 @@
<BLANKLINE>
[debug]
__buildout_installed__ =
- __buildout_signature__ = recipes-IX/o5hMSw90MtZVxRpjz0Q==
+ __buildout_signature__ = recipes-PiIFiO8ny5yNZ1S3JfT0xg==
op1 = 1
op7 = 7
recipe = recipes:debug
<BLANKLINE>
[d2]
- __buildout_installed__ = d2
- __buildout_signature__ = recipes-IX/o5hMSw90MtZVxRpjz0Q==
- path = d2
+ __buildout_installed__ = /tmp/sample-buildout/d2
+ __buildout_signature__ = recipes-PiIFiO8ny5yNZ1S3JfT0xg==
+ path = /tmp/sample-buildout/d2
recipe = recipes:mkdir
<BLANKLINE>
[d3]
- __buildout_installed__ = data3
- __buildout_signature__ = recipes-IX/o5hMSw90MtZVxRpjz0Q==
- path = data3
+ __buildout_installed__ = /tmp/sample-buildout/data3
+ __buildout_signature__ = recipes-PiIFiO8ny5yNZ1S3JfT0xg==
+ path = /tmp/sample-buildout/data3
recipe = recipes:mkdir
<BLANKLINE>
[d4]
- __buildout_installed__ = data4
- __buildout_signature__ = recipes-IX/o5hMSw90MtZVxRpjz0Q==
- path = data4
+ __buildout_installed__ = /tmp/sample-buildout/data4
+ __buildout_signature__ = recipes-PiIFiO8ny5yNZ1S3JfT0xg==
+ path = /tmp/sample-buildout/data4
recipe = recipes:mkdir
<BLANKLINE>
[d1]
- __buildout_installed__ = d1
- __buildout_signature__ = recipes-IX/o5hMSw90MtZVxRpjz0Q==
- path = d1
+ __buildout_installed__ = /tmp/sample-buildout/d1
+ __buildout_signature__ = recipes-PiIFiO8ny5yNZ1S3JfT0xg==
+ path = /tmp/sample-buildout/d1
recipe = recipes:mkdir
Note that the installed data for debug, d1, and d2 haven't changed,
Modified: zc.buildout/trunk/src/zc/buildout/testing.py
===================================================================
--- zc.buildout/trunk/src/zc/buildout/testing.py 2006-06-14 16:30:52 UTC (rev 68633)
+++ zc.buildout/trunk/src/zc/buildout/testing.py 2006-06-14 18:34:19 UTC (rev 68634)
@@ -53,7 +53,7 @@
return o.read()
def buildoutSetUp(test):
- sample = tempfile.mkdtemp('buildout-tests')
+ sample = tempfile.mkdtemp('sample-buildout')
for name in ('bin', 'eggs', 'parts'):
os.mkdir(os.path.join(sample, name))
@@ -118,7 +118,7 @@
os.chdir(here)
def create_sample_eggs(test):
- sample = tempfile.mkdtemp('eggtest')
+ sample = tempfile.mkdtemp('sample-eggs')
test.globs['_sample_eggs_container'] = sample
test.globs['sample_eggs'] = os.path.join(sample, 'dist')
write(sample, 'README.txt', '')
Modified: zc.buildout/trunk/src/zc/buildout/tests.py
===================================================================
--- zc.buildout/trunk/src/zc/buildout/tests.py 2006-06-14 16:30:52 UTC (rev 68633)
+++ zc.buildout/trunk/src/zc/buildout/tests.py 2006-06-14 18:34:19 UTC (rev 68634)
@@ -94,7 +94,9 @@
checker=renormalizing.RENormalizing([
(re.compile('__buildout_signature__ = recipes-\S+'),
'__buildout_signature__ = recipes-SSSSSSSSSSS'),
- ])
+ (re.compile('\S+sample-(\w+)%s(\S+)' % os.path.sep),
+ r'/sample-\1/\2'),
+ ])
),
doctest.DocFileSuite(
'egglinker.txt', 'easy_install.txt',
Modified: zc.buildout/trunk/testrunnerrecipe/src/zc/recipe/testrunner/__init__.py
===================================================================
--- zc.buildout/trunk/testrunnerrecipe/src/zc/recipe/testrunner/__init__.py 2006-06-14 16:30:52 UTC (rev 68633)
+++ zc.buildout/trunk/testrunnerrecipe/src/zc/recipe/testrunner/__init__.py 2006-06-14 18:34:19 UTC (rev 68634)
@@ -25,7 +25,12 @@
self.buildout = buildout
self.name = name
self.options = options
+ options['script'] = os.path.join(buildout['buildout']['bin-directory'],
+ options.get('script', self.name),
+ )
+ options['_e'] = buildout['buildout']['eggs-directory']
+
def install(self):
distributions = [
req.strip()
@@ -34,13 +39,12 @@
]
path = zc.buildout.egglinker.path(
distributions+['zope.testing'],
- [self.buildout.eggs],
+ [self.options['_e']],
)
locations = [zc.buildout.egglinker.location(distribution,
- [self.buildout.eggs])
+ [self.options['_e']])
for distribution in distributions]
- script = self.options.get('script', self.name)
- script = self.buildout.buildout_path('bin', script)
+ script = self.options['script']
open(script, 'w').write(tests_template % dict(
PYTHON=sys.executable,
PATH="',\n '".join(path),
More information about the Checkins
mailing list