[Checkins] SVN: zc.buildout/trunk/ - Changed the way the installed
database (.installed.cfg) is handled
Jim Fulton
jim at zope.com
Wed Mar 14 18:44:00 EDT 2007
Log message for revision 73178:
- Changed the way the installed database (.installed.cfg) is handled
to avoid database corruption when a user breaks out of a buildout
with control-c.
- Don't save an installed database if there are no installed parts or
develop egg links.
Changed:
U zc.buildout/trunk/CHANGES.txt
U zc.buildout/trunk/src/zc/buildout/buildout.py
U zc.buildout/trunk/src/zc/buildout/buildout.txt
U zc.buildout/trunk/src/zc/buildout/tests.py
-=-
Modified: zc.buildout/trunk/CHANGES.txt
===================================================================
--- zc.buildout/trunk/CHANGES.txt 2007-03-14 22:39:24 UTC (rev 73177)
+++ zc.buildout/trunk/CHANGES.txt 2007-03-14 22:44:00 UTC (rev 73178)
@@ -17,19 +17,26 @@
Feature Changes
---------------
-Improved error reporting and debugging support:
+- Improved error reporting and debugging support:
-- Added "logical tracebacks" that show functionally what the buildout
- was doing when an error occurs. Don't show a Python traceback
- unless the -D option is used.
+ - Added "logical tracebacks" that show functionally what the buildout
+ was doing when an error occurs. Don't show a Python traceback
+ unless the -D option is used.
-- Added a -D option that causes the buildout to print a traceback and
- start the pdb post-mortem debugger when an error occurs.
+ - Added a -D option that causes the buildout to print a traceback and
+ start the pdb post-mortem debugger when an error occurs.
-- Warnings are printed for unused options in the buildout section and
- installed-part sections. This should make it easier to catch option
- misspellings.
+ - Warnings are printed for unused options in the buildout section and
+ installed-part sections. This should make it easier to catch option
+ misspellings.
+- Changed the way the installed database (.installed.cfg) is handled
+ to avoid database corruption when a user breaks out of a buildout
+ with control-c.
+
+- Don't save an installed database if there are no installed parts or
+ develop egg links.
+
1.0.0b21 (2007-03-06)
=====================
Modified: zc.buildout/trunk/src/zc/buildout/buildout.py
===================================================================
--- zc.buildout/trunk/src/zc/buildout/buildout.py 2007-03-14 22:39:24 UTC (rev 73177)
+++ zc.buildout/trunk/src/zc/buildout/buildout.py 2007-03-14 22:44:00 UTC (rev 73178)
@@ -202,7 +202,8 @@
self._maybe_upgrade()
# load installed data
- installed_part_options = self._read_installed_part_options()
+ (installed_part_options, installed_exists
+ )= self._read_installed_part_options()
# Remove old develop eggs
self._uninstall(
@@ -212,6 +213,12 @@
# Build develop eggs
installed_develop_eggs = self._develop()
+ installed_part_options['buildout']['installed_develop_eggs'
+ ] = installed_develop_eggs
+
+ if installed_exists:
+ self._update_installed(
+ installed_develop_eggs=installed_develop_eggs)
# get configured and installed part lists
conf_parts = self['buildout']['parts']
@@ -244,119 +251,148 @@
# compute new part recipe signatures
self._compute_part_signatures(install_parts)
- try:
- # uninstall parts that are no-longer used or who's configs
- # have changed
- for part in reversed(installed_parts):
- if part in install_parts:
- old_options = installed_part_options[part].copy()
- installed_files = old_options.pop('__buildout_installed__')
- new_options = self.get(part)
- if old_options == new_options:
- # The options are the same, but are all of the
- # installed files still there? If not, we should
- # reinstall.
- if not installed_files:
- continue
- for f in installed_files.split('\n'):
- if not os.path.exists(self._buildout_path(f)):
- break
- else:
- continue
+ # uninstall parts that are no-longer used or who's configs
+ # have changed
+ for part in reversed(installed_parts):
+ if part in install_parts:
+ old_options = installed_part_options[part].copy()
+ installed_files = old_options.pop('__buildout_installed__')
+ new_options = self.get(part)
+ if old_options == new_options:
+ # The options are the same, but are all of the
+ # installed files still there? If not, we should
+ # reinstall.
+ if not installed_files:
+ continue
+ for f in installed_files.split('\n'):
+ if not os.path.exists(self._buildout_path(f)):
+ break
+ else:
+ continue
- # output debugging info
- for k in old_options:
- if k not in new_options:
- self._logger.debug("Part: %s, dropped option %s",
- part, k)
- elif old_options[k] != new_options[k]:
- self._logger.debug(
- "Part: %s, option %s, %r != %r",
- part, k, new_options[k], old_options[k],
- )
- for k in new_options:
- if k not in old_options:
- self._logger.debug("Part: %s, new option %s",
- part, k)
+ # output debugging info
+ for k in old_options:
+ if k not in new_options:
+ self._logger.debug("Part: %s, dropped option %s",
+ part, k)
+ elif old_options[k] != new_options[k]:
+ self._logger.debug(
+ "Part: %s, option %s, %r != %r",
+ part, k, new_options[k], old_options[k],
+ )
+ for k in new_options:
+ if k not in old_options:
+ self._logger.debug("Part: %s, new option %s",
+ part, k)
- elif not uninstall_missing:
- continue
+ elif not uninstall_missing:
+ continue
- self._uninstall_part(part, installed_part_options)
- installed_parts = [p for p in installed_parts if p != part]
+ self._uninstall_part(part, installed_part_options)
+ installed_parts = [p for p in installed_parts if p != part]
- # Check for unused buildout options:
- _check_for_unused_options_in_section(self, 'buildout')
+ if installed_exists:
+ self._update_installed(parts=' '.join(installed_parts))
- # install new parts
- for part in install_parts:
- signature = self[part].pop('__buildout_signature__')
- saved_options = self[part].copy()
- recipe = self[part].recipe
- if part in installed_parts:
- __doing__ = 'Updating %s', part
- self._logger.info(*__doing__)
- old_options = installed_part_options[part]
- old_installed_files = old_options['__buildout_installed__']
- try:
- update = recipe.update
- except AttributeError:
- update = recipe.install
- self._logger.warning(
- "The recipe for %s doesn't define an update "
- "method. Using its install method.",
- part)
+ # Check for unused buildout options:
+ _check_for_unused_options_in_section(self, 'buildout')
- try:
- installed_files = update()
- except:
- installed_parts.remove(part)
- self._uninstall(old_installed_files)
- raise
-
- if installed_files is None:
- installed_files = old_installed_files.split('\n')
+ # install new parts
+ for part in install_parts:
+ signature = self[part].pop('__buildout_signature__')
+ saved_options = self[part].copy()
+ recipe = self[part].recipe
+ if part in installed_parts: # update
+ need_to_save_installed = False
+ __doing__ = 'Updating %s', part
+ self._logger.info(*__doing__)
+ old_options = installed_part_options[part]
+ old_installed_files = old_options['__buildout_installed__']
+
+ try:
+ update = recipe.update
+ except AttributeError:
+ update = recipe.install
+ self._logger.warning(
+ "The recipe for %s doesn't define an update "
+ "method. Using its install method.",
+ part)
+
+ try:
+ installed_files = update()
+ except:
+ installed_parts.remove(part)
+ self._uninstall(old_installed_files)
+ if installed_exists:
+ self._update_installed(
+ parts=' '.join(installed_parts))
+ raise
+
+ old_installed_files = old_installed_files.split('\n')
+ if installed_files is None:
+ installed_files = old_installed_files
+ else:
+ if isinstance(installed_files, str):
+ installed_files = [installed_files]
else:
- if isinstance(installed_files, str):
- installed_files = [installed_files]
- else:
- installed_files = list(installed_files)
+ installed_files = list(installed_files)
- installed_files += [
- p for p in old_installed_files.split('\n')
- if p and p not in installed_files]
+ need_to_save_installed = [
+ p for p in installed_files
+ if p not in old_installed_files]
+ if need_to_save_installed:
+ installed_files = (old_installed_files
+ + need_to_save_installed)
+
+ else: # install
+ need_to_save_installed = True
+ __doing__ = 'Installing %s', part
+ self._logger.info(*__doing__)
+ installed_files = recipe.install()
+ if installed_files is None:
+ self._logger.warning(
+ "The %s install returned None. A path or "
+ "iterable os paths should be returned.",
+ part)
+ installed_files = ()
+ elif isinstance(installed_files, str):
+ installed_files = [installed_files]
else:
- __doing__ = 'Installing %s', part
- self._logger.info(*__doing__)
- installed_files = recipe.install()
- if installed_files is None:
- self._logger.warning(
- "The %s install returned None. A path or "
- "iterable os paths should be returned.",
- part)
- installed_files = ()
-
- if isinstance(installed_files, str):
- installed_files = [installed_files]
+ installed_files = list(installed_files)
- installed_part_options[part] = saved_options
- saved_options['__buildout_installed__'
- ] = '\n'.join(installed_files)
- saved_options['__buildout_signature__'] = signature
+ installed_part_options[part] = saved_options
+ saved_options['__buildout_installed__'
+ ] = '\n'.join(installed_files)
+ saved_options['__buildout_signature__'] = signature
- installed_parts = [p for p in installed_parts if p != part]
- installed_parts.append(part)
- _check_for_unused_options_in_section(self, part)
+ installed_parts = [p for p in installed_parts if p != part]
+ installed_parts.append(part)
+ _check_for_unused_options_in_section(self, part)
- finally:
- installed_part_options['buildout']['parts'] = (
- ' '.join(installed_parts))
- installed_part_options['buildout']['installed_develop_eggs'
- ] = installed_develop_eggs
-
- self._save_installed_options(installed_part_options)
+ if need_to_save_installed:
+ installed_part_options['buildout']['parts'] = (
+ ' '.join(installed_parts))
+ self._save_installed_options(installed_part_options)
+ installed_exists = True
+ else:
+ assert installed_exists
+ self._update_installed(parts=' '.join(installed_parts))
+ if installed_develop_eggs:
+ if not installed_exists:
+ self._save_installed_options(installed_part_options)
+ elif (not installed_parts) and installed_exists:
+ os.remove(self['buildout']['installed'])
+
+ def _update_installed(self, **buildout_options):
+ installed = self['buildout']['installed']
+ f = open(installed, 'a')
+ f.write('\n[buildout]\n')
+ for option, value in buildout_options.items():
+ _save_option(option, value, f)
+ f.close()
+
def _uninstall_part(self, part, installed_part_options):
# ununstall part
__doing__ = 'Uninstalling %s', part
@@ -466,9 +502,11 @@
options[option] = value
result[section] = Options(self, section, options)
- return result
+ return result, True
else:
- return {'buildout': Options(self, 'buildout', {'parts': ''})}
+ return ({'buildout': Options(self, 'buildout', {'parts': ''})},
+ False,
+ )
def _uninstall(self, installed):
for f in installed.split('\n'):
@@ -891,17 +929,20 @@
)
return result
+def _save_option(option, value, f):
+ value = _spacey_nl.sub(_quote_spacey_nl, value)
+ if value.startswith('\n\t'):
+ value = '%(__buildout_space_n__)s' + value[2:]
+ if value.endswith('\n\t'):
+ value = value[:-2] + '%(__buildout_space_n__)s'
+ print >>f, option, '=', value
+
def _save_options(section, options, f):
print >>f, '[%s]' % section
items = options.items()
items.sort()
for option, value in items:
- value = _spacey_nl.sub(_quote_spacey_nl, value)
- if value.startswith('\n\t'):
- value = '%(__buildout_space_n__)s' + value[2:]
- if value.endswith('\n\t'):
- value = value[:-2] + '%(__buildout_space_n__)s'
- print >>f, option, '=', value
+ _save_option(option, value, f)
def _open(base, filename, seen):
"""Open a configuration file and return the result as a dictionary,
Modified: zc.buildout/trunk/src/zc/buildout/buildout.txt
===================================================================
--- zc.buildout/trunk/src/zc/buildout/buildout.txt 2007-03-14 22:39:24 UTC (rev 73177)
+++ zc.buildout/trunk/src/zc/buildout/buildout.txt 2007-03-14 22:44:00 UTC (rev 73178)
@@ -1756,8 +1756,20 @@
".installed.cfg", but it can be overridded in the configuration file
or on the command line:
- >>> os.remove('.installed.cfg')
+ >>> write('buildout.cfg',
+ ... """
+ ... [buildout]
+ ... develop = recipes
+ ... parts = debug
+ ...
+ ... [debug]
+ ... recipe = recipes:debug
+ ... """)
+
>>> print system(buildout+' buildout:installed=inst.cfg'),
+ buildout: Develop: /sample-buildout/recipes
+ buildout: Installing debug
+ recipe recipes:debug
>>> ls(sample_buildout)
- b1.cfg
@@ -1771,12 +1783,14 @@
d parts
d recipes
-
The installation database can be disabled by supplying an empty
buildout installed opttion:
>>> os.remove('inst.cfg')
>>> print system(buildout+' buildout:installed='),
+ buildout: Develop: /sample-buildout/recipes
+ buildout: Installing debug
+ recipe recipes:debug
>>> ls(sample_buildout)
- b1.cfg
@@ -1790,6 +1804,28 @@
d recipes
+Note that there will be no installation database if there are no
+parts:
+
+ >>> write('buildout.cfg',
+ ... """
+ ... [buildout]
+ ... parts =
+ ... """)
+
+ >>> print system(buildout+' buildout:installed=inst.cfg'),
+
+ >>> ls(sample_buildout)
+ - b1.cfg
+ - b2.cfg
+ - base.cfg
+ d bin
+ - buildout.cfg
+ d develop-eggs
+ d eggs
+ d parts
+ d recipes
+
Extensions
----------
Modified: zc.buildout/trunk/src/zc/buildout/tests.py
===================================================================
--- zc.buildout/trunk/src/zc/buildout/tests.py 2007-03-14 22:39:24 UTC (rev 73177)
+++ zc.buildout/trunk/src/zc/buildout/tests.py 2007-03-14 22:44:00 UTC (rev 73178)
@@ -1240,11 +1240,11 @@
[buildout]
...
[foo]
- __buildout_installed__ = c
+ __buildout_installed__ = a
+ b
+ c
d
e
- a
- b
__buildout_signature__ = ...
"""
@@ -1386,6 +1386,208 @@
buildout: Unused options for foo: 'z'
'''
+def abnormal_exit():
+ """
+People sometimes hit control-c while running a builout. We need to make
+sure that the installed database Isn't corrupted. To test this, we'll create
+some evil recipes that exit uncleanly:
+
+ >>> mkdir('recipes')
+ >>> write('recipes', 'recipes.py',
+ ... '''
+ ... import os
+ ...
+ ... class Clean:
+ ... def __init__(*_): pass
+ ... def install(_): return ()
+ ... def update(_): pass
+ ...
+ ... class EvilInstall(Clean):
+ ... def install(_): os._exit(1)
+ ...
+ ... class EvilUpdate(Clean):
+ ... def update(_): os._exit(1)
+ ... ''')
+
+ >>> write('recipes', 'setup.py',
+ ... '''
+ ... import setuptools
+ ... setuptools.setup(name='recipes',
+ ... entry_points = {
+ ... 'zc.buildout': [
+ ... 'clean = recipes:Clean',
+ ... 'evil_install = recipes:EvilInstall',
+ ... 'evil_update = recipes:EvilUpdate',
+ ... 'evil_uninstall = recipes:Clean',
+ ... ],
+ ... },
+ ... )
+ ... ''')
+
+Now let's look at 3 cases:
+
+1. We exit during installation after installing some other parts:
+
+ >>> write('buildout.cfg',
+ ... '''
+ ... [buildout]
+ ... develop = recipes
+ ... parts = p1 p2 p3 p4
+ ...
+ ... [p1]
+ ... recipe = recipes:clean
+ ...
+ ... [p2]
+ ... recipe = recipes:clean
+ ...
+ ... [p3]
+ ... recipe = recipes:evil_install
+ ...
+ ... [p4]
+ ... recipe = recipes:clean
+ ... ''')
+
+ >>> print system(buildout),
+ buildout: Develop: /sample-buildout/recipes
+ buildout: Installing p1
+ buildout: Installing p2
+ buildout: Installing p3
+
+ >>> print system(buildout),
+ buildout: Develop: /sample-buildout/recipes
+ buildout: Updating p1
+ buildout: Updating p2
+ buildout: Installing p3
+
+ >>> print system(buildout+' buildout:parts='),
+ buildout: Develop: /sample-buildout/recipes
+ buildout: Uninstalling p2
+ buildout: Uninstalling p1
+
+2. We exit while updating:
+
+ >>> write('buildout.cfg',
+ ... '''
+ ... [buildout]
+ ... develop = recipes
+ ... parts = p1 p2 p3 p4
+ ...
+ ... [p1]
+ ... recipe = recipes:clean
+ ...
+ ... [p2]
+ ... recipe = recipes:clean
+ ...
+ ... [p3]
+ ... recipe = recipes:evil_update
+ ...
+ ... [p4]
+ ... recipe = recipes:clean
+ ... ''')
+
+ >>> print system(buildout),
+ buildout: Develop: /sample-buildout/recipes
+ buildout: Installing p1
+ buildout: Installing p2
+ buildout: Installing p3
+ buildout: Installing p4
+
+ >>> print system(buildout),
+ buildout: Develop: /sample-buildout/recipes
+ buildout: Updating p1
+ buildout: Updating p2
+ buildout: Updating p3
+
+ >>> print system(buildout+' buildout:parts='),
+ buildout: Develop: /sample-buildout/recipes
+ buildout: Uninstalling p2
+ buildout: Uninstalling p1
+ buildout: Uninstalling p4
+ buildout: Uninstalling p3
+
+3. We exit while installing or updating after uninstalling:
+
+ >>> write('buildout.cfg',
+ ... '''
+ ... [buildout]
+ ... develop = recipes
+ ... parts = p1 p2 p3 p4
+ ...
+ ... [p1]
+ ... recipe = recipes:evil_update
+ ...
+ ... [p2]
+ ... recipe = recipes:clean
+ ...
+ ... [p3]
+ ... recipe = recipes:clean
+ ...
+ ... [p4]
+ ... recipe = recipes:clean
+ ... ''')
+
+ >>> print system(buildout),
+ buildout: Develop: /sample-buildout/recipes
+ buildout: Installing p1
+ buildout: Installing p2
+ buildout: Installing p3
+ buildout: Installing p4
+
+ >>> write('buildout.cfg',
+ ... '''
+ ... [buildout]
+ ... develop = recipes
+ ... parts = p1 p2 p3 p4
+ ...
+ ... [p1]
+ ... recipe = recipes:evil_update
+ ...
+ ... [p2]
+ ... recipe = recipes:clean
+ ...
+ ... [p3]
+ ... recipe = recipes:clean
+ ...
+ ... [p4]
+ ... recipe = recipes:clean
+ ... x = 1
+ ... ''')
+
+ >>> print system(buildout),
+ buildout: Develop: /sample-buildout/recipes
+ buildout: Uninstalling p4
+ buildout: Updating p1
+
+ >>> write('buildout.cfg',
+ ... '''
+ ... [buildout]
+ ... develop = recipes
+ ... parts = p1 p2 p3 p4
+ ...
+ ... [p1]
+ ... recipe = recipes:clean
+ ...
+ ... [p2]
+ ... recipe = recipes:clean
+ ...
+ ... [p3]
+ ... recipe = recipes:clean
+ ...
+ ... [p4]
+ ... recipe = recipes:clean
+ ... ''')
+
+ >>> print system(buildout),
+ buildout: Develop: /sample-buildout/recipes
+ buildout: Uninstalling p1
+ buildout: Installing p1
+ buildout: Updating p2
+ buildout: Updating p3
+ buildout: Installing p4
+
+ """
+
+
######################################################################
def create_sample_eggs(test, executable=sys.executable):
More information about the Checkins
mailing list