[Checkins] SVN: zc.buildout/branches/gary-4/ simplify resulting code a bit more and try again to remove warnings
Gary Poster
gary.poster at canonical.com
Fri Feb 12 10:58:10 EST 2010
Log message for revision 108953:
simplify resulting code a bit more and try again to remove warnings
Changed:
U zc.buildout/branches/gary-4/src/zc/buildout/easy_install.py
U zc.buildout/branches/gary-4/src/zc/buildout/easy_install.txt
U zc.buildout/branches/gary-4/src/zc/buildout/testing.py
U zc.buildout/branches/gary-4/z3c.recipe.scripts_/src/z3c/recipe/scripts/README.txt
U zc.buildout/branches/gary-4/z3c.recipe.scripts_/src/z3c/recipe/scripts/tests.py
-=-
Modified: zc.buildout/branches/gary-4/src/zc/buildout/easy_install.py
===================================================================
--- zc.buildout/branches/gary-4/src/zc/buildout/easy_install.py 2010-02-12 15:33:00 UTC (rev 108952)
+++ zc.buildout/branches/gary-4/src/zc/buildout/easy_install.py 2010-02-12 15:58:10 UTC (rev 108953)
@@ -1063,7 +1063,8 @@
return generated
def _relative_path_and_setup(sname, path,
- relative_paths=False, indent_level=1):
+ relative_paths=False, indent_level=1,
+ omit_os_import=False):
"""Return a string of code of paths and of setup if appropriate.
- sname is the full path to the script name to be created.
@@ -1080,8 +1081,10 @@
[_relativitize(os.path.normcase(path_item), sname, relative_paths)
for path_item in path], indent_level=indent_level)
rpsetup = relative_paths_setup
+ if not omit_os_import:
+ rpsetup = '\n\nimport os\n' + rpsetup
for i in range(_relative_depth(relative_paths, sname)):
- rpsetup += "base = os.path.dirname(base)\n"
+ rpsetup += "\nbase = os.path.dirname(base)"
else:
spath = _format_paths((repr(p) for p in path),
indent_level=indent_level)
@@ -1143,11 +1146,8 @@
return repr(path)
relative_paths_setup = """
-import os
-
join = os.path.join
-base = os.path.dirname(os.path.abspath(os.path.realpath(__file__)))
-"""
+base = os.path.dirname(os.path.abspath(os.path.realpath(__file__)))"""
def _write_script(full_name, contents, logged_type):
"""Write contents of script in full_name, logging the action.
@@ -1214,8 +1214,8 @@
'''
script_template = script_header + '''\
-
%(relative_paths_setup)s
+
import sys
sys.path[0:0] = [
%(path)s,
@@ -1240,8 +1240,8 @@
return _write_script(dest, contents, 'interpreter')
py_script_template = script_header + '''\
-
%(relative_paths_setup)s
+
import sys
sys.path[0:0] = [
@@ -1384,29 +1384,32 @@
"""
path = _get_path(working_set, extra_paths)
site_path = os.path.join(dest, 'site.py')
- path_string, rpsetup = _relative_path_and_setup(
- site_path, path, relative_paths, indent_level=2)
- if rpsetup:
- rpsetup = '\n'.join(
+ egg_path_string, preamble = _relative_path_and_setup(
+ site_path, path, relative_paths, indent_level=2, omit_os_import=True)
+ if preamble:
+ preamble = '\n'.join(
[(line and ' %s' % (line,) or line)
- for line in rpsetup.split('\n')])
- namespace_setup = ''
- addsitedir = addsitedir_snippet
+ for line in preamble.split('\n')])
+ original_path_setup = ''
if add_site_packages:
stdlib, site_paths = _get_system_paths(executable)
- path_string = ''.join([
- path_string,
- (",\n"
- " # These are the underlying Python's site-packages.\n"
- " "),
- _format_paths((repr(p) for p in site_paths), 2)])
+ original_path_setup = original_path_snippet % (
+ _format_paths((repr(p) for p in site_paths), 2),)
distribution = working_set.find(
pkg_resources.Requirement.parse('setuptools'))
if distribution is not None:
# We need to worry about namespace packages.
- namespace_setup = namespace_add_site_packages_setup % (
- distribution.location,)
- addsitedir = addsitedir_namespace_add_site_packages_snippet
+ if relative_paths:
+ location = _relativitize(
+ distribution.location,
+ os.path.normcase(os.path.abspath(site_path)),
+ relative_paths)
+ else:
+ location = repr(distribution.location)
+ preamble += namespace_add_site_packages_setup % (location,)
+ original_path_setup = (
+ addsitedir_namespace_originalpackages_snippet +
+ original_path_setup)
addsitepackages_marker = 'def addsitepackages('
enableusersite_marker = 'ENABLE_USER_SITE = '
successful_rewrite = False
@@ -1420,7 +1423,7 @@
site.write('False # buildout does not support user sites.\n')
elif line.startswith(addsitepackages_marker):
site.write(addsitepackages_script % (
- namespace_setup, rpsetup, path_string, addsitedir))
+ preamble, egg_path_string, original_path_setup))
site.write(line[len(addsitepackages_marker):])
successful_rewrite = True
else:
@@ -1433,49 +1436,41 @@
return site_path
namespace_add_site_packages_setup = '''
- setuptools_path = %r
+ setuptools_path = %s
sys.path.append(setuptools_path)
- known_paths.add(setuptools_path)
+ known_paths.add(os.path.normcase(setuptools_path))
import pkg_resources'''
-addsitedir_snippet = '''
- for path in paths:
+addsitedir_namespace_originalpackages_snippet = '''
+ for dist in pkg_resources.find_distributions(sitedir, True):
+ pkg_resources.fixup_namespace_packages(dist.location)
+ if dist.has_metadata('namespace_packages.txt'):
+ for namespace in dist.get_metadata_lines(
+ 'namespace_packages.txt'):
+ pkg_resources.declare_namespace(namespace)'''
+
+original_path_snippet = '''
+ original_paths = [
+ %s
+ ]
+ for path in original_paths:
addsitedir(path, known_paths)'''
-addsitedir_namespace_add_site_packages_snippet = '''
- dotpth = os.extsep + "pth"
- for path in paths:
- # This duplicates addsitedir except for adding the pkg_resources call.
- sitedir, sitedircase = makepath(path)
- if not sitedircase in known_paths and os.path.exists(sitedir):
- sys.path.append(sitedir)
- pkg_resources.working_set.add_entry(sitedir)
- known_paths.add(sitedircase)
- try:
- names = os.listdir(sitedir)
- except os.error:
- continue
- names = [name for name in names if name.endswith(dotpth)]
- names.sort()
- for name in names:
- addpackage(sitedir, name, known_paths)'''
-
addsitepackages_script = '''\
def addsitepackages(known_paths):
- """Add site packages.
+ """Add site packages, as determined by zc.buildout.
- This function is written by buildout. See original_addsitepackages,
- below, for the original version."""%s
-%s paths = [
- # Eggs.
+ See original_addsitepackages, below, for the original version."""%s
+ buildout_paths = [
%s
- ]%s
- global addsitepackages
- addsitepackages = original_addsitepackages
+ ]
+ for path in buildout_paths:
+ sitedir, sitedircase = makepath(path)
+ if not sitedircase in known_paths and os.path.exists(sitedir):
+ sys.path.append(sitedir)
+ known_paths.add(sitedircase)%s
return known_paths
-buildout_addsitepackages = addsitepackages
-
def original_addsitepackages('''
def _generate_interpreter(name, dest, executable, site_py_dest,
@@ -1483,7 +1478,9 @@
"""Write an interpreter script, using the site.py approach."""
full_name = os.path.join(dest, name)
site_py_dest_string, rpsetup = _relative_path_and_setup(
- full_name, [site_py_dest], relative_paths)
+ full_name, [site_py_dest], relative_paths, omit_os_import=True)
+ if rpsetup:
+ rpsetup += "\n"
if sys.platform == 'win32':
windows_import = '\nimport subprocess'
# os.exec* is a mess on Windows, particularly if the path
@@ -1534,12 +1531,10 @@
)
return _write_script(full_name, contents, 'interpreter')
-interpreter_template = script_header + '''\
-
-%(relative_paths_setup)s
+interpreter_template = script_header + '''
import os
import sys%(windows_import)s
-
+%(relative_paths_setup)s
argv = [sys.executable] + sys.argv[1:]
environ = os.environ.copy()
path = %(site_dest)s
Modified: zc.buildout/branches/gary-4/src/zc/buildout/easy_install.txt
===================================================================
--- zc.buildout/branches/gary-4/src/zc/buildout/easy_install.txt 2010-02-12 15:33:00 UTC (rev 108952)
+++ zc.buildout/branches/gary-4/src/zc/buildout/easy_install.txt 2010-02-12 15:58:10 UTC (rev 108953)
@@ -997,8 +997,7 @@
site.py and sitecustomize.py on the PYTHONPATH and (re)starts Python.
>>> cat(py_path)
- #!/usr/bin/python2.4 -S
- <BLANKLINE>
+ #!/usr/bin/python -S
import os
import sys
<BLANKLINE>
@@ -1012,31 +1011,28 @@
The site.py file is a modified version of the underlying Python's site.py.
The most important modification is that it has a different version of the
-addsitepackages function. It has all of the trickier bits, and sets up the
-Python path, similarly to the behavior of the function it replaces. The
-following shows the part that buildout inserts.
+addsitepackages function. It sets up the Python path, similarly to the
+behavior of the function it replaces. The following shows the part that
+buildout inserts, in the simplest case.
>>> sys.stdout.write('#\n'); cat(site_path)
... # doctest: +ELLIPSIS +NORMALIZE_WHITESPACE
#...
def addsitepackages(known_paths):
- """Add site packages.
+ """Add site packages, as determined by zc.buildout.
<BLANKLINE>
- This function is written by buildout. See original_addsitepackages,
- below, for the original version."""
- paths = [
- # Eggs.
+ See original_addsitepackages, below, for the original version."""
+ buildout_paths = [
'/interpreter/eggs/demo-0.3-pyN.N.egg',
'/interpreter/eggs/demoneeded-1.1-pyN.N.egg'
]
- for path in paths:
- addsitedir(path, known_paths)
- global addsitepackages
- addsitepackages = original_addsitepackages
+ for path in buildout_paths:
+ sitedir, sitedircase = makepath(path)
+ if not sitedircase in known_paths and os.path.exists(sitedir):
+ sys.path.append(sitedir)
+ known_paths.add(sitedircase)
return known_paths
<BLANKLINE>
- buildout_addsitepackages = addsitepackages
- <BLANKLINE>
def original_addsitepackages(known_paths):...
Here are some examples of the interpreter in use.
@@ -1079,24 +1075,23 @@
bar baz bing shazam
<BLANKLINE>
-If you use relative paths, this affects the interpreter and site.py.
+If you use relative paths, this affects the interpreter and site.py. (This is
+again the UNIX version; the Windows version uses subprocess instead of
+os.execve.)
>>> reset_interpreter()
>>> generated = zc.buildout.easy_install.generate_scripts(
... interpreter_bin_dir, ws, sys.executable, interpreter_parts_dir,
... interpreter='py', relative_paths=interpreter_dir)
>>> cat(py_path)
- #!/usr/bin/python2.4 -S
- <BLANKLINE>
+ #!/usr/bin/python -S
import os
+ import sys
<BLANKLINE>
join = os.path.join
base = os.path.dirname(os.path.abspath(os.path.realpath(__file__)))
base = os.path.dirname(base)
<BLANKLINE>
- import os
- import sys
- <BLANKLINE>
argv = [sys.executable] + sys.argv[1:]
environ = os.environ.copy()
path = join(base, 'parts/interpreter')
@@ -1112,19 +1107,14 @@
>>> sys.stdout.write('#\n'); cat(site_path) # doctest: +ELLIPSIS
#...
def addsitepackages(known_paths):
- """Add site packages.
+ """Add site packages, as determined by zc.buildout.
<BLANKLINE>
- This function is written by buildout. See original_addsitepackages,
- below, for the original version."""
- <BLANKLINE>
- import os
- <BLANKLINE>
+ See original_addsitepackages, below, for the original version."""
join = os.path.join
base = os.path.dirname(os.path.abspath(os.path.realpath(__file__)))
base = os.path.dirname(base)
base = os.path.dirname(base)
- paths = [
- # Eggs.
+ buildout_paths = [
join(base, 'eggs/demo-0.3-pyN.N.egg'),
join(base, 'eggs/demoneeded-1.1-pyN.N.egg')
]...
@@ -1152,12 +1142,10 @@
>>> sys.stdout.write('#\n'); cat(site_path) # doctest: +ELLIPSIS
#...
def addsitepackages(known_paths):
- """Add site packages.
+ """Add site packages, as determined by zc.buildout.
<BLANKLINE>
- This function is written by buildout. See original_addsitepackages,
- below, for the original version."""
- paths = [
- # Eggs.
+ See original_addsitepackages, below, for the original version."""
+ buildout_paths = [
'/interpreter/eggs/demo-0.3-pyN.N.egg',
'/interpreter/eggs/demoneeded-1.1-pyN.N.egg',
'/interpreter/other'
@@ -1213,19 +1201,29 @@
... # doctest: +ELLIPSIS +NORMALIZE_WHITESPACE
#...
def addsitepackages(known_paths):
- """Add site packages.
+ """Add site packages, as determined by zc.buildout.
<BLANKLINE>
- This function is written by buildout. See original_addsitepackages,
- below, for the original version."""
- paths = [
- # Eggs.
+ See original_addsitepackages, below, for the original version."""
+ buildout_paths = [
'/interpreter/eggs/demo-0.3-pyN.N.egg',
- '/interpreter/eggs/demoneeded-1.1-pyN.N.egg',
- # These are the underlying Python's site-packages.
+ '/interpreter/eggs/demoneeded-1.1-pyN.N.egg'
+ ]
+ for path in buildout_paths:
+ sitedir, sitedircase = makepath(path)
+ if not sitedircase in known_paths and os.path.exists(sitedir):
+ sys.path.append(sitedir)
+ known_paths.add(sitedircase)
+ original_paths = [
...
- ]...
+ ]
+ for path in original_paths:
+ addsitedir(path, known_paths)
+ return known_paths
+ <BLANKLINE>
+ def original_addsitepackages(known_paths):...
-It simply adds the site-packages after the eggs.
+It simply adds the original paths using addsitedir after the code to add the
+buildout paths.
Here's an example of the new script in use. Other documents and tests in
this package give the feature a more thorough workout, but this should
@@ -1271,45 +1269,37 @@
... # doctest: +ELLIPSIS +NORMALIZE_WHITESPACE
#...
def addsitepackages(known_paths):
- """Add site packages.
+ """Add site packages, as determined by zc.buildout.
<BLANKLINE>
- This function is written by buildout. See original_addsitepackages,
- below, for the original version."""
+ See original_addsitepackages, below, for the original version."""
setuptools_path = '...setuptools...'
sys.path.append(setuptools_path)
- known_paths.add(setuptools_path)
+ known_paths.add(os.path.normcase(setuptools_path))
import pkg_resources
- paths = [
- # Eggs.
+ buildout_paths = [
'/interpreter/eggs/demo-0.3-pyN.N.egg',
'/interpreter/eggs/tellmy.fortune-1.0-pyN.N.egg',
'...setuptools...',
- '/interpreter/eggs/demoneeded-1.1-pyN.N.egg',
- # These are the underlying Python's site-packages.
- ...
+ '/interpreter/eggs/demoneeded-1.1-pyN.N.egg'
]
- dotpth = os.extsep + "pth"
- for path in paths:
- # This duplicates addsitedir except for adding the pkg_resources call.
+ for path in buildout_paths:
sitedir, sitedircase = makepath(path)
if not sitedircase in known_paths and os.path.exists(sitedir):
sys.path.append(sitedir)
- pkg_resources.working_set.add_entry(sitedir)
known_paths.add(sitedircase)
- try:
- names = os.listdir(sitedir)
- except os.error:
- continue
- names = [name for name in names if name.endswith(dotpth)]
- names.sort()
- for name in names:
- addpackage(sitedir, name, known_paths)
- global addsitepackages
- addsitepackages = original_addsitepackages
+ for dist in pkg_resources.find_distributions(sitedir, True):
+ pkg_resources.fixup_namespace_packages(dist.location)
+ if dist.has_metadata('namespace_packages.txt'):
+ for namespace in dist.get_metadata_lines(
+ 'namespace_packages.txt'):
+ pkg_resources.declare_namespace(namespace)
+ original_paths = [
+ ...
+ ]
+ for path in original_paths:
+ addsitedir(path, known_paths)
return known_paths
<BLANKLINE>
- buildout_addsitepackages = addsitepackages
- <BLANKLINE>
def original_addsitepackages(known_paths):...
>>> print call_py(interpreter_path, "import sys; print sys.path")
@@ -1330,6 +1320,66 @@
implementation for system packages, as mentioned above, and that must
come after processing egg namespaces.
+The most complex that this function gets is if you use namespace packages,
+include site-packages, and use relative paths. For completeness, we'll look
+at that result.
+
+ >>> generated = zc.buildout.easy_install.generate_scripts(
+ ... interpreter_bin_dir, ws, sys.executable, interpreter_parts_dir,
+ ... interpreter='py', add_site_packages=True,
+ ... relative_paths=interpreter_dir)
+ >>> sys.stdout.write('#\n'); cat(site_path)
+ ... # doctest: +ELLIPSIS +NORMALIZE_WHITESPACE
+ #...
+ def addsitepackages(known_paths):
+ """Add site packages, as determined by zc.buildout.
+ <BLANKLINE>
+ See original_addsitepackages, below, for the original version."""
+ join = os.path.join
+ base = os.path.dirname(os.path.abspath(os.path.realpath(__file__)))
+ base = os.path.dirname(base)
+ base = os.path.dirname(base)
+ setuptools_path = '...setuptools...'
+ sys.path.append(setuptools_path)
+ known_paths.add(os.path.normcase(setuptools_path))
+ import pkg_resources
+ buildout_paths = [
+ join(base, 'eggs/demo-0.3-pyN.N.egg'),
+ join(base, 'eggs/tellmy.fortune-1.0-pyN.N.egg'),
+ '...setuptools...',
+ join(base, 'eggs/demoneeded-1.1-pyN.N.egg')
+ ]
+ for path in buildout_paths:
+ sitedir, sitedircase = makepath(path)
+ if not sitedircase in known_paths and os.path.exists(sitedir):
+ sys.path.append(sitedir)
+ known_paths.add(sitedircase)
+ for dist in pkg_resources.find_distributions(sitedir, True):
+ pkg_resources.fixup_namespace_packages(dist.location)
+ if dist.has_metadata('namespace_packages.txt'):
+ for namespace in dist.get_metadata_lines(
+ 'namespace_packages.txt'):
+ pkg_resources.declare_namespace(namespace)
+ original_paths = [
+ ...
+ ]
+ for path in original_paths:
+ addsitedir(path, known_paths)
+ return known_paths
+ <BLANKLINE>
+ def original_addsitepackages(known_paths):...
+
+ >>> print call_py(interpreter_path, "import sys; print sys.path")
+ ... # doctest: +ELLIPSIS +NORMALIZE_WHITESPACE
+ ['',
+ '/interpreter/parts/interpreter',
+ ...,
+ '...setuptools...',
+ '/interpreter/eggs/demo-0.3-pyN.N.egg',
+ '/interpreter/eggs/tellmy.fortune-1.0-pyN.N.egg',
+ '/interpreter/eggs/demoneeded-1.1-pyN.N.egg',
+ ...]
+
The ``exec_sitecustomize`` argument does the same thing for the
sitecustomize module--it allows you to include the code from the
sitecustomize module in the underlying Python if you set the argument to
Modified: zc.buildout/branches/gary-4/src/zc/buildout/testing.py
===================================================================
--- zc.buildout/branches/gary-4/src/zc/buildout/testing.py 2010-02-12 15:33:00 UTC (rev 108952)
+++ zc.buildout/branches/gary-4/src/zc/buildout/testing.py 2010-02-12 15:58:10 UTC (rev 108953)
@@ -310,6 +310,16 @@
old_wd = os.getcwd()
os.chdir(buildout)
make_buildout()
+ # Normally we don't process .pth files in extra-paths. We want to
+ # in this case so that we can test with setuptools system installs
+ # (--single-version-externally-managed), which use .pth files.
+ initialization = (
+ ('import sys\n'
+ 'import site\n'
+ 'known_paths = set(sys.path)\n'
+ 'site_packages_dir = %r\n'
+ 'site.addsitedir(site_packages_dir, known_paths)\n'
+ ) % (site_packages_dir,)) + initialization
initialization = '\n'.join(
' ' + line for line in initialization.split('\n'))
install_develop(
Modified: zc.buildout/branches/gary-4/z3c.recipe.scripts_/src/z3c/recipe/scripts/README.txt
===================================================================
--- zc.buildout/branches/gary-4/z3c.recipe.scripts_/src/z3c/recipe/scripts/README.txt 2010-02-12 15:33:00 UTC (rev 108952)
+++ zc.buildout/branches/gary-4/z3c.recipe.scripts_/src/z3c/recipe/scripts/README.txt 2010-02-12 15:58:10 UTC (rev 108953)
@@ -273,11 +273,11 @@
Generated interpreter '/sample-buildout/bin/py'.
>>> cat(sample_buildout, 'parts', 'py', 'sitecustomize.py')
- ... # doctest: +NORMALIZE_WHITESPACE
+ ... # doctest: +NORMALIZE_WHITESPACE +ELLIPSIS
<BLANKLINE>
# The following is from
# /executable_buildout/parts/py/sitecustomize.py
- <BLANKLINE>
+ ...
import os
os.environ['zc.buildout'] = 'foo bar baz shazam'
Modified: zc.buildout/branches/gary-4/z3c.recipe.scripts_/src/z3c/recipe/scripts/tests.py
===================================================================
--- zc.buildout/branches/gary-4/z3c.recipe.scripts_/src/z3c/recipe/scripts/tests.py 2010-02-12 15:33:00 UTC (rev 108952)
+++ zc.buildout/branches/gary-4/z3c.recipe.scripts_/src/z3c/recipe/scripts/tests.py 2010-02-12 15:58:10 UTC (rev 108953)
@@ -175,14 +175,14 @@
Generated interpreter '/sample-buildout/bin/py'.
>>> cat(sample_buildout, 'parts', 'py', 'sitecustomize.py')
- ... # doctest: +NORMALIZE_WHITESPACE
+ ... # doctest: +NORMALIZE_WHITESPACE +ELLIPSIS
<BLANKLINE>
import os
os.environ['zc.recipe.egg'] = 'baLOOba'
<BLANKLINE>
# The following is from
# /executable_buildout/parts/py/sitecustomize.py
- <BLANKLINE>
+ ...
import os
os.environ['zc.buildout'] = 'foo bar baz shazam'
@@ -227,15 +227,11 @@
#...
def addsitepackages(known_paths):
"..."
- <BLANKLINE>
- import os
- <BLANKLINE>
join = os.path.join
base = os.path.dirname(os.path.abspath(os.path.realpath(__file__)))
base = os.path.dirname(base)
base = os.path.dirname(base)
- paths = [
- # Eggs.
+ buildout_paths = [
'/foo/bar',
join(base, 'spam')
]...
More information about the checkins
mailing list