[Checkins] SVN: zc.buildout/branches/ctheune-buildout-parallelfetch/ something that works with 4 threads
Christian Theune
ct at gocept.com
Fri Apr 8 09:41:37 EDT 2011
Log message for revision 121350:
something that works with 4 threads
Changed:
U zc.buildout/branches/ctheune-buildout-parallelfetch/buildout.cfg
U zc.buildout/branches/ctheune-buildout-parallelfetch/src/zc/buildout/download.py
U zc.buildout/branches/ctheune-buildout-parallelfetch/src/zc/buildout/easy_install.py
U zc.buildout/branches/ctheune-buildout-parallelfetch/src/zc/buildout/testselectingpython.py
-=-
Modified: zc.buildout/branches/ctheune-buildout-parallelfetch/buildout.cfg
===================================================================
--- zc.buildout/branches/ctheune-buildout-parallelfetch/buildout.cfg 2011-04-08 12:19:26 UTC (rev 121349)
+++ zc.buildout/branches/ctheune-buildout-parallelfetch/buildout.cfg 2011-04-08 13:41:36 UTC (rev 121350)
@@ -1,7 +1,13 @@
[buildout]
develop = zc.recipe.egg_ z3c.recipe.scripts_ .
-parts = test oltest py
+parts = test oltest py
+eggs-dir = eggs
+[plone]
+recipe = zc.recipe.egg
+eggs = Plone
+interpreter = plonepy
+
[py]
recipe = z3c.recipe.scripts
eggs = zc.buildout
Modified: zc.buildout/branches/ctheune-buildout-parallelfetch/src/zc/buildout/download.py
===================================================================
--- zc.buildout/branches/ctheune-buildout-parallelfetch/src/zc/buildout/download.py 2011-04-08 12:19:26 UTC (rev 121349)
+++ zc.buildout/branches/ctheune-buildout-parallelfetch/src/zc/buildout/download.py 2011-04-08 13:41:36 UTC (rev 121350)
@@ -175,17 +175,18 @@
urllib._urlopener = url_opener
handle, tmp_path = tempfile.mkstemp(prefix='buildout-')
try:
- tmp_path, headers = urllib.urlretrieve(url, tmp_path)
- if not check_md5sum(tmp_path, md5sum):
- raise ChecksumError(
- 'MD5 checksum mismatch downloading %r' % url)
- except IOError, e:
- os.remove(tmp_path)
- raise zc.buildout.UserError("Error downloading extends for URL "
- "%s: %r" % (url, e[1:3]))
- except Exception, e:
- os.remove(tmp_path)
- raise
+ try:
+ tmp_path, headers = urllib.urlretrieve(url, tmp_path)
+ if not check_md5sum(tmp_path, md5sum):
+ raise ChecksumError(
+ 'MD5 checksum mismatch downloading %r' % url)
+ except IOError, e:
+ os.remove(tmp_path)
+ raise zc.buildout.UserError("Error downloading extends for URL "
+ "%s: %r" % (url, e[1:3]))
+ except Exception, e:
+ os.remove(tmp_path)
+ raise
finally:
os.close(handle)
Modified: zc.buildout/branches/ctheune-buildout-parallelfetch/src/zc/buildout/easy_install.py
===================================================================
--- zc.buildout/branches/ctheune-buildout-parallelfetch/src/zc/buildout/easy_install.py 2011-04-08 12:19:26 UTC (rev 121349)
+++ zc.buildout/branches/ctheune-buildout-parallelfetch/src/zc/buildout/easy_install.py 2011-04-08 13:41:36 UTC (rev 121350)
@@ -18,6 +18,9 @@
installed.
"""
+import time
+import Queue
+import threading
import distutils.errors
import fnmatch
import glob
@@ -715,7 +718,6 @@
return dist.clone(location=new_location)
def _get_dist(self, requirement, ws, always_unzip):
-
__doing__ = 'Getting distribution for %r.', str(requirement)
# Maybe an existing dist is already the best dist that satisfies the
@@ -910,38 +912,69 @@
# Note that we don't use the existing environment, because we want
# to look for new eggs unless what we have is the best that
# matches the requirement.
+ get_queue = Queue.Queue()
+ response_queue = Queue.Queue()
+
+ worker_state = {'busy': 0}
+ seen = set()
+
+ def worker():
+ while True:
+ req, ws, unzip = get_queue.get()
+ worker_state['busy'] += 1
+ response_queue.put((req, self._get_dist(req, ws, unzip)))
+ worker_state['busy'] -= 1
+ get_queue.task_done()
+
+ for i in range(4):
+ t = threading.Thread(target=worker)
+ t.daemon = True
+ t.start()
+
env = pkg_resources.Environment(ws.entries)
- while requirements:
- # Process dependencies breadth-first.
- req = self._constrain(requirements.pop(0))
- if req in processed:
- # Ignore cyclic or redundant dependencies.
- continue
- dist = best.get(req.key)
- if dist is None:
- # Find the best distribution and add it to the map.
- dist = ws.by_key.get(req.key)
+ last = time.time()
+ while (requirements or get_queue.unfinished_tasks or
+ response_queue.unfinished_tasks):
+ if time.time()-last > 1:
+ print "Workers:", worker_state['busy'], "Queue:", get_queue.qsize()
+ last = time.time()
+ if requirements:
+ req = self._constrain(requirements.pop(0))
+ if req in processed:
+ # Ignore cyclic or redundant dependencies.
+ continue
+ dist = best.get(req.key)
if dist is None:
- try:
- dist = best[req.key] = env.best_match(req, ws)
- except pkg_resources.VersionConflict, err:
- raise VersionConflict(err, ws)
- if dist is None or (
- dist.location in self._site_packages and not
- self.allow_site_package_egg(dist.project_name)):
- # If we didn't find a distribution in the
- # environment, or what we found is from site
- # packages and not allowed to be there, try
- # again.
- if destination:
- logger.debug('Getting required %r', str(req))
- else:
- logger.debug('Adding required %r', str(req))
- _log_requirement(ws, req)
- for dist in self._get_dist(req,
- ws, self._always_unzip):
- ws.add(dist)
- self._maybe_add_setuptools(ws, dist)
+ # Find the best distribution and add it to the map.
+ dist = ws.by_key.get(req.key)
+ if dist is None:
+ try:
+ dist = best[req.key] = env.best_match(req, ws)
+ except pkg_resources.VersionConflict, err:
+ raise VersionConflict(err, ws)
+ if dist is None or (
+ dist.location in self._site_packages and not
+ self.allow_site_package_egg(dist.project_name)):
+ # If we didn't find a distribution in the
+ # environment, or what we found is from site
+ # packages and not allowed to be there, try
+ # again.
+ if destination:
+ logger.debug('Getting required %r', str(req))
+ else:
+ logger.debug('Adding required %r', str(req))
+ _log_requirement(ws, req)
+ if req not in seen:
+ seen.add(req)
+ get_queue.put((req, ws, self._always_unzip))
+ continue
+ else:
+ req, dists = response_queue.get()
+ for dist in dists:
+ ws.add(dist)
+ self._maybe_add_setuptools(ws, dist)
+ response_queue.task_done()
+
if dist not in req:
# Oops, the "best" so far conflicts with a dependency.
raise VersionConflict(
Modified: zc.buildout/branches/ctheune-buildout-parallelfetch/src/zc/buildout/testselectingpython.py
===================================================================
--- zc.buildout/branches/ctheune-buildout-parallelfetch/src/zc/buildout/testselectingpython.py 2011-04-08 12:19:26 UTC (rev 121349)
+++ zc.buildout/branches/ctheune-buildout-parallelfetch/src/zc/buildout/testselectingpython.py 2011-04-08 13:41:36 UTC (rev 121350)
@@ -15,8 +15,8 @@
from zope.testing import renormalizing
import zc.buildout.tests
import zc.buildout.testing
+import pkg_resources
-
if sys.version_info[:2] == (2, 5):
other_version = "2.6"
else:
@@ -59,13 +59,36 @@
executable_dir = test.globs['tmpdir']('executable_dir')
executable_parts = os.path.join(executable_dir, 'parts')
test.globs['mkdir'](executable_parts)
- ws = zc.buildout.easy_install.install(
- ['setuptools'], executable_dir,
- index='http://www.python.org/pypi/',
- always_unzip=True, executable=other_executable)
- zc.buildout.easy_install.sitepackage_safe_scripts(
- executable_dir, ws, other_executable, executable_parts,
- reqs=['setuptools'], interpreter='py')
+
+ current_buildout_ws = [
+ x for x in pkg_resources.WorkingSet()
+ if x.project_name == 'zc.buildout'][0].location
+ ez_setup_other_path = os.path.join(test.globs['tmpdir']('ez_setup'), 'ez_setup_other.py')
+ ez_setup_other = open(ez_setup_other_path, 'w')
+ ez_setup_other.write(textwrap.dedent('''\
+import urllib2
+ez_code = urllib2.urlopen(
+ 'http://python-distribute.org/distribute_setup.py').read().replace('\\r\\n', '\\n')
+ez = {}
+exec ez_code in ez
+ez['use_setuptools'](to_dir='%(executable_dir)s', download_delay=0, no_fake=True)
+import pkg_resources
+print list(pkg_resources.WorkingSet())[0].__dict__
+import sys
+#ws = pkg_resources.WorkingSet()
+sys.path.insert(0, '%(buildout_location)s')
+print '%(buildout_location)s'
+import pdb; pdb.set_trace()
+import zc.buildout.easy_install
+zc.buildout.easy_install.sitepackage_safe_scripts(
+ '%(executable_dir)s', ws, sys.executable, '%(parts)s',
+ reqs=['setuptools'], interpreter='py')
+ ''' % dict(executable_dir=executable_dir,
+ buildout_location=current_buildout_ws,
+ parts=executable_parts)))
+ ez_setup_other.close()
+ assert not subprocess.call([other_executable, ez_setup_other_path])
+ import pdb; pdb.set_trace()
original_executable = other_executable
other_executable = os.path.join(executable_dir, 'py')
assert not subprocess.call(
More information about the checkins
mailing list