[Checkins] SVN: zc.buildout/branches/ctheune-buildout-parallelfetch/ something that works with 4 threads

Christian Theune ct at gocept.com
Fri Apr 8 09:41:37 EDT 2011


Log message for revision 121350:
  something that works with 4 threads
  

Changed:
  U   zc.buildout/branches/ctheune-buildout-parallelfetch/buildout.cfg
  U   zc.buildout/branches/ctheune-buildout-parallelfetch/src/zc/buildout/download.py
  U   zc.buildout/branches/ctheune-buildout-parallelfetch/src/zc/buildout/easy_install.py
  U   zc.buildout/branches/ctheune-buildout-parallelfetch/src/zc/buildout/testselectingpython.py

-=-
Modified: zc.buildout/branches/ctheune-buildout-parallelfetch/buildout.cfg
===================================================================
--- zc.buildout/branches/ctheune-buildout-parallelfetch/buildout.cfg	2011-04-08 12:19:26 UTC (rev 121349)
+++ zc.buildout/branches/ctheune-buildout-parallelfetch/buildout.cfg	2011-04-08 13:41:36 UTC (rev 121350)
@@ -1,7 +1,13 @@
 [buildout]
 develop = zc.recipe.egg_ z3c.recipe.scripts_ .
-parts = test oltest py
+parts = test oltest py 
+eggs-dir = eggs
 
+[plone]
+recipe = zc.recipe.egg
+eggs = Plone
+interpreter = plonepy
+
 [py]
 recipe = z3c.recipe.scripts
 eggs = zc.buildout

Modified: zc.buildout/branches/ctheune-buildout-parallelfetch/src/zc/buildout/download.py
===================================================================
--- zc.buildout/branches/ctheune-buildout-parallelfetch/src/zc/buildout/download.py	2011-04-08 12:19:26 UTC (rev 121349)
+++ zc.buildout/branches/ctheune-buildout-parallelfetch/src/zc/buildout/download.py	2011-04-08 13:41:36 UTC (rev 121350)
@@ -175,17 +175,18 @@
         urllib._urlopener = url_opener
         handle, tmp_path = tempfile.mkstemp(prefix='buildout-')
         try:
-            tmp_path, headers = urllib.urlretrieve(url, tmp_path)
-            if not check_md5sum(tmp_path, md5sum):
-                raise ChecksumError(
-                    'MD5 checksum mismatch downloading %r' % url)
-        except IOError, e:
-            os.remove(tmp_path)
-            raise zc.buildout.UserError("Error downloading extends for URL "
-                              "%s: %r" % (url, e[1:3]))
-        except Exception, e:
-            os.remove(tmp_path)
-            raise
+            try:
+                tmp_path, headers = urllib.urlretrieve(url, tmp_path)
+                if not check_md5sum(tmp_path, md5sum):
+                    raise ChecksumError(
+                        'MD5 checksum mismatch downloading %r' % url)
+            except IOError, e:
+                os.remove(tmp_path)
+                raise zc.buildout.UserError("Error downloading extends for URL "
+                                  "%s: %r" % (url, e[1:3]))
+            except Exception, e:
+                os.remove(tmp_path)
+                raise
         finally:
             os.close(handle)
 

Modified: zc.buildout/branches/ctheune-buildout-parallelfetch/src/zc/buildout/easy_install.py
===================================================================
--- zc.buildout/branches/ctheune-buildout-parallelfetch/src/zc/buildout/easy_install.py	2011-04-08 12:19:26 UTC (rev 121349)
+++ zc.buildout/branches/ctheune-buildout-parallelfetch/src/zc/buildout/easy_install.py	2011-04-08 13:41:36 UTC (rev 121350)
@@ -18,6 +18,9 @@
 installed.
 """
 
+import time
+import Queue
+import threading
 import distutils.errors
 import fnmatch
 import glob
@@ -715,7 +718,6 @@
         return dist.clone(location=new_location)
 
     def _get_dist(self, requirement, ws, always_unzip):
-
         __doing__ = 'Getting distribution for %r.', str(requirement)
 
         # Maybe an existing dist is already the best dist that satisfies the
@@ -910,38 +912,69 @@
         # Note that we don't use the existing environment, because we want
         # to look for new eggs unless what we have is the best that
         # matches the requirement.
+        get_queue = Queue.Queue()
+        response_queue = Queue.Queue()
+
+        worker_state = {'busy': 0}
+        seen = set()
+
+        def worker():
+            while True:
+                req, ws, unzip = get_queue.get()
+                worker_state['busy'] += 1
+                response_queue.put((req, self._get_dist(req, ws, unzip)))
+                worker_state['busy'] -= 1
+                get_queue.task_done()
+
+        for i in range(4):
+             t = threading.Thread(target=worker)
+             t.daemon = True
+             t.start()
+
         env = pkg_resources.Environment(ws.entries)
-        while requirements:
-            # Process dependencies breadth-first.
-            req = self._constrain(requirements.pop(0))
-            if req in processed:
-                # Ignore cyclic or redundant dependencies.
-                continue
-            dist = best.get(req.key)
-            if dist is None:
-                # Find the best distribution and add it to the map.
-                dist = ws.by_key.get(req.key)
+        last = time.time()
+        while (requirements or get_queue.unfinished_tasks or
+            response_queue.unfinished_tasks):
+            if time.time()-last > 1:
+                print "Workers:", worker_state['busy'], "Queue:", get_queue.qsize()
+                last = time.time()
+            if requirements:
+                req = self._constrain(requirements.pop(0))
+                if req in processed:
+                    # Ignore cyclic or redundant dependencies.
+                    continue
+                dist = best.get(req.key)
                 if dist is None:
-                    try:
-                        dist = best[req.key] = env.best_match(req, ws)
-                    except pkg_resources.VersionConflict, err:
-                        raise VersionConflict(err, ws)
-                    if dist is None or (
-                        dist.location in self._site_packages and not
-                        self.allow_site_package_egg(dist.project_name)):
-                        # If we didn't find a distribution in the
-                        # environment, or what we found is from site
-                        # packages and not allowed to be there, try
-                        # again.
-                        if destination:
-                            logger.debug('Getting required %r', str(req))
-                        else:
-                            logger.debug('Adding required %r', str(req))
-                        _log_requirement(ws, req)
-                        for dist in self._get_dist(req,
-                                                   ws, self._always_unzip):
-                            ws.add(dist)
-                            self._maybe_add_setuptools(ws, dist)
+                    # Find the best distribution and add it to the map.
+                    dist = ws.by_key.get(req.key)
+                    if dist is None:
+                        try:
+                            dist = best[req.key] = env.best_match(req, ws)
+                        except pkg_resources.VersionConflict, err:
+                            raise VersionConflict(err, ws)
+                        if dist is None or (
+                            dist.location in self._site_packages and not
+                            self.allow_site_package_egg(dist.project_name)):
+                            # If we didn't find a distribution in the
+                            # environment, or what we found is from site
+                            # packages and not allowed to be there, try
+                            # again.
+                            if destination:
+                                logger.debug('Getting required %r', str(req))
+                            else:
+                                logger.debug('Adding required %r', str(req))
+                            _log_requirement(ws, req)
+                            if req not in seen:
+                                seen.add(req)
+                                get_queue.put((req, ws, self._always_unzip))
+                            continue
+            else:
+                req, dists = response_queue.get()
+                for dist in dists:
+                    ws.add(dist)
+                    self._maybe_add_setuptools(ws, dist)
+                response_queue.task_done()
+
             if dist not in req:
                 # Oops, the "best" so far conflicts with a dependency.
                 raise VersionConflict(

Modified: zc.buildout/branches/ctheune-buildout-parallelfetch/src/zc/buildout/testselectingpython.py
===================================================================
--- zc.buildout/branches/ctheune-buildout-parallelfetch/src/zc/buildout/testselectingpython.py	2011-04-08 12:19:26 UTC (rev 121349)
+++ zc.buildout/branches/ctheune-buildout-parallelfetch/src/zc/buildout/testselectingpython.py	2011-04-08 13:41:36 UTC (rev 121350)
@@ -15,8 +15,8 @@
 from zope.testing import renormalizing
 import zc.buildout.tests
 import zc.buildout.testing
+import pkg_resources
 
-
 if sys.version_info[:2] == (2, 5):
     other_version = "2.6"
 else:
@@ -59,13 +59,36 @@
         executable_dir = test.globs['tmpdir']('executable_dir')
         executable_parts = os.path.join(executable_dir, 'parts')
         test.globs['mkdir'](executable_parts)
-        ws = zc.buildout.easy_install.install(
-            ['setuptools'], executable_dir,
-            index='http://www.python.org/pypi/',
-            always_unzip=True, executable=other_executable)
-        zc.buildout.easy_install.sitepackage_safe_scripts(
-            executable_dir, ws, other_executable, executable_parts,
-            reqs=['setuptools'], interpreter='py')
+
+        current_buildout_ws = [
+            x for x in pkg_resources.WorkingSet()
+            if x.project_name == 'zc.buildout'][0].location
+        ez_setup_other_path = os.path.join(test.globs['tmpdir']('ez_setup'), 'ez_setup_other.py')
+        ez_setup_other = open(ez_setup_other_path, 'w')
+        ez_setup_other.write(textwrap.dedent('''\
+import urllib2
+ez_code = urllib2.urlopen(
+    'http://python-distribute.org/distribute_setup.py').read().replace('\\r\\n', '\\n')
+ez = {}
+exec ez_code in ez
+ez['use_setuptools'](to_dir='%(executable_dir)s', download_delay=0, no_fake=True)
+import pkg_resources
+print list(pkg_resources.WorkingSet())[0].__dict__
+import sys
+#ws = pkg_resources.WorkingSet()
+sys.path.insert(0, '%(buildout_location)s')
+print '%(buildout_location)s'
+import pdb; pdb.set_trace()
+import zc.buildout.easy_install
+zc.buildout.easy_install.sitepackage_safe_scripts(
+    '%(executable_dir)s', ws, sys.executable, '%(parts)s',
+    reqs=['setuptools'], interpreter='py')
+            ''' % dict(executable_dir=executable_dir,
+                       buildout_location=current_buildout_ws,
+                       parts=executable_parts)))
+        ez_setup_other.close()
+        assert not subprocess.call([other_executable, ez_setup_other_path])
+        import pdb; pdb.set_trace() 
         original_executable = other_executable
         other_executable = os.path.join(executable_dir, 'py')
         assert not subprocess.call(



More information about the checkins mailing list