[Checkins] SVN: zc.buildout/branches/env-cache/src/zc/buildout/easy_install.py Add a pkg_resources.Environment cache, which are very expensive to

[Ross Patterson]

Log message for revision 124114:
  Add a pkg_resources.Environment cache, which are very expensive to
  instantiate.
  
  They're very expensive to instantiate since they scan the filesystem
  for possible dists every time.  This optimization helps a great deal
  where buildouts have multiple parts with identical required dists, but
  may also help in cases where multiple parts have overlapping required
  dists.

Changed:
  U   zc.buildout/branches/env-cache/src/zc/buildout/easy_install.py

-=-
Modified: zc.buildout/branches/env-cache/src/zc/buildout/easy_install.py
===================================================================
--- zc.buildout/branches/env-cache/src/zc/buildout/easy_install.py	2012-01-21 08:42:38 UTC (rev 124113)
+++ zc.buildout/branches/env-cache/src/zc/buildout/easy_install.py	2012-01-21 08:54:36 UTC (rev 124114)
@@ -231,8 +231,23 @@
     _indexes[key] = index
     return index
 
-clear_index_cache = _indexes.clear
 
+_envs = {}
+def _get_env(executable, path=None):
+    key = executable, tuple(path)
+    env = _envs.get(key)
+    python = _get_version(executable)
+    if env is None:
+        _envs[key] = env = pkg_resources.Environment(
+            search_path=path, python=python)
+    return env
+
+
+def clear_index_cache():
+    _indexes.clear()
+    _envs.clear()
+
+
 if is_win32:
     # work around spawn lamosity on windows
     # XXX need safe quoting (see the subprocess.list2cmdline) and test
@@ -395,8 +410,7 @@
         if self._dest is None:
             newest = False
         self._newest = newest
-        self._env = pkg_resources.Environment(path,
-                                              python=_get_version(executable))
+        self._env = _get_env(executable, path)
         self._index = _get_index(executable, index, links, self._allow_hosts,
                                  self._path)
 
@@ -526,10 +540,7 @@
         return best_we_have, None
 
     def _load_dist(self, dist):
-        dists = pkg_resources.Environment(
-            dist.location,
-            python=_get_version(self._executable),
-            )[dist.project_name]
+        dists = _get_env(self._executable, dist.location,)[dist.project_name]
         assert len(dists) == 1
         return dists[0]
 
@@ -573,10 +584,7 @@
                     *args)
 
             dists = []
-            env = pkg_resources.Environment(
-                [tmp],
-                python=_get_version(self._executable),
-                )
+            env = _get_env(self._executable, [tmp])
             for project in env:
                 dists.extend(env[project])
 
@@ -619,12 +627,7 @@
                     else:
                         os.remove(newloc)
                 os.rename(d.location, newloc)
-
-                [d] = pkg_resources.Environment(
-                    [newloc],
-                    python=_get_version(self._executable),
-                    )[d.project_name]
-
+                [d] = _get_env(self._executable, [newloc])[d.project_name]
                 result.append(d)
 
             return result
@@ -780,10 +783,8 @@
                     # Getting the dist from the environment causes the
                     # distribution meta data to be read.  Cloning isn't
                     # good enough.
-                    dists = pkg_resources.Environment(
-                        [newloc],
-                        python=_get_version(self._executable),
-                        )[dist.project_name]
+                    dists = _get_env(self._executable, [newloc]
+                                     )[dist.project_name]
                 else:
                     # It's some other kind of dist.  We'll let easy_install
                     # deal with it:
@@ -910,7 +911,7 @@
         # Note that we don't use the existing environment, because we want
         # to look for new eggs unless what we have is the best that
         # matches the requirement.
-        env = pkg_resources.Environment(ws.entries)
+        env = _get_env(self._executable, ws.entries)
         while requirements:
             # Process dependencies breadth-first.
             req = self._constrain(requirements.pop(0))