[Checkins] SVN: zc.buildout/branches/downloadcache/ - snapshot of
the first little work to get the first download cache in
Christian Theune
ct at gocept.com
Wed Mar 7 15:59:23 EST 2007
Log message for revision 73039:
- snapshot of the first little work to get the first download cache in
Changed:
U zc.buildout/branches/downloadcache/src/zc/buildout/easy_install.py
U zc.buildout/branches/downloadcache/todo.txt
-=-
Modified: zc.buildout/branches/downloadcache/src/zc/buildout/easy_install.py
===================================================================
--- zc.buildout/branches/downloadcache/src/zc/buildout/easy_install.py 2007-03-07 20:57:23 UTC (rev 73038)
+++ zc.buildout/branches/downloadcache/src/zc/buildout/easy_install.py 2007-03-07 20:59:22 UTC (rev 73039)
@@ -117,7 +117,9 @@
path=None,
newest=True,
versions=None,
+ download_cache=None,
):
+ self._download_cache = download_cache
self._dest = dest
self._links = list(links)
self._index_url = index
@@ -257,6 +259,16 @@
if self._dest is not None:
logger.info("Getting new distribution for %s", requirement)
+ if self._download_cache is None:
+ # There is no persistent download cache defined,
+ # so we create one that will be removed later on again.
+ download_cache = tempfile.mkdtemp('download_cache')
+ cleanup_download_cache = (
+ lambda:shutil.rmtree(download_cache))
+ else:
+ download_cache = self._download_cache
+ cleanup_download_cache = lambda:None
+
# Retrieve the dist:grokonepage
index = self._index
dist = index.obtain(requirement)
@@ -265,15 +277,23 @@
"Couldn't find a distribution for %s."
% requirement)
- fname = dist.location
- if url_match(fname):
- fname = urlparse.urlparse(fname)[2]
+ try:
+ # Retrieve the dist:
+ index = self._index
+ dist = index.obtain(requirement)
+ if dist is None:
+ raise zc.buildout.UserError(
+ "Couldn't find a distribution for %s."
+ % requirement)
- if fname.endswith('.egg'):
- # It's already an egg, just fetch it into the dest
- tmp = tempfile.mkdtemp('get_dist')
- try:
- dist = index.fetch_distribution(requirement, tmp)
+ fname = dist.location
+ if url_match(fname):
+ fname = urlparse.urlparse(fname)[2]
+
+ if fname.endswith('.egg'):
+ # It's already an egg, just fetch it into the dest
+ dist = index.fetch_distribution(requirement,
+ download_cache)
if dist is None:
raise zc.buildout.UserError(
"Couln't download a distribution for %s."
@@ -304,24 +324,18 @@
dist.location, newloc)
else:
shutil.copyfile(dist.location, newloc)
+ else:
+ # It's some other kind of dist. We'll download it to
+ # a temporary directory and let easy_install have it's
+ # way with it:
+ dist = index.fetch_distribution(requirement,
+ download_cache)
- finally:
- shutil.rmtree(tmp)
-
- else:
- # It's some other kind of dist. We'll download it to
- # a temporary directory and let easy_install have it's
- # way with it:
- tmp = tempfile.mkdtemp('get_dist')
- try:
- dist = index.fetch_distribution(requirement, tmp)
-
# May need a new one. Call easy_install
self._call_easy_install(dist.location, ws, self._dest)
- finally:
- shutil.rmtree(tmp)
+ finally:
+ cleanup_download_cache()
-
# Because we have added a new egg, we need to rescan
# the destination directory.
@@ -507,9 +521,11 @@
def install(specs, dest,
links=(), index=None,
executable=sys.executable, always_unzip=False,
- path=None, working_set=None, newest=True, versions=None):
- installer = Installer(dest, links, index, executable, always_unzip, path,
- newest, versions)
+ path=None, working_set=None, newest=True, versions=None,
+ download_cache=None):
+ installer = Installer(dest, links, index, executable,
+ always_unzip, path, newest,
+ versions, download_cache)
return installer.install(specs, working_set)
@@ -521,7 +537,6 @@
versions)
return installer.build(spec, build_ext)
-
def _rm(*paths):
for path in paths:
Modified: zc.buildout/branches/downloadcache/todo.txt
===================================================================
--- zc.buildout/branches/downloadcache/todo.txt 2007-03-07 20:57:23 UTC (rev 73038)
+++ zc.buildout/branches/downloadcache/todo.txt 2007-03-07 20:59:22 UTC (rev 73039)
@@ -1,3 +1,6 @@
+====
+TODO
+====
- Report error if repeated parts
@@ -53,10 +56,40 @@
- document recipe initialization order
+Download Cache
+==============
+ a) cache for distutils fetches (buildout cache)
+ - the cache should be in the 'find-links' so that distutils picks them up
+ enabling the cache feature
+
+ b) cache for recipes that download things (file cache)
+
+ - simple library (z3c.filecache) that doesn't depend on buildout and is
+ not a part of buildout but a library with a simple API (e.g. get(url,
+ directory)->local file or None; set(url, directory, already downloaded
+ file) -> return path of new file (move given file to download
+ directory))
+
+ - recipes that want to use this mechanism would depend on z3c.filecache
+
+ - files of a) and b) probably should be held in separate areas
+
+ - check work that fred did
+
+ - a persistent cache should be optional, temporary directories will be used
+ if no cache is defined
+
+ - implementations for a) and b) have to create a directory in the download
+ cache directory to store their files. case a) will be called "zc.buildout",
+ case be will be called like the package that provides the cache feature
+ (e.g. z3c.filecache).
+
Issues
+======
+
- Should we include setuptools and buildout eggs for buildout process
in environment when searching for requirements?
More information about the Checkins
mailing list