[Zope-CVS] CVS: Packages/zpkgtools/zpkgtools - app.py:1.36 include.py:1.30 loader.py:1.4

Fred L. Drake, Jr. fred at zope.com
Fri Apr 23 15:56:43 EDT 2004


Update of /cvs-repository/Packages/zpkgtools/zpkgtools
In directory cvs.zope.org:/tmp/cvs-serv17724/zpkgtools

Modified Files:
	app.py include.py loader.py 
Log Message:
change the PACKAGE.cfg file to include three sections instead of two; the
<load> section is used to load files from external sources into the
collection, the <collection> section describes what files to include and
exclude from the component, and the <distribution> section describes what
to include in the distribution root

this required a lot of internal changes


=== Packages/zpkgtools/zpkgtools/app.py 1.35 => 1.36 ===
--- Packages/zpkgtools/zpkgtools/app.py:1.35	Tue Apr 20 15:01:30 2004
+++ Packages/zpkgtools/zpkgtools/app.py	Fri Apr 23 15:56:12 2004
@@ -42,43 +42,52 @@
         returned by `parse_args()`.
         """
         self.logger = logging.getLogger(options.program)
-        self.ip = None
         self.options = options
-        self.resource = locationmap.normalizeResourceId(options.resource)
-        self.resource_type, self.resource_name = self.resource.split(":", 1)
-        if not options.release_name:
-            options.release_name = self.resource_name
-        # Create a new directory for all temporary files to go in:
-        self.tmpdir = tempfile.mkdtemp(prefix=options.program + "-")
-        tempfile.tempdir = self.tmpdir
-        if options.revision_tag:
-            self.loader = loader.Loader(tag=options.revision_tag)
-        else:
-            self.loader = loader.Loader()
         cf = config.Configuration()
         cf.location_maps.extend(options.location_maps)
         path = options.configfile
         if path is None:
             path = config.defaultConfigurationPath()
             if os.path.exists(path):
+                self.logger.debug("loading configuration file %s", path)
                 cf.loadPath(path)
         elif path:
+            self.logger.debug("loading configuration file %s", path)
             cf.loadPath(path)
-
         cf.finalize()
         self.locations = cf.locations
+
+        # XXX Hack: This should be part of BuilderApplication
         if options.include_support_code is None:
             options.include_support_code = cf.include_support_code
 
+    def error(self, message, rc=1):
+        print >>sys.stderr, message
+        sys.exit(rc)
+
+
+class BuilderApplication(Application):
+
+    def __init__(self, options):
+        Application.__init__(self, options)
+        self.ip = None
+        self.resource = locationmap.normalizeResourceId(options.resource)
+        self.resource_type, self.resource_name = self.resource.split(":", 1)
+        if not options.release_name:
+            options.release_name = self.resource_name
+        # Create a new directory for all temporary files to go in:
+        self.tmpdir = tempfile.mkdtemp(prefix=options.program + "-")
+        tempfile.tempdir = self.tmpdir
+        if options.revision_tag:
+            self.loader = loader.Loader(tag=options.revision_tag)
+        else:
+            self.loader = loader.Loader()
+
         if self.resource not in self.locations:
             self.error("unknown resource: %s" % self.resource)
         self.resource_url = self.locations[self.resource]
         self.handled_resources = sets.Set()
 
-    def error(self, message, rc=1):
-        print >>sys.stderr, message
-        sys.exit(rc)
-
     def build_distribution(self):
         """Create the distribution tree.
 
@@ -92,7 +101,7 @@
         # distribution; it's the former if there's an __init__.py in
         # the source directory.
         os.mkdir(self.destination)
-        self.ip = include.InclusionProcessor(self.source, loader=self.loader)
+        self.ip = include.InclusionProcessor(self.source, self.loader)
         self.ip.add_manifest(self.destination)
         self.handled_resources.add(self.resource)
         name = "build_%s_distribution" % self.resource_type
@@ -102,12 +111,14 @@
     def build_package_distribution(self):
         pkgname = self.metadata.name
         pkgdest = os.path.join(self.destination, pkgname)
-        spec, dist = include.load(self.source)
+        specs = include.load(self.source)
+        self.ip.addIncludes(self.source, specs.loads)
+        specs.collection.cook()
         try:
-            self.ip.createDistributionTree(pkgdest, spec)
+            self.ip.createDistributionTree(pkgdest, specs.collection)
         except cvsloader.CvsLoadingError, e:
             self.error(str(e))
-        self.ip.addIncludes(self.destination, dist)
+        self.ip.addIncludes(self.destination, specs.distribution)
         pkgdir = os.path.join(self.destination, pkgname)
         pkginfo = package.loadPackageInfo(pkgname, pkgdir, pkgname)
         setup_cfg = os.path.join(self.destination, "setup.cfg")
@@ -189,7 +200,10 @@
                 unhandled_resources.add(resource)
                 continue
             #
-            source = self.loader.load(self.locations[resource])
+            location = self.locations[resource]
+            self.logger.debug("loading resource %r from %s",
+                              resource, location)
+            source = self.loader.load_mutable_copy(location)
             self.handled_resources.add(resource)
             deps = self.add_component(type, name, source)
             if type == "package" and "." in name:
@@ -230,15 +244,17 @@
         """
         destination = os.path.join(self.destination, name)
         self.ip.add_manifest(destination)
-        spec, dist = include.load(source)
+        specs = include.load(source)
+        self.ip.addIncludes(source, specs.loads)
+        specs.collection.cook()
 
         if type == "package":
-            self.add_package_component(name, destination, spec)
+            self.add_package_component(name, destination, specs.collection)
         elif type == "collection":
-            self.add_collection_component(name, destination, spec)
+            self.add_collection_component(name, destination, specs.collection)
 
         if distribution:
-            self.ip.addIncludes(self.destination, dist)
+            self.ip.addIncludes(self.destination, specs.distribution)
 
         self.create_manifest(destination)
         deps_file = os.path.join(source, "DEPENDENCIES.cfg")
@@ -299,7 +315,9 @@
 
     def load_resource(self):
         """Load the primary resource and initialize internal metadata."""
-        self.source = self.loader.load(self.resource_url)
+        self.logger.debug("loading resource %r from %s",
+                          self.resource, self.resource_url)
+        self.source = self.loader.load_mutable_copy(self.resource_url)
         self.load_metadata()
         release_name = self.options.release_name
         self.target_name = "%s-%s" % (release_name, self.options.version)
@@ -442,6 +460,8 @@
                 mod = sys.modules[name]
                 source = os.path.abspath(mod.__path__[0])
         if source is None:
+            self.logger.debug("loading resource 'package:%s' from %s",
+                              name, url)
             source = self.loader.load(url)
 
         tests_dir = os.path.join(source, "tests")
@@ -493,7 +513,7 @@
 
     def run(self):
         """Run the application, using the other methods of the
-        ``Application`` object.
+        ``BuilderApplication`` object.
         """
         try:
             try:
@@ -553,6 +573,8 @@
         prog=prog,
         usage="usage: %prog [options] resource",
         version="%prog 0.1")
+
+    # "global" options:
     parser.add_option(
         "-C", "--configure", dest="configfile",
         help="path or URL to the configuration file", metavar="FILE")
@@ -565,6 +587,8 @@
         action="append", default=[],
         help=("specify an additional location map to load before"
               " maps specified in the configuration"), metavar="MAP")
+
+    # options specific to building a package:
     parser.add_option(
         "-n", "--name", dest="release_name",
         help="base name of the distribution file", metavar="NAME")
@@ -581,6 +605,7 @@
     parser.add_option(
         "-v", dest="version",
         help="version label for the new distribution")
+
     options, args = parser.parse_args(argv[1:])
     if len(args) != 1:
         parser.error("wrong number of arguments")
@@ -612,7 +637,7 @@
         return 2
 
     try:
-        app = Application(options)
+        app = BuilderApplication(options)
         app.run()
     except SystemExit, e:
         return e.code


=== Packages/zpkgtools/zpkgtools/include.py 1.29 => 1.30 ===
--- Packages/zpkgtools/zpkgtools/include.py:1.29	Mon Apr 19 11:54:30 2004
+++ Packages/zpkgtools/zpkgtools/include.py	Fri Apr 23 15:56:12 2004
@@ -22,6 +22,9 @@
     that are copied.  Any file with a name matching these patterns
     will be ignored.
 
+  - `PACKAGE_CONF`: The name of the file that specifies how the
+    package is assembled.
+
 """
 
 import fnmatch
@@ -35,7 +38,7 @@
 from zpkgtools import Error
 
 from zpkgtools import cfgparser
-from zpkgtools import cvsloader
+from zpkgtools import loader
 from zpkgtools import publication
 
 
@@ -74,10 +77,10 @@
             config = parser.load()
         finally:
             f.close()
-        config.collection.excludes[package_conf] = package_conf
+        config.collection.add_exclusion(package_conf)
     else:
         config = schema.getConfiguration()
-    return config.collection, config.distribution
+    return config
 
 
 def filter_names(names):
@@ -93,7 +96,7 @@
     return names
 
 
-def normalize_path(path, type):
+def normalize_path(path, type, group):
     if ":" in path:
         scheme, rest = urllib.splittype(path)
         if len(scheme) == 1:
@@ -101,6 +104,9 @@
             # 'cause that's not allowable:
             raise InclusionSpecificationError(
                 "drive letters are not allowed in inclusions: %r" % path)
+        else:
+            raise InclusionSpecificationError(
+                "URLs are not allowed in inclusions")
     np = posixpath.normpath(path)
     if posixpath.isabs(np) or np[:1] == ".":
         raise InclusionSpecificationError(
@@ -110,17 +116,17 @@
     return np.replace("/", os.sep)
 
 
-def normalize_path_or_url(path, type):
+def normalize_path_or_url(path, type, group):
     if ":" in path:
         scheme, rest = urllib.splittype(path)
         if len(scheme) != 1:
             # should normalize the URL, but skip that for now
             return path
-    return normalize_path(path, type)
+    return normalize_path(path, type, group)
 
 
 class SpecificationSchema(cfgparser.Schema):
-    """Specialized schema that handles populating a pair of Specifications.
+    """Specialized schema that handles populating a set of Specifications.
     """
 
     def __init__(self, source, filename):
@@ -129,8 +135,12 @@
 
     def getConfiguration(self):
         conf = cfgparser.SectionValue(None, None, None)
-        conf.collection = self.collection = Specification(self.source)
-        conf.distribution = self.distribution = Specification(self.source)
+        conf.loads = Specification(
+            self.source, self.filename, "load")
+        conf.collection = Specification(
+            self.source, self.filename, "collection")
+        conf.distribution = Specification(
+            self.source, self.filename, "distribution")
         return conf
 
     def startSection(self, parent, typename, name):
@@ -140,9 +150,11 @@
             return parent.collection
         elif typename == "distribution":
             return parent.distribution
+        elif typename == "load":
+            return parent.loads
         raise cfgparser.ConfigurationError("unknown section type: %s"
                                            % typename)
-                                           
+
     def endSection(self, parent, typename, name, child):
         pass
 
@@ -161,21 +173,18 @@
         if not src:
             raise InclusionSpecificationError("source information omitted",
                                               self.filename)
-        dest = normalize_path(dest, "destination")
-        src = normalize_path_or_url(src, "source")
+        dest = normalize_path(dest, "destination", section.group)
+        if section.group == "load":
+            f = normalize_path_or_url
+        else:
+            f = normalize_path
+        src = f(src, "source", section.group)
         if src == "-":
-            if section is self.distribution:
-                raise InclusionSpecificationError(
-                    "cannot exclude files from the distribution root",
-                    self.filename)
-            path = os.path.join(self.source, dest)
-            expansions = filter_names(glob.glob(path))
-            if not expansions:
+            if section.group != "collection":
                 raise InclusionSpecificationError(
-                    "exclusion %r doesn't match any files" % dest,
+                    "can only exclude files from the collection group",
                     self.filename)
-            for fn in expansions:
-                section.excludes[fn] = fn
+            section.add_exclusion(dest)
         else:
             section.includes[dest] = src
 
@@ -194,16 +203,7 @@
 
     """
 
-    # XXX Needing to pass the source directory to the constructor is a
-    # bit of a hack, ...  A
-    # better approach may be to have "raw" and "cooked" versions of
-    # the specification object; the raw version would only have the
-    # information loaded from a specification file, and the cooked
-    # version would be (essentially) a list of directory creation and
-    # file copy operations.  The input source directory would be a
-    # parameter to the "cook" operation.
-
-    def __init__(self, source):
+    def __init__(self, source, filename, group):
         """Initialize the Specification object.
 
         :Parameters:
@@ -213,9 +213,26 @@
         """
         # The source directory is needed since globbing is performed
         # to locate files if the spec includes wildcards.
-        self.excludes = {}
+        self.excludes = []
         self.includes = {}
         self.source = source
+        self.filename = filename
+        self.group = group
+
+    def add_exclusion(self, path):
+        self.excludes.append(path)
+
+    def cook(self):
+        patterns = self.excludes
+        self.excludes = []
+        for pat in patterns:
+            path = os.path.join(self.source, pat)
+            expansions = filter_names(glob.glob(path))
+            if not expansions:
+                raise InclusionSpecificationError(
+                    "exclusion %r doesn't match any files" % pat,
+                    self.filename)
+            self.excludes.extend(expansions)
 
 
 class InclusionProcessor:
@@ -225,15 +242,13 @@
     the output tree.
 
     """
-    def __init__(self, source, loader=None):
+    def __init__(self, source, loader):
         if not os.path.exists(source):
             raise InclusionError("source directory does not exist: %r"
                                  % source)
         self.source = os.path.abspath(source)
         self.manifests = []
-        if loader is None:
-            loader = cvsloader.CvsLoader()
-        self.cvs_loader = loader
+        self.loader = loader
 
     def createDistributionTree(self, destination, spec=None):
         """Create the output tree according to `spec`.
@@ -248,9 +263,9 @@
 
         """
         if spec is None:
-            spec = Specification(self.source)
+            spec = Specification(self.source, None, "collection")
         destination = os.path.abspath(destination)
-        self.copyTree(spec.source, destination, spec.excludes)
+        self.copyTree(self.source, destination, spec.excludes)
         self.addIncludes(destination, spec)
 
     def copyTree(self, source, destination, excludes={}):
@@ -381,24 +396,18 @@
         # This is what we want to create:
         destdir = os.path.join(destdir, basename)
 
+        type = urllib.splittype(source)[0] or ''
+        if len(type) in (0, 1):
+            # figure it's a path ref, possibly w/ a Windows drive letter
+            source = os.path.join(self.source, source)
+            source = "file://" + urllib.pathname2url(source)
+            type = "file"
         try:
-            cvsurl = cvsloader.parse(source)
+            path = self.loader.load(source)
         except ValueError:
-            # not a cvs: or repository: URL
-            type, rest = urllib.splittype(source)
-            if type:
-                # some sort of URL
-                self.includeFromUrl(source, destdir)
-            else:
-                # local path; perhaps this join should be handled by
-                # the Specification to avoid having to keep
-                # self.source around?
-                self.includeFromLocalTree(os.path.join(self.source, source),
-                                          destdir)
-        else:
-            if isinstance(cvsurl, cvsloader.RepositoryUrl):
-                raise InclusionError("can't load from repository: URL")
-            self.includeFromCvs(cvsurl, destdir)
+            # not a supported URL type
+            raise InclusionError("cannot load from a %r URL" % type)
+        self.includeFromLocalTree(path, destdir)
 
     def includeFromLocalTree(self, source, destination):
         # Check for file-ness here since copyTree() doesn't handle
@@ -407,20 +416,3 @@
             self.copy_file(source, destination)
         else:
             self.copyTree(source, destination)
-
-    def includeFromUrl(self, source, destination):
-        # XXX treat FTP URLs specially to get permission bits and directories?
-        inf = urllib2.urlopen(source)
-        try:
-            outf = open(destination, "w")
-            try:
-                shutil.copyfileobj(inf, outf)
-            finally:
-                outf.close()
-            self.add_output(destination)
-        finally:
-            inf.close()
-
-    def includeFromCvs(self, cvsurl, destination):
-        source = self.cvs_loader.load(cvsurl.getUrl())
-        self.includeFromLocalTree(source, destination)


=== Packages/zpkgtools/zpkgtools/loader.py 1.3 => 1.4 ===
--- Packages/zpkgtools/zpkgtools/loader.py:1.3	Wed Apr 21 12:03:16 2004
+++ Packages/zpkgtools/zpkgtools/loader.py	Fri Apr 23 15:56:12 2004
@@ -42,23 +42,46 @@
 
     def __init__(self, tag=None):
         self.tag = tag or None
-        self.workdirs = {}  # URL -> (directory, path, temporary)
+        self.workdirs = {}  # URL -> (tmp.directory, path, istemporary)
         self.cvsloader = None
 
-    def add_working_dir(self, url, directory, path, temporary):
-        self.workdirs[url] = (directory, path, temporary)
+    def add_working_dir(self, url, directory, path, istemporary):
+        self.workdirs[url] = (directory, path, istemporary)
 
     def cleanup(self):
         """Remove all checkouts that are present."""
         while self.workdirs:
-            url, (directory, path, temporary) = self.workdirs.popitem()
-            if temporary:
+            url, (directory, path, istemporary) = self.workdirs.popitem()
+            if istemporary:
                 if directory:
                     shutil.rmtree(directory)
                 else:
                     os.unlink(path)
 
+    def transform_url(self, url):
+        """Transform a URL to encode the tag passed to the constructor.
+
+        :param url:  URL that may not be associated with a particular tag.
+
+        If `url` can be modified to encode the tag associated with the
+        loader, a modified URL that does so is returned.  If not, the
+        original URL is returned unmodified.
+        """
+        if self.tag:
+            try:
+                parsed_url = cvsloader.parse(url)
+            except ValueError:
+                pass
+            else:
+                if not parsed_url.tag:
+                    parsed_url.tag = self.tag
+                    url = parsed_url.getUrl()
+        return url
+
     def load(self, url):
+        url = self.transform_url(url)
+        if url in self.workdirs:
+            return self.workdirs[url][1]
         if ":" in url and url.find(":") != 1:
             type, rest = urllib.splittype(url)
             # the replace() is to support svn+ssh: URLs
@@ -67,12 +90,48 @@
             if method is None:
                 method = self.unknown_load
         else:
-            method = self.file_load
-        return method(url)
+            raise ValueError("can only load from URLs, not path references")
+        path = method(url)
+        assert path == self.workdirs[url][1]
+        return path
 
-    def file_load(self, path):
-        """Load using a local path."""
-        raise NotImplementedError("is this ever used?")
+    def load_mutable_copy(self, url):
+        """Load the resource referenced by `url` so the application
+        can modify it.
+
+        If the copy provided by the `load()` method isn't 'owned' by
+        the application (because it's a temporary copy), a copy will
+        be made and used instead.
+        """
+        url = self.transform_url(url)
+        self.load(url)
+        directory, path, istemporary = self.workdirs[url]
+        if not istemporary:
+            # We need a copy we can mutate and throw away later:
+            p = self.create_copy(url, path)
+            assert p != path
+            path = p
+        assert path == self.workdirs[url][1]
+        assert self.workdirs[url][2]
+        return path
+
+    def create_copy(self, url, path):
+        """Create a copy of the tree rooted at `path`.
+
+        The copy must be 'owned' by the application, and can be
+        mutated freely without affecting the original.
+        """
+        tmpdir = tempfile.mkdtemp(prefix="loader-")
+        # we have to normalize in case there's a trailing slash
+        path = os.path.normpath(path)
+        basename = os.path.basename(path)
+        filename = os.path.join(tmpdir, basename)
+        if os.path.isfile(path):
+            shutil.copy2(path, filename)
+        else:
+            shutil.copytree(path, filename, symlinks=False)
+        self.add_working_dir(url, tmpdir, filename, True)
+        return filename
 
     def load_file(self, url):
         # XXX This doesn't deal with file: URLs that Subversion uses,
@@ -89,10 +148,6 @@
     def load_cvs(self, url):
         if self.cvsloader is None:
             self.cvsloader = cvsloader.CvsLoader()
-        parsed_url = cvsloader.parse(url)
-        if not parsed_url.tag:
-            parsed_url.tag = self.tag
-            url = parsed_url.getUrl()
         # If we've already loaded this, use that copy.  This doesn't
         # consider fetching something with a different path that's
         # represented by a previous load():
@@ -100,6 +155,7 @@
             return self.workdirs[url][1]
 
         tmp = tempfile.mkdtemp(prefix="loader-")
+        parsed_url = cvsloader.parse(url)
         path = self.cvsloader.load(parsed_url, tmp)
         self.add_working_dir(url, tmp, path, True)
         return path
@@ -118,7 +174,7 @@
         f = urllib2.urlopen(url)
         fd, tmp = tempfile.mkstemp(prefix="loader-")
         try:
-            os.fwrite(fd, f.read())
+            os.write(fd, f.read())
         except:
             os.close(fd)
             os.unlink(tmp)




More information about the Zope-CVS mailing list