[Checkins] SVN: z3c.repoexternals/trunk/ Generate externals from a repository

Ross Patterson me at rpatterson.net
Thu Sep 27 04:37:32 EDT 2007


Log message for revision 80198:
  Generate externals from a repository
  

Changed:
  _U  z3c.repoexternals/trunk/
  A   z3c.repoexternals/trunk/.cvsignore
  A   z3c.repoexternals/trunk/EXTERNALS.txt
  A   z3c.repoexternals/trunk/README.txt
  A   z3c.repoexternals/trunk/buildout.cfg
  A   z3c.repoexternals/trunk/paster.cfg
  A   z3c.repoexternals/trunk/setup.cfg
  A   z3c.repoexternals/trunk/setup.py
  A   z3c.repoexternals/trunk/z3c/
  A   z3c.repoexternals/trunk/z3c/__init__.py
  A   z3c.repoexternals/trunk/z3c/repoexternals/
  A   z3c.repoexternals/trunk/z3c/repoexternals/__init__.py
  A   z3c.repoexternals/trunk/z3c/repoexternals/tag.py
  A   z3c.repoexternals/trunk/z3c/repoexternals/tests/
  A   z3c.repoexternals/trunk/z3c/repoexternals/tests/EXTERNALS.txt
  A   z3c.repoexternals/trunk/z3c/repoexternals/tests/__init__.py
  A   z3c.repoexternals/trunk/z3c/repoexternals/tests/functional.txt
  A   z3c.repoexternals/trunk/z3c/repoexternals/tests/repo/
  A   z3c.repoexternals/trunk/z3c/repoexternals/tests/repo/bar/
  A   z3c.repoexternals/trunk/z3c/repoexternals/tests/repo/bar/branches/
  A   z3c.repoexternals/trunk/z3c/repoexternals/tests/repo/bar/trunk/
  A   z3c.repoexternals/trunk/z3c/repoexternals/tests/repo/baz/
  A   z3c.repoexternals/trunk/z3c/repoexternals/tests/repo/baz/branches/
  A   z3c.repoexternals/trunk/z3c/repoexternals/tests/repo/baz/trunk/
  A   z3c.repoexternals/trunk/z3c/repoexternals/tests/repo/foo/
  A   z3c.repoexternals/trunk/z3c/repoexternals/tests/repo/foo/bar/
  A   z3c.repoexternals/trunk/z3c/repoexternals/tests/repo/foo/bar/Trunk/
  A   z3c.repoexternals/trunk/z3c/repoexternals/tests/repo/foo/bar/baz/
  A   z3c.repoexternals/trunk/z3c/repoexternals/tests/repo/foo/bar/baz/qux/
  A   z3c.repoexternals/trunk/z3c/repoexternals/tests/repo/foo/bar/baz/qux/quux/
  A   z3c.repoexternals/trunk/z3c/repoexternals/tests/repo/foo/bar/baz/qux/trunk/
  A   z3c.repoexternals/trunk/z3c/repoexternals/tests/repo/foo/bar/branches/
  A   z3c.repoexternals/trunk/z3c/repoexternals/tests/repo/foo/branches/
  A   z3c.repoexternals/trunk/z3c/repoexternals/tests/repo/foo/branchesjunk/
  A   z3c.repoexternals/trunk/z3c/repoexternals/tests/repo/foo/trunk/
  A   z3c.repoexternals/trunk/z3c/repoexternals/tests/repo/foo/trunkjunk/
  A   z3c.repoexternals/trunk/z3c/repoexternals/tests/script.txt
  A   z3c.repoexternals/trunk/z3c/repoexternals/tests/test.py

-=-

Property changes on: z3c.repoexternals/trunk
___________________________________________________________________
Name: svn:ignore
   + .installed.cfg
bin
develop-eggs
parts
build
dist

Name: svn:externals
   + #
# applied by: svn propset svn:externals -F ./EXTERNALS.txt .
#
bootstrap svn://svn.zope.org/repos/main/zc.buildout/trunk/bootstrap


Copied: z3c.repoexternals/trunk/.cvsignore (from rev 80119, z3c.offlinepack/trunk/.cvsignore)
===================================================================
--- z3c.repoexternals/trunk/.cvsignore	                        (rev 0)
+++ z3c.repoexternals/trunk/.cvsignore	2007-09-27 08:37:31 UTC (rev 80198)
@@ -0,0 +1,6 @@
+.installed.cfg
+bin
+develop-eggs
+parts
+build
+dist
\ No newline at end of file

Copied: z3c.repoexternals/trunk/EXTERNALS.txt (from rev 80119, z3c.offlinepack/trunk/EXTERNALS.txt)
===================================================================
--- z3c.repoexternals/trunk/EXTERNALS.txt	                        (rev 0)
+++ z3c.repoexternals/trunk/EXTERNALS.txt	2007-09-27 08:37:31 UTC (rev 80198)
@@ -0,0 +1,4 @@
+#
+# applied by: svn propset svn:externals -F ./EXTERNALS.txt .
+#
+bootstrap svn://svn.zope.org/repos/main/zc.buildout/trunk/bootstrap

Added: z3c.repoexternals/trunk/README.txt
===================================================================
--- z3c.repoexternals/trunk/README.txt	                        (rev 0)
+++ z3c.repoexternals/trunk/README.txt	2007-09-27 08:37:31 UTC (rev 80198)
@@ -0,0 +1,64 @@
+usage: repoexternals [options] url_or_path
+
+Recursively retrieves subversion directory listings from the url or
+path and matches directories against a previous set of svn:externals
+if provided then against regular expressions and generates qualifying
+svn:externals lines.  The defaults generate a set of svn:externals for
+all the trunks in a repository and keeps them up to date with the
+repository as new trunks are added when the previous externals are
+provided thereafter.
+
+options:
+  -h, --help            show this help message and exit
+  -v, --verbose         Output logging to stdandard error. Set twice
+                        to log debugging mesages.
+  -p FILE, --previous=FILE
+                        If provided, only URLs in the repository not
+                        included in the previous externals will be
+                        included. If the filename is '-', use standard
+                        input.  Valid svn:externals lines beginning
+                        with one comment character, '#', will also
+                        affect output.  This is useful, for example,
+                        to prevent lengthy recursions into directories
+                        that are known not to contain any desired
+                        matches.  The file is read completely and
+                        closed before anything is output, so it is
+                        safe to append output to the previous file:
+                        "repoexternals -p EXTERNALS.txt
+                        http://svn.foo.org/repos/main
+                        >>EXTERNALS.txt".
+  -i REGEXP, --include=REGEXP
+                        Directory names matching this python regular
+                        expression will be included in output and will
+                        not be descended into.
+                        [default: (?i)^((.*)/.+?|.*)/trunk$]
+  -e REGEXP, --exclude=REGEXP
+                        Directory names matching this python regular
+                        expression will be excluded from output and
+                        will not be descended into. Include overrides
+                        exclude.  [default:
+                        (?i)^.*/(branch(es)?|tags?|releases?|vendor|bundles?|sandbox|build|dist)$]
+  -m TEMPLATE, --matched-template=TEMPLATE
+                        The result of expanding previous file URL
+                        matches with the include regular expression
+                        through this template is added to the set of
+                        previous URLs excluded from output and
+                        descending.  The default will add the parents
+                        of trunks to the set of previous URLs
+                        excluded.  [default: \1]
+  -t TEMPLATE, --parent-template=TEMPLATE
+                        The result of expanding previous file URL
+                        matches with the include regular expression
+                        through this template is removed from the set
+                        of matched previous URLs excluded from output
+                        and descending. The default ensures that
+                        directories containing trunks within a
+                        directory that contains a trunk are not
+                        excluded.  [default: \2]
+  -d INT, --depth=INT   The maximum directory depth to descend to.
+                        WARNING: large values can greatly increase run
+                        time.  [default: 5]
+  -s INT, --pool-size=INT
+                        The number of concurrent svn clients.
+                        WARNING: large values can DOS the repository.
+                        [default: 5]

Copied: z3c.repoexternals/trunk/buildout.cfg (from rev 80119, z3c.offlinepack/trunk/buildout.cfg)
===================================================================
--- z3c.repoexternals/trunk/buildout.cfg	                        (rev 0)
+++ z3c.repoexternals/trunk/buildout.cfg	2007-09-27 08:37:31 UTC (rev 80198)
@@ -0,0 +1,11 @@
+[buildout]
+develop = .
+parts = repoexternals test
+
+[repoexternals]
+recipe = zc.recipe.egg:scripts
+eggs = z3c.repoexternals
+
+[test]
+recipe = zc.recipe.testrunner
+eggs = z3c.repoexternals [test]

Added: z3c.repoexternals/trunk/paster.cfg
===================================================================
--- z3c.repoexternals/trunk/paster.cfg	                        (rev 0)
+++ z3c.repoexternals/trunk/paster.cfg	2007-09-27 08:37:31 UTC (rev 80198)
@@ -0,0 +1,15 @@
+[pastescript]
+namespace_package = z3c
+description = Generate externals from a repository
+author = Ross Patterson
+author_email = me at rpatteron.net
+license_name = GPL
+url = http://cheeseshop.python.org/pypi/z3c.repoexternals
+version = 0.1
+plus = +
+zip_safe = True
+keywords = 
+egg = z3c.repoexternals
+long_description = 
+dot = .
+

Added: z3c.repoexternals/trunk/setup.cfg
===================================================================
--- z3c.repoexternals/trunk/setup.cfg	                        (rev 0)
+++ z3c.repoexternals/trunk/setup.cfg	2007-09-27 08:37:31 UTC (rev 80198)
@@ -0,0 +1,3 @@
+[egg_info]
+tag_build = dev
+tag_svn_revision = true

Added: z3c.repoexternals/trunk/setup.py
===================================================================
--- z3c.repoexternals/trunk/setup.py	                        (rev 0)
+++ z3c.repoexternals/trunk/setup.py	2007-09-27 08:37:31 UTC (rev 80198)
@@ -0,0 +1,39 @@
+from setuptools import setup, find_packages
+import sys, os
+
+version = '0.1'
+
+setup(name='z3c.repoexternals',
+      version=version,
+      description="Generate externals from a repository",
+      long_description=open(os.path.join(os.path.dirname(__file__),
+                                         'README.txt')).read(),
+      # Get more strings from http://www.python.org/pypi?%3Aaction=list_classifiers
+      classifiers=[
+        "Framework :: Plone",
+        "Framework :: Zope2",
+        "Framework :: Zope3",
+        "Programming Language :: Python",
+        "Topic :: Software Development :: Libraries :: Python Modules",
+        ],
+      keywords='',
+      author='Ross Patterson',
+      author_email='me at rpatteron.net',
+      url='http://cheeseshop.python.org/pypi/z3c.repoexternals',
+      license='GPL',
+      packages=find_packages(exclude=['ez_setup']),
+      namespace_packages=['z3c'],
+      include_package_data=True,
+      zip_safe=True,
+      install_requires=[
+          'setuptools',
+          # -*- Extra requirements: -*-
+          # 'pysvn',
+      ],
+      extras_require=dict(test=['zc.buildout', 'zc.recipe.egg']),
+      entry_points="""
+      # -*- Entry points: -*-
+      [console_scripts]
+      repoexternals = z3c.repoexternals:main
+      """,
+      )

Added: z3c.repoexternals/trunk/z3c/__init__.py
===================================================================
--- z3c.repoexternals/trunk/z3c/__init__.py	                        (rev 0)
+++ z3c.repoexternals/trunk/z3c/__init__.py	2007-09-27 08:37:31 UTC (rev 80198)
@@ -0,0 +1,6 @@
+# See http://peak.telecommunity.com/DevCenter/setuptools#namespace-packages
+try:
+    __import__('pkg_resources').declare_namespace(__name__)
+except ImportError:
+    from pkgutil import extend_path
+    __path__ = extend_path(__path__, __name__)

Added: z3c.repoexternals/trunk/z3c/repoexternals/__init__.py
===================================================================
--- z3c.repoexternals/trunk/z3c/repoexternals/__init__.py	                        (rev 0)
+++ z3c.repoexternals/trunk/z3c/repoexternals/__init__.py	2007-09-27 08:37:31 UTC (rev 80198)
@@ -0,0 +1,344 @@
+#!/usr/bin/python
+"""Recursively retrieves subversion directory listings from the url or
+path and matches directories against a previous set of svn:externals
+if provided then against regular expressions and generates qualifying
+svn:externals lines.  The defaults generate a set of svn:externals for
+all the trunks in a repository and keeps them up to date with the
+repository as new trunks are added when the previous externals are
+provided thereafter."""
+
+import sys, os, re, logging, thread, threading, Queue, optparse
+import pysvn
+
+usage = "usage: %prog [options] url_or_path"
+
+parser = optparse.OptionParser(usage=usage, description=__doc__)
+
+parser.add_option(
+    "-v", "--verbose", action="count",
+    help=("Output logging to stdandard error. Set twice to log "
+          "debugging mesages.")) 
+
+parser.add_option(
+    "-p", "--previous", metavar='FILE',
+    help=("""If provided, only URLs in the repository not 
+included in the previous externals will be included. If the filename
+is '-', use standard input.  Valid svn:externals lines beginning with
+one comment character, '#', will also affect output.  This is useful,
+for example, to prevent lengthy recursions into directories that are
+known not to contain any desired matches.  The file is read completely
+and closed before anything is output, so it is safe to append output
+to the previous file: "repoexternals -p EXTERNALS.txt
+http://svn.foo.org/repos/main >>EXTERNALS.txt"."""))
+
+include = r'(?i)^((.*)/.+?|.*)/trunk$'
+parser.add_option(
+    "-i", "--include", default=include, metavar='REGEXP',
+    help=("Directory names matching this python regular expression "
+          "will be included in output and will not be descended into."
+          "  [default: %default]"))
+
+exclude = (r'(?i)^.*/(branch(es)?|tags?|releases?|vendor|bundles?'
+           r'|sandbox|build|dist)$')
+parser.add_option(
+    "-e", "--exclude", default=exclude, metavar='REGEXP',
+    help=("Directory names matching this python regular expression "
+          "will be excluded from output and will not be descended "
+          "into. Include overrides exclude.  [default: %default]"))
+
+matched_template = r'\1'
+parser.add_option(
+    "-m", "--matched-template", default=matched_template,
+    metavar='TEMPLATE',
+    help=("""The result of expanding previous file URL matches with
+the include regular expression through this template is added to the
+set of previous URLs excluded from output and descending.  The default
+will add the parents of trunks to the set of previous URLs
+excluded.  [default: %default]"""))
+
+parent_template = r'\2'
+parser.add_option(
+    "-t", "--parent-template", default=parent_template,
+    metavar='TEMPLATE',
+    help=("""The result of expanding previous file URL matches with
+the include regular expression through this template is removed from
+the set of matched previous URLs excluded from output and descending.
+The default ensures that directories containing trunks within a
+directory that contains a trunk are not excluded.
+[default: %default]"""))
+
+depth = 5
+parser.add_option(
+    "-d", "--depth", type="int", default=depth, metavar='INT',
+    help=("The maximum directory depth to descend to.  WARNING: "
+          "large values can greatly increase run time.  "
+          "[default: %default]"))
+
+pool_size = 5
+parser.add_option(
+    "-s", "--pool-size", type="int", default=pool_size, metavar='INT',
+    help=("The number of concurrent svn clients.  WARNING: large "
+          "values can DOS the repository.  [default: %default]"))
+
+shutdown = object()
+class Thread(threading.Thread):
+
+    def __init__(self, queue=None, results=None, **kwargs):
+        super(Thread, self).__init__(**kwargs)
+        if queue is None:
+            queue = Queue.Queue()
+        if results is None:
+            results = Queue.Queue()
+        self.queue = queue
+        self.results = results
+
+    def run(self):
+        try:
+            payload = self.queue.get()
+            while payload is not shutdown:
+                payload(self)
+                payload = self.queue.get()
+        except:
+            thread.interrupt_main()
+            raise
+
+    def shutdown(self):
+        self.queue.put(shutdown)
+        self.join()
+
+    def interrupt(self):
+        self.queue.mutex.acquire()
+        self.queue._init(self.queue.maxsize)
+        self.queue._put(shutdown)
+        self.queue.not_empty.notify()
+        self.queue.mutex.release()
+        self.join()
+
+class ClientThread(Thread):
+
+    def __init__(self, *args, **kwargs):
+        super(ClientThread, self).__init__(*args, **kwargs)
+        self.client = pysvn.Client()
+
+class ClientPool(Queue.Queue):
+
+    def __init__(self, size=pool_size, results=None,
+                 **kwargs):
+        Queue.Queue.__init__(self, **kwargs)
+        if results is None:
+            results = Queue.Queue()
+        self.results = results
+        self.threads = [
+            ClientThread(queue=self, results=self.results)
+            for ignored in xrange(size)]
+
+    def start(self):
+        for thread in self.threads:
+            thread.start()
+        logging.getLogger('repoexternals').debug(
+            'Started %s client threads' % len(self.threads))
+
+    def shutdown(self):
+        for thread in self.threads:
+            self.put(shutdown)
+        for thread in self.threads:
+            thread.join()
+
+    def interrupt(self):
+        self.mutex.acquire()
+        self._init(self.maxsize)
+        for thread in self.threads:
+            self._put(shutdown)
+        self.not_empty.notify()
+        self.mutex.release()
+        for thread in self.threads:
+            if thread.isAlive():
+                thread.join()
+
+class Root(object):
+    """Return self unless overriden in a child instance"""
+
+    def __get__(self, instance, owner):
+        return instance
+
+class Line(object):
+
+    def __init__(self, path, dirent):
+        self.path = path
+        self.dirent = dirent
+
+    def __str__(self):
+        return '%s %s' % (self.path, self.dirent.path)
+
+class Listing(object):
+    """A single svn listing"""
+
+    def __init__(self, url, include=include, exclude=exclude,
+                 depth=depth):
+        self.url = url
+        self.include = re.compile(include)
+        self.exclude = re.compile(exclude)
+        self.depth = depth
+
+        self.results = Queue.Queue()
+
+    def list(self, thread):
+        """Retrieve the svn listing from the repository"""
+        try:
+            self.listing = thread.client.list(
+                self.url, dirent_fields=pysvn.SVN_DIRENT_KIND)
+        except pysvn.ClientError, e:
+            logging.getLogger('repoexternals').exception(
+                'pysvn.ClientError %s' % self.url)
+            self.listing = []
+
+        # Queue for processing
+        thread.results.put(self.process)
+
+    root = Root()
+
+    def getchild(self, url):
+        """Create and return a child listing setting it's root"""
+        child = Listing(url, include=self.include,
+                        exclude=self.exclude, depth=self.depth)
+        child.root = self.root
+        return child
+    
+    def process(self, thread):
+        """Process the results of the svn listing"""
+
+        # Use local names in the inner loop
+        dir_node_kind = pysvn.node_kind.dir
+        root_url = self.root.url
+        include_match = self.include.match
+        exclude_match = self.exclude.match
+        results_put = self.results.put
+        lLine = Line
+        info = logging.getLogger('repoexternals').info
+        depth = self.depth
+        getchild = self.getchild
+        thread_results_put = thread.results.put
+        previous = self.root.previous
+
+        for dirent, ignored in self.listing[1:]:
+
+            if dirent.kind != dir_node_kind:
+                # externals can only be directories
+                continue
+
+            dirent_path = dirent.path
+
+            if dirent_path in previous:
+                info('In previous, skipping %s' % dirent_path)
+                continue
+
+            # No previous line, use matching
+            path = dirent_path[len(root_url):].lstrip('/')
+
+            if include_match(dirent_path) is not None:
+                # Include this line in the results
+                results_put(lLine(path=path, dirent=dirent))
+
+            elif exclude_match(dirent_path) is not None:
+                info('Excluding %s' % dirent_path)
+            elif len(path.split('/')) >= depth:
+                info('Too deep, skipping %s' % dirent_path)
+
+            else:
+                child = getchild(dirent_path)
+                info('Descending into %s' % dirent_path)
+                thread_results_put(child.list)
+                results_put(child)
+
+        # Let the iterator know we're done
+        results_put(shutdown)
+
+    def __iter__(self):
+        item = self.results.get()
+        while item is not shutdown:
+            if isinstance(item, Line):
+                yield item
+            else:
+                for child in item:
+                    yield child
+            item = self.results.get()
+
+# Match valid externals definitions in existing externals
+external = re.compile(r'^\s*#?\s*([^#\s]+)\s(.*\s|)(\S+)\s*$')
+
+def run(url, previous=(), include=include, exclude=exclude,
+        matched_template=matched_template,
+        parent_template=parent_template,
+        depth=depth, pool_size=pool_size):
+    pool = ClientPool(size=pool_size)
+    thread = Thread(queue=pool.results, results=pool)
+
+    pool.start()
+    thread.start()
+
+    try:
+        root = Listing(url, include=include, exclude=exclude,
+                       depth=depth)
+
+        # Build the set of previous URLs
+        include_match = root.include.match
+        root.previous = set()
+        raw_add = root.previous.add
+        matched = set()
+        matched_add = matched.add
+        parents = set()
+        parents_add = parents.add
+        external_match = external.match
+        for line in previous:
+            match = external_match(line)
+            if match is not None:
+                # TODO: also use pysvn.Cliens.is_url to verify url
+                # validity?
+                line_url = match.group(3)
+                raw_add(line_url)
+                include_matched = include_match(line_url)
+                if include_matched is not None:
+                    matched_add(
+                        include_matched.expand(matched_template))
+                    parents_add(
+                        include_matched.expand(parent_template))
+        root.previous.update(matched.difference(parents))
+
+        pool.put(root.list)
+
+        for line in root:
+            yield line
+    except:
+        # TODO: can't find a way to test interruptability
+        pool.interrupt()
+        thread.interrupt()
+        raise
+    else:
+        pool.shutdown()
+        thread.shutdown()
+
+def main():
+    options, args = parser.parse_args()
+
+    if len(args) != 1:
+        parser.error("requires one url_or_path")
+    url, = args
+
+    if options.verbose is not None:
+        verbose = options.verbose <= 2 and options.verbose or 2
+        logging.basicConfig(level=logging.WARN - (verbose * 10))
+
+    previous = ()
+    if options.previous:
+        if options.previous == '-':
+            previous = sys.stdin
+        else:
+            previous = file(options.previous)
+
+    for line in run(url, previous, options.include, options.exclude,
+                    options.matched_template, options.parent_template,
+                    options.depth, options.pool_size):
+        print line
+
+if __name__ == '__main__':
+    main()

Added: z3c.repoexternals/trunk/z3c/repoexternals/tag.py
===================================================================
--- z3c.repoexternals/trunk/z3c/repoexternals/tag.py	                        (rev 0)
+++ z3c.repoexternals/trunk/z3c/repoexternals/tag.py	2007-09-27 08:37:31 UTC (rev 80198)
@@ -0,0 +1,47 @@
+#!/usr/bin/python
+
+import sys, re, optparse
+import pysvn
+
+usage = "usage: %prog [options] externals"
+parser = optparse.OptionParser(usage=usage)
+
+external = re.compile(
+    r'^(\s*#?\s*)([^#\s]+)(\s*)(.*?)(\s*)(\S+)(\s*)$')
+
+def run(externals):
+    client = pysvn.Client()
+    external_match = external.match
+    for line in externals:
+        match = external_match(line)
+        if match is not None and client.is_url(match.group(6)):
+            try:
+                info = client.info(match.group(2))
+            except pysvn.ClientError:
+                info = None
+            if info is not None:
+                yield match.expand(
+                    r'\1\2\3-r %s%s\6\7' % (info.revision.number,
+                                            match.group(5) or ' '))
+            else:
+                yield line
+        else:
+            yield line
+
+def main():
+    options, args = parser.parse_args()
+
+    if len(args) != 1:
+        parser.error("requires externals")
+
+    externals, = args
+    if externals == '-':
+        externals = sys.stdin
+    else:
+        externals = file(externals)
+
+    for line in run(externals):
+        print line,
+
+if __name__ == '__main__':
+    main()

Added: z3c.repoexternals/trunk/z3c/repoexternals/tests/EXTERNALS.txt
===================================================================
--- z3c.repoexternals/trunk/z3c/repoexternals/tests/EXTERNALS.txt	                        (rev 0)
+++ z3c.repoexternals/trunk/z3c/repoexternals/tests/EXTERNALS.txt	2007-09-27 08:37:31 UTC (rev 80198)
@@ -0,0 +1,12 @@
+## Test externals
+## Second line
+bar/trunk file://%(tmpdir)s/repo/bar/trunk
+## Skip deep directory
+#foo/bar/bar file://%(tmpdir)s/repo/foo/bar/bar
+foo/bar/Trunk file://%(tmpdir)s/repo/foo/bar/Trunk
+other/foo http://svn.other.org/svn/main/other/foo
+# other/bar http://svn.other.org/svn/main/other/bar
+## Two line comment
+##Two line comment
+foo/trunk file://%(tmpdir)s/repo/foo/trunk
+## End comment

Added: z3c.repoexternals/trunk/z3c/repoexternals/tests/__init__.py
===================================================================
--- z3c.repoexternals/trunk/z3c/repoexternals/tests/__init__.py	                        (rev 0)
+++ z3c.repoexternals/trunk/z3c/repoexternals/tests/__init__.py	2007-09-27 08:37:31 UTC (rev 80198)
@@ -0,0 +1 @@
+"""z3c.repoexternals.tests"""

Added: z3c.repoexternals/trunk/z3c/repoexternals/tests/functional.txt
===================================================================
--- z3c.repoexternals/trunk/z3c/repoexternals/tests/functional.txt	                        (rev 0)
+++ z3c.repoexternals/trunk/z3c/repoexternals/tests/functional.txt	2007-09-27 08:37:31 UTC (rev 80198)
@@ -0,0 +1,86 @@
+;-*-Doctest-*-
+====================
+Repository Externals
+====================
+
+Walks a repository looking for trunks and outputting externals for
+those trunks.
+
+    >>> import logging
+    >>> logging.getLogger('repoexternals').setLevel(logging.INFO)
+
+    >>> for line in run(url): print line
+    bar/trunk file://.../repo/bar/trunk
+    baz/trunk file://.../repo/baz/trunk
+    foo/bar/Trunk file://.../repo/foo/bar/Trunk
+    foo/bar/baz/qux/trunk file://.../repo/foo/bar/baz/qux/trunk
+    foo/trunk file://.../repo/foo/trunk
+
+Warnings are issued for all directories skipped over.
+
+    >>> log.seek(0)
+    >>> for line in sorted(log): print line
+    Descending into file://.../repo/bar
+    Descending into file://.../repo/baz
+    Descending into file://.../repo/foo
+    Descending into file://.../repo/foo/bar
+    Descending into file://.../repo/foo/bar/baz
+    Descending into file://.../repo/foo/bar/baz/qux
+    Descending into file://.../repo/foo/branchesjunk
+    Descending into file://.../repo/foo/trunkjunk
+    Excluding file://.../repo/bar/branches
+    Excluding file://.../repo/baz/branches
+    Excluding file://.../repo/foo/bar/branches
+    Excluding file://.../repo/foo/branches
+    Too deep, skipping file://.../repo/foo/bar/baz/qux/quux
+    >>> position = log.tell()
+
+A previous externals definitions can be provided.
+
+    >>> externals_file = file(externals)
+    >>> for line in run(url, externals_file): print line
+    baz/trunk file://.../repo/baz/trunk
+    >>> externals_file.close()
+
+    >>> log.seek(position)
+    >>> for line in sorted(log): print line
+    Descending into file://.../repo/baz
+    Descending into file://.../repo/foo
+    Descending into file://.../repo/foo/branchesjunk
+    Descending into file://.../repo/foo/trunkjunk
+    Excluding file://.../repo/baz/branches
+    Excluding file://.../repo/foo/branches
+    In previous, skipping file://.../repo/bar
+    In previous, skipping file://.../repo/foo/bar
+    In previous, skipping file://.../repo/foo/trunk
+    >>> position = log.tell()
+
+different including and excluding regexps can be provided.
+
+    >>> externals_file = file(externals)
+    >>> for line in run(
+    ...     url, externals_file, r'(?i)^(.*)/branches$',
+    ...     r'(?i)^.*/trunk$'): print line 
+    bar/branches file://.../repo/bar/branches
+    baz/branches file://.../repo/baz/branches
+    foo/bar/branches file://.../repo/foo/bar/branches
+    foo/branches file://.../repo/foo/branches
+    >>> externals_file.close()
+
+    >>> log.seek(position)
+    >>> for line in sorted(log): print line
+    Descending into file://.../repo/bar
+    Descending into file://.../repo/baz
+    Descending into file://.../repo/foo
+    Descending into file://.../repo/foo/bar
+    Descending into file://.../repo/foo/bar/baz
+    Descending into file://.../repo/foo/bar/baz/qux
+    Descending into file://.../repo/foo/branchesjunk
+    Descending into file://.../repo/foo/trunkjunk
+    Excluding file://.../repo/baz/trunk
+    Excluding file://.../repo/foo/bar/baz/qux/trunk
+    In previous, skipping file://.../repo/bar/trunk
+    In previous, skipping file://.../repo/foo/bar/Trunk
+    In previous, skipping file://.../repo/foo/trunk
+    Too deep, skipping file://.../repo/foo/bar/baz/qux/quux
+    >>> position = log.tell()

Added: z3c.repoexternals/trunk/z3c/repoexternals/tests/script.txt
===================================================================
--- z3c.repoexternals/trunk/z3c/repoexternals/tests/script.txt	                        (rev 0)
+++ z3c.repoexternals/trunk/z3c/repoexternals/tests/script.txt	2007-09-27 08:37:31 UTC (rev 80198)
@@ -0,0 +1,273 @@
+;-*-Doctest-*-
+===================
+Commande Line Usage
+===================
+
+Get the usage and help.
+
+    >>> import os
+    >>> stdin, stdout, stderr = os.popen3(' '.join([script, '-h']))
+
+    >>> print stdout.read()
+    usage: repoexternals [options] url_or_path
+    <BLANKLINE>
+    ...
+    <BLANKLINE>
+    options:
+      -h, --help            show this help message and exit
+      -v, --verbose         Output logging to stdandard error. Set
+                            twice to log debugging mesages...
+
+    >>> print stderr.read()
+    >>> stdin.close()
+    >>> stdout.close()
+    >>> stderr.close()
+
+Errors
+------
+
+When invoked with other than one positional args an error is returned.
+
+    >>> stdin, stdout, stderr = os.popen3(script)
+
+    >>> print stderr.read()
+    usage: repoexternals [options] url_or_path
+    repoexternals: error: requires one url_or_path
+
+    >>> print stdout.read()
+    >>> stdin.close()
+    >>> stdout.close()
+    >>> stderr.close()
+        
+    >>> stdin, stdout, stderr = os.popen3(
+    ...     ' '.join([script, url, url]))
+
+    >>> print stderr.read()
+    usage: repoexternals [options] url_or_path
+    repoexternals: error: requires one url_or_path
+
+    >>> print stdout.read()
+    >>> stdin.close()
+    >>> stdout.close()
+    >>> stderr.close()
+
+---------
+Arguments
+---------
+
+When invoked with just one url, externals are returned based on the
+defaults.
+        
+    >>> stdin, stdout, stderr = os.popen3(' '.join([script, url]))
+
+    >>> print stdout.read()
+    bar/trunk file://.../repo/bar/trunk
+    baz/trunk file://.../repo/baz/trunk
+    foo/bar/Trunk file://.../repo/foo/bar/Trunk
+    foo/bar/baz/qux/trunk file://.../repo/foo/bar/baz/qux/trunk
+    foo/trunk file://.../repo/foo/trunk
+
+    >>> print stderr.read()
+    >>> stdin.close()
+    >>> stdout.close()
+    >>> stderr.close()
+
+Verbose
+-------
+
+When verbose is set twice, the number of threads will be logged as
+well as the urls being excluded.
+        
+    >>> stdin, stdout, stderr = os.popen3(
+    ...     ' '.join([script, '-vv', url]))
+
+    >>> print stdout.read()
+    bar/trunk file://.../repo/bar/trunk
+    baz/trunk file://.../repo/baz/trunk
+    foo/bar/Trunk file://.../repo/foo/bar/Trunk
+    foo/bar/baz/qux/trunk file://.../repo/foo/bar/baz/qux/trunk
+    foo/trunk file://.../repo/foo/trunk
+
+    >>> for line in sorted(stderr): print line
+    DEBUG:repoexternals:Started 5 client threads
+    INFO:repoexternals:Descending into file://.../repo/bar
+    INFO:repoexternals:Descending into file://.../repo/baz
+    INFO:repoexternals:Descending into file://.../repo/foo
+    INFO:repoexternals:Descending into file://.../repo/foo/bar
+    INFO:repoexternals:Descending into file://.../repo/foo/bar/baz
+    INFO:repoexternals:Descending into file://.../repo/foo/bar/baz/qux
+    INFO:repoexternals:Descending into
+    file://.../repo/foo/branchesjunk
+    INFO:repoexternals:Descending into file://.../repo/foo/trunkjunk
+    INFO:repoexternals:Excluding file://.../repo/bar/branches
+    INFO:repoexternals:Excluding file://.../repo/baz/branches
+    INFO:repoexternals:Excluding file://.../repo/foo/bar/branches
+    INFO:repoexternals:Excluding file://.../repo/foo/branches
+    INFO:repoexternals:Too deep, skipping
+    file://.../repo/foo/bar/baz/qux/quux
+
+    >>> stdin.close()
+    >>> stdout.close()
+    >>> stderr.close()
+
+Previous
+--------
+
+When an externals file is provided, the output is filterd through it.
+        
+    >>> stdin, stdout, stderr = os.popen3(
+    ...     ' '.join([script, '-vv', '-p', externals, url]))
+
+    >>> print stdout.read()
+    baz/trunk file://.../repo/baz/trunk
+
+    >>> for line in sorted(stderr): print line
+    DEBUG:repoexternals:Started 5 client threads
+    INFO:repoexternals:Descending into file://.../repo/baz
+    INFO:repoexternals:Descending into file://.../repo/foo
+    INFO:repoexternals:Descending into file://.../repo/foo/branchesjunk
+    INFO:repoexternals:Descending into file://.../repo/foo/trunkjunk
+    INFO:repoexternals:Excluding file://.../repo/baz/branches
+    INFO:repoexternals:Excluding file://.../repo/foo/branches
+    INFO:repoexternals:In previous, skipping file://.../repo/bar
+    INFO:repoexternals:In previous, skipping file://.../repo/foo/bar
+    INFO:repoexternals:In previous, skipping file://.../repo/foo/trunk
+
+    >>> stdin.close()
+    >>> stdout.close()
+    >>> stderr.close()
+
+If the filename is '-', stdin will be used.
+        
+    >>> stdin, stdout, stderr = os.popen3(
+    ...     ' '.join([script, '-vv', '-p -', url]))
+
+    >>> externals_file = file(externals)
+    >>> stdin.write(externals_file.read())
+    >>> externals_file.close()
+    >>> stdin.close()
+
+    >>> print stdout.read()
+    baz/trunk file://.../repo/baz/trunk
+
+    >>> for line in sorted(stderr): print line
+    DEBUG:repoexternals:Started 5 client threads
+    INFO:repoexternals:Descending into file://.../repo/baz
+    INFO:repoexternals:Descending into file://.../repo/foo
+    INFO:repoexternals:Descending into file://.../repo/foo/branchesjunk
+    INFO:repoexternals:Descending into file://.../repo/foo/trunkjunk
+    INFO:repoexternals:Excluding file://.../repo/baz/branches
+    INFO:repoexternals:Excluding file://.../repo/foo/branches
+    INFO:repoexternals:In previous, skipping file://.../repo/bar
+    INFO:repoexternals:In previous, skipping file://.../repo/foo/bar
+    INFO:repoexternals:In previous, skipping file://.../repo/foo/trunk
+
+    >>> stdout.close()
+    >>> stderr.close()
+
+Include and Exclude
+-------------------
+
+different including and excluding regexps can be provided.
+        
+    >>> stdin, stdout, stderr = os.popen3(' '.join([
+    ...     script, '-vv', '-p', externals, '-i',
+    ...     "'(?i)^(.*)/branches$'", '-e', "'(?i)^.*/trunk$'", url]))
+
+    >>> print stdout.read()
+    bar/branches file://.../repo/bar/branches
+    baz/branches file://.../repo/baz/branches
+    foo/bar/branches file://.../repo/foo/bar/branches
+    foo/branches file://.../repo/foo/branches
+
+    >>> for line in sorted(stderr): print line
+    DEBUG:repoexternals:Started 5 client threads
+    INFO:repoexternals:Descending into file://.../repo/bar
+    INFO:repoexternals:Descending into file://.../repo/baz
+    INFO:repoexternals:Descending into file://.../repo/foo
+    INFO:repoexternals:Descending into file://.../repo/foo/bar
+    INFO:repoexternals:Descending into file://.../repo/foo/bar/baz
+    INFO:repoexternals:Descending into file://.../repo/foo/bar/baz/qux
+    INFO:repoexternals:Descending into
+    file://.../repo/foo/branchesjunk
+    INFO:repoexternals:Descending into file://.../repo/foo/trunkjunk
+    INFO:repoexternals:Excluding file://.../repo/baz/trunk
+    INFO:repoexternals:Excluding file://.../repo/foo/bar/baz/qux/trunk
+    INFO:repoexternals:In previous, skipping file://.../repo/bar/trunk
+    INFO:repoexternals:In previous, skipping
+    file://.../repo/foo/bar/Trunk
+    INFO:repoexternals:In previous, skipping file://.../repo/foo/trunk
+    INFO:repoexternals:Too deep, skipping
+    file://.../repo/foo/bar/baz/qux/quux
+
+    >>> stdin.close()
+    >>> stdout.close()
+    >>> stderr.close()
+
+Depth
+-----
+        
+    >>> stdin, stdout, stderr = os.popen3(
+    ...     ' '.join([script, '-vv', '-d 4', url]))
+
+    >>> print stdout.read()
+    bar/trunk file://.../repo/bar/trunk
+    baz/trunk file://.../repo/baz/trunk
+    foo/bar/Trunk file://.../repo/foo/bar/Trunk
+    foo/trunk file://.../repo/foo/trunk
+
+    >>> for line in sorted(stderr): print line
+    DEBUG:repoexternals:Started 5 client threads
+    INFO:repoexternals:Descending into file://.../repo/bar
+    INFO:repoexternals:Descending into file://.../repo/baz
+    INFO:repoexternals:Descending into file://.../repo/foo
+    INFO:repoexternals:Descending into file://.../repo/foo/bar
+    INFO:repoexternals:Descending into file://.../repo/foo/bar/baz
+    INFO:repoexternals:Descending into
+    file://.../repo/foo/branchesjunk
+    INFO:repoexternals:Descending into file://.../repo/foo/trunkjunk
+    INFO:repoexternals:Excluding file://.../repo/bar/branches
+    INFO:repoexternals:Excluding file://.../repo/baz/branches
+    INFO:repoexternals:Excluding file://.../repo/foo/bar/branches
+    INFO:repoexternals:Excluding file://.../repo/foo/branches
+    INFO:repoexternals:Too deep, skipping
+    file://.../repo/foo/bar/baz/qux
+
+    >>> stdin.close()
+    >>> stdout.close()
+    >>> stderr.close()
+
+Pool Size
+---------
+        
+    >>> stdin, stdout, stderr = os.popen3(
+    ...     ' '.join([script, '-vv', '-s 4', url]))
+
+    >>> print stdout.read()
+    bar/trunk file://.../repo/bar/trunk
+    baz/trunk file://.../repo/baz/trunk
+    foo/bar/Trunk file://.../repo/foo/bar/Trunk
+    foo/bar/baz/qux/trunk file://.../repo/foo/bar/baz/qux/trunk
+    foo/trunk file://.../repo/foo/trunk
+
+    >>> for line in sorted(stderr): print line
+    DEBUG:repoexternals:Started 4 client threads
+    INFO:repoexternals:Descending into file://.../repo/bar
+    INFO:repoexternals:Descending into file://.../repo/baz
+    INFO:repoexternals:Descending into file://.../repo/foo
+    INFO:repoexternals:Descending into file://.../repo/foo/bar
+    INFO:repoexternals:Descending into file://.../repo/foo/bar/baz
+    INFO:repoexternals:Descending into file://.../repo/foo/bar/baz/qux
+    INFO:repoexternals:Descending into
+    file://.../repo/foo/branchesjunk
+    INFO:repoexternals:Descending into file://.../repo/foo/trunkjunk
+    INFO:repoexternals:Excluding file://.../repo/bar/branches
+    INFO:repoexternals:Excluding file://.../repo/baz/branches
+    INFO:repoexternals:Excluding file://.../repo/foo/bar/branches
+    INFO:repoexternals:Excluding file://.../repo/foo/branches
+    INFO:repoexternals:Too deep, skipping
+    file://.../repo/foo/bar/baz/qux/quux
+
+    >>> stdin.close()
+    >>> stdout.close()
+    >>> stderr.close()

Added: z3c.repoexternals/trunk/z3c/repoexternals/tests/test.py
===================================================================
--- z3c.repoexternals/trunk/z3c/repoexternals/tests/test.py	                        (rev 0)
+++ z3c.repoexternals/trunk/z3c/repoexternals/tests/test.py	2007-09-27 08:37:31 UTC (rev 80198)
@@ -0,0 +1,60 @@
+#!/usr/bin/python
+
+import os
+
+def setUp(test):
+    import os.path, tempfile, StringIO, logging
+    import pysvn
+
+    # Create a repository in a temporary directory
+    test.tmpdir = tempfile.mkdtemp()
+    repo = os.path.join(test.tmpdir, 'repo')
+    stdin, stdouterr = os.popen4('svnadmin create ' + repo)
+    assert stdouterr.read() == ''
+    assert stdin.close() is None
+    assert stdouterr.close() is None
+
+    # Import out test repository layout into the temporary repository
+    url = 'file://' + repo
+    testdir = os.path.dirname(__file__)
+    pysvn.Client().import_(os.path.join(testdir, 'repo'), url, '_')
+
+    # Construct an externals file from the template
+    template = file(os.path.join(testdir, 'EXTERNALS.txt'))
+    fd, externals_path = tempfile.mkstemp()
+    externals = file(externals_path, 'w')
+    externals.write(template.read() % {'tmpdir': test.tmpdir})
+    externals.close()
+    template.close()
+
+    # Collect log
+    log = StringIO.StringIO()
+    logger = logging.getLogger('repoexternals')
+    logger.addHandler(logging.StreamHandler(log))
+
+    # Add names from setup to the test globals
+    script = os.path.join(
+        os.path.dirname(os.path.dirname(os.getcwd())),
+        'bin', 'repoexternals')
+    test.globs.update(script=script, url=url, log=log,
+                      externals=externals_path)
+
+    import z3c.repoexternals
+    test.globs.update(z3c.repoexternals.__dict__)
+
+def tearDown(test):
+    import shutil
+    shutil.rmtree(test.tmpdir)
+    os.remove(test.globs['externals'])
+
+def test_suite():
+    import doctest
+    return doctest.DocFileSuite(
+        'functional.txt',
+        'script.txt',
+        setUp=setUp, tearDown=tearDown,
+        optionflags=doctest.ELLIPSIS|doctest.NORMALIZE_WHITESPACE)
+
+if __name__ == '__main__':
+    import unittest
+    unittest.main(defaultTest='test_suite')


Property changes on: z3c.repoexternals/trunk/z3c/repoexternals/tests/test.py
___________________________________________________________________
Name: svn:executable
   + *



More information about the Checkins mailing list