[Checkins] SVN: z3c.repoexternals/trunk/ Generate externals from a
repository
Ross Patterson
me at rpatterson.net
Thu Sep 27 04:37:32 EDT 2007
Log message for revision 80198:
Generate externals from a repository
Changed:
_U z3c.repoexternals/trunk/
A z3c.repoexternals/trunk/.cvsignore
A z3c.repoexternals/trunk/EXTERNALS.txt
A z3c.repoexternals/trunk/README.txt
A z3c.repoexternals/trunk/buildout.cfg
A z3c.repoexternals/trunk/paster.cfg
A z3c.repoexternals/trunk/setup.cfg
A z3c.repoexternals/trunk/setup.py
A z3c.repoexternals/trunk/z3c/
A z3c.repoexternals/trunk/z3c/__init__.py
A z3c.repoexternals/trunk/z3c/repoexternals/
A z3c.repoexternals/trunk/z3c/repoexternals/__init__.py
A z3c.repoexternals/trunk/z3c/repoexternals/tag.py
A z3c.repoexternals/trunk/z3c/repoexternals/tests/
A z3c.repoexternals/trunk/z3c/repoexternals/tests/EXTERNALS.txt
A z3c.repoexternals/trunk/z3c/repoexternals/tests/__init__.py
A z3c.repoexternals/trunk/z3c/repoexternals/tests/functional.txt
A z3c.repoexternals/trunk/z3c/repoexternals/tests/repo/
A z3c.repoexternals/trunk/z3c/repoexternals/tests/repo/bar/
A z3c.repoexternals/trunk/z3c/repoexternals/tests/repo/bar/branches/
A z3c.repoexternals/trunk/z3c/repoexternals/tests/repo/bar/trunk/
A z3c.repoexternals/trunk/z3c/repoexternals/tests/repo/baz/
A z3c.repoexternals/trunk/z3c/repoexternals/tests/repo/baz/branches/
A z3c.repoexternals/trunk/z3c/repoexternals/tests/repo/baz/trunk/
A z3c.repoexternals/trunk/z3c/repoexternals/tests/repo/foo/
A z3c.repoexternals/trunk/z3c/repoexternals/tests/repo/foo/bar/
A z3c.repoexternals/trunk/z3c/repoexternals/tests/repo/foo/bar/Trunk/
A z3c.repoexternals/trunk/z3c/repoexternals/tests/repo/foo/bar/baz/
A z3c.repoexternals/trunk/z3c/repoexternals/tests/repo/foo/bar/baz/qux/
A z3c.repoexternals/trunk/z3c/repoexternals/tests/repo/foo/bar/baz/qux/quux/
A z3c.repoexternals/trunk/z3c/repoexternals/tests/repo/foo/bar/baz/qux/trunk/
A z3c.repoexternals/trunk/z3c/repoexternals/tests/repo/foo/bar/branches/
A z3c.repoexternals/trunk/z3c/repoexternals/tests/repo/foo/branches/
A z3c.repoexternals/trunk/z3c/repoexternals/tests/repo/foo/branchesjunk/
A z3c.repoexternals/trunk/z3c/repoexternals/tests/repo/foo/trunk/
A z3c.repoexternals/trunk/z3c/repoexternals/tests/repo/foo/trunkjunk/
A z3c.repoexternals/trunk/z3c/repoexternals/tests/script.txt
A z3c.repoexternals/trunk/z3c/repoexternals/tests/test.py
-=-
Property changes on: z3c.repoexternals/trunk
___________________________________________________________________
Name: svn:ignore
+ .installed.cfg
bin
develop-eggs
parts
build
dist
Name: svn:externals
+ #
# applied by: svn propset svn:externals -F ./EXTERNALS.txt .
#
bootstrap svn://svn.zope.org/repos/main/zc.buildout/trunk/bootstrap
Copied: z3c.repoexternals/trunk/.cvsignore (from rev 80119, z3c.offlinepack/trunk/.cvsignore)
===================================================================
--- z3c.repoexternals/trunk/.cvsignore (rev 0)
+++ z3c.repoexternals/trunk/.cvsignore 2007-09-27 08:37:31 UTC (rev 80198)
@@ -0,0 +1,6 @@
+.installed.cfg
+bin
+develop-eggs
+parts
+build
+dist
\ No newline at end of file
Copied: z3c.repoexternals/trunk/EXTERNALS.txt (from rev 80119, z3c.offlinepack/trunk/EXTERNALS.txt)
===================================================================
--- z3c.repoexternals/trunk/EXTERNALS.txt (rev 0)
+++ z3c.repoexternals/trunk/EXTERNALS.txt 2007-09-27 08:37:31 UTC (rev 80198)
@@ -0,0 +1,4 @@
+#
+# applied by: svn propset svn:externals -F ./EXTERNALS.txt .
+#
+bootstrap svn://svn.zope.org/repos/main/zc.buildout/trunk/bootstrap
Added: z3c.repoexternals/trunk/README.txt
===================================================================
--- z3c.repoexternals/trunk/README.txt (rev 0)
+++ z3c.repoexternals/trunk/README.txt 2007-09-27 08:37:31 UTC (rev 80198)
@@ -0,0 +1,64 @@
+usage: repoexternals [options] url_or_path
+
+Recursively retrieves subversion directory listings from the url or
+path and matches directories against a previous set of svn:externals
+if provided then against regular expressions and generates qualifying
+svn:externals lines. The defaults generate a set of svn:externals for
+all the trunks in a repository and keeps them up to date with the
+repository as new trunks are added when the previous externals are
+provided thereafter.
+
+options:
+ -h, --help show this help message and exit
+ -v, --verbose Output logging to stdandard error. Set twice
+ to log debugging mesages.
+ -p FILE, --previous=FILE
+ If provided, only URLs in the repository not
+ included in the previous externals will be
+ included. If the filename is '-', use standard
+ input. Valid svn:externals lines beginning
+ with one comment character, '#', will also
+ affect output. This is useful, for example,
+ to prevent lengthy recursions into directories
+ that are known not to contain any desired
+ matches. The file is read completely and
+ closed before anything is output, so it is
+ safe to append output to the previous file:
+ "repoexternals -p EXTERNALS.txt
+ http://svn.foo.org/repos/main
+ >>EXTERNALS.txt".
+ -i REGEXP, --include=REGEXP
+ Directory names matching this python regular
+ expression will be included in output and will
+ not be descended into.
+ [default: (?i)^((.*)/.+?|.*)/trunk$]
+ -e REGEXP, --exclude=REGEXP
+ Directory names matching this python regular
+ expression will be excluded from output and
+ will not be descended into. Include overrides
+ exclude. [default:
+ (?i)^.*/(branch(es)?|tags?|releases?|vendor|bundles?|sandbox|build|dist)$]
+ -m TEMPLATE, --matched-template=TEMPLATE
+ The result of expanding previous file URL
+ matches with the include regular expression
+ through this template is added to the set of
+ previous URLs excluded from output and
+ descending. The default will add the parents
+ of trunks to the set of previous URLs
+ excluded. [default: \1]
+ -t TEMPLATE, --parent-template=TEMPLATE
+ The result of expanding previous file URL
+ matches with the include regular expression
+ through this template is removed from the set
+ of matched previous URLs excluded from output
+ and descending. The default ensures that
+ directories containing trunks within a
+ directory that contains a trunk are not
+ excluded. [default: \2]
+ -d INT, --depth=INT The maximum directory depth to descend to.
+ WARNING: large values can greatly increase run
+ time. [default: 5]
+ -s INT, --pool-size=INT
+ The number of concurrent svn clients.
+ WARNING: large values can DOS the repository.
+ [default: 5]
Copied: z3c.repoexternals/trunk/buildout.cfg (from rev 80119, z3c.offlinepack/trunk/buildout.cfg)
===================================================================
--- z3c.repoexternals/trunk/buildout.cfg (rev 0)
+++ z3c.repoexternals/trunk/buildout.cfg 2007-09-27 08:37:31 UTC (rev 80198)
@@ -0,0 +1,11 @@
+[buildout]
+develop = .
+parts = repoexternals test
+
+[repoexternals]
+recipe = zc.recipe.egg:scripts
+eggs = z3c.repoexternals
+
+[test]
+recipe = zc.recipe.testrunner
+eggs = z3c.repoexternals [test]
Added: z3c.repoexternals/trunk/paster.cfg
===================================================================
--- z3c.repoexternals/trunk/paster.cfg (rev 0)
+++ z3c.repoexternals/trunk/paster.cfg 2007-09-27 08:37:31 UTC (rev 80198)
@@ -0,0 +1,15 @@
+[pastescript]
+namespace_package = z3c
+description = Generate externals from a repository
+author = Ross Patterson
+author_email = me at rpatteron.net
+license_name = GPL
+url = http://cheeseshop.python.org/pypi/z3c.repoexternals
+version = 0.1
+plus = +
+zip_safe = True
+keywords =
+egg = z3c.repoexternals
+long_description =
+dot = .
+
Added: z3c.repoexternals/trunk/setup.cfg
===================================================================
--- z3c.repoexternals/trunk/setup.cfg (rev 0)
+++ z3c.repoexternals/trunk/setup.cfg 2007-09-27 08:37:31 UTC (rev 80198)
@@ -0,0 +1,3 @@
+[egg_info]
+tag_build = dev
+tag_svn_revision = true
Added: z3c.repoexternals/trunk/setup.py
===================================================================
--- z3c.repoexternals/trunk/setup.py (rev 0)
+++ z3c.repoexternals/trunk/setup.py 2007-09-27 08:37:31 UTC (rev 80198)
@@ -0,0 +1,39 @@
+from setuptools import setup, find_packages
+import sys, os
+
+version = '0.1'
+
+setup(name='z3c.repoexternals',
+ version=version,
+ description="Generate externals from a repository",
+ long_description=open(os.path.join(os.path.dirname(__file__),
+ 'README.txt')).read(),
+ # Get more strings from http://www.python.org/pypi?%3Aaction=list_classifiers
+ classifiers=[
+ "Framework :: Plone",
+ "Framework :: Zope2",
+ "Framework :: Zope3",
+ "Programming Language :: Python",
+ "Topic :: Software Development :: Libraries :: Python Modules",
+ ],
+ keywords='',
+ author='Ross Patterson',
+ author_email='me at rpatteron.net',
+ url='http://cheeseshop.python.org/pypi/z3c.repoexternals',
+ license='GPL',
+ packages=find_packages(exclude=['ez_setup']),
+ namespace_packages=['z3c'],
+ include_package_data=True,
+ zip_safe=True,
+ install_requires=[
+ 'setuptools',
+ # -*- Extra requirements: -*-
+ # 'pysvn',
+ ],
+ extras_require=dict(test=['zc.buildout', 'zc.recipe.egg']),
+ entry_points="""
+ # -*- Entry points: -*-
+ [console_scripts]
+ repoexternals = z3c.repoexternals:main
+ """,
+ )
Added: z3c.repoexternals/trunk/z3c/__init__.py
===================================================================
--- z3c.repoexternals/trunk/z3c/__init__.py (rev 0)
+++ z3c.repoexternals/trunk/z3c/__init__.py 2007-09-27 08:37:31 UTC (rev 80198)
@@ -0,0 +1,6 @@
+# See http://peak.telecommunity.com/DevCenter/setuptools#namespace-packages
+try:
+ __import__('pkg_resources').declare_namespace(__name__)
+except ImportError:
+ from pkgutil import extend_path
+ __path__ = extend_path(__path__, __name__)
Added: z3c.repoexternals/trunk/z3c/repoexternals/__init__.py
===================================================================
--- z3c.repoexternals/trunk/z3c/repoexternals/__init__.py (rev 0)
+++ z3c.repoexternals/trunk/z3c/repoexternals/__init__.py 2007-09-27 08:37:31 UTC (rev 80198)
@@ -0,0 +1,344 @@
+#!/usr/bin/python
+"""Recursively retrieves subversion directory listings from the url or
+path and matches directories against a previous set of svn:externals
+if provided then against regular expressions and generates qualifying
+svn:externals lines. The defaults generate a set of svn:externals for
+all the trunks in a repository and keeps them up to date with the
+repository as new trunks are added when the previous externals are
+provided thereafter."""
+
+import sys, os, re, logging, thread, threading, Queue, optparse
+import pysvn
+
+usage = "usage: %prog [options] url_or_path"
+
+parser = optparse.OptionParser(usage=usage, description=__doc__)
+
+parser.add_option(
+ "-v", "--verbose", action="count",
+ help=("Output logging to stdandard error. Set twice to log "
+ "debugging mesages."))
+
+parser.add_option(
+ "-p", "--previous", metavar='FILE',
+ help=("""If provided, only URLs in the repository not
+included in the previous externals will be included. If the filename
+is '-', use standard input. Valid svn:externals lines beginning with
+one comment character, '#', will also affect output. This is useful,
+for example, to prevent lengthy recursions into directories that are
+known not to contain any desired matches. The file is read completely
+and closed before anything is output, so it is safe to append output
+to the previous file: "repoexternals -p EXTERNALS.txt
+http://svn.foo.org/repos/main >>EXTERNALS.txt"."""))
+
+include = r'(?i)^((.*)/.+?|.*)/trunk$'
+parser.add_option(
+ "-i", "--include", default=include, metavar='REGEXP',
+ help=("Directory names matching this python regular expression "
+ "will be included in output and will not be descended into."
+ " [default: %default]"))
+
+exclude = (r'(?i)^.*/(branch(es)?|tags?|releases?|vendor|bundles?'
+ r'|sandbox|build|dist)$')
+parser.add_option(
+ "-e", "--exclude", default=exclude, metavar='REGEXP',
+ help=("Directory names matching this python regular expression "
+ "will be excluded from output and will not be descended "
+ "into. Include overrides exclude. [default: %default]"))
+
+matched_template = r'\1'
+parser.add_option(
+ "-m", "--matched-template", default=matched_template,
+ metavar='TEMPLATE',
+ help=("""The result of expanding previous file URL matches with
+the include regular expression through this template is added to the
+set of previous URLs excluded from output and descending. The default
+will add the parents of trunks to the set of previous URLs
+excluded. [default: %default]"""))
+
+parent_template = r'\2'
+parser.add_option(
+ "-t", "--parent-template", default=parent_template,
+ metavar='TEMPLATE',
+ help=("""The result of expanding previous file URL matches with
+the include regular expression through this template is removed from
+the set of matched previous URLs excluded from output and descending.
+The default ensures that directories containing trunks within a
+directory that contains a trunk are not excluded.
+[default: %default]"""))
+
+depth = 5
+parser.add_option(
+ "-d", "--depth", type="int", default=depth, metavar='INT',
+ help=("The maximum directory depth to descend to. WARNING: "
+ "large values can greatly increase run time. "
+ "[default: %default]"))
+
+pool_size = 5
+parser.add_option(
+ "-s", "--pool-size", type="int", default=pool_size, metavar='INT',
+ help=("The number of concurrent svn clients. WARNING: large "
+ "values can DOS the repository. [default: %default]"))
+
+shutdown = object()
+class Thread(threading.Thread):
+
+ def __init__(self, queue=None, results=None, **kwargs):
+ super(Thread, self).__init__(**kwargs)
+ if queue is None:
+ queue = Queue.Queue()
+ if results is None:
+ results = Queue.Queue()
+ self.queue = queue
+ self.results = results
+
+ def run(self):
+ try:
+ payload = self.queue.get()
+ while payload is not shutdown:
+ payload(self)
+ payload = self.queue.get()
+ except:
+ thread.interrupt_main()
+ raise
+
+ def shutdown(self):
+ self.queue.put(shutdown)
+ self.join()
+
+ def interrupt(self):
+ self.queue.mutex.acquire()
+ self.queue._init(self.queue.maxsize)
+ self.queue._put(shutdown)
+ self.queue.not_empty.notify()
+ self.queue.mutex.release()
+ self.join()
+
+class ClientThread(Thread):
+
+ def __init__(self, *args, **kwargs):
+ super(ClientThread, self).__init__(*args, **kwargs)
+ self.client = pysvn.Client()
+
+class ClientPool(Queue.Queue):
+
+ def __init__(self, size=pool_size, results=None,
+ **kwargs):
+ Queue.Queue.__init__(self, **kwargs)
+ if results is None:
+ results = Queue.Queue()
+ self.results = results
+ self.threads = [
+ ClientThread(queue=self, results=self.results)
+ for ignored in xrange(size)]
+
+ def start(self):
+ for thread in self.threads:
+ thread.start()
+ logging.getLogger('repoexternals').debug(
+ 'Started %s client threads' % len(self.threads))
+
+ def shutdown(self):
+ for thread in self.threads:
+ self.put(shutdown)
+ for thread in self.threads:
+ thread.join()
+
+ def interrupt(self):
+ self.mutex.acquire()
+ self._init(self.maxsize)
+ for thread in self.threads:
+ self._put(shutdown)
+ self.not_empty.notify()
+ self.mutex.release()
+ for thread in self.threads:
+ if thread.isAlive():
+ thread.join()
+
+class Root(object):
+ """Return self unless overriden in a child instance"""
+
+ def __get__(self, instance, owner):
+ return instance
+
+class Line(object):
+
+ def __init__(self, path, dirent):
+ self.path = path
+ self.dirent = dirent
+
+ def __str__(self):
+ return '%s %s' % (self.path, self.dirent.path)
+
+class Listing(object):
+ """A single svn listing"""
+
+ def __init__(self, url, include=include, exclude=exclude,
+ depth=depth):
+ self.url = url
+ self.include = re.compile(include)
+ self.exclude = re.compile(exclude)
+ self.depth = depth
+
+ self.results = Queue.Queue()
+
+ def list(self, thread):
+ """Retrieve the svn listing from the repository"""
+ try:
+ self.listing = thread.client.list(
+ self.url, dirent_fields=pysvn.SVN_DIRENT_KIND)
+ except pysvn.ClientError, e:
+ logging.getLogger('repoexternals').exception(
+ 'pysvn.ClientError %s' % self.url)
+ self.listing = []
+
+ # Queue for processing
+ thread.results.put(self.process)
+
+ root = Root()
+
+ def getchild(self, url):
+ """Create and return a child listing setting it's root"""
+ child = Listing(url, include=self.include,
+ exclude=self.exclude, depth=self.depth)
+ child.root = self.root
+ return child
+
+ def process(self, thread):
+ """Process the results of the svn listing"""
+
+ # Use local names in the inner loop
+ dir_node_kind = pysvn.node_kind.dir
+ root_url = self.root.url
+ include_match = self.include.match
+ exclude_match = self.exclude.match
+ results_put = self.results.put
+ lLine = Line
+ info = logging.getLogger('repoexternals').info
+ depth = self.depth
+ getchild = self.getchild
+ thread_results_put = thread.results.put
+ previous = self.root.previous
+
+ for dirent, ignored in self.listing[1:]:
+
+ if dirent.kind != dir_node_kind:
+ # externals can only be directories
+ continue
+
+ dirent_path = dirent.path
+
+ if dirent_path in previous:
+ info('In previous, skipping %s' % dirent_path)
+ continue
+
+ # No previous line, use matching
+ path = dirent_path[len(root_url):].lstrip('/')
+
+ if include_match(dirent_path) is not None:
+ # Include this line in the results
+ results_put(lLine(path=path, dirent=dirent))
+
+ elif exclude_match(dirent_path) is not None:
+ info('Excluding %s' % dirent_path)
+ elif len(path.split('/')) >= depth:
+ info('Too deep, skipping %s' % dirent_path)
+
+ else:
+ child = getchild(dirent_path)
+ info('Descending into %s' % dirent_path)
+ thread_results_put(child.list)
+ results_put(child)
+
+ # Let the iterator know we're done
+ results_put(shutdown)
+
+ def __iter__(self):
+ item = self.results.get()
+ while item is not shutdown:
+ if isinstance(item, Line):
+ yield item
+ else:
+ for child in item:
+ yield child
+ item = self.results.get()
+
+# Match valid externals definitions in existing externals
+external = re.compile(r'^\s*#?\s*([^#\s]+)\s(.*\s|)(\S+)\s*$')
+
+def run(url, previous=(), include=include, exclude=exclude,
+ matched_template=matched_template,
+ parent_template=parent_template,
+ depth=depth, pool_size=pool_size):
+ pool = ClientPool(size=pool_size)
+ thread = Thread(queue=pool.results, results=pool)
+
+ pool.start()
+ thread.start()
+
+ try:
+ root = Listing(url, include=include, exclude=exclude,
+ depth=depth)
+
+ # Build the set of previous URLs
+ include_match = root.include.match
+ root.previous = set()
+ raw_add = root.previous.add
+ matched = set()
+ matched_add = matched.add
+ parents = set()
+ parents_add = parents.add
+ external_match = external.match
+ for line in previous:
+ match = external_match(line)
+ if match is not None:
+ # TODO: also use pysvn.Cliens.is_url to verify url
+ # validity?
+ line_url = match.group(3)
+ raw_add(line_url)
+ include_matched = include_match(line_url)
+ if include_matched is not None:
+ matched_add(
+ include_matched.expand(matched_template))
+ parents_add(
+ include_matched.expand(parent_template))
+ root.previous.update(matched.difference(parents))
+
+ pool.put(root.list)
+
+ for line in root:
+ yield line
+ except:
+ # TODO: can't find a way to test interruptability
+ pool.interrupt()
+ thread.interrupt()
+ raise
+ else:
+ pool.shutdown()
+ thread.shutdown()
+
+def main():
+ options, args = parser.parse_args()
+
+ if len(args) != 1:
+ parser.error("requires one url_or_path")
+ url, = args
+
+ if options.verbose is not None:
+ verbose = options.verbose <= 2 and options.verbose or 2
+ logging.basicConfig(level=logging.WARN - (verbose * 10))
+
+ previous = ()
+ if options.previous:
+ if options.previous == '-':
+ previous = sys.stdin
+ else:
+ previous = file(options.previous)
+
+ for line in run(url, previous, options.include, options.exclude,
+ options.matched_template, options.parent_template,
+ options.depth, options.pool_size):
+ print line
+
+if __name__ == '__main__':
+ main()
Added: z3c.repoexternals/trunk/z3c/repoexternals/tag.py
===================================================================
--- z3c.repoexternals/trunk/z3c/repoexternals/tag.py (rev 0)
+++ z3c.repoexternals/trunk/z3c/repoexternals/tag.py 2007-09-27 08:37:31 UTC (rev 80198)
@@ -0,0 +1,47 @@
+#!/usr/bin/python
+
+import sys, re, optparse
+import pysvn
+
+usage = "usage: %prog [options] externals"
+parser = optparse.OptionParser(usage=usage)
+
+external = re.compile(
+ r'^(\s*#?\s*)([^#\s]+)(\s*)(.*?)(\s*)(\S+)(\s*)$')
+
+def run(externals):
+ client = pysvn.Client()
+ external_match = external.match
+ for line in externals:
+ match = external_match(line)
+ if match is not None and client.is_url(match.group(6)):
+ try:
+ info = client.info(match.group(2))
+ except pysvn.ClientError:
+ info = None
+ if info is not None:
+ yield match.expand(
+ r'\1\2\3-r %s%s\6\7' % (info.revision.number,
+ match.group(5) or ' '))
+ else:
+ yield line
+ else:
+ yield line
+
+def main():
+ options, args = parser.parse_args()
+
+ if len(args) != 1:
+ parser.error("requires externals")
+
+ externals, = args
+ if externals == '-':
+ externals = sys.stdin
+ else:
+ externals = file(externals)
+
+ for line in run(externals):
+ print line,
+
+if __name__ == '__main__':
+ main()
Added: z3c.repoexternals/trunk/z3c/repoexternals/tests/EXTERNALS.txt
===================================================================
--- z3c.repoexternals/trunk/z3c/repoexternals/tests/EXTERNALS.txt (rev 0)
+++ z3c.repoexternals/trunk/z3c/repoexternals/tests/EXTERNALS.txt 2007-09-27 08:37:31 UTC (rev 80198)
@@ -0,0 +1,12 @@
+## Test externals
+## Second line
+bar/trunk file://%(tmpdir)s/repo/bar/trunk
+## Skip deep directory
+#foo/bar/bar file://%(tmpdir)s/repo/foo/bar/bar
+foo/bar/Trunk file://%(tmpdir)s/repo/foo/bar/Trunk
+other/foo http://svn.other.org/svn/main/other/foo
+# other/bar http://svn.other.org/svn/main/other/bar
+## Two line comment
+##Two line comment
+foo/trunk file://%(tmpdir)s/repo/foo/trunk
+## End comment
Added: z3c.repoexternals/trunk/z3c/repoexternals/tests/__init__.py
===================================================================
--- z3c.repoexternals/trunk/z3c/repoexternals/tests/__init__.py (rev 0)
+++ z3c.repoexternals/trunk/z3c/repoexternals/tests/__init__.py 2007-09-27 08:37:31 UTC (rev 80198)
@@ -0,0 +1 @@
+"""z3c.repoexternals.tests"""
Added: z3c.repoexternals/trunk/z3c/repoexternals/tests/functional.txt
===================================================================
--- z3c.repoexternals/trunk/z3c/repoexternals/tests/functional.txt (rev 0)
+++ z3c.repoexternals/trunk/z3c/repoexternals/tests/functional.txt 2007-09-27 08:37:31 UTC (rev 80198)
@@ -0,0 +1,86 @@
+;-*-Doctest-*-
+====================
+Repository Externals
+====================
+
+Walks a repository looking for trunks and outputting externals for
+those trunks.
+
+ >>> import logging
+ >>> logging.getLogger('repoexternals').setLevel(logging.INFO)
+
+ >>> for line in run(url): print line
+ bar/trunk file://.../repo/bar/trunk
+ baz/trunk file://.../repo/baz/trunk
+ foo/bar/Trunk file://.../repo/foo/bar/Trunk
+ foo/bar/baz/qux/trunk file://.../repo/foo/bar/baz/qux/trunk
+ foo/trunk file://.../repo/foo/trunk
+
+Warnings are issued for all directories skipped over.
+
+ >>> log.seek(0)
+ >>> for line in sorted(log): print line
+ Descending into file://.../repo/bar
+ Descending into file://.../repo/baz
+ Descending into file://.../repo/foo
+ Descending into file://.../repo/foo/bar
+ Descending into file://.../repo/foo/bar/baz
+ Descending into file://.../repo/foo/bar/baz/qux
+ Descending into file://.../repo/foo/branchesjunk
+ Descending into file://.../repo/foo/trunkjunk
+ Excluding file://.../repo/bar/branches
+ Excluding file://.../repo/baz/branches
+ Excluding file://.../repo/foo/bar/branches
+ Excluding file://.../repo/foo/branches
+ Too deep, skipping file://.../repo/foo/bar/baz/qux/quux
+ >>> position = log.tell()
+
+A previous externals definitions can be provided.
+
+ >>> externals_file = file(externals)
+ >>> for line in run(url, externals_file): print line
+ baz/trunk file://.../repo/baz/trunk
+ >>> externals_file.close()
+
+ >>> log.seek(position)
+ >>> for line in sorted(log): print line
+ Descending into file://.../repo/baz
+ Descending into file://.../repo/foo
+ Descending into file://.../repo/foo/branchesjunk
+ Descending into file://.../repo/foo/trunkjunk
+ Excluding file://.../repo/baz/branches
+ Excluding file://.../repo/foo/branches
+ In previous, skipping file://.../repo/bar
+ In previous, skipping file://.../repo/foo/bar
+ In previous, skipping file://.../repo/foo/trunk
+ >>> position = log.tell()
+
+different including and excluding regexps can be provided.
+
+ >>> externals_file = file(externals)
+ >>> for line in run(
+ ... url, externals_file, r'(?i)^(.*)/branches$',
+ ... r'(?i)^.*/trunk$'): print line
+ bar/branches file://.../repo/bar/branches
+ baz/branches file://.../repo/baz/branches
+ foo/bar/branches file://.../repo/foo/bar/branches
+ foo/branches file://.../repo/foo/branches
+ >>> externals_file.close()
+
+ >>> log.seek(position)
+ >>> for line in sorted(log): print line
+ Descending into file://.../repo/bar
+ Descending into file://.../repo/baz
+ Descending into file://.../repo/foo
+ Descending into file://.../repo/foo/bar
+ Descending into file://.../repo/foo/bar/baz
+ Descending into file://.../repo/foo/bar/baz/qux
+ Descending into file://.../repo/foo/branchesjunk
+ Descending into file://.../repo/foo/trunkjunk
+ Excluding file://.../repo/baz/trunk
+ Excluding file://.../repo/foo/bar/baz/qux/trunk
+ In previous, skipping file://.../repo/bar/trunk
+ In previous, skipping file://.../repo/foo/bar/Trunk
+ In previous, skipping file://.../repo/foo/trunk
+ Too deep, skipping file://.../repo/foo/bar/baz/qux/quux
+ >>> position = log.tell()
Added: z3c.repoexternals/trunk/z3c/repoexternals/tests/script.txt
===================================================================
--- z3c.repoexternals/trunk/z3c/repoexternals/tests/script.txt (rev 0)
+++ z3c.repoexternals/trunk/z3c/repoexternals/tests/script.txt 2007-09-27 08:37:31 UTC (rev 80198)
@@ -0,0 +1,273 @@
+;-*-Doctest-*-
+===================
+Commande Line Usage
+===================
+
+Get the usage and help.
+
+ >>> import os
+ >>> stdin, stdout, stderr = os.popen3(' '.join([script, '-h']))
+
+ >>> print stdout.read()
+ usage: repoexternals [options] url_or_path
+ <BLANKLINE>
+ ...
+ <BLANKLINE>
+ options:
+ -h, --help show this help message and exit
+ -v, --verbose Output logging to stdandard error. Set
+ twice to log debugging mesages...
+
+ >>> print stderr.read()
+ >>> stdin.close()
+ >>> stdout.close()
+ >>> stderr.close()
+
+Errors
+------
+
+When invoked with other than one positional args an error is returned.
+
+ >>> stdin, stdout, stderr = os.popen3(script)
+
+ >>> print stderr.read()
+ usage: repoexternals [options] url_or_path
+ repoexternals: error: requires one url_or_path
+
+ >>> print stdout.read()
+ >>> stdin.close()
+ >>> stdout.close()
+ >>> stderr.close()
+
+ >>> stdin, stdout, stderr = os.popen3(
+ ... ' '.join([script, url, url]))
+
+ >>> print stderr.read()
+ usage: repoexternals [options] url_or_path
+ repoexternals: error: requires one url_or_path
+
+ >>> print stdout.read()
+ >>> stdin.close()
+ >>> stdout.close()
+ >>> stderr.close()
+
+---------
+Arguments
+---------
+
+When invoked with just one url, externals are returned based on the
+defaults.
+
+ >>> stdin, stdout, stderr = os.popen3(' '.join([script, url]))
+
+ >>> print stdout.read()
+ bar/trunk file://.../repo/bar/trunk
+ baz/trunk file://.../repo/baz/trunk
+ foo/bar/Trunk file://.../repo/foo/bar/Trunk
+ foo/bar/baz/qux/trunk file://.../repo/foo/bar/baz/qux/trunk
+ foo/trunk file://.../repo/foo/trunk
+
+ >>> print stderr.read()
+ >>> stdin.close()
+ >>> stdout.close()
+ >>> stderr.close()
+
+Verbose
+-------
+
+When verbose is set twice, the number of threads will be logged as
+well as the urls being excluded.
+
+ >>> stdin, stdout, stderr = os.popen3(
+ ... ' '.join([script, '-vv', url]))
+
+ >>> print stdout.read()
+ bar/trunk file://.../repo/bar/trunk
+ baz/trunk file://.../repo/baz/trunk
+ foo/bar/Trunk file://.../repo/foo/bar/Trunk
+ foo/bar/baz/qux/trunk file://.../repo/foo/bar/baz/qux/trunk
+ foo/trunk file://.../repo/foo/trunk
+
+ >>> for line in sorted(stderr): print line
+ DEBUG:repoexternals:Started 5 client threads
+ INFO:repoexternals:Descending into file://.../repo/bar
+ INFO:repoexternals:Descending into file://.../repo/baz
+ INFO:repoexternals:Descending into file://.../repo/foo
+ INFO:repoexternals:Descending into file://.../repo/foo/bar
+ INFO:repoexternals:Descending into file://.../repo/foo/bar/baz
+ INFO:repoexternals:Descending into file://.../repo/foo/bar/baz/qux
+ INFO:repoexternals:Descending into
+ file://.../repo/foo/branchesjunk
+ INFO:repoexternals:Descending into file://.../repo/foo/trunkjunk
+ INFO:repoexternals:Excluding file://.../repo/bar/branches
+ INFO:repoexternals:Excluding file://.../repo/baz/branches
+ INFO:repoexternals:Excluding file://.../repo/foo/bar/branches
+ INFO:repoexternals:Excluding file://.../repo/foo/branches
+ INFO:repoexternals:Too deep, skipping
+ file://.../repo/foo/bar/baz/qux/quux
+
+ >>> stdin.close()
+ >>> stdout.close()
+ >>> stderr.close()
+
+Previous
+--------
+
+When an externals file is provided, the output is filterd through it.
+
+ >>> stdin, stdout, stderr = os.popen3(
+ ... ' '.join([script, '-vv', '-p', externals, url]))
+
+ >>> print stdout.read()
+ baz/trunk file://.../repo/baz/trunk
+
+ >>> for line in sorted(stderr): print line
+ DEBUG:repoexternals:Started 5 client threads
+ INFO:repoexternals:Descending into file://.../repo/baz
+ INFO:repoexternals:Descending into file://.../repo/foo
+ INFO:repoexternals:Descending into file://.../repo/foo/branchesjunk
+ INFO:repoexternals:Descending into file://.../repo/foo/trunkjunk
+ INFO:repoexternals:Excluding file://.../repo/baz/branches
+ INFO:repoexternals:Excluding file://.../repo/foo/branches
+ INFO:repoexternals:In previous, skipping file://.../repo/bar
+ INFO:repoexternals:In previous, skipping file://.../repo/foo/bar
+ INFO:repoexternals:In previous, skipping file://.../repo/foo/trunk
+
+ >>> stdin.close()
+ >>> stdout.close()
+ >>> stderr.close()
+
+If the filename is '-', stdin will be used.
+
+ >>> stdin, stdout, stderr = os.popen3(
+ ... ' '.join([script, '-vv', '-p -', url]))
+
+ >>> externals_file = file(externals)
+ >>> stdin.write(externals_file.read())
+ >>> externals_file.close()
+ >>> stdin.close()
+
+ >>> print stdout.read()
+ baz/trunk file://.../repo/baz/trunk
+
+ >>> for line in sorted(stderr): print line
+ DEBUG:repoexternals:Started 5 client threads
+ INFO:repoexternals:Descending into file://.../repo/baz
+ INFO:repoexternals:Descending into file://.../repo/foo
+ INFO:repoexternals:Descending into file://.../repo/foo/branchesjunk
+ INFO:repoexternals:Descending into file://.../repo/foo/trunkjunk
+ INFO:repoexternals:Excluding file://.../repo/baz/branches
+ INFO:repoexternals:Excluding file://.../repo/foo/branches
+ INFO:repoexternals:In previous, skipping file://.../repo/bar
+ INFO:repoexternals:In previous, skipping file://.../repo/foo/bar
+ INFO:repoexternals:In previous, skipping file://.../repo/foo/trunk
+
+ >>> stdout.close()
+ >>> stderr.close()
+
+Include and Exclude
+-------------------
+
+different including and excluding regexps can be provided.
+
+ >>> stdin, stdout, stderr = os.popen3(' '.join([
+ ... script, '-vv', '-p', externals, '-i',
+ ... "'(?i)^(.*)/branches$'", '-e', "'(?i)^.*/trunk$'", url]))
+
+ >>> print stdout.read()
+ bar/branches file://.../repo/bar/branches
+ baz/branches file://.../repo/baz/branches
+ foo/bar/branches file://.../repo/foo/bar/branches
+ foo/branches file://.../repo/foo/branches
+
+ >>> for line in sorted(stderr): print line
+ DEBUG:repoexternals:Started 5 client threads
+ INFO:repoexternals:Descending into file://.../repo/bar
+ INFO:repoexternals:Descending into file://.../repo/baz
+ INFO:repoexternals:Descending into file://.../repo/foo
+ INFO:repoexternals:Descending into file://.../repo/foo/bar
+ INFO:repoexternals:Descending into file://.../repo/foo/bar/baz
+ INFO:repoexternals:Descending into file://.../repo/foo/bar/baz/qux
+ INFO:repoexternals:Descending into
+ file://.../repo/foo/branchesjunk
+ INFO:repoexternals:Descending into file://.../repo/foo/trunkjunk
+ INFO:repoexternals:Excluding file://.../repo/baz/trunk
+ INFO:repoexternals:Excluding file://.../repo/foo/bar/baz/qux/trunk
+ INFO:repoexternals:In previous, skipping file://.../repo/bar/trunk
+ INFO:repoexternals:In previous, skipping
+ file://.../repo/foo/bar/Trunk
+ INFO:repoexternals:In previous, skipping file://.../repo/foo/trunk
+ INFO:repoexternals:Too deep, skipping
+ file://.../repo/foo/bar/baz/qux/quux
+
+ >>> stdin.close()
+ >>> stdout.close()
+ >>> stderr.close()
+
+Depth
+-----
+
+ >>> stdin, stdout, stderr = os.popen3(
+ ... ' '.join([script, '-vv', '-d 4', url]))
+
+ >>> print stdout.read()
+ bar/trunk file://.../repo/bar/trunk
+ baz/trunk file://.../repo/baz/trunk
+ foo/bar/Trunk file://.../repo/foo/bar/Trunk
+ foo/trunk file://.../repo/foo/trunk
+
+ >>> for line in sorted(stderr): print line
+ DEBUG:repoexternals:Started 5 client threads
+ INFO:repoexternals:Descending into file://.../repo/bar
+ INFO:repoexternals:Descending into file://.../repo/baz
+ INFO:repoexternals:Descending into file://.../repo/foo
+ INFO:repoexternals:Descending into file://.../repo/foo/bar
+ INFO:repoexternals:Descending into file://.../repo/foo/bar/baz
+ INFO:repoexternals:Descending into
+ file://.../repo/foo/branchesjunk
+ INFO:repoexternals:Descending into file://.../repo/foo/trunkjunk
+ INFO:repoexternals:Excluding file://.../repo/bar/branches
+ INFO:repoexternals:Excluding file://.../repo/baz/branches
+ INFO:repoexternals:Excluding file://.../repo/foo/bar/branches
+ INFO:repoexternals:Excluding file://.../repo/foo/branches
+ INFO:repoexternals:Too deep, skipping
+ file://.../repo/foo/bar/baz/qux
+
+ >>> stdin.close()
+ >>> stdout.close()
+ >>> stderr.close()
+
+Pool Size
+---------
+
+ >>> stdin, stdout, stderr = os.popen3(
+ ... ' '.join([script, '-vv', '-s 4', url]))
+
+ >>> print stdout.read()
+ bar/trunk file://.../repo/bar/trunk
+ baz/trunk file://.../repo/baz/trunk
+ foo/bar/Trunk file://.../repo/foo/bar/Trunk
+ foo/bar/baz/qux/trunk file://.../repo/foo/bar/baz/qux/trunk
+ foo/trunk file://.../repo/foo/trunk
+
+ >>> for line in sorted(stderr): print line
+ DEBUG:repoexternals:Started 4 client threads
+ INFO:repoexternals:Descending into file://.../repo/bar
+ INFO:repoexternals:Descending into file://.../repo/baz
+ INFO:repoexternals:Descending into file://.../repo/foo
+ INFO:repoexternals:Descending into file://.../repo/foo/bar
+ INFO:repoexternals:Descending into file://.../repo/foo/bar/baz
+ INFO:repoexternals:Descending into file://.../repo/foo/bar/baz/qux
+ INFO:repoexternals:Descending into
+ file://.../repo/foo/branchesjunk
+ INFO:repoexternals:Descending into file://.../repo/foo/trunkjunk
+ INFO:repoexternals:Excluding file://.../repo/bar/branches
+ INFO:repoexternals:Excluding file://.../repo/baz/branches
+ INFO:repoexternals:Excluding file://.../repo/foo/bar/branches
+ INFO:repoexternals:Excluding file://.../repo/foo/branches
+ INFO:repoexternals:Too deep, skipping
+ file://.../repo/foo/bar/baz/qux/quux
+
+ >>> stdin.close()
+ >>> stdout.close()
+ >>> stderr.close()
Added: z3c.repoexternals/trunk/z3c/repoexternals/tests/test.py
===================================================================
--- z3c.repoexternals/trunk/z3c/repoexternals/tests/test.py (rev 0)
+++ z3c.repoexternals/trunk/z3c/repoexternals/tests/test.py 2007-09-27 08:37:31 UTC (rev 80198)
@@ -0,0 +1,60 @@
+#!/usr/bin/python
+
+import os
+
+def setUp(test):
+ import os.path, tempfile, StringIO, logging
+ import pysvn
+
+ # Create a repository in a temporary directory
+ test.tmpdir = tempfile.mkdtemp()
+ repo = os.path.join(test.tmpdir, 'repo')
+ stdin, stdouterr = os.popen4('svnadmin create ' + repo)
+ assert stdouterr.read() == ''
+ assert stdin.close() is None
+ assert stdouterr.close() is None
+
+ # Import out test repository layout into the temporary repository
+ url = 'file://' + repo
+ testdir = os.path.dirname(__file__)
+ pysvn.Client().import_(os.path.join(testdir, 'repo'), url, '_')
+
+ # Construct an externals file from the template
+ template = file(os.path.join(testdir, 'EXTERNALS.txt'))
+ fd, externals_path = tempfile.mkstemp()
+ externals = file(externals_path, 'w')
+ externals.write(template.read() % {'tmpdir': test.tmpdir})
+ externals.close()
+ template.close()
+
+ # Collect log
+ log = StringIO.StringIO()
+ logger = logging.getLogger('repoexternals')
+ logger.addHandler(logging.StreamHandler(log))
+
+ # Add names from setup to the test globals
+ script = os.path.join(
+ os.path.dirname(os.path.dirname(os.getcwd())),
+ 'bin', 'repoexternals')
+ test.globs.update(script=script, url=url, log=log,
+ externals=externals_path)
+
+ import z3c.repoexternals
+ test.globs.update(z3c.repoexternals.__dict__)
+
+def tearDown(test):
+ import shutil
+ shutil.rmtree(test.tmpdir)
+ os.remove(test.globs['externals'])
+
+def test_suite():
+ import doctest
+ return doctest.DocFileSuite(
+ 'functional.txt',
+ 'script.txt',
+ setUp=setUp, tearDown=tearDown,
+ optionflags=doctest.ELLIPSIS|doctest.NORMALIZE_WHITESPACE)
+
+if __name__ == '__main__':
+ import unittest
+ unittest.main(defaultTest='test_suite')
Property changes on: z3c.repoexternals/trunk/z3c/repoexternals/tests/test.py
___________________________________________________________________
Name: svn:executable
+ *
More information about the Checkins
mailing list