[Zope3-checkins]
SVN: zope.testing/trunk/src/zope/testing/testrunner
Added the capability to run remaining layers in a subprocess when a
Jim Fulton
jim at zope.com
Fri Jul 1 13:23:13 EDT 2005
Log message for revision 30971:
Added the capability to run remaining layers in a subprocess when a
layer can't be torn down.
Changed:
A zope.testing/trunk/src/zope/testing/testrunner-ex/sample1/sampletests_ntd.py
A zope.testing/trunk/src/zope/testing/testrunner-ex/sample2/sampletests_ntd.py
A zope.testing/trunk/src/zope/testing/testrunner-ex/sample3/sampletests_ntd.py
U zope.testing/trunk/src/zope/testing/testrunner.py
U zope.testing/trunk/src/zope/testing/testrunner.txt
-=-
Added: zope.testing/trunk/src/zope/testing/testrunner-ex/sample1/sampletests_ntd.py
===================================================================
--- zope.testing/trunk/src/zope/testing/testrunner-ex/sample1/sampletests_ntd.py 2005-07-01 12:54:32 UTC (rev 30970)
+++ zope.testing/trunk/src/zope/testing/testrunner-ex/sample1/sampletests_ntd.py 2005-07-01 17:23:12 UTC (rev 30971)
@@ -0,0 +1,61 @@
+##############################################################################
+#
+# Copyright (c) 2003 Zope Corporation and Contributors.
+# All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.0 (ZPL). A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE.
+#
+##############################################################################
+"""Sample tests with a layer that can't be torn down
+
+$Id$
+"""
+
+import unittest
+
+class Layer:
+
+ def setUp(self):
+ pass
+ setUp = classmethod(setUp)
+
+ def tearDown(self):
+ raise NotImplementedError
+ tearDown = classmethod(tearDown)
+
+class TestSomething(unittest.TestCase):
+
+ layer = Layer
+
+ def test_something(self):
+ pass
+
+ def test_something_else(self):
+ pass
+
+ def test_error1(self):
+ raise TypeError("Can we see errors")
+
+ def test_error2(self):
+ raise TypeError("I hope so")
+
+ def test_fail1(self):
+ self.assertEqual(1, 2)
+
+ def test_fail2(self):
+ self.assertEqual(1, 3)
+
+
+def test_suite():
+ suite = unittest.TestSuite()
+ suite.addTest(unittest.makeSuite(TestSomething))
+ return suite
+
+
+if __name__ == '__main__':
+ unittest.main()
Property changes on: zope.testing/trunk/src/zope/testing/testrunner-ex/sample1/sampletests_ntd.py
___________________________________________________________________
Name: svn:keywords
+ Id
Name: svn:eol-style
+ native
Added: zope.testing/trunk/src/zope/testing/testrunner-ex/sample2/sampletests_ntd.py
===================================================================
--- zope.testing/trunk/src/zope/testing/testrunner-ex/sample2/sampletests_ntd.py 2005-07-01 12:54:32 UTC (rev 30970)
+++ zope.testing/trunk/src/zope/testing/testrunner-ex/sample2/sampletests_ntd.py 2005-07-01 17:23:12 UTC (rev 30971)
@@ -0,0 +1,46 @@
+##############################################################################
+#
+# Copyright (c) 2003 Zope Corporation and Contributors.
+# All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.0 (ZPL). A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE.
+#
+##############################################################################
+"""Sample tests with a layer that can't be torn down
+
+$Id$
+"""
+
+import unittest
+
+class Layer:
+
+ def setUp(self):
+ pass
+ setUp = classmethod(setUp)
+
+ def tearDown(self):
+ raise NotImplementedError
+ tearDown = classmethod(tearDown)
+
+class TestSomething(unittest.TestCase):
+
+ layer = Layer
+
+ def test_something(self):
+ pass
+
+
+def test_suite():
+ suite = unittest.TestSuite()
+ suite.addTest(unittest.makeSuite(TestSomething))
+ return suite
+
+
+if __name__ == '__main__':
+ unittest.main()
Property changes on: zope.testing/trunk/src/zope/testing/testrunner-ex/sample2/sampletests_ntd.py
___________________________________________________________________
Name: svn:keywords
+ Id
Name: svn:eol-style
+ native
Added: zope.testing/trunk/src/zope/testing/testrunner-ex/sample3/sampletests_ntd.py
===================================================================
--- zope.testing/trunk/src/zope/testing/testrunner-ex/sample3/sampletests_ntd.py 2005-07-01 12:54:32 UTC (rev 30970)
+++ zope.testing/trunk/src/zope/testing/testrunner-ex/sample3/sampletests_ntd.py 2005-07-01 17:23:12 UTC (rev 30971)
@@ -0,0 +1,46 @@
+##############################################################################
+#
+# Copyright (c) 2003 Zope Corporation and Contributors.
+# All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.0 (ZPL). A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE.
+#
+##############################################################################
+"""Sample tests with a layer that can't be torn down
+
+$Id$
+"""
+
+import unittest
+
+class Layer:
+
+ def setUp(self):
+ pass
+ setUp = classmethod(setUp)
+
+ def tearDown(self):
+ raise NotImplementedError
+ tearDown = classmethod(tearDown)
+
+class TestSomething(unittest.TestCase):
+
+ layer = Layer
+
+ def test_something(self):
+ pass
+
+
+def test_suite():
+ suite = unittest.TestSuite()
+ suite.addTest(unittest.makeSuite(TestSomething))
+ return suite
+
+
+if __name__ == '__main__':
+ unittest.main()
Property changes on: zope.testing/trunk/src/zope/testing/testrunner-ex/sample3/sampletests_ntd.py
___________________________________________________________________
Name: svn:keywords
+ Id
Name: svn:eol-style
+ native
Modified: zope.testing/trunk/src/zope/testing/testrunner.py
===================================================================
--- zope.testing/trunk/src/zope/testing/testrunner.py 2005-07-01 12:54:32 UTC (rev 30970)
+++ zope.testing/trunk/src/zope/testing/testrunner.py 2005-07-01 17:23:12 UTC (rev 30971)
@@ -19,7 +19,6 @@
# Too bad: For now, we depend on zope.testing. This is because
# we want to use the latest, greatest doctest, which zope.testing
# provides. Then again, zope.testing is generally useful.
-from zope.testing import doctest
import gc
import logging
import optparse
@@ -41,14 +40,32 @@
"""
def run(defaults=None, args=None):
+ if args is None:
+ args = sys.argv
+
# Control reporting flags during run
old_reporting_flags = doctest.set_unittest_reportflags(0)
# Make sure we start with real pdb.set_trace. This is needed
# to make tests of the test runner work properly. :)
pdb.set_trace = real_pdb_set_trace
+
+ # Check to see if we are being run as a subprocess. If we are,
+ # then use the resume-layer and defaults passed in.
+ if len(args) > 1 and args[1] == '--resume-layer':
+ args.pop(1)
+ resume_layer = args.pop(1)
+ defaults = []
+ while args[1] == '--default':
+ args.pop(1)
+ defaults.append(args.pop(1))
+ else:
+ resume_layer = None
options = get_options(args, defaults)
+ options.testrunner_defaults = defaults
+ options.resume_layer = resume_layer
+
try:
run_with_options(options)
except EndRun:
@@ -58,7 +75,10 @@
def run_with_options(options):
- if options.verbose:
+ if options.resume_layer:
+ original_stderr = sys.stderr
+ sys.stderr = sys.stdout
+ elif options.verbose:
if options.all:
print "Running tests at all levels"
else:
@@ -78,7 +98,6 @@
tests_by_layer_name = find_tests(options)
-
ran = 0
failures = []
errors = []
@@ -94,7 +113,7 @@
if 'unit' in tests_by_layer_name:
tests = tests_by_layer_name.pop('unit')
- if not options.non_unit:
+ if (not options.non_unit) and not options.resume_layer:
if options.layer:
should_run = False
for pat in options.layer:
@@ -109,6 +128,11 @@
nlayers += 1
ran += run_tests(options, tests, 'unit', failures, errors)
+ if options.resume_layer:
+ resume = False
+ else:
+ resume = True
+
setup_layers = {}
for layer_name, layer, tests in ordered_layers(tests_by_layer_name):
if options.layer:
@@ -121,36 +145,54 @@
should_run = True
if should_run:
+ if (not resume) and (layer_name == options.resume_layer):
+ resume = True
+ if not resume:
+ continue
nlayers += 1
- ran += run_layer(options, layer_name, layer, tests, setup_layers,
- failures, errors)
+ try:
+ ran += run_layer(options, layer_name, layer, tests,
+ setup_layers, failures, errors)
+ except CanNotTearDown:
+ setup_layers = None
+ ran += resume_tests(options, layer_name, failures, errors)
+ break
if setup_layers:
print "Tearing down left over layers:"
- tear_down_unneeded((), setup_layers)
+ tear_down_unneeded((), setup_layers, True)
- if options.verbose > 1:
- if errors:
- print
- print "Tests with errors:"
- for test, exc_info in errors:
- print " ", test
+ if options.resume_layer:
+ sys.stdout.close()
+ print >> original_stderr, ran, len(failures), len(errors)
+ for test, exc_info in failures:
+ print >> original_stderr, ' '.join(str(test).strip().split('\n'))
+ for test, exc_info in errors:
+ print >> original_stderr, ' '.join(str(test).strip().split('\n'))
- if failures:
- print
- print "Tests with failures:"
- for test, exc_info in failures:
- print " ", test
+ else:
+ if options.verbose > 1:
+ if errors:
+ print
+ print "Tests with errors:"
+ for test, exc_info in errors:
+ print " ", test
- if nlayers != 1:
- print "Total: %s tests, %s failures, %s errors" % (
- ran, len(failures), len(errors))
+ if failures:
+ print
+ print "Tests with failures:"
+ for test, exc_info in failures:
+ print " ", test
- if import_errors:
- print
- print "Test-modules with import problems:"
- for test in import_errors:
- print " " + test.module
+ if nlayers != 1:
+ print "Total: %s tests, %s failures, %s errors" % (
+ ran, len(failures), len(errors))
+
+ if import_errors:
+ print
+ print "Test-modules with import problems:"
+ for test in import_errors:
+ print " " + test.module
def run_tests(options, tests, name, failures, errors):
@@ -209,7 +251,39 @@
setup_layer(layer, setup_layers)
return run_tests(options, tests, layer_name, failures, errors)
-def tear_down_unneeded(needed, setup_layers):
+def resume_tests(options, layer_name, failures, errors):
+ args = [sys.executable,
+ options.original_testrunner_args[0],
+ '--resume-layer', layer_name,
+ ]
+ for d in options.testrunner_defaults:
+ args.extend(['--default', d])
+
+ args.extend(options.original_testrunner_args[1:])
+
+ if sys.platform.startswith('win'):
+ args = ' '.join([
+ ('"' + a.replace('\\', '\\\\').replace('"', '\\"') + '"')
+ for a in args
+ ])
+
+ subin, subout, suberr = os.popen3(args)
+ for l in subout:
+ sys.stdout.write(l)
+ ran, nfail, nerr = map(int, suberr.readline().strip().split())
+ while nfail > 0:
+ nfail -= 1
+ failures.append((suberr.readline().strip(), None))
+ while nerr > 0:
+ nerr -= 1
+ errors.append((suberr.readline().strip(), None))
+ return ran
+
+
+class CanNotTearDown(Exception):
+ "Couldn't tear down a test"
+
+def tear_down_unneeded(needed, setup_layers, optional=False):
# Tear down any layers not needed for these tests. The unneeded
# layers might interfere.
unneeded = [l for l in setup_layers if l not in needed]
@@ -218,11 +292,18 @@
for l in unneeded:
print " Tear down %s" % name_from_layer(l),
t = time.time()
- l.tearDown()
+ try:
+ l.tearDown()
+ except NotImplementedError:
+ print "... not supported"
+ if not optional:
+ raise CanNotTearDown(l)
+ else:
+ print "in %.3f seconds." % (time.time() - t)
del setup_layers[l]
- print "in %.3f seconds." % (time.time() - t)
+
def setup_layer(layer, setup_layers):
if layer not in setup_layers:
for base in layer.__bases__:
@@ -951,10 +1032,13 @@
defaults = default_setup
if args is None:
- args = sys.argv[1:]
+ args = sys.argv
+ original_testrunner_args = args
+ args = args[1:]
options, positional = parser.parse_args(args)
merge_options(options, defaults)
-
+ options.original_testrunner_args = original_testrunner_args
+
if positional:
module_filter = positional.pop()
if module_filter != '.':
@@ -1039,6 +1123,7 @@
def setUp(test):
test.globs['saved-sys-info'] = sys.path, sys.argv
test.globs['this_directory'] = os.path.split(__file__)[0]
+ test.globs['testrunner_script'] = __file__
def tearDown(test):
sys.path, sys.argv = test.globs['saved-sys-info']
@@ -1057,9 +1142,29 @@
if __name__ == '__main__':
# Hm, when run as a script, we need to import the testrunner under
# it's own name, so that there's the imported flavor has the right
- # real_pdb_set_trace.
- import zope.testing.testrunner
+ # real_pdb_set_trace. We also want to make sure we can import it.
+ # If we can't, we should try fixing the path and trying again
+ try:
+ import zope.testing.testrunner
+ except ImportError:
+ sys.path.append(
+ os.path.split(
+ os.path.split(
+ os.path.split(
+ os.path.abspath(sys.argv[0])
+ )[0]
+ )[0]
+ )[0]
+ )
+ import zope.testing.testrunner
+
+
+ from zope.testing import doctest
main()
# Test the testrunner
###############################################################################
+
+# Delay import to give main an opportunity to fix up the path if
+# necessary
+from zope.testing import doctest
Modified: zope.testing/trunk/src/zope/testing/testrunner.txt
===================================================================
--- zope.testing/trunk/src/zope/testing/testrunner.txt 2005-07-01 12:54:32 UTC (rev 30970)
+++ zope.testing/trunk/src/zope/testing/testrunner.txt 2005-07-01 17:23:12 UTC (rev 30971)
@@ -227,6 +227,26 @@
Tear down samplelayers.Layerx in 0.000 seconds.
Total: 213 tests, 0 failures, 0 errors
+Passing arguments explicitly
+----------------------------
+
+In most of the examples here, we set up `sys.argv`. In normal usage,
+the testrunner just uses `sys.argv`. It is possible to pass athiments
+explicitly.
+
+ >>> testrunner.run(defaults, 'test --layer 111'.split())
+ Running samplelayers.Layer111 tests:
+ Set up samplelayers.Layerx in 0.000 seconds.
+ Set up samplelayers.Layer1 in 0.000 seconds.
+ Set up samplelayers.Layer11 in 0.000 seconds.
+ Set up samplelayers.Layer111 in 0.000 seconds.
+ Ran 34 tests with 0 failures and 0 errors in 0.007 seconds.
+ Tearing down left over layers:
+ Tear down samplelayers.Layer111 in 0.000 seconds.
+ Tear down samplelayers.Layer11 in 0.000 seconds.
+ Tear down samplelayers.Layer1 in 0.000 seconds.
+ Tear down samplelayers.Layerx in 0.000 seconds.
+
Verbose Output
--------------
@@ -387,7 +407,7 @@
You can specify multiple packages:
>>> sys.argv = 'test -u -vv -ssample1 -ssample2'.split()
- >>> testrunner.run(defaults)
+ >>> testrunner.run(defaults) # doctest: +REPORT_NDIFF
Running tests at level 1
Running unit tests:
Running:
@@ -1497,6 +1517,89 @@
sample2.sample22.sampletests_i
sample2.sample23.sampletests_i
+Layers that can't be torn down
+------------------------------
+
+If a layer can have a tearDown method that raises
+NotImplementedError. If this is the case and there are no remaining
+tests to run, the test runner will just note that the tear down
+couldn't be done:
+
+ >>> sys.argv = 'test -ssample2 --tests-pattern sampletests_ntd'.split()
+ >>> testrunner.run(defaults)
+ Running sample2.sampletests_ntd.Layer tests:
+ Set up sample2.sampletests_ntd.Layer in 0.000 seconds.
+ Ran 1 tests with 0 failures and 0 errors in 0.000 seconds.
+ Tearing down left over layers:
+ Tear down sample2.sampletests_ntd.Layer ... not supported
+
+If the tearDown method raises NotImplementedError and there are remaining
+tests to run, the test runner will restart itself as a new process,
+resuming tests where it left off:
+
+ >>> sys.argv = [testrunner_script, '--tests-pattern', 'sampletests_ntd']
+ >>> testrunner.run(defaults)
+ Running sample3.sampletests_ntd.Layer tests:
+ Set up sample3.sampletests_ntd.Layer in 0.000 seconds.
+ Ran 1 tests with 0 failures and 0 errors in 0.000 seconds.
+ Running sample2.sampletests_ntd.Layer tests:
+ Tear down sample3.sampletests_ntd.Layer ... not supported
+ Running sample2.sampletests_ntd.Layer tests:
+ Set up sample2.sampletests_ntd.Layer in 0.000 seconds.
+ Ran 1 tests with 0 failures and 0 errors in 0.000 seconds.
+ Running sample1.sampletests_ntd.Layer tests:
+ Tear down sample2.sampletests_ntd.Layer ... not supported
+ Running sample1.sampletests_ntd.Layer tests:
+ Set up sample1.sampletests_ntd.Layer in 0.000 seconds.
+ <BLANKLINE>
+ <BLANKLINE>
+ Error in test test_error1 (sample1.sampletests_ntd.TestSomething)
+ Traceback (most recent call last):
+ File ".../unittest.py", line 260, in run
+ testMethod()
+ File "testrunner-ex/sample1/sampletests_ntd.py", line 42, in test_error1
+ raise TypeError("Can we see errors")
+ TypeError: Can we see errors
+ <BLANKLINE>
+ <BLANKLINE>
+ <BLANKLINE>
+ Error in test test_error2 (sample1.sampletests_ntd.TestSomething)
+ Traceback (most recent call last):
+ File ".../unittest.py", line 260, in run
+ testMethod()
+ File "testrunner-ex/sample1/sampletests_ntd.py", line 45, in test_error2
+ raise TypeError("I hope so")
+ TypeError: I hope so
+ <BLANKLINE>
+ <BLANKLINE>
+ <BLANKLINE>
+ Failure in test test_fail1 (sample1.sampletests_ntd.TestSomething)
+ Traceback (most recent call last):
+ File ".../unittest.py", line 260, in run
+ testMethod()
+ File "testrunner-ex/sample1/sampletests_ntd.py", line 48, in test_fail1
+ self.assertEqual(1, 2)
+ File ".../unittest.py", line 332, in failUnlessEqual
+ raise self.failureException, \
+ AssertionError: 1 != 2
+ <BLANKLINE>
+ <BLANKLINE>
+ <BLANKLINE>
+ Failure in test test_fail2 (sample1.sampletests_ntd.TestSomething)
+ Traceback (most recent call last):
+ File ".../unittest.py", line 260, in run
+ testMethod()
+ File "testrunner-ex/sample1/sampletests_ntd.py", line 51, in test_fail2
+ self.assertEqual(1, 3)
+ File ".../unittest.py", line 332, in failUnlessEqual
+ raise self.failureException, \
+ AssertionError: 1 != 3
+ <BLANKLINE>
+ Ran 6 tests with 2 failures and 2 errors in 0.002 seconds.
+ Tearing down left over layers:
+ Tear down sample1.sampletests_ntd.Layer ... not supported
+ Total: 8 tests, 2 failures, 2 errors
+
Debugging
---------
More information about the Zope3-Checkins
mailing list