[Checkins] SVN: zope.testing/branches/sdouche-shuffle/ Initial version of new option shuffle (from repository http://bitbucket.org/multani/zope.testing-shuffle).
Sébastien Douche
sdouche at free.fr
Fri Oct 16 08:55:12 EDT 2009
Log message for revision 105102:
Initial version of new option shuffle (from repository http://bitbucket.org/multani/zope.testing-shuffle).
Changed:
U zope.testing/branches/sdouche-shuffle/CHANGES.txt
U zope.testing/branches/sdouche-shuffle/src/zope/testing/testrunner/options.py
U zope.testing/branches/sdouche-shuffle/src/zope/testing/testrunner/runner.py
A zope.testing/branches/sdouche-shuffle/src/zope/testing/testrunner/shuffle.py
A zope.testing/branches/sdouche-shuffle/src/zope/testing/testrunner/testrunner-shuffle.txt
U zope.testing/branches/sdouche-shuffle/src/zope/testing/testrunner/tests.py
-=-
Modified: zope.testing/branches/sdouche-shuffle/CHANGES.txt
===================================================================
--- zope.testing/branches/sdouche-shuffle/CHANGES.txt 2009-10-16 12:52:13 UTC (rev 105101)
+++ zope.testing/branches/sdouche-shuffle/CHANGES.txt 2009-10-16 12:55:12 UTC (rev 105102)
@@ -9,6 +9,9 @@
- Cleaned up unused imports reported by pyflakes.
+- Added two new options to generate randomly ordered list of tests and to
+ select a specific order of tests.
+
3.8.3 (2009-09-21)
==================
Modified: zope.testing/branches/sdouche-shuffle/src/zope/testing/testrunner/options.py
===================================================================
--- zope.testing/branches/sdouche-shuffle/src/zope/testing/testrunner/options.py 2009-10-16 12:52:13 UTC (rev 105101)
+++ zope.testing/branches/sdouche-shuffle/src/zope/testing/testrunner/options.py 2009-10-16 12:55:12 UTC (rev 105102)
@@ -398,6 +398,19 @@
Specifies the name of a directory to ignore when looking for tests.
""")
+setup.add_option(
+ '--shuffle', action="store_true", dest='shuffle',
+ help="""\
+Shuffles the order in which tests are ran.
+""")
+
+setup.add_option(
+ '--shuffle-seed', action="store", dest='shuffle_seed', type="int",
+ help="""\
+Value used to initialize the tests shuffler. Specify a value to create
+repeatable random ordered tests.
+""")
+
parser.add_option_group(setup)
######################################################################
Modified: zope.testing/branches/sdouche-shuffle/src/zope/testing/testrunner/runner.py
===================================================================
--- zope.testing/branches/sdouche-shuffle/src/zope/testing/testrunner/runner.py 2009-10-16 12:52:13 UTC (rev 105101)
+++ zope.testing/branches/sdouche-shuffle/src/zope/testing/testrunner/runner.py 2009-10-16 12:55:12 UTC (rev 105102)
@@ -46,6 +46,7 @@
import zope.testing.testrunner.interfaces
import zope.testing.testrunner.debug
import zope.testing.testrunner.tb_format
+import zope.testing.testrunner.shuffle
PYREFCOUNT_PATTERN = re.compile('\[[0-9]+ refs\]')
@@ -188,6 +189,7 @@
zope.testing.testrunner.garbagecollection.Debug(self))
self.features.append(zope.testing.testrunner.find.Find(self))
+ self.features.append(zope.testing.testrunner.shuffle.Shuffle(self))
self.features.append(zope.testing.testrunner.process.SubProcess(self))
self.features.append(zope.testing.testrunner.filter.Filter(self))
self.features.append(zope.testing.testrunner.listing.Listing(self))
Added: zope.testing/branches/sdouche-shuffle/src/zope/testing/testrunner/shuffle.py
===================================================================
--- zope.testing/branches/sdouche-shuffle/src/zope/testing/testrunner/shuffle.py (rev 0)
+++ zope.testing/branches/sdouche-shuffle/src/zope/testing/testrunner/shuffle.py 2009-10-16 12:55:12 UTC (rev 105102)
@@ -0,0 +1,53 @@
+##############################################################################
+#
+# Copyright (c) 2004-2008 Zope Corporation and Contributors.
+# All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE.
+#
+##############################################################################
+"""Shuffle tests discovered before executing them.
+
+$Id$
+"""
+
+import time
+import random
+import unittest
+import zope.testing.testrunner.feature
+
+
+class Shuffle(zope.testing.testrunner.feature.Feature):
+ """Shuffle tests found."""
+
+ def __init__(self, runner):
+ super(Shuffle, self).__init__(runner)
+ self.active = runner.options.shuffle
+
+ self.seed = runner.options.shuffle_seed
+ if self.seed is None:
+ # Imported from stdlib's random.py
+ # This is not the best way to obtain a random value, but in our
+ # case, this is very sufficient.
+ self.seed = long(time.time() * 256) # use fractional seconds
+
+ def global_setup(self):
+ shuffler = random.Random(self.seed)
+
+ # Recreate the list of tests and shuffle them
+ shuffled_tests = {}
+ for layer_name, tests in self.runner.tests_by_layer_name.iteritems():
+ tests = list(tests) # Make a copy
+ shuffler.shuffle(tests)
+ shuffled_tests[layer_name] = unittest.TestSuite(tests)
+
+ self.runner.tests_by_layer_name = shuffled_tests
+
+ def report(self):
+ msg = "Tests were shuffled using seed number %d" % self.seed
+ self.runner.options.output.info(msg)
Added: zope.testing/branches/sdouche-shuffle/src/zope/testing/testrunner/testrunner-shuffle.txt
===================================================================
--- zope.testing/branches/sdouche-shuffle/src/zope/testing/testrunner/testrunner-shuffle.txt (rev 0)
+++ zope.testing/branches/sdouche-shuffle/src/zope/testing/testrunner/testrunner-shuffle.txt 2009-10-16 12:55:12 UTC (rev 105102)
@@ -0,0 +1,106 @@
+Shuffling tests
+===============
+
+By default, every time you launch the testrunner, it will run every tests in a
+specific order. However, if you want to be sure that your setup/teardown
+functions are correctly written, and that tests aren't depending on the result
+or effect of another test, you might want to launch those tests in another,
+completely different order.
+
+The ``--shuffle`` option allows to create a shuffled list of tests.
+
+
+ >>> import os.path, sys
+ >>> directory_with_tests = os.path.join(this_directory, 'testrunner-ex')
+ >>> defaults = [
+ ... '--path', directory_with_tests,
+ ... '--tests-pattern', '^sampletestsf?$',
+ ... ]
+
+ >>> from zope.testing import testrunner
+ >>> default_argv = 'test -u -m sample1 -t test_y0 --list-tests '
+
+Launching shuffled tests
+------------------------
+
+By default, tests are shuffled differently each time you launch tests using the
+``--shuffle`` option:
+
+ >>> argv = (default_argv + '--shuffle').split()
+ >>> testrunner.run_internal(defaults, argv)
+ Tests were shuffled using seed number ...
+ Listing zope.testing.testrunner.layer.UnitTests tests:
+ ...
+ False
+
+Note that the runner prints out a new information, which is the seed number used
+to generate the shuffled list of tests. Using this seed number allows to repeat
+again and again the same randomly shuffled list of tests.
+
+Specifying a seed number to control tests shuffling
+---------------------------------------------------
+
+Along with the ``--shuffle`` option comes the ``--shuffle-seed`` option, which
+takes a seed number as argument. If you spot a specific shuffled list of tests
+which is failing, you can retrieve the seed number used to generate this list,
+and replay it again: you have the guarantee that, specifying a seed number, the
+random list of tests will always be the same.
+
+For example, using the seed number 0 will give us the following, stable, list of
+tests:
+
+ >>> argv = (default_argv + '--shuffle --shuffle-seed 0').split()
+ >>> testrunner.run_internal(defaults, argv)
+ Tests were shuffled using seed number 0
+ Listing zope.testing.testrunner.layer.UnitTests tests:
+ test_y0 (sample1.sampletestsf.TestA)
+ test_y0 (sample1.sampletests.test_one)
+ test_y0 (sample1.sampletests.test1.TestA)
+ test_y0 (sample1.sampletestsf)
+ test_y0 (sample1.sampletests.test_one.TestA)
+ test_y0 (sample1.sample13.sampletests)
+ test_y0 (sample1.sample13.sampletests.TestA)
+ test_y0 (sample1.sample11.sampletests)
+ test_y0 (sample1.sample11.sampletests.TestA)
+ test_y0 (sample1.sampletests.test1)
+ False
+
+Whereas using the seed number 42 will give us the following, different but
+stable, list of tests:
+
+ >>> argv = (default_argv + '--shuffle --shuffle-seed 42').split()
+ >>> testrunner.run_internal(defaults, argv)
+ Tests were shuffled using seed number 42
+ Listing zope.testing.testrunner.layer.UnitTests tests:
+ test_y0 (sample1.sample13.sampletests.TestA)
+ test_y0 (sample1.sample13.sampletests)
+ test_y0 (sample1.sampletests.test1)
+ test_y0 (sample1.sampletests.test1.TestA)
+ test_y0 (sample1.sample11.sampletests.TestA)
+ test_y0 (sample1.sampletestsf)
+ test_y0 (sample1.sampletests.test_one)
+ test_y0 (sample1.sample11.sampletests)
+ test_y0 (sample1.sampletestsf.TestA)
+ test_y0 (sample1.sampletests.test_one.TestA)
+ False
+
+Selecting a seed number without ``--shuffle``
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Note that the ``--shuffle-seed`` option must be used along with ``--shuffle``
+option, or tests will not be re-ordered:
+
+ >>> argv = (default_argv + '--shuffle-seed 42').split()
+ >>> testrunner.run_internal(defaults, argv)
+ Listing zope.testing.testrunner.layer.UnitTests tests:
+ test_y0 (sample1.sampletestsf.TestA)
+ test_y0 (sample1.sampletestsf)
+ test_y0 (sample1.sample11.sampletests.TestA)
+ test_y0 (sample1.sample11.sampletests)
+ test_y0 (sample1.sample13.sampletests.TestA)
+ test_y0 (sample1.sample13.sampletests)
+ test_y0 (sample1.sampletests.test1.TestA)
+ test_y0 (sample1.sampletests.test1)
+ test_y0 (sample1.sampletests.test_one.TestA)
+ test_y0 (sample1.sampletests.test_one)
+ False
Modified: zope.testing/branches/sdouche-shuffle/src/zope/testing/testrunner/tests.py
===================================================================
--- zope.testing/branches/sdouche-shuffle/src/zope/testing/testrunner/tests.py 2009-10-16 12:52:13 UTC (rev 105101)
+++ zope.testing/branches/sdouche-shuffle/src/zope/testing/testrunner/tests.py 2009-10-16 12:55:12 UTC (rev 105102)
@@ -163,6 +163,7 @@
'testrunner-repeat.txt',
'testrunner-gc.txt',
'testrunner-knit.txt',
+ 'testrunner-shuffle.txt',
setUp=setUp, tearDown=tearDown,
optionflags=doctest.ELLIPSIS+doctest.NORMALIZE_WHITESPACE,
checker=checker),
More information about the checkins
mailing list