gtest_shuffle_test.py
Go to the documentation of this file.
00001 #!/usr/bin/env python
00002 #
00003 # Copyright 2009 Google Inc. All Rights Reserved.
00004 #
00005 # Redistribution and use in source and binary forms, with or without
00006 # modification, are permitted provided that the following conditions are
00007 # met:
00008 #
00009 #     * Redistributions of source code must retain the above copyright
00010 # notice, this list of conditions and the following disclaimer.
00011 #     * Redistributions in binary form must reproduce the above
00012 # copyright notice, this list of conditions and the following disclaimer
00013 # in the documentation and/or other materials provided with the
00014 # distribution.
00015 #     * Neither the name of Google Inc. nor the names of its
00016 # contributors may be used to endorse or promote products derived from
00017 # this software without specific prior written permission.
00018 #
00019 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
00020 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
00021 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
00022 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
00023 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
00024 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
00025 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
00026 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
00027 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
00028 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
00029 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
00030 
00031 """Verifies that test shuffling works."""
00032 
00033 __author__ = 'wan@google.com (Zhanyong Wan)'
00034 
00035 import os
00036 import gtest_test_utils
00037 
00038 # Command to run the gtest_shuffle_test_ program.
00039 COMMAND = gtest_test_utils.GetTestExecutablePath('gtest_shuffle_test_')
00040 
00041 # The environment variables for test sharding.
00042 TOTAL_SHARDS_ENV_VAR = 'GTEST_TOTAL_SHARDS'
00043 SHARD_INDEX_ENV_VAR = 'GTEST_SHARD_INDEX'
00044 
00045 TEST_FILTER = 'A*.A:A*.B:C*'
00046 
00047 ALL_TESTS = []
00048 ACTIVE_TESTS = []
00049 FILTERED_TESTS = []
00050 SHARDED_TESTS = []
00051 
00052 SHUFFLED_ALL_TESTS = []
00053 SHUFFLED_ACTIVE_TESTS = []
00054 SHUFFLED_FILTERED_TESTS = []
00055 SHUFFLED_SHARDED_TESTS = []
00056 
00057 
00058 def AlsoRunDisabledTestsFlag():
00059   return '--gtest_also_run_disabled_tests'
00060 
00061 
00062 def FilterFlag(test_filter):
00063   return '--gtest_filter=%s' % (test_filter,)
00064 
00065 
00066 def RepeatFlag(n):
00067   return '--gtest_repeat=%s' % (n,)
00068 
00069 
00070 def ShuffleFlag():
00071   return '--gtest_shuffle'
00072 
00073 
00074 def RandomSeedFlag(n):
00075   return '--gtest_random_seed=%s' % (n,)
00076 
00077 
00078 def RunAndReturnOutput(extra_env, args):
00079   """Runs the test program and returns its output."""
00080 
00081   environ_copy = os.environ.copy()
00082   environ_copy.update(extra_env)
00083 
00084   return gtest_test_utils.Subprocess([COMMAND] + args, env=environ_copy).output
00085 
00086 
00087 def GetTestsForAllIterations(extra_env, args):
00088   """Runs the test program and returns a list of test lists.
00089 
00090   Args:
00091     extra_env: a map from environment variables to their values
00092     args: command line flags to pass to gtest_shuffle_test_
00093 
00094   Returns:
00095     A list where the i-th element is the list of tests run in the i-th
00096     test iteration.
00097   """
00098 
00099   test_iterations = []
00100   for line in RunAndReturnOutput(extra_env, args).split('\n'):
00101     if line.startswith('----'):
00102       tests = []
00103       test_iterations.append(tests)
00104     elif line.strip():
00105       tests.append(line.strip())  # 'TestCaseName.TestName'
00106 
00107   return test_iterations
00108 
00109 
00110 def GetTestCases(tests):
00111   """Returns a list of test cases in the given full test names.
00112 
00113   Args:
00114     tests: a list of full test names
00115 
00116   Returns:
00117     A list of test cases from 'tests', in their original order.
00118     Consecutive duplicates are removed.
00119   """
00120 
00121   test_cases = []
00122   for test in tests:
00123     test_case = test.split('.')[0]
00124     if not test_case in test_cases:
00125       test_cases.append(test_case)
00126 
00127   return test_cases
00128 
00129 
00130 def CalculateTestLists():
00131   """Calculates the list of tests run under different flags."""
00132 
00133   if not ALL_TESTS:
00134     ALL_TESTS.extend(
00135         GetTestsForAllIterations({}, [AlsoRunDisabledTestsFlag()])[0])
00136 
00137   if not ACTIVE_TESTS:
00138     ACTIVE_TESTS.extend(GetTestsForAllIterations({}, [])[0])
00139 
00140   if not FILTERED_TESTS:
00141     FILTERED_TESTS.extend(
00142         GetTestsForAllIterations({}, [FilterFlag(TEST_FILTER)])[0])
00143 
00144   if not SHARDED_TESTS:
00145     SHARDED_TESTS.extend(
00146         GetTestsForAllIterations({TOTAL_SHARDS_ENV_VAR: '3',
00147                                   SHARD_INDEX_ENV_VAR: '1'},
00148                                  [])[0])
00149 
00150   if not SHUFFLED_ALL_TESTS:
00151     SHUFFLED_ALL_TESTS.extend(GetTestsForAllIterations(
00152         {}, [AlsoRunDisabledTestsFlag(), ShuffleFlag(), RandomSeedFlag(1)])[0])
00153 
00154   if not SHUFFLED_ACTIVE_TESTS:
00155     SHUFFLED_ACTIVE_TESTS.extend(GetTestsForAllIterations(
00156         {}, [ShuffleFlag(), RandomSeedFlag(1)])[0])
00157 
00158   if not SHUFFLED_FILTERED_TESTS:
00159     SHUFFLED_FILTERED_TESTS.extend(GetTestsForAllIterations(
00160         {}, [ShuffleFlag(), RandomSeedFlag(1), FilterFlag(TEST_FILTER)])[0])
00161 
00162   if not SHUFFLED_SHARDED_TESTS:
00163     SHUFFLED_SHARDED_TESTS.extend(
00164         GetTestsForAllIterations({TOTAL_SHARDS_ENV_VAR: '3',
00165                                   SHARD_INDEX_ENV_VAR: '1'},
00166                                  [ShuffleFlag(), RandomSeedFlag(1)])[0])
00167 
00168 
00169 class GTestShuffleUnitTest(gtest_test_utils.TestCase):
00170   """Tests test shuffling."""
00171 
00172   def setUp(self):
00173     CalculateTestLists()
00174 
00175   def testShufflePreservesNumberOfTests(self):
00176     self.assertEqual(len(ALL_TESTS), len(SHUFFLED_ALL_TESTS))
00177     self.assertEqual(len(ACTIVE_TESTS), len(SHUFFLED_ACTIVE_TESTS))
00178     self.assertEqual(len(FILTERED_TESTS), len(SHUFFLED_FILTERED_TESTS))
00179     self.assertEqual(len(SHARDED_TESTS), len(SHUFFLED_SHARDED_TESTS))
00180 
00181   def testShuffleChangesTestOrder(self):
00182     self.assert_(SHUFFLED_ALL_TESTS != ALL_TESTS, SHUFFLED_ALL_TESTS)
00183     self.assert_(SHUFFLED_ACTIVE_TESTS != ACTIVE_TESTS, SHUFFLED_ACTIVE_TESTS)
00184     self.assert_(SHUFFLED_FILTERED_TESTS != FILTERED_TESTS,
00185                  SHUFFLED_FILTERED_TESTS)
00186     self.assert_(SHUFFLED_SHARDED_TESTS != SHARDED_TESTS,
00187                  SHUFFLED_SHARDED_TESTS)
00188 
00189   def testShuffleChangesTestCaseOrder(self):
00190     self.assert_(GetTestCases(SHUFFLED_ALL_TESTS) != GetTestCases(ALL_TESTS),
00191                  GetTestCases(SHUFFLED_ALL_TESTS))
00192     self.assert_(
00193         GetTestCases(SHUFFLED_ACTIVE_TESTS) != GetTestCases(ACTIVE_TESTS),
00194         GetTestCases(SHUFFLED_ACTIVE_TESTS))
00195     self.assert_(
00196         GetTestCases(SHUFFLED_FILTERED_TESTS) != GetTestCases(FILTERED_TESTS),
00197         GetTestCases(SHUFFLED_FILTERED_TESTS))
00198     self.assert_(
00199         GetTestCases(SHUFFLED_SHARDED_TESTS) != GetTestCases(SHARDED_TESTS),
00200         GetTestCases(SHUFFLED_SHARDED_TESTS))
00201 
00202   def testShuffleDoesNotRepeatTest(self):
00203     for test in SHUFFLED_ALL_TESTS:
00204       self.assertEqual(1, SHUFFLED_ALL_TESTS.count(test),
00205                        '%s appears more than once' % (test,))
00206     for test in SHUFFLED_ACTIVE_TESTS:
00207       self.assertEqual(1, SHUFFLED_ACTIVE_TESTS.count(test),
00208                        '%s appears more than once' % (test,))
00209     for test in SHUFFLED_FILTERED_TESTS:
00210       self.assertEqual(1, SHUFFLED_FILTERED_TESTS.count(test),
00211                        '%s appears more than once' % (test,))
00212     for test in SHUFFLED_SHARDED_TESTS:
00213       self.assertEqual(1, SHUFFLED_SHARDED_TESTS.count(test),
00214                        '%s appears more than once' % (test,))
00215 
00216   def testShuffleDoesNotCreateNewTest(self):
00217     for test in SHUFFLED_ALL_TESTS:
00218       self.assert_(test in ALL_TESTS, '%s is an invalid test' % (test,))
00219     for test in SHUFFLED_ACTIVE_TESTS:
00220       self.assert_(test in ACTIVE_TESTS, '%s is an invalid test' % (test,))
00221     for test in SHUFFLED_FILTERED_TESTS:
00222       self.assert_(test in FILTERED_TESTS, '%s is an invalid test' % (test,))
00223     for test in SHUFFLED_SHARDED_TESTS:
00224       self.assert_(test in SHARDED_TESTS, '%s is an invalid test' % (test,))
00225 
00226   def testShuffleIncludesAllTests(self):
00227     for test in ALL_TESTS:
00228       self.assert_(test in SHUFFLED_ALL_TESTS, '%s is missing' % (test,))
00229     for test in ACTIVE_TESTS:
00230       self.assert_(test in SHUFFLED_ACTIVE_TESTS, '%s is missing' % (test,))
00231     for test in FILTERED_TESTS:
00232       self.assert_(test in SHUFFLED_FILTERED_TESTS, '%s is missing' % (test,))
00233     for test in SHARDED_TESTS:
00234       self.assert_(test in SHUFFLED_SHARDED_TESTS, '%s is missing' % (test,))
00235 
00236   def testShuffleLeavesDeathTestsAtFront(self):
00237     non_death_test_found = False
00238     for test in SHUFFLED_ACTIVE_TESTS:
00239       if 'DeathTest.' in test:
00240         self.assert_(not non_death_test_found,
00241                      '%s appears after a non-death test' % (test,))
00242       else:
00243         non_death_test_found = True
00244 
00245   def _VerifyTestCasesDoNotInterleave(self, tests):
00246     test_cases = []
00247     for test in tests:
00248       [test_case, _] = test.split('.')
00249       if test_cases and test_cases[-1] != test_case:
00250         test_cases.append(test_case)
00251         self.assertEqual(1, test_cases.count(test_case),
00252                          'Test case %s is not grouped together in %s' %
00253                          (test_case, tests))
00254 
00255   def testShuffleDoesNotInterleaveTestCases(self):
00256     self._VerifyTestCasesDoNotInterleave(SHUFFLED_ALL_TESTS)
00257     self._VerifyTestCasesDoNotInterleave(SHUFFLED_ACTIVE_TESTS)
00258     self._VerifyTestCasesDoNotInterleave(SHUFFLED_FILTERED_TESTS)
00259     self._VerifyTestCasesDoNotInterleave(SHUFFLED_SHARDED_TESTS)
00260 
00261   def testShuffleRestoresOrderAfterEachIteration(self):
00262     # Get the test lists in all 3 iterations, using random seed 1, 2,
00263     # and 3 respectively.  Google Test picks a different seed in each
00264     # iteration, and this test depends on the current implementation
00265     # picking successive numbers.  This dependency is not ideal, but
00266     # makes the test much easier to write.
00267     [tests_in_iteration1, tests_in_iteration2, tests_in_iteration3] = (
00268         GetTestsForAllIterations(
00269             {}, [ShuffleFlag(), RandomSeedFlag(1), RepeatFlag(3)]))
00270 
00271     # Make sure running the tests with random seed 1 gets the same
00272     # order as in iteration 1 above.
00273     [tests_with_seed1] = GetTestsForAllIterations(
00274         {}, [ShuffleFlag(), RandomSeedFlag(1)])
00275     self.assertEqual(tests_in_iteration1, tests_with_seed1)
00276 
00277     # Make sure running the tests with random seed 2 gets the same
00278     # order as in iteration 2 above.  Success means that Google Test
00279     # correctly restores the test order before re-shuffling at the
00280     # beginning of iteration 2.
00281     [tests_with_seed2] = GetTestsForAllIterations(
00282         {}, [ShuffleFlag(), RandomSeedFlag(2)])
00283     self.assertEqual(tests_in_iteration2, tests_with_seed2)
00284 
00285     # Make sure running the tests with random seed 3 gets the same
00286     # order as in iteration 3 above.  Success means that Google Test
00287     # correctly restores the test order before re-shuffling at the
00288     # beginning of iteration 3.
00289     [tests_with_seed3] = GetTestsForAllIterations(
00290         {}, [ShuffleFlag(), RandomSeedFlag(3)])
00291     self.assertEqual(tests_in_iteration3, tests_with_seed3)
00292 
00293   def testShuffleGeneratesNewOrderInEachIteration(self):
00294     [tests_in_iteration1, tests_in_iteration2, tests_in_iteration3] = (
00295         GetTestsForAllIterations(
00296             {}, [ShuffleFlag(), RandomSeedFlag(1), RepeatFlag(3)]))
00297 
00298     self.assert_(tests_in_iteration1 != tests_in_iteration2,
00299                  tests_in_iteration1)
00300     self.assert_(tests_in_iteration1 != tests_in_iteration3,
00301                  tests_in_iteration1)
00302     self.assert_(tests_in_iteration2 != tests_in_iteration3,
00303                  tests_in_iteration2)
00304 
00305   def testShuffleShardedTestsPreservesPartition(self):
00306     # If we run M tests on N shards, the same M tests should be run in
00307     # total, regardless of the random seeds used by the shards.
00308     [tests1] = GetTestsForAllIterations({TOTAL_SHARDS_ENV_VAR: '3',
00309                                          SHARD_INDEX_ENV_VAR: '0'},
00310                                         [ShuffleFlag(), RandomSeedFlag(1)])
00311     [tests2] = GetTestsForAllIterations({TOTAL_SHARDS_ENV_VAR: '3',
00312                                          SHARD_INDEX_ENV_VAR: '1'},
00313                                         [ShuffleFlag(), RandomSeedFlag(20)])
00314     [tests3] = GetTestsForAllIterations({TOTAL_SHARDS_ENV_VAR: '3',
00315                                          SHARD_INDEX_ENV_VAR: '2'},
00316                                         [ShuffleFlag(), RandomSeedFlag(25)])
00317     sorted_sharded_tests = tests1 + tests2 + tests3
00318     sorted_sharded_tests.sort()
00319     sorted_active_tests = []
00320     sorted_active_tests.extend(ACTIVE_TESTS)
00321     sorted_active_tests.sort()
00322     self.assertEqual(sorted_active_tests, sorted_sharded_tests)
00323 
00324 if __name__ == '__main__':
00325   gtest_test_utils.Main()


rc_visard_driver
Author(s): Heiko Hirschmueller , Christian Emmerich , Felix Ruess
autogenerated on Thu Jun 6 2019 20:43:04