31 """Unit test for Google Test test filters. 33 A user can specify which test(s) in a Google Test program to run via either 34 the GTEST_FILTER environment variable or the --gtest_filter flag. 35 This script tests such functionality by invoking 36 gtest_filter_unittest_ (a program written with Google Test) with different 37 environments and command line flags. 39 Note that test sharding may also influence which tests are filtered. Therefore, 40 we test that here also. 43 __author__ =
'wan@google.com (Zhanyong Wan)' 50 import gtest_test_utils
59 os.environ[
'EMPTY_VAR'] =
'' 61 [sys.executable,
'-c',
'import os; print \'EMPTY_VAR\' in os.environ'])
62 CAN_PASS_EMPTY_ENV = eval(child.output)
71 os.environ[
'UNSET_VAR'] =
'X' 72 del os.environ[
'UNSET_VAR']
74 [sys.executable,
'-c',
'import os; print \'UNSET_VAR\' not in os.environ'])
75 CAN_UNSET_ENV = eval(child.output)
82 CAN_TEST_EMPTY_FILTER = (CAN_PASS_EMPTY_ENV
and CAN_UNSET_ENV)
86 FILTER_ENV_VAR =
'GTEST_FILTER' 89 TOTAL_SHARDS_ENV_VAR =
'GTEST_TOTAL_SHARDS' 90 SHARD_INDEX_ENV_VAR =
'GTEST_SHARD_INDEX' 91 SHARD_STATUS_FILE_ENV_VAR =
'GTEST_SHARD_STATUS_FILE' 94 FILTER_FLAG =
'gtest_filter' 97 ALSO_RUN_DISABED_TESTS_FLAG =
'gtest_also_run_disabled_tests' 103 PARAM_TEST_REGEX = re.compile(
r'/ParamTest')
106 TEST_CASE_REGEX = re.compile(
r'^\[\-+\] \d+ tests? from (\w+(/\w+)?)')
109 TEST_REGEX = re.compile(
r'^\[\s*RUN\s*\].*\.(\w+(/\w+)?)')
113 LIST_TESTS_FLAG =
'--gtest_list_tests' 117 [COMMAND, LIST_TESTS_FLAG]).output
121 'SeqP/ParamTest.TestX/0',
122 'SeqP/ParamTest.TestX/1',
123 'SeqP/ParamTest.TestY/0',
124 'SeqP/ParamTest.TestY/1',
125 'SeqQ/ParamTest.TestX/0',
126 'SeqQ/ParamTest.TestX/1',
127 'SeqQ/ParamTest.TestY/0',
128 'SeqQ/ParamTest.TestY/1',
132 'BarTest.DISABLED_TestFour',
133 'BarTest.DISABLED_TestFive',
134 'BazTest.DISABLED_TestC',
135 'DISABLED_FoobarTest.Test1',
136 'DISABLED_FoobarTest.DISABLED_Test2',
137 'DISABLED_FoobarbazTest.TestA',
140 if SUPPORTS_DEATH_TESTS:
142 'HasDeathTest.Test1',
143 'HasDeathTest.Test2',
160 ] + DEATH_TESTS + PARAM_TESTS
162 param_tests_present =
None 166 environ = os.environ.copy()
170 """Sets the env variable to 'value'; unsets it when 'value' is None.""" 172 if value
is not None:
173 environ[env_var] = value
174 elif env_var
in environ:
179 """Runs the test program and returns its output.""" 186 """Runs the test program and returns its exit code and a list of tests run.""" 192 for line
in p.output.split(
'\n'):
193 match = TEST_CASE_REGEX.match(line)
194 if match
is not None:
195 test_case = match.group(1)
197 match = TEST_REGEX.match(line)
198 if match
is not None:
199 test = match.group(1)
200 tests_run.append(test_case +
'.' + test)
201 return (tests_run, p.exit_code)
205 """Runs the given function and arguments in a modified environment.""" 207 original_env = environ.copy()
208 environ.update(extra_env)
209 return function(*args, **kwargs)
212 environ.update(original_env)
216 """Runs a test program shard and returns exit code and a list of tests run.""" 218 extra_env = {SHARD_INDEX_ENV_VAR: str(shard_index),
219 TOTAL_SHARDS_ENV_VAR: str(total_shards)}
226 """Tests the env variable or the command line flag to filter tests.""" 231 """Asserts that two sets are equal.""" 234 self.assertTrue(elem
in rhs,
'%s in %s' % (elem, rhs))
237 self.assertTrue(elem
in lhs,
'%s in %s' % (elem, lhs))
240 """Asserts that list_of_sets is a valid partition of set_var.""" 243 for slice_var
in list_of_sets:
244 full_partition.extend(slice_var)
245 self.assertEqual(len(set_var), len(full_partition))
246 self.assertEqual(sets.Set(set_var), sets.Set(full_partition))
249 """Adjust tests_to_run in case value parameterized tests are disabled.""" 251 global param_tests_present
252 if not param_tests_present:
253 return list(sets.Set(tests_to_run) - sets.Set(PARAM_TESTS))
258 """Checks that the binary runs correct set of tests for a given filter.""" 270 if CAN_TEST_EMPTY_FILTER
or gtest_filter !=
'':
279 if gtest_filter
is None:
282 args = [
'--%s=%s' % (FILTER_FLAG, gtest_filter)]
288 args=
None, check_exit_0=
False):
289 """Checks that binary runs correct tests for the given filter and shard. 291 Runs all shards of gtest_filter_unittest_ with the given filter, and 292 verifies that the right set of tests were run. The union of tests run 293 on each shard should be identical to tests_to_run, without duplicates. 296 gtest_filter: A filter to apply to the tests. 297 total_shards: A total number of shards to split test run into. 298 tests_to_run: A set of tests expected to run. 299 args : Arguments to pass to the to the test binary. 300 check_exit_0: When set to a true value, make sure that all shards 312 if CAN_TEST_EMPTY_FILTER
or gtest_filter !=
'':
315 for i
in range(0, total_shards):
318 self.assertEqual(0, exit_code)
319 partition.append(tests_run)
326 """Checks that the binary runs correct set of tests for the given filter. 328 Runs gtest_filter_unittest_ with the given filter, and enables 329 disabled tests. Verifies that the right set of tests were run. 332 gtest_filter: A filter to apply to the tests. 333 tests_to_run: A set of tests expected to run. 339 args = [
'--%s' % ALSO_RUN_DISABED_TESTS_FLAG]
340 if gtest_filter
is not None:
341 args.append(
'--%s=%s' % (FILTER_FLAG, gtest_filter))
347 """Sets up test case. 349 Determines whether value-parameterized tests are enabled in the binary and 350 sets the flags accordingly. 353 global param_tests_present
354 if param_tests_present
is None:
355 param_tests_present = PARAM_TEST_REGEX.search(
359 """Tests the behavior of not specifying the filter.""" 364 """Tests the behavior without the filter, with sharding enabled.""" 373 """Tests an empty filter.""" 380 """Tests a filter that matches nothing.""" 386 """Tests filtering by full name.""" 393 """Tests filters that match everything.""" 402 """Tests filtering by test case name.""" 404 self.
RunAndVerify(
'FooTest.*', [
'FooTest.Abc',
'FooTest.Xyz'])
406 BAZ_TESTS = [
'BazTest.TestOne',
'BazTest.TestA',
'BazTest.TestB']
409 BAZ_TESTS + [
'BazTest.DISABLED_TestC'])
412 """Tests filtering by test name.""" 414 self.
RunAndVerify(
'*.TestOne', [
'BarTest.TestOne',
'BazTest.TestOne'])
417 """Select only the disabled tests to run.""" 421 [
'DISABLED_FoobarTest.Test1'])
428 'BarTest.DISABLED_TestFour',
429 'BarTest.DISABLED_TestFive',
430 'BazTest.DISABLED_TestC',
431 'DISABLED_FoobarTest.DISABLED_Test2',
436 'DISABLED_FoobarTest.Test1',
437 'DISABLED_FoobarTest.DISABLED_Test2',
438 'DISABLED_FoobarbazTest.TestA',
442 """Tests using wildcard in the test case name.""" 451 'BazTest.TestB', ] + DEATH_TESTS + PARAM_TESTS)
454 """Tests using wildcard in the test name.""" 456 self.
RunAndVerify(
'*.*A*', [
'FooTest.Abc',
'BazTest.TestA'])
459 """Tests a filter that has no '.' in it.""" 470 """Tests filters that consist of two patterns.""" 480 self.
RunAndVerify(
':*A*', [
'FooTest.Abc',
'BazTest.TestA'])
483 """Tests filters that consist of three patterns.""" 522 ] + DEATH_TESTS + PARAM_TESTS)
530 ] + DEATH_TESTS + PARAM_TESTS)
538 self.
RunAndVerify(
'-FooTest.Abc:FooTest.Xyz:BazTest.*', [
542 ] + DEATH_TESTS + PARAM_TESTS)
549 'SeqP/ParamTest.TestX/0',
550 'SeqP/ParamTest.TestX/1',
551 'SeqP/ParamTest.TestY/0',
552 'SeqP/ParamTest.TestY/1',
557 'SeqP/ParamTest.TestX/0',
558 'SeqP/ParamTest.TestY/0',
559 'SeqQ/ParamTest.TestX/0',
560 'SeqQ/ParamTest.TestY/0',
564 """Tests that the filter flag overrides the filtering env. variable.""" 567 args = [
'--%s=%s' % (FILTER_FLAG,
'*One')]
571 self.
AssertSetEqual(tests_run, [
'BarTest.TestOne',
'BazTest.TestOne'])
574 """Tests that the shard file is created if specified in the environment.""" 578 self.assertTrue(
not os.path.exists(shard_status_file))
580 extra_env = {SHARD_STATUS_FILE_ENV_VAR: shard_status_file}
584 self.assertTrue(os.path.exists(shard_status_file))
585 os.remove(shard_status_file)
588 """Tests that the shard file is created with the "list_tests" flag.""" 591 'shard_status_file2')
592 self.assertTrue(
not os.path.exists(shard_status_file))
594 extra_env = {SHARD_STATUS_FILE_ENV_VAR: shard_status_file}
602 self.assertTrue(
'[==========]' not in output,
603 'Unexpected output during test enumeration.\n' 604 'Please ensure that LIST_TESTS_FLAG is assigned the\n' 605 'correct flag value for listing Google Test tests.')
607 self.assertTrue(os.path.exists(shard_status_file))
608 os.remove(shard_status_file)
610 if SUPPORTS_DEATH_TESTS:
612 """Tests integration with death tests and sharding.""" 614 gtest_filter =
'HasDeathTest.*:SeqP/*' 616 'HasDeathTest.Test1',
617 'HasDeathTest.Test2',
619 'SeqP/ParamTest.TestX/0',
620 'SeqP/ParamTest.TestX/1',
621 'SeqP/ParamTest.TestY/0',
622 'SeqP/ParamTest.TestY/1',
625 for flag
in [
'--gtest_death_test_style=threadsafe',
626 '--gtest_death_test_style=fast']:
628 check_exit_0=
True, args=[flag])
630 check_exit_0=
True, args=[flag])
632 if __name__ ==
'__main__':
def testShardStatusFileIsCreatedWithListTests(self)
def testShardingWorksWithDeathTests(self)
def testFilterByTestCase(self)
def testNegativeFilters(self)
def testShardStatusFileIsCreated(self)
def GetTestExecutablePath(executable_name, build_dir=None)
def testWildcardInTestName(self)
def testUniversalFilters(self)
def testTwoPatterns(self)
def testDefaultBehaviorWithShards(self)
def RunAndExtractTestList(args=None)
def RunWithSharding(total_shards, shard_index, command)
def InvokeWithModifiedEnv(extra_env, function, args, kwargs)
def RunAndVerify(self, gtest_filter, tests_to_run)
def AssertSetEqual(self, lhs, rhs)
def testWildcardInTestCaseName(self)
def SetEnvVar(env_var, value)
def testDefaultBehavior(self)
def testFlagOverridesEnvVar(self)
def testFilterWithoutDot(self)
def testFilterDisabledTests(self)
def RunAndVerifyAllowingDisabled(self, gtest_filter, tests_to_run)
def RunAndReturnOutput(args=None)
def testEmptyFilter(self)
def testThreePatterns(self)
def AdjustForParameterizedTests(self, tests_to_run)
def RunAndVerifyWithSharding(self, gtest_filter, total_shards, tests_to_run, args=None, check_exit_0=False)
def AssertPartitionIsValid(self, set_var, list_of_sets)
def testFilterByTest(self)