00001
00002
00003
00004
00005
00006
00007
00008
00009
00010
00011
00012
00013
00014
00015
00016
00017
00018
00019
00020
00021
00022
00023
00024
00025
00026
00027
00028
00029
00030
00031 """Unit test for Google Test test filters.
00032
00033 A user can specify which test(s) in a Google Test program to run via either
00034 the GTEST_FILTER environment variable or the --gtest_filter flag.
00035 This script tests such functionality by invoking
00036 gtest_filter_unittest_ (a program written with Google Test) with different
00037 environments and command line flags.
00038
00039 Note that test sharding may also influence which tests are filtered. Therefore,
00040 we test that here also.
00041 """
00042
00043 __author__ = 'wan@google.com (Zhanyong Wan)'
00044
00045 import os
00046 import re
00047 import sets
00048 import sys
00049
00050 import gtest_test_utils
00051
00052
00053
00054
00055
00056
00057
00058
00059 os.environ['EMPTY_VAR'] = ''
00060 child = gtest_test_utils.Subprocess(
00061 [sys.executable, '-c', 'import os; print \'EMPTY_VAR\' in os.environ'])
00062 CAN_PASS_EMPTY_ENV = eval(child.output)
00063
00064
00065
00066
00067
00068
00069
00070
00071 os.environ['UNSET_VAR'] = 'X'
00072 del os.environ['UNSET_VAR']
00073 child = gtest_test_utils.Subprocess(
00074 [sys.executable, '-c', 'import os; print \'UNSET_VAR\' not in os.environ'])
00075 CAN_UNSET_ENV = eval(child.output)
00076
00077
00078
00079
00080
00081
00082 CAN_TEST_EMPTY_FILTER = (CAN_PASS_EMPTY_ENV and CAN_UNSET_ENV)
00083
00084
00085
00086 FILTER_ENV_VAR = 'GTEST_FILTER'
00087
00088
00089 TOTAL_SHARDS_ENV_VAR = 'GTEST_TOTAL_SHARDS'
00090 SHARD_INDEX_ENV_VAR = 'GTEST_SHARD_INDEX'
00091 SHARD_STATUS_FILE_ENV_VAR = 'GTEST_SHARD_STATUS_FILE'
00092
00093
00094 FILTER_FLAG = 'gtest_filter'
00095
00096
00097 ALSO_RUN_DISABED_TESTS_FLAG = 'gtest_also_run_disabled_tests'
00098
00099
00100 COMMAND = gtest_test_utils.GetTestExecutablePath('gtest_filter_unittest_')
00101
00102
00103 PARAM_TEST_REGEX = re.compile(r'/ParamTest')
00104
00105
00106 TEST_CASE_REGEX = re.compile(r'^\[\-+\] \d+ tests? from (\w+(/\w+)?)')
00107
00108
00109 TEST_REGEX = re.compile(r'^\[\s*RUN\s*\].*\.(\w+(/\w+)?)')
00110
00111
00112
00113 LIST_TESTS_FLAG = '--gtest_list_tests'
00114
00115
00116 SUPPORTS_DEATH_TESTS = 'HasDeathTest' in gtest_test_utils.Subprocess(
00117 [COMMAND, LIST_TESTS_FLAG]).output
00118
00119
00120 PARAM_TESTS = [
00121 'SeqP/ParamTest.TestX/0',
00122 'SeqP/ParamTest.TestX/1',
00123 'SeqP/ParamTest.TestY/0',
00124 'SeqP/ParamTest.TestY/1',
00125 'SeqQ/ParamTest.TestX/0',
00126 'SeqQ/ParamTest.TestX/1',
00127 'SeqQ/ParamTest.TestY/0',
00128 'SeqQ/ParamTest.TestY/1',
00129 ]
00130
00131 DISABLED_TESTS = [
00132 'BarTest.DISABLED_TestFour',
00133 'BarTest.DISABLED_TestFive',
00134 'BazTest.DISABLED_TestC',
00135 'DISABLED_FoobarTest.Test1',
00136 'DISABLED_FoobarTest.DISABLED_Test2',
00137 'DISABLED_FoobarbazTest.TestA',
00138 ]
00139
00140 if SUPPORTS_DEATH_TESTS:
00141 DEATH_TESTS = [
00142 'HasDeathTest.Test1',
00143 'HasDeathTest.Test2',
00144 ]
00145 else:
00146 DEATH_TESTS = []
00147
00148
00149 ACTIVE_TESTS = [
00150 'FooTest.Abc',
00151 'FooTest.Xyz',
00152
00153 'BarTest.TestOne',
00154 'BarTest.TestTwo',
00155 'BarTest.TestThree',
00156
00157 'BazTest.TestOne',
00158 'BazTest.TestA',
00159 'BazTest.TestB',
00160 ] + DEATH_TESTS + PARAM_TESTS
00161
00162 param_tests_present = None
00163
00164
00165
00166 environ = os.environ.copy()
00167
00168
00169 def SetEnvVar(env_var, value):
00170 """Sets the env variable to 'value'; unsets it when 'value' is None."""
00171
00172 if value is not None:
00173 environ[env_var] = value
00174 elif env_var in environ:
00175 del environ[env_var]
00176
00177
00178 def RunAndReturnOutput(args = None):
00179 """Runs the test program and returns its output."""
00180
00181 return gtest_test_utils.Subprocess([COMMAND] + (args or []),
00182 env=environ).output
00183
00184
00185 def RunAndExtractTestList(args = None):
00186 """Runs the test program and returns its exit code and a list of tests run."""
00187
00188 p = gtest_test_utils.Subprocess([COMMAND] + (args or []), env=environ)
00189 tests_run = []
00190 test_case = ''
00191 test = ''
00192 for line in p.output.split('\n'):
00193 match = TEST_CASE_REGEX.match(line)
00194 if match is not None:
00195 test_case = match.group(1)
00196 else:
00197 match = TEST_REGEX.match(line)
00198 if match is not None:
00199 test = match.group(1)
00200 tests_run.append(test_case + '.' + test)
00201 return (tests_run, p.exit_code)
00202
00203
00204 def InvokeWithModifiedEnv(extra_env, function, *args, **kwargs):
00205 """Runs the given function and arguments in a modified environment."""
00206 try:
00207 original_env = environ.copy()
00208 environ.update(extra_env)
00209 return function(*args, **kwargs)
00210 finally:
00211 environ.clear()
00212 environ.update(original_env)
00213
00214
00215 def RunWithSharding(total_shards, shard_index, command):
00216 """Runs a test program shard and returns exit code and a list of tests run."""
00217
00218 extra_env = {SHARD_INDEX_ENV_VAR: str(shard_index),
00219 TOTAL_SHARDS_ENV_VAR: str(total_shards)}
00220 return InvokeWithModifiedEnv(extra_env, RunAndExtractTestList, command)
00221
00222
00223
00224
00225 class GTestFilterUnitTest(gtest_test_utils.TestCase):
00226 """Tests the env variable or the command line flag to filter tests."""
00227
00228
00229
00230 def AssertSetEqual(self, lhs, rhs):
00231 """Asserts that two sets are equal."""
00232
00233 for elem in lhs:
00234 self.assert_(elem in rhs, '%s in %s' % (elem, rhs))
00235
00236 for elem in rhs:
00237 self.assert_(elem in lhs, '%s in %s' % (elem, lhs))
00238
00239 def AssertPartitionIsValid(self, set_var, list_of_sets):
00240 """Asserts that list_of_sets is a valid partition of set_var."""
00241
00242 full_partition = []
00243 for slice_var in list_of_sets:
00244 full_partition.extend(slice_var)
00245 self.assertEqual(len(set_var), len(full_partition))
00246 self.assertEqual(sets.Set(set_var), sets.Set(full_partition))
00247
00248 def AdjustForParameterizedTests(self, tests_to_run):
00249 """Adjust tests_to_run in case value parameterized tests are disabled."""
00250
00251 global param_tests_present
00252 if not param_tests_present:
00253 return list(sets.Set(tests_to_run) - sets.Set(PARAM_TESTS))
00254 else:
00255 return tests_to_run
00256
00257 def RunAndVerify(self, gtest_filter, tests_to_run):
00258 """Checks that the binary runs correct set of tests for a given filter."""
00259
00260 tests_to_run = self.AdjustForParameterizedTests(tests_to_run)
00261
00262
00263
00264
00265
00266
00267
00268
00269
00270 if CAN_TEST_EMPTY_FILTER or gtest_filter != '':
00271 SetEnvVar(FILTER_ENV_VAR, gtest_filter)
00272 tests_run = RunAndExtractTestList()[0]
00273 SetEnvVar(FILTER_ENV_VAR, None)
00274 self.AssertSetEqual(tests_run, tests_to_run)
00275
00276
00277
00278
00279 if gtest_filter is None:
00280 args = []
00281 else:
00282 args = ['--%s=%s' % (FILTER_FLAG, gtest_filter)]
00283
00284 tests_run = RunAndExtractTestList(args)[0]
00285 self.AssertSetEqual(tests_run, tests_to_run)
00286
00287 def RunAndVerifyWithSharding(self, gtest_filter, total_shards, tests_to_run,
00288 args=None, check_exit_0=False):
00289 """Checks that binary runs correct tests for the given filter and shard.
00290
00291 Runs all shards of gtest_filter_unittest_ with the given filter, and
00292 verifies that the right set of tests were run. The union of tests run
00293 on each shard should be identical to tests_to_run, without duplicates.
00294
00295 Args:
00296 gtest_filter: A filter to apply to the tests.
00297 total_shards: A total number of shards to split test run into.
00298 tests_to_run: A set of tests expected to run.
00299 args : Arguments to pass to the to the test binary.
00300 check_exit_0: When set to a true value, make sure that all shards
00301 return 0.
00302 """
00303
00304 tests_to_run = self.AdjustForParameterizedTests(tests_to_run)
00305
00306
00307
00308
00309
00310
00311
00312 if CAN_TEST_EMPTY_FILTER or gtest_filter != '':
00313 SetEnvVar(FILTER_ENV_VAR, gtest_filter)
00314 partition = []
00315 for i in range(0, total_shards):
00316 (tests_run, exit_code) = RunWithSharding(total_shards, i, args)
00317 if check_exit_0:
00318 self.assertEqual(0, exit_code)
00319 partition.append(tests_run)
00320
00321 self.AssertPartitionIsValid(tests_to_run, partition)
00322 SetEnvVar(FILTER_ENV_VAR, None)
00323
00324
00325 def RunAndVerifyAllowingDisabled(self, gtest_filter, tests_to_run):
00326 """Checks that the binary runs correct set of tests for the given filter.
00327
00328 Runs gtest_filter_unittest_ with the given filter, and enables
00329 disabled tests. Verifies that the right set of tests were run.
00330
00331 Args:
00332 gtest_filter: A filter to apply to the tests.
00333 tests_to_run: A set of tests expected to run.
00334 """
00335
00336 tests_to_run = self.AdjustForParameterizedTests(tests_to_run)
00337
00338
00339 args = ['--%s' % ALSO_RUN_DISABED_TESTS_FLAG]
00340 if gtest_filter is not None:
00341 args.append('--%s=%s' % (FILTER_FLAG, gtest_filter))
00342
00343 tests_run = RunAndExtractTestList(args)[0]
00344 self.AssertSetEqual(tests_run, tests_to_run)
00345
00346 def setUp(self):
00347 """Sets up test case.
00348
00349 Determines whether value-parameterized tests are enabled in the binary and
00350 sets the flags accordingly.
00351 """
00352
00353 global param_tests_present
00354 if param_tests_present is None:
00355 param_tests_present = PARAM_TEST_REGEX.search(
00356 RunAndReturnOutput()) is not None
00357
00358 def testDefaultBehavior(self):
00359 """Tests the behavior of not specifying the filter."""
00360
00361 self.RunAndVerify(None, ACTIVE_TESTS)
00362
00363 def testDefaultBehaviorWithShards(self):
00364 """Tests the behavior without the filter, with sharding enabled."""
00365
00366 self.RunAndVerifyWithSharding(None, 1, ACTIVE_TESTS)
00367 self.RunAndVerifyWithSharding(None, 2, ACTIVE_TESTS)
00368 self.RunAndVerifyWithSharding(None, len(ACTIVE_TESTS) - 1, ACTIVE_TESTS)
00369 self.RunAndVerifyWithSharding(None, len(ACTIVE_TESTS), ACTIVE_TESTS)
00370 self.RunAndVerifyWithSharding(None, len(ACTIVE_TESTS) + 1, ACTIVE_TESTS)
00371
00372 def testEmptyFilter(self):
00373 """Tests an empty filter."""
00374
00375 self.RunAndVerify('', [])
00376 self.RunAndVerifyWithSharding('', 1, [])
00377 self.RunAndVerifyWithSharding('', 2, [])
00378
00379 def testBadFilter(self):
00380 """Tests a filter that matches nothing."""
00381
00382 self.RunAndVerify('BadFilter', [])
00383 self.RunAndVerifyAllowingDisabled('BadFilter', [])
00384
00385 def testFullName(self):
00386 """Tests filtering by full name."""
00387
00388 self.RunAndVerify('FooTest.Xyz', ['FooTest.Xyz'])
00389 self.RunAndVerifyAllowingDisabled('FooTest.Xyz', ['FooTest.Xyz'])
00390 self.RunAndVerifyWithSharding('FooTest.Xyz', 5, ['FooTest.Xyz'])
00391
00392 def testUniversalFilters(self):
00393 """Tests filters that match everything."""
00394
00395 self.RunAndVerify('*', ACTIVE_TESTS)
00396 self.RunAndVerify('*.*', ACTIVE_TESTS)
00397 self.RunAndVerifyWithSharding('*.*', len(ACTIVE_TESTS) - 3, ACTIVE_TESTS)
00398 self.RunAndVerifyAllowingDisabled('*', ACTIVE_TESTS + DISABLED_TESTS)
00399 self.RunAndVerifyAllowingDisabled('*.*', ACTIVE_TESTS + DISABLED_TESTS)
00400
00401 def testFilterByTestCase(self):
00402 """Tests filtering by test case name."""
00403
00404 self.RunAndVerify('FooTest.*', ['FooTest.Abc', 'FooTest.Xyz'])
00405
00406 BAZ_TESTS = ['BazTest.TestOne', 'BazTest.TestA', 'BazTest.TestB']
00407 self.RunAndVerify('BazTest.*', BAZ_TESTS)
00408 self.RunAndVerifyAllowingDisabled('BazTest.*',
00409 BAZ_TESTS + ['BazTest.DISABLED_TestC'])
00410
00411 def testFilterByTest(self):
00412 """Tests filtering by test name."""
00413
00414 self.RunAndVerify('*.TestOne', ['BarTest.TestOne', 'BazTest.TestOne'])
00415
00416 def testFilterDisabledTests(self):
00417 """Select only the disabled tests to run."""
00418
00419 self.RunAndVerify('DISABLED_FoobarTest.Test1', [])
00420 self.RunAndVerifyAllowingDisabled('DISABLED_FoobarTest.Test1',
00421 ['DISABLED_FoobarTest.Test1'])
00422
00423 self.RunAndVerify('*DISABLED_*', [])
00424 self.RunAndVerifyAllowingDisabled('*DISABLED_*', DISABLED_TESTS)
00425
00426 self.RunAndVerify('*.DISABLED_*', [])
00427 self.RunAndVerifyAllowingDisabled('*.DISABLED_*', [
00428 'BarTest.DISABLED_TestFour',
00429 'BarTest.DISABLED_TestFive',
00430 'BazTest.DISABLED_TestC',
00431 'DISABLED_FoobarTest.DISABLED_Test2',
00432 ])
00433
00434 self.RunAndVerify('DISABLED_*', [])
00435 self.RunAndVerifyAllowingDisabled('DISABLED_*', [
00436 'DISABLED_FoobarTest.Test1',
00437 'DISABLED_FoobarTest.DISABLED_Test2',
00438 'DISABLED_FoobarbazTest.TestA',
00439 ])
00440
00441 def testWildcardInTestCaseName(self):
00442 """Tests using wildcard in the test case name."""
00443
00444 self.RunAndVerify('*a*.*', [
00445 'BarTest.TestOne',
00446 'BarTest.TestTwo',
00447 'BarTest.TestThree',
00448
00449 'BazTest.TestOne',
00450 'BazTest.TestA',
00451 'BazTest.TestB', ] + DEATH_TESTS + PARAM_TESTS)
00452
00453 def testWildcardInTestName(self):
00454 """Tests using wildcard in the test name."""
00455
00456 self.RunAndVerify('*.*A*', ['FooTest.Abc', 'BazTest.TestA'])
00457
00458 def testFilterWithoutDot(self):
00459 """Tests a filter that has no '.' in it."""
00460
00461 self.RunAndVerify('*z*', [
00462 'FooTest.Xyz',
00463
00464 'BazTest.TestOne',
00465 'BazTest.TestA',
00466 'BazTest.TestB',
00467 ])
00468
00469 def testTwoPatterns(self):
00470 """Tests filters that consist of two patterns."""
00471
00472 self.RunAndVerify('Foo*.*:*A*', [
00473 'FooTest.Abc',
00474 'FooTest.Xyz',
00475
00476 'BazTest.TestA',
00477 ])
00478
00479
00480 self.RunAndVerify(':*A*', ['FooTest.Abc', 'BazTest.TestA'])
00481
00482 def testThreePatterns(self):
00483 """Tests filters that consist of three patterns."""
00484
00485 self.RunAndVerify('*oo*:*A*:*One', [
00486 'FooTest.Abc',
00487 'FooTest.Xyz',
00488
00489 'BarTest.TestOne',
00490
00491 'BazTest.TestOne',
00492 'BazTest.TestA',
00493 ])
00494
00495
00496 self.RunAndVerify('*oo*::*One', [
00497 'FooTest.Abc',
00498 'FooTest.Xyz',
00499
00500 'BarTest.TestOne',
00501
00502 'BazTest.TestOne',
00503 ])
00504
00505
00506 self.RunAndVerify('*oo*::', [
00507 'FooTest.Abc',
00508 'FooTest.Xyz',
00509 ])
00510
00511 def testNegativeFilters(self):
00512 self.RunAndVerify('*-BazTest.TestOne', [
00513 'FooTest.Abc',
00514 'FooTest.Xyz',
00515
00516 'BarTest.TestOne',
00517 'BarTest.TestTwo',
00518 'BarTest.TestThree',
00519
00520 'BazTest.TestA',
00521 'BazTest.TestB',
00522 ] + DEATH_TESTS + PARAM_TESTS)
00523
00524 self.RunAndVerify('*-FooTest.Abc:BazTest.*', [
00525 'FooTest.Xyz',
00526
00527 'BarTest.TestOne',
00528 'BarTest.TestTwo',
00529 'BarTest.TestThree',
00530 ] + DEATH_TESTS + PARAM_TESTS)
00531
00532 self.RunAndVerify('BarTest.*-BarTest.TestOne', [
00533 'BarTest.TestTwo',
00534 'BarTest.TestThree',
00535 ])
00536
00537
00538 self.RunAndVerify('-FooTest.Abc:FooTest.Xyz:BazTest.*', [
00539 'BarTest.TestOne',
00540 'BarTest.TestTwo',
00541 'BarTest.TestThree',
00542 ] + DEATH_TESTS + PARAM_TESTS)
00543
00544
00545 self.RunAndVerify('*/*', PARAM_TESTS)
00546
00547
00548 self.RunAndVerify('SeqP/*', [
00549 'SeqP/ParamTest.TestX/0',
00550 'SeqP/ParamTest.TestX/1',
00551 'SeqP/ParamTest.TestY/0',
00552 'SeqP/ParamTest.TestY/1',
00553 ])
00554
00555
00556 self.RunAndVerify('*/0', [
00557 'SeqP/ParamTest.TestX/0',
00558 'SeqP/ParamTest.TestY/0',
00559 'SeqQ/ParamTest.TestX/0',
00560 'SeqQ/ParamTest.TestY/0',
00561 ])
00562
00563 def testFlagOverridesEnvVar(self):
00564 """Tests that the filter flag overrides the filtering env. variable."""
00565
00566 SetEnvVar(FILTER_ENV_VAR, 'Foo*')
00567 args = ['--%s=%s' % (FILTER_FLAG, '*One')]
00568 tests_run = RunAndExtractTestList(args)[0]
00569 SetEnvVar(FILTER_ENV_VAR, None)
00570
00571 self.AssertSetEqual(tests_run, ['BarTest.TestOne', 'BazTest.TestOne'])
00572
00573 def testShardStatusFileIsCreated(self):
00574 """Tests that the shard file is created if specified in the environment."""
00575
00576 shard_status_file = os.path.join(gtest_test_utils.GetTempDir(),
00577 'shard_status_file')
00578 self.assert_(not os.path.exists(shard_status_file))
00579
00580 extra_env = {SHARD_STATUS_FILE_ENV_VAR: shard_status_file}
00581 try:
00582 InvokeWithModifiedEnv(extra_env, RunAndReturnOutput)
00583 finally:
00584 self.assert_(os.path.exists(shard_status_file))
00585 os.remove(shard_status_file)
00586
00587 def testShardStatusFileIsCreatedWithListTests(self):
00588 """Tests that the shard file is created with the "list_tests" flag."""
00589
00590 shard_status_file = os.path.join(gtest_test_utils.GetTempDir(),
00591 'shard_status_file2')
00592 self.assert_(not os.path.exists(shard_status_file))
00593
00594 extra_env = {SHARD_STATUS_FILE_ENV_VAR: shard_status_file}
00595 try:
00596 output = InvokeWithModifiedEnv(extra_env,
00597 RunAndReturnOutput,
00598 [LIST_TESTS_FLAG])
00599 finally:
00600
00601
00602 self.assert_('[==========]' not in output,
00603 'Unexpected output during test enumeration.\n'
00604 'Please ensure that LIST_TESTS_FLAG is assigned the\n'
00605 'correct flag value for listing Google Test tests.')
00606
00607 self.assert_(os.path.exists(shard_status_file))
00608 os.remove(shard_status_file)
00609
00610 if SUPPORTS_DEATH_TESTS:
00611 def testShardingWorksWithDeathTests(self):
00612 """Tests integration with death tests and sharding."""
00613
00614 gtest_filter = 'HasDeathTest.*:SeqP/*'
00615 expected_tests = [
00616 'HasDeathTest.Test1',
00617 'HasDeathTest.Test2',
00618
00619 'SeqP/ParamTest.TestX/0',
00620 'SeqP/ParamTest.TestX/1',
00621 'SeqP/ParamTest.TestY/0',
00622 'SeqP/ParamTest.TestY/1',
00623 ]
00624
00625 for flag in ['--gtest_death_test_style=threadsafe',
00626 '--gtest_death_test_style=fast']:
00627 self.RunAndVerifyWithSharding(gtest_filter, 3, expected_tests,
00628 check_exit_0=True, args=[flag])
00629 self.RunAndVerifyWithSharding(gtest_filter, 5, expected_tests,
00630 check_exit_0=True, args=[flag])
00631
00632 if __name__ == '__main__':
00633 gtest_test_utils.Main()