run_tests_matrix.py
Go to the documentation of this file.
1 #!/usr/bin/env python3
2 # Copyright 2015 gRPC authors.
3 #
4 # Licensed under the Apache License, Version 2.0 (the "License");
5 # you may not use this file except in compliance with the License.
6 # You may obtain a copy of the License at
7 #
8 # http://www.apache.org/licenses/LICENSE-2.0
9 #
10 # Unless required by applicable law or agreed to in writing, software
11 # distributed under the License is distributed on an "AS IS" BASIS,
12 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 # See the License for the specific language governing permissions and
14 # limitations under the License.
15 """Run test matrix."""
16 
17 from __future__ import print_function
18 
19 import argparse
20 import multiprocessing
21 import os
22 import sys
23 
24 from python_utils.filter_pull_request_tests import filter_tests
25 import python_utils.jobset as jobset
26 import python_utils.report_utils as report_utils
27 
28 _ROOT = os.path.abspath(os.path.join(os.path.dirname(sys.argv[0]), '../..'))
29 os.chdir(_ROOT)
30 
31 _DEFAULT_RUNTESTS_TIMEOUT = 1 * 60 * 60
32 
33 # C/C++ tests can take long time
34 _CPP_RUNTESTS_TIMEOUT = 4 * 60 * 60
35 
36 # Set timeout high for ObjC for Cocoapods to install pods
37 _OBJC_RUNTESTS_TIMEOUT = 2 * 60 * 60
38 
39 # Number of jobs assigned to each run_tests.py instance
40 _DEFAULT_INNER_JOBS = 2
41 
42 # Name of the top-level umbrella report that includes all the run_tests.py invocations
43 # Note that the starting letter 't' matters so that the targets are listed AFTER
44 # the per-test breakdown items that start with 'run_tests/' (it is more readable that way)
45 _MATRIX_REPORT_NAME = 'toplevel_run_tests_invocations'
46 
47 
48 def _safe_report_name(name):
49  """Reports with '+' in target name won't show correctly in ResultStore"""
50  return name.replace('+', 'p')
51 
52 
53 def _report_filename(name):
54  """Generates report file name with directory structure that leads to better presentation by internal CI"""
55  # 'sponge_log.xml' suffix must be there for results to get recognized by kokoro.
56  return '%s/%s' % (_safe_report_name(name), 'sponge_log.xml')
57 
58 
59 def _matrix_job_logfilename(shortname_for_multi_target):
60  """Generate location for log file that will match the sponge_log.xml from the top-level matrix report."""
61  # 'sponge_log.log' suffix must be there for log to get recognized as "target log"
62  # for the corresponding 'sponge_log.xml' report.
63  # the shortname_for_multi_target component must be set to match the sponge_log.xml location
64  # because the top-level render_junit_xml_report is called with multi_target=True
65  sponge_log_name = '%s/%s/%s' % (
66  _MATRIX_REPORT_NAME, shortname_for_multi_target, 'sponge_log.log')
67  # env variable can be used to override the base location for the reports
68  # so we need to match that behavior here too
69  base_dir = os.getenv('GRPC_TEST_REPORT_BASE_DIR', None)
70  if base_dir:
71  sponge_log_name = os.path.join(base_dir, sponge_log_name)
72  return sponge_log_name
73 
74 
75 def _docker_jobspec(name,
76  runtests_args=[],
77  runtests_envs={},
78  inner_jobs=_DEFAULT_INNER_JOBS,
79  timeout_seconds=None):
80  """Run a single instance of run_tests.py in a docker container"""
81  if not timeout_seconds:
82  timeout_seconds = _DEFAULT_RUNTESTS_TIMEOUT
83  shortname = 'run_tests_%s' % name
84  test_job = jobset.JobSpec(cmdline=[
85  'python3', 'tools/run_tests/run_tests.py', '--use_docker', '-t', '-j',
86  str(inner_jobs), '-x',
87  'run_tests/%s' % _report_filename(name), '--report_suite_name',
88  '%s' % _safe_report_name(name)
89  ] + runtests_args,
90  environ=runtests_envs,
91  shortname=shortname,
92  timeout_seconds=timeout_seconds,
93  logfilename=_matrix_job_logfilename(shortname))
94  return test_job
95 
96 
97 def _workspace_jobspec(name,
98  runtests_args=[],
99  workspace_name=None,
100  runtests_envs={},
101  inner_jobs=_DEFAULT_INNER_JOBS,
102  timeout_seconds=None):
103  """Run a single instance of run_tests.py in a separate workspace"""
104  if not workspace_name:
105  workspace_name = 'workspace_%s' % name
106  if not timeout_seconds:
107  timeout_seconds = _DEFAULT_RUNTESTS_TIMEOUT
108  shortname = 'run_tests_%s' % name
109  env = {'WORKSPACE_NAME': workspace_name}
110  env.update(runtests_envs)
111  # if report base dir is set, we don't need to ".." to come out of the workspace dir
112  report_dir_prefix = '' if os.getenv('GRPC_TEST_REPORT_BASE_DIR',
113  None) else '../'
114  test_job = jobset.JobSpec(cmdline=[
115  'bash', 'tools/run_tests/helper_scripts/run_tests_in_workspace.sh',
116  '-t', '-j',
117  str(inner_jobs), '-x',
118  '%srun_tests/%s' %
119  (report_dir_prefix, _report_filename(name)), '--report_suite_name',
120  '%s' % _safe_report_name(name)
121  ] + runtests_args,
122  environ=env,
123  shortname=shortname,
124  timeout_seconds=timeout_seconds,
125  logfilename=_matrix_job_logfilename(shortname))
126  return test_job
127 
128 
129 def _generate_jobs(languages,
130  configs,
131  platforms,
132  iomgr_platforms=['native'],
133  arch=None,
134  compiler=None,
135  labels=[],
136  extra_args=[],
137  extra_envs={},
138  inner_jobs=_DEFAULT_INNER_JOBS,
139  timeout_seconds=None):
140  result = []
141  for language in languages:
142  for platform in platforms:
143  for iomgr_platform in iomgr_platforms:
144  for config in configs:
145  name = '%s_%s_%s_%s' % (language, platform, config,
146  iomgr_platform)
147  runtests_args = [
148  '-l', language, '-c', config, '--iomgr_platform',
149  iomgr_platform
150  ]
151  if arch or compiler:
152  name += '_%s_%s' % (arch, compiler)
153  runtests_args += [
154  '--arch', arch, '--compiler', compiler
155  ]
156  if '--build_only' in extra_args:
157  name += '_buildonly'
158  for extra_env in extra_envs:
159  name += '_%s_%s' % (extra_env, extra_envs[extra_env])
160 
161  runtests_args += extra_args
162  if platform == 'linux':
163  job = _docker_jobspec(name=name,
164  runtests_args=runtests_args,
165  runtests_envs=extra_envs,
166  inner_jobs=inner_jobs,
167  timeout_seconds=timeout_seconds)
168  else:
169  job = _workspace_jobspec(
170  name=name,
171  runtests_args=runtests_args,
172  runtests_envs=extra_envs,
173  inner_jobs=inner_jobs,
174  timeout_seconds=timeout_seconds)
175 
176  job.labels = [platform, config, language, iomgr_platform
177  ] + labels
178  result.append(job)
179  return result
180 
181 
182 def _create_test_jobs(extra_args=[], inner_jobs=_DEFAULT_INNER_JOBS):
183  test_jobs = []
184  # sanity tests
185  test_jobs += _generate_jobs(languages=['sanity'],
186  configs=['dbg'],
187  platforms=['linux'],
188  labels=['basictests'],
189  extra_args=extra_args +
190  ['--report_multi_target'],
191  inner_jobs=inner_jobs)
192 
193  # supported on all platforms.
194  test_jobs += _generate_jobs(
195  languages=['c'],
196  configs=['dbg', 'opt'],
197  platforms=['linux', 'macos', 'windows'],
198  labels=['basictests', 'corelang'],
199  extra_args=
200  extra_args, # don't use multi_target report because C has too many test cases
201  inner_jobs=inner_jobs,
202  timeout_seconds=_CPP_RUNTESTS_TIMEOUT)
203 
204  # C# tests (both on .NET desktop/mono and .NET core)
205  test_jobs += _generate_jobs(languages=['csharp'],
206  configs=['dbg', 'opt'],
207  platforms=['linux', 'macos', 'windows'],
208  labels=['basictests', 'multilang'],
209  extra_args=extra_args +
210  ['--report_multi_target'],
211  inner_jobs=inner_jobs)
212 
213  # ARM64 Linux C# tests
214  test_jobs += _generate_jobs(languages=['csharp'],
215  configs=['dbg', 'opt'],
216  platforms=['linux'],
217  arch='arm64',
218  compiler='default',
219  labels=['basictests_arm64'],
220  extra_args=extra_args +
221  ['--report_multi_target'],
222  inner_jobs=inner_jobs)
223 
224  test_jobs += _generate_jobs(languages=['python'],
225  configs=['opt'],
226  platforms=['linux', 'macos', 'windows'],
227  iomgr_platforms=['native'],
228  labels=['basictests', 'multilang'],
229  extra_args=extra_args +
230  ['--report_multi_target'],
231  inner_jobs=inner_jobs)
232 
233  # ARM64 Linux Python tests
234  test_jobs += _generate_jobs(languages=['python'],
235  configs=['opt'],
236  platforms=['linux'],
237  arch='arm64',
238  compiler='default',
239  iomgr_platforms=['native'],
240  labels=['basictests_arm64'],
241  extra_args=extra_args +
242  ['--report_multi_target'],
243  inner_jobs=inner_jobs)
244 
245  # supported on linux and mac.
246  test_jobs += _generate_jobs(
247  languages=['c++'],
248  configs=['dbg', 'opt'],
249  platforms=['linux', 'macos'],
250  labels=['basictests', 'corelang'],
251  extra_args=
252  extra_args, # don't use multi_target report because C++ has too many test cases
253  inner_jobs=inner_jobs,
254  timeout_seconds=_CPP_RUNTESTS_TIMEOUT)
255 
256  test_jobs += _generate_jobs(languages=['ruby', 'php7'],
257  configs=['dbg', 'opt'],
258  platforms=['linux', 'macos'],
259  labels=['basictests', 'multilang'],
260  extra_args=extra_args +
261  ['--report_multi_target'],
262  inner_jobs=inner_jobs)
263 
264  # ARM64 Linux Ruby and PHP tests
265  test_jobs += _generate_jobs(languages=['ruby', 'php7'],
266  configs=['dbg', 'opt'],
267  platforms=['linux'],
268  arch='arm64',
269  compiler='default',
270  labels=['basictests_arm64'],
271  extra_args=extra_args +
272  ['--report_multi_target'],
273  inner_jobs=inner_jobs)
274 
275  # supported on mac only.
276  test_jobs += _generate_jobs(languages=['objc'],
277  configs=['opt'],
278  platforms=['macos'],
279  labels=['basictests', 'multilang'],
280  extra_args=extra_args +
281  ['--report_multi_target'],
282  inner_jobs=inner_jobs,
283  timeout_seconds=_OBJC_RUNTESTS_TIMEOUT)
284 
285  return test_jobs
286 
287 
288 def _create_portability_test_jobs(extra_args=[],
289  inner_jobs=_DEFAULT_INNER_JOBS):
290  test_jobs = []
291  # portability C x86
292  test_jobs += _generate_jobs(languages=['c'],
293  configs=['dbg'],
294  platforms=['linux'],
295  arch='x86',
296  compiler='default',
297  labels=['portability', 'corelang'],
298  extra_args=extra_args,
299  inner_jobs=inner_jobs)
300 
301  # portability C and C++ on x64
302  for compiler in [
303  'gcc6', 'gcc10.2_openssl102', 'gcc11', 'gcc_musl', 'clang6',
304  'clang13'
305  ]:
306  test_jobs += _generate_jobs(languages=['c', 'c++'],
307  configs=['dbg'],
308  platforms=['linux'],
309  arch='x64',
310  compiler=compiler,
311  labels=['portability', 'corelang'],
312  extra_args=extra_args,
313  inner_jobs=inner_jobs,
314  timeout_seconds=_CPP_RUNTESTS_TIMEOUT)
315 
316  # portability C on Windows 64-bit (x86 is the default)
317  test_jobs += _generate_jobs(languages=['c'],
318  configs=['dbg'],
319  platforms=['windows'],
320  arch='x64',
321  compiler='default',
322  labels=['portability', 'corelang'],
323  extra_args=extra_args,
324  inner_jobs=inner_jobs)
325 
326  # portability C on Windows with the "Visual Studio" cmake
327  # generator, i.e. not using Ninja (to verify that we can still build with msbuild)
328  test_jobs += _generate_jobs(languages=['c'],
329  configs=['dbg'],
330  platforms=['windows'],
331  arch='default',
332  compiler='cmake_vs2017',
333  labels=['portability', 'corelang'],
334  extra_args=extra_args,
335  inner_jobs=inner_jobs)
336 
337  # portability C++ on Windows
338  # TODO(jtattermusch): some of the tests are failing, so we force --build_only
339  test_jobs += _generate_jobs(languages=['c++'],
340  configs=['dbg'],
341  platforms=['windows'],
342  arch='default',
343  compiler='default',
344  labels=['portability', 'corelang'],
345  extra_args=extra_args + ['--build_only'],
346  inner_jobs=inner_jobs,
347  timeout_seconds=_CPP_RUNTESTS_TIMEOUT)
348 
349  # portability C and C++ on Windows using VS2017 (build only)
350  # TODO(jtattermusch): some of the tests are failing, so we force --build_only
351  test_jobs += _generate_jobs(languages=['c', 'c++'],
352  configs=['dbg'],
353  platforms=['windows'],
354  arch='x64',
355  compiler='cmake_ninja_vs2017',
356  labels=['portability', 'corelang'],
357  extra_args=extra_args + ['--build_only'],
358  inner_jobs=inner_jobs,
359  timeout_seconds=_CPP_RUNTESTS_TIMEOUT)
360 
361  # C and C++ with no-exceptions on Linux
362  test_jobs += _generate_jobs(languages=['c', 'c++'],
363  configs=['noexcept'],
364  platforms=['linux'],
365  labels=['portability', 'corelang'],
366  extra_args=extra_args,
367  inner_jobs=inner_jobs,
368  timeout_seconds=_CPP_RUNTESTS_TIMEOUT)
369 
370  test_jobs += _generate_jobs(languages=['python'],
371  configs=['dbg'],
372  platforms=['linux'],
373  arch='default',
374  compiler='python_alpine',
375  labels=['portability', 'multilang'],
376  extra_args=extra_args +
377  ['--report_multi_target'],
378  inner_jobs=inner_jobs)
379 
380  return test_jobs
381 
382 
383 def _allowed_labels():
384  """Returns a list of existing job labels."""
385  all_labels = set()
386  for job in _create_test_jobs() + _create_portability_test_jobs():
387  for label in job.labels:
388  all_labels.add(label)
389  return sorted(all_labels)
390 
391 
392 def _runs_per_test_type(arg_str):
393  """Auxiliary function to parse the "runs_per_test" flag."""
394  try:
395  n = int(arg_str)
396  if n <= 0:
397  raise ValueError
398  return n
399  except:
400  msg = '\'{}\' is not a positive integer'.format(arg_str)
401  raise argparse.ArgumentTypeError(msg)
402 
403 
404 if __name__ == "__main__":
405  argp = argparse.ArgumentParser(
406  description='Run a matrix of run_tests.py tests.')
407  argp.add_argument('-j',
408  '--jobs',
409  default=multiprocessing.cpu_count() / _DEFAULT_INNER_JOBS,
410  type=int,
411  help='Number of concurrent run_tests.py instances.')
412  argp.add_argument('-f',
413  '--filter',
414  choices=_allowed_labels(),
415  nargs='+',
416  default=[],
417  help='Filter targets to run by label with AND semantics.')
418  argp.add_argument('--exclude',
419  choices=_allowed_labels(),
420  nargs='+',
421  default=[],
422  help='Exclude targets with any of given labels.')
423  argp.add_argument('--build_only',
424  default=False,
425  action='store_const',
426  const=True,
427  help='Pass --build_only flag to run_tests.py instances.')
428  argp.add_argument(
429  '--force_default_poller',
430  default=False,
431  action='store_const',
432  const=True,
433  help='Pass --force_default_poller to run_tests.py instances.')
434  argp.add_argument('--dry_run',
435  default=False,
436  action='store_const',
437  const=True,
438  help='Only print what would be run.')
439  argp.add_argument(
440  '--filter_pr_tests',
441  default=False,
442  action='store_const',
443  const=True,
444  help='Filters out tests irrelevant to pull request changes.')
445  argp.add_argument(
446  '--base_branch',
447  default='origin/master',
448  type=str,
449  help='Branch that pull request is requesting to merge into')
450  argp.add_argument('--inner_jobs',
451  default=_DEFAULT_INNER_JOBS,
452  type=int,
453  help='Number of jobs in each run_tests.py instance')
454  argp.add_argument(
455  '-n',
456  '--runs_per_test',
457  default=1,
458  type=_runs_per_test_type,
459  help='How many times to run each tests. >1 runs implies ' +
460  'omitting passing test from the output & reports.')
461  argp.add_argument('--max_time',
462  default=-1,
463  type=int,
464  help='Maximum amount of time to run tests for' +
465  '(other tests will be skipped)')
466  argp.add_argument(
467  '--internal_ci',
468  default=False,
469  action='store_const',
470  const=True,
471  help=
472  '(Deprecated, has no effect) Put reports into subdirectories to improve presentation of '
473  'results by Kokoro.')
474  argp.add_argument('--bq_result_table',
475  default='',
476  type=str,
477  nargs='?',
478  help='Upload test results to a specified BQ table.')
479  argp.add_argument('--extra_args',
480  default='',
481  type=str,
482  nargs=argparse.REMAINDER,
483  help='Extra test args passed to each sub-script.')
484  args = argp.parse_args()
485 
486  extra_args = []
487  if args.build_only:
488  extra_args.append('--build_only')
489  if args.force_default_poller:
490  extra_args.append('--force_default_poller')
491  if args.runs_per_test > 1:
492  extra_args.append('-n')
493  extra_args.append('%s' % args.runs_per_test)
494  extra_args.append('--quiet_success')
495  if args.max_time > 0:
496  extra_args.extend(('--max_time', '%d' % args.max_time))
497  if args.bq_result_table:
498  extra_args.append('--bq_result_table')
499  extra_args.append('%s' % args.bq_result_table)
500  extra_args.append('--measure_cpu_costs')
501  if args.extra_args:
502  extra_args.extend(args.extra_args)
503 
504  all_jobs = _create_test_jobs(extra_args=extra_args, inner_jobs=args.inner_jobs) + \
505  _create_portability_test_jobs(extra_args=extra_args, inner_jobs=args.inner_jobs)
506 
507  jobs = []
508  for job in all_jobs:
509  if not args.filter or all(
510  filter in job.labels for filter in args.filter):
511  if not any(exclude_label in job.labels
512  for exclude_label in args.exclude):
513  jobs.append(job)
514 
515  if not jobs:
516  jobset.message('FAILED',
517  'No test suites match given criteria.',
518  do_newline=True)
519  sys.exit(1)
520 
521  print('IMPORTANT: The changes you are testing need to be locally committed')
522  print('because only the committed changes in the current branch will be')
523  print('copied to the docker environment or into subworkspaces.')
524 
525  skipped_jobs = []
526 
527  if args.filter_pr_tests:
528  print('Looking for irrelevant tests to skip...')
529  relevant_jobs = filter_tests(jobs, args.base_branch)
530  if len(relevant_jobs) == len(jobs):
531  print('No tests will be skipped.')
532  else:
533  print('These tests will be skipped:')
534  skipped_jobs = list(set(jobs) - set(relevant_jobs))
535  # Sort by shortnames to make printing of skipped tests consistent
536  skipped_jobs.sort(key=lambda job: job.shortname)
537  for job in list(skipped_jobs):
538  print(' %s' % job.shortname)
539  jobs = relevant_jobs
540 
541  print('Will run these tests:')
542  for job in jobs:
543  print(' %s: "%s"' % (job.shortname, ' '.join(job.cmdline)))
544  print('')
545 
546  if args.dry_run:
547  print('--dry_run was used, exiting')
548  sys.exit(1)
549 
550  jobset.message('START', 'Running test matrix.', do_newline=True)
551  num_failures, resultset = jobset.run(jobs,
552  newline_on_success=True,
553  travis=True,
554  maxjobs=args.jobs)
555  # Merge skipped tests into results to show skipped tests on report.xml
556  if skipped_jobs:
557  ignored_num_skipped_failures, skipped_results = jobset.run(
558  skipped_jobs, skip_jobs=True)
559  resultset.update(skipped_results)
560  report_utils.render_junit_xml_report(resultset,
561  _report_filename(_MATRIX_REPORT_NAME),
562  suite_name=_MATRIX_REPORT_NAME,
563  multi_target=True)
564 
565  if num_failures == 0:
566  jobset.message('SUCCESS',
567  'All run_tests.py instances finished successfully.',
568  do_newline=True)
569  else:
570  jobset.message('FAILED',
571  'Some run_tests.py instances have failed.',
572  do_newline=True)
573  sys.exit(1)
xds_interop_client.str
str
Definition: xds_interop_client.py:487
test_group_name.all
all
Definition: test_group_name.py:241
python_utils.filter_pull_request_tests
Definition: filter_pull_request_tests.py:1
http2_test_server.format
format
Definition: http2_test_server.py:118
python_utils.report_utils
Definition: report_utils.py:1
run_tests_matrix._workspace_jobspec
def _workspace_jobspec(name, runtests_args=[], workspace_name=None, runtests_envs={}, inner_jobs=_DEFAULT_INNER_JOBS, timeout_seconds=None)
Definition: run_tests_matrix.py:97
run_tests_matrix._generate_jobs
def _generate_jobs(languages, configs, platforms, iomgr_platforms=['native'], arch=None, compiler=None, labels=[], extra_args=[], extra_envs={}, inner_jobs=_DEFAULT_INNER_JOBS, timeout_seconds=None)
Definition: run_tests_matrix.py:129
xds_interop_client.int
int
Definition: xds_interop_client.py:113
python_utils.jobset
Definition: jobset.py:1
run_tests_matrix._docker_jobspec
def _docker_jobspec(name, runtests_args=[], runtests_envs={}, inner_jobs=_DEFAULT_INNER_JOBS, timeout_seconds=None)
Definition: run_tests_matrix.py:75
run_tests_matrix._allowed_labels
def _allowed_labels()
Definition: run_tests_matrix.py:383
run_tests_matrix._matrix_job_logfilename
def _matrix_job_logfilename(shortname_for_multi_target)
Definition: run_tests_matrix.py:59
run_tests_matrix._runs_per_test_type
def _runs_per_test_type(arg_str)
Definition: run_tests_matrix.py:392
run_tests_matrix._create_test_jobs
def _create_test_jobs(extra_args=[], inner_jobs=_DEFAULT_INNER_JOBS)
Definition: run_tests_matrix.py:182
cpp.gmock_class.set
set
Definition: bloaty/third_party/googletest/googlemock/scripts/generator/cpp/gmock_class.py:44
run_tests_matrix._create_portability_test_jobs
def _create_portability_test_jobs(extra_args=[], inner_jobs=_DEFAULT_INNER_JOBS)
Definition: run_tests_matrix.py:288
run_tests_matrix._report_filename
def _report_filename(name)
Definition: run_tests_matrix.py:53
len
int len
Definition: abseil-cpp/absl/base/internal/low_level_alloc_test.cc:46
python_utils.filter_pull_request_tests.filter_tests
def filter_tests(tests, base_branch)
Definition: filter_pull_request_tests.py:179
run_tests_matrix._safe_report_name
def _safe_report_name(name)
Definition: run_tests_matrix.py:48

grpc
Author(s):
autogenerated on Thu Mar 13 2025 03:01:14