run_microbenchmark.py
Go to the documentation of this file.
1 #!/usr/bin/env python3
2 # Copyright 2017 gRPC authors.
3 #
4 # Licensed under the Apache License, Version 2.0 (the "License");
5 # you may not use this file except in compliance with the License.
6 # You may obtain a copy of the License at
7 #
8 # http://www.apache.org/licenses/LICENSE-2.0
9 #
10 # Unless required by applicable law or agreed to in writing, software
11 # distributed under the License is distributed on an "AS IS" BASIS,
12 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 # See the License for the specific language governing permissions and
14 # limitations under the License.
15 
16 import argparse
17 import html
18 import multiprocessing
19 import os
20 import subprocess
21 import sys
22 
23 import python_utils.jobset as jobset
24 import python_utils.start_port_server as start_port_server
25 
26 sys.path.append(
27  os.path.join(os.path.dirname(sys.argv[0]), '..', 'profiling',
28  'microbenchmarks', 'bm_diff'))
29 import bm_constants
30 
31 flamegraph_dir = os.path.join(os.path.expanduser('~'), 'FlameGraph')
32 
33 os.chdir(os.path.join(os.path.dirname(sys.argv[0]), '../..'))
34 if not os.path.exists('reports'):
35  os.makedirs('reports')
36 
37 start_port_server.start_port_server()
38 
39 
40 def fnize(s):
41  out = ''
42  for c in s:
43  if c in '<>, /':
44  if len(out) and out[-1] == '_':
45  continue
46  out += '_'
47  else:
48  out += c
49  return out
50 
51 
52 # index html
53 index_html = """
54 <html>
55 <head>
56 <title>Microbenchmark Results</title>
57 </head>
58 <body>
59 """
60 
61 
62 def heading(name):
63  global index_html
64  index_html += "<h1>%s</h1>\n" % name
65 
66 
67 def link(txt, tgt):
68  global index_html
69  index_html += "<p><a href=\"%s\">%s</a></p>\n" % (html.escape(
70  tgt, quote=True), html.escape(txt))
71 
72 
73 def text(txt):
74  global index_html
75  index_html += "<p><pre>%s</pre></p>\n" % html.escape(txt)
76 
77 
78 def _bazel_build_benchmark(bm_name, cfg):
79  """Build given benchmark with bazel"""
80  subprocess.check_call([
81  'tools/bazel', 'build',
82  '--config=%s' % cfg,
83  '//test/cpp/microbenchmarks:%s' % bm_name
84  ])
85 
86 
87 def collect_latency(bm_name, args):
88  """generate latency profiles"""
89  benchmarks = []
90  profile_analysis = []
91  cleanup = []
92 
93  heading('Latency Profiles: %s' % bm_name)
94  _bazel_build_benchmark(bm_name, 'basicprof')
95  for line in subprocess.check_output([
96  'bazel-bin/test/cpp/microbenchmarks/%s' % bm_name,
97  '--benchmark_list_tests'
98  ]).decode('UTF-8').splitlines():
99  link(line, '%s.txt' % fnize(line))
100  benchmarks.append(
101  jobset.JobSpec([
102  'bazel-bin/test/cpp/microbenchmarks/%s' % bm_name,
103  '--benchmark_filter=^%s$' % line, '--benchmark_min_time=0.05'
104  ],
105  environ={
106  'GRPC_LATENCY_TRACE': '%s.trace' % fnize(line)
107  },
108  shortname='profile-%s' % fnize(line)))
109  profile_analysis.append(
110  jobset.JobSpec([
111  sys.executable,
112  'tools/profiling/latency_profile/profile_analyzer.py',
113  '--source',
114  '%s.trace' % fnize(line), '--fmt', 'simple', '--out',
115  'reports/%s.txt' % fnize(line)
116  ],
117  timeout_seconds=20 * 60,
118  shortname='analyze-%s' % fnize(line)))
119  cleanup.append(jobset.JobSpec(['rm', '%s.trace' % fnize(line)]))
120  # periodically flush out the list of jobs: profile_analysis jobs at least
121  # consume upwards of five gigabytes of ram in some cases, and so analysing
122  # hundreds of them at once is impractical -- but we want at least some
123  # concurrency or the work takes too long
124  if len(benchmarks) >= min(16, multiprocessing.cpu_count()):
125  # run up to half the cpu count: each benchmark can use up to two cores
126  # (one for the microbenchmark, one for the data flush)
127  jobset.run(benchmarks,
128  maxjobs=max(1,
129  multiprocessing.cpu_count() / 2))
130  jobset.run(profile_analysis, maxjobs=multiprocessing.cpu_count())
131  jobset.run(cleanup, maxjobs=multiprocessing.cpu_count())
132  benchmarks = []
133  profile_analysis = []
134  cleanup = []
135  # run the remaining benchmarks that weren't flushed
136  if len(benchmarks):
137  jobset.run(benchmarks, maxjobs=max(1, multiprocessing.cpu_count() / 2))
138  jobset.run(profile_analysis, maxjobs=multiprocessing.cpu_count())
139  jobset.run(cleanup, maxjobs=multiprocessing.cpu_count())
140 
141 
142 def collect_perf(bm_name, args):
143  """generate flamegraphs"""
144  heading('Flamegraphs: %s' % bm_name)
145  _bazel_build_benchmark(bm_name, 'mutrace')
146  benchmarks = []
147  profile_analysis = []
148  cleanup = []
149  for line in subprocess.check_output([
150  'bazel-bin/test/cpp/microbenchmarks/%s' % bm_name,
151  '--benchmark_list_tests'
152  ]).decode('UTF-8').splitlines():
153  link(line, '%s.svg' % fnize(line))
154  benchmarks.append(
155  jobset.JobSpec([
156  'perf', 'record', '-o',
157  '%s-perf.data' % fnize(line), '-g', '-F', '997',
158  'bazel-bin/test/cpp/microbenchmarks/%s' % bm_name,
159  '--benchmark_filter=^%s$' % line, '--benchmark_min_time=10'
160  ],
161  shortname='perf-%s' % fnize(line)))
162  profile_analysis.append(
163  jobset.JobSpec(
164  [
165  'tools/run_tests/performance/process_local_perf_flamegraphs.sh'
166  ],
167  environ={
168  'PERF_BASE_NAME': fnize(line),
169  'OUTPUT_DIR': 'reports',
170  'OUTPUT_FILENAME': fnize(line),
171  },
172  shortname='flame-%s' % fnize(line)))
173  cleanup.append(jobset.JobSpec(['rm', '%s-perf.data' % fnize(line)]))
174  cleanup.append(jobset.JobSpec(['rm', '%s-out.perf' % fnize(line)]))
175  # periodically flush out the list of jobs: temporary space required for this
176  # processing is large
177  if len(benchmarks) >= 20:
178  # run up to half the cpu count: each benchmark can use up to two cores
179  # (one for the microbenchmark, one for the data flush)
180  jobset.run(benchmarks, maxjobs=1)
181  jobset.run(profile_analysis, maxjobs=multiprocessing.cpu_count())
182  jobset.run(cleanup, maxjobs=multiprocessing.cpu_count())
183  benchmarks = []
184  profile_analysis = []
185  cleanup = []
186  # run the remaining benchmarks that weren't flushed
187  if len(benchmarks):
188  jobset.run(benchmarks, maxjobs=1)
189  jobset.run(profile_analysis, maxjobs=multiprocessing.cpu_count())
190  jobset.run(cleanup, maxjobs=multiprocessing.cpu_count())
191 
192 
193 def run_summary(bm_name, cfg, base_json_name):
194  _bazel_build_benchmark(bm_name, cfg)
195  cmd = [
196  'bazel-bin/test/cpp/microbenchmarks/%s' % bm_name,
197  '--benchmark_out=%s.%s.json' % (base_json_name, cfg),
198  '--benchmark_out_format=json'
199  ]
200  if args.summary_time is not None:
201  cmd += ['--benchmark_min_time=%d' % args.summary_time]
202  return subprocess.check_output(cmd).decode('UTF-8')
203 
204 
205 def collect_summary(bm_name, args):
206  # no counters, run microbenchmark and add summary
207  # both to HTML report and to console.
208  nocounters_heading = 'Summary: %s [no counters]' % bm_name
209  nocounters_summary = run_summary(bm_name, 'opt', bm_name)
210  heading(nocounters_heading)
211  text(nocounters_summary)
212  print(nocounters_heading)
213  print(nocounters_summary)
214 
215  # with counters, run microbenchmark and add summary
216  # both to HTML report and to console.
217  counters_heading = 'Summary: %s [with counters]' % bm_name
218  counters_summary = run_summary(bm_name, 'counters', bm_name)
219  heading(counters_heading)
220  text(counters_summary)
221  print(counters_heading)
222  print(counters_summary)
223 
224  if args.bq_result_table:
225  with open('%s.csv' % bm_name, 'w') as f:
226  f.write(
227  subprocess.check_output([
228  'tools/profiling/microbenchmarks/bm2bq.py',
229  '%s.counters.json' % bm_name,
230  '%s.opt.json' % bm_name
231  ]).decode('UTF-8'))
232  subprocess.check_call(
233  ['bq', 'load',
234  '%s' % args.bq_result_table,
235  '%s.csv' % bm_name])
236 
237 
238 collectors = {
239  'latency': collect_latency,
240  'perf': collect_perf,
241  'summary': collect_summary,
242 }
243 
244 argp = argparse.ArgumentParser(description='Collect data from microbenchmarks')
245 argp.add_argument('-c',
246  '--collect',
247  choices=sorted(collectors.keys()),
248  nargs='*',
249  default=sorted(collectors.keys()),
250  help='Which collectors should be run against each benchmark')
251 argp.add_argument('-b',
252  '--benchmarks',
253  choices=bm_constants._AVAILABLE_BENCHMARK_TESTS,
254  default=bm_constants._AVAILABLE_BENCHMARK_TESTS,
255  nargs='+',
256  type=str,
257  help='Which microbenchmarks should be run')
258 argp.add_argument(
259  '--bq_result_table',
260  default='',
261  type=str,
262  help='Upload results from summary collection to a specified bigquery table.'
263 )
264 argp.add_argument(
265  '--summary_time',
266  default=None,
267  type=int,
268  help='Minimum time to run benchmarks for the summary collection')
269 args = argp.parse_args()
270 
271 try:
272  for collect in args.collect:
273  for bm_name in args.benchmarks:
274  collectors[collect](bm_name, args)
275 finally:
276  if not os.path.exists('reports'):
277  os.makedirs('reports')
278  index_html += "</body>\n</html>\n"
279  with open('reports/index.html', 'w') as f:
280  f.write(index_html)
run_microbenchmark.collect_latency
def collect_latency(bm_name, args)
Definition: run_microbenchmark.py:87
run_microbenchmark._bazel_build_benchmark
def _bazel_build_benchmark(bm_name, cfg)
Definition: run_microbenchmark.py:78
run_microbenchmark.run_summary
def run_summary(bm_name, cfg, base_json_name)
Definition: run_microbenchmark.py:193
max
int max
Definition: bloaty/third_party/zlib/examples/enough.c:170
python_utils.jobset
Definition: jobset.py:1
min
#define min(a, b)
Definition: qsort.h:83
run_microbenchmark.collect_perf
def collect_perf(bm_name, args)
Definition: run_microbenchmark.py:142
run_microbenchmark.text
def text(txt)
Definition: run_microbenchmark.py:73
run_microbenchmark.heading
def heading(name)
Definition: run_microbenchmark.py:62
run_microbenchmark.fnize
def fnize(s)
Definition: run_microbenchmark.py:40
python_utils.start_port_server
Definition: python_utils/start_port_server.py:1
grpc._common.decode
def decode(b)
Definition: grpc/_common.py:75
open
#define open
Definition: test-fs.c:46
len
int len
Definition: abseil-cpp/absl/base/internal/low_level_alloc_test.cc:46
run_microbenchmark.link
def link(txt, tgt)
Definition: run_microbenchmark.py:67
run_microbenchmark.collect_summary
def collect_summary(bm_name, args)
Definition: run_microbenchmark.py:205

grpc
Author(s):
autogenerated on Fri May 16 2025 03:00:08