scenario_config.py
Go to the documentation of this file.
1 # Copyright 2016 gRPC authors.
2 #
3 # Licensed under the Apache License, Version 2.0 (the "License");
4 # you may not use this file except in compliance with the License.
5 # You may obtain a copy of the License at
6 #
7 # http://www.apache.org/licenses/LICENSE-2.0
8 #
9 # Unless required by applicable law or agreed to in writing, software
10 # distributed under the License is distributed on an "AS IS" BASIS,
11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 # See the License for the specific language governing permissions and
13 # limitations under the License.
14 
15 # performance scenario configuration for various languages
16 
17 import math
18 
19 WARMUP_SECONDS = 5
20 JAVA_WARMUP_SECONDS = 15 # Java needs more warmup time for JIT to kick in.
21 BENCHMARK_SECONDS = 30
22 
23 SMOKETEST = 'smoketest'
24 SCALABLE = 'scalable'
25 INPROC = 'inproc'
26 SWEEP = 'sweep'
27 PSM = 'psm'
28 DEFAULT_CATEGORIES = (SCALABLE, SMOKETEST)
29 
30 SECURE_SECARGS = {
31  'use_test_ca': True,
32  'server_host_override': 'foo.test.google.fr'
33 }
34 
35 HISTOGRAM_PARAMS = {
36  'resolution': 0.01,
37  'max_possible': 60e9,
38 }
39 
40 # target number of RPCs outstanding on across all client channels in
41 # non-ping-pong tests (since we can only specify per-channel numbers, the
42 # actual target will be slightly higher)
43 OUTSTANDING_REQUESTS = {'async': 6400, 'async-limited': 800, 'sync': 1000}
44 
45 # wide is the number of client channels in multi-channel tests (1 otherwise)
46 WIDE = 64
47 
48 
49 def _get_secargs(is_secure):
50  if is_secure:
51  return SECURE_SECARGS
52  else:
53  return None
54 
55 
56 def remove_nonproto_fields(scenario):
57  """Removes special-purpose fields that don't belong in the protobuf.
58 
59  This function removes additional information about the scenario that is not
60  included in the ScenarioConfig protobuf message.
61  """
62  scenario.pop('CATEGORIES', None)
63  scenario.pop('CLIENT_LANGUAGE', None)
64  scenario.pop('SERVER_LANGUAGE', None)
65  scenario.pop('EXCLUDED_POLL_ENGINES', None)
66  return scenario
67 
68 
69 def geometric_progression(start, stop, step):
70  n = start
71  while n < stop:
72  yield int(round(n))
73  n *= step
74 
75 
76 def _payload_type(use_generic_payload, req_size, resp_size):
77  r = {}
78  sizes = {
79  'req_size': req_size,
80  'resp_size': resp_size,
81  }
82  if use_generic_payload:
83  r['bytebuf_params'] = sizes
84  else:
85  r['simple_params'] = sizes
86  return r
87 
88 
89 def _load_params(offered_load):
90  r = {}
91  if offered_load is None:
92  r['closed_loop'] = {}
93  else:
94  load = {}
95  load['offered_load'] = offered_load
96  r['poisson'] = load
97  return r
98 
99 
100 def _add_channel_arg(config, key, value):
101  if 'channel_args' in config:
102  channel_args = config['channel_args']
103  else:
104  channel_args = []
105  config['channel_args'] = channel_args
106  arg = {'name': key}
107  if isinstance(value, int):
108  arg['int_value'] = value
109  else:
110  arg['str_value'] = value
111  channel_args.append(arg)
112 
113 
114 def _ping_pong_scenario(name,
115  rpc_type,
116  client_type,
117  server_type,
118  secure=True,
119  use_generic_payload=False,
120  req_size=0,
121  resp_size=0,
122  unconstrained_client=None,
123  client_language=None,
124  server_language=None,
125  async_server_threads=0,
126  client_processes=0,
127  server_processes=0,
128  server_threads_per_cq=0,
129  client_threads_per_cq=0,
130  warmup_seconds=WARMUP_SECONDS,
131  categories=None,
132  channels=None,
133  outstanding=None,
134  num_clients=None,
135  resource_quota_size=None,
136  messages_per_stream=None,
137  excluded_poll_engines=None,
138  minimal_stack=False,
139  offered_load=None):
140  """Creates a basic ping pong scenario."""
141  scenario = {
142  'name': name,
143  'num_servers': 1,
144  'num_clients': 1,
145  'client_config': {
146  'client_type': client_type,
147  'security_params': _get_secargs(secure),
148  'outstanding_rpcs_per_channel': 1,
149  'client_channels': 1,
150  'async_client_threads': 1,
151  'client_processes': client_processes,
152  'threads_per_cq': client_threads_per_cq,
153  'rpc_type': rpc_type,
154  'histogram_params': HISTOGRAM_PARAMS,
155  'channel_args': [],
156  },
157  'server_config': {
158  'server_type': server_type,
159  'security_params': _get_secargs(secure),
160  'async_server_threads': async_server_threads,
161  'server_processes': server_processes,
162  'threads_per_cq': server_threads_per_cq,
163  'channel_args': [],
164  },
165  'warmup_seconds': warmup_seconds,
166  'benchmark_seconds': BENCHMARK_SECONDS,
167  'CATEGORIES': list(DEFAULT_CATEGORIES),
168  'EXCLUDED_POLL_ENGINES': [],
169  }
170  if resource_quota_size:
171  scenario['server_config']['resource_quota_size'] = resource_quota_size
172  if use_generic_payload:
173  if server_type != 'ASYNC_GENERIC_SERVER':
174  raise Exception('Use ASYNC_GENERIC_SERVER for generic payload.')
175  scenario['server_config']['payload_config'] = _payload_type(
176  use_generic_payload, req_size, resp_size)
177 
178  scenario['client_config']['payload_config'] = _payload_type(
179  use_generic_payload, req_size, resp_size)
180 
181  # Optimization target of 'throughput' does not work well with epoll1 polling
182  # engine. Use the default value of 'blend'
183  optimization_target = 'throughput'
184 
185  if unconstrained_client:
186  outstanding_calls = outstanding if outstanding is not None else OUTSTANDING_REQUESTS[
187  unconstrained_client]
188  # clamp buffer usage to something reasonable (16 gig for now)
189  MAX_MEMORY_USE = 16 * 1024 * 1024 * 1024
190  if outstanding_calls * max(req_size, resp_size) > MAX_MEMORY_USE:
191  outstanding_calls = max(1,
192  MAX_MEMORY_USE / max(req_size, resp_size))
193  wide = channels if channels is not None else WIDE
194  deep = int(math.ceil(1.0 * outstanding_calls / wide))
195 
196  scenario[
197  'num_clients'] = num_clients if num_clients is not None else 0 # use as many clients as available.
198  scenario['client_config']['outstanding_rpcs_per_channel'] = deep
199  scenario['client_config']['client_channels'] = wide
200  scenario['client_config']['async_client_threads'] = 0
201  if offered_load is not None:
202  optimization_target = 'latency'
203  else:
204  scenario['client_config']['outstanding_rpcs_per_channel'] = 1
205  scenario['client_config']['client_channels'] = 1
206  scenario['client_config']['async_client_threads'] = 1
207  optimization_target = 'latency'
208 
209  scenario['client_config']['load_params'] = _load_params(offered_load)
210 
211  optimization_channel_arg = {
212  'name': 'grpc.optimization_target',
213  'str_value': optimization_target
214  }
215  scenario['client_config']['channel_args'].append(optimization_channel_arg)
216  scenario['server_config']['channel_args'].append(optimization_channel_arg)
217 
218  if minimal_stack:
219  _add_channel_arg(scenario['client_config'], 'grpc.minimal_stack', 1)
220  _add_channel_arg(scenario['server_config'], 'grpc.minimal_stack', 1)
221 
222  if messages_per_stream:
223  scenario['client_config']['messages_per_stream'] = messages_per_stream
224  if client_language:
225  # the CLIENT_LANGUAGE field is recognized by run_performance_tests.py
226  scenario['CLIENT_LANGUAGE'] = client_language
227  if server_language:
228  # the SERVER_LANGUAGE field is recognized by run_performance_tests.py
229  scenario['SERVER_LANGUAGE'] = server_language
230  if categories:
231  scenario['CATEGORIES'] = categories
232  if excluded_poll_engines:
233  # The polling engines for which this scenario is excluded
234  scenario['EXCLUDED_POLL_ENGINES'] = excluded_poll_engines
235  return scenario
236 
237 
238 class Language(object):
239 
240  @property
241  def safename(self):
242  return str(self)
243 
244 
245 class CXXLanguage(Language):
246 
247  @property
248  def safename(self):
249  return 'cxx'
250 
251  def worker_cmdline(self):
252  return ['cmake/build/qps_worker']
253 
254  def worker_port_offset(self):
255  return 0
256 
257  def scenarios(self):
258  yield _ping_pong_scenario('cpp_protobuf_async_unary_5000rpcs_1KB_psm',
259  rpc_type='UNARY',
260  client_type='ASYNC_CLIENT',
261  server_type='ASYNC_SERVER',
262  req_size=1024,
263  resp_size=1024,
264  outstanding=5000,
265  channels=1,
266  num_clients=1,
267  secure=False,
268  async_server_threads=1,
269  categories=[PSM])
270 
271  # TODO(ctiller): add 70% load latency test
272  yield _ping_pong_scenario(
273  'cpp_protobuf_async_unary_1channel_100rpcs_1MB',
274  rpc_type='UNARY',
275  client_type='ASYNC_CLIENT',
276  server_type='ASYNC_SERVER',
277  req_size=1024 * 1024,
278  resp_size=1024 * 1024,
279  unconstrained_client='async',
280  outstanding=100,
281  channels=1,
282  num_clients=1,
283  secure=False,
284  categories=[SWEEP])
285 
286  yield _ping_pong_scenario(
287  'cpp_protobuf_async_streaming_from_client_1channel_1MB',
288  rpc_type='STREAMING_FROM_CLIENT',
289  client_type='ASYNC_CLIENT',
290  server_type='ASYNC_SERVER',
291  req_size=1024 * 1024,
292  resp_size=1024 * 1024,
293  unconstrained_client='async',
294  outstanding=1,
295  channels=1,
296  num_clients=1,
297  secure=False,
298  categories=[SWEEP])
299 
300  # Scenario was added in https://github.com/grpc/grpc/pull/12987, but its purpose is unclear
301  # (beyond excercising some params that other scenarios don't)
302  yield _ping_pong_scenario(
303  'cpp_protobuf_async_unary_75Kqps_600channel_60Krpcs_300Breq_50Bresp',
304  rpc_type='UNARY',
305  client_type='ASYNC_CLIENT',
306  server_type='ASYNC_SERVER',
307  req_size=300,
308  resp_size=50,
309  unconstrained_client='async',
310  outstanding=30000,
311  channels=300,
312  offered_load=37500,
313  secure=False,
314  async_server_threads=16,
315  server_threads_per_cq=1,
316  categories=[SCALABLE])
317 
318  for secure in [True, False]:
319  secstr = 'secure' if secure else 'insecure'
320  smoketest_categories = ([SMOKETEST] if secure else [])
321  inproc_categories = ([INPROC] if not secure else [])
322 
323  yield _ping_pong_scenario(
324  'cpp_generic_async_streaming_ping_pong_%s' % secstr,
325  rpc_type='STREAMING',
326  client_type='ASYNC_CLIENT',
327  server_type='ASYNC_GENERIC_SERVER',
328  use_generic_payload=True,
329  async_server_threads=1,
330  secure=secure,
331  categories=smoketest_categories + inproc_categories +
332  [SCALABLE])
333 
334  yield _ping_pong_scenario(
335  'cpp_generic_async_streaming_qps_unconstrained_%s' % secstr,
336  rpc_type='STREAMING',
337  client_type='ASYNC_CLIENT',
338  server_type='ASYNC_GENERIC_SERVER',
339  unconstrained_client='async',
340  use_generic_payload=True,
341  secure=secure,
342  client_threads_per_cq=2,
343  server_threads_per_cq=2,
344  minimal_stack=not secure,
345  categories=smoketest_categories + inproc_categories +
346  [SCALABLE])
347 
348  for mps in geometric_progression(10, 20, 10):
349  yield _ping_pong_scenario(
350  'cpp_generic_async_streaming_qps_unconstrained_%smps_%s' %
351  (mps, secstr),
352  rpc_type='STREAMING',
353  client_type='ASYNC_CLIENT',
354  server_type='ASYNC_GENERIC_SERVER',
355  unconstrained_client='async',
356  use_generic_payload=True,
357  secure=secure,
358  messages_per_stream=mps,
359  minimal_stack=not secure,
360  categories=smoketest_categories + inproc_categories +
361  [SCALABLE])
362 
363  for mps in geometric_progression(1, 200, math.sqrt(10)):
364  yield _ping_pong_scenario(
365  'cpp_generic_async_streaming_qps_unconstrained_%smps_%s' %
366  (mps, secstr),
367  rpc_type='STREAMING',
368  client_type='ASYNC_CLIENT',
369  server_type='ASYNC_GENERIC_SERVER',
370  unconstrained_client='async',
371  use_generic_payload=True,
372  secure=secure,
373  messages_per_stream=mps,
374  minimal_stack=not secure,
375  categories=[SWEEP])
376 
377  yield _ping_pong_scenario(
378  'cpp_generic_async_streaming_qps_1channel_1MBmsg_%s' % secstr,
379  rpc_type='STREAMING',
380  req_size=1024 * 1024,
381  resp_size=1024 * 1024,
382  client_type='ASYNC_CLIENT',
383  server_type='ASYNC_GENERIC_SERVER',
384  unconstrained_client='async',
385  use_generic_payload=True,
386  secure=secure,
387  minimal_stack=not secure,
388  categories=inproc_categories + [SCALABLE],
389  channels=1,
390  outstanding=100)
391 
392  yield _ping_pong_scenario(
393  'cpp_generic_async_streaming_qps_unconstrained_64KBmsg_%s' %
394  secstr,
395  rpc_type='STREAMING',
396  req_size=64 * 1024,
397  resp_size=64 * 1024,
398  client_type='ASYNC_CLIENT',
399  server_type='ASYNC_GENERIC_SERVER',
400  unconstrained_client='async',
401  use_generic_payload=True,
402  secure=secure,
403  minimal_stack=not secure,
404  categories=inproc_categories + [SCALABLE])
405 
406  yield _ping_pong_scenario(
407  'cpp_generic_async_streaming_qps_unconstrained_1cq_%s' % secstr,
408  rpc_type='STREAMING',
409  client_type='ASYNC_CLIENT',
410  server_type='ASYNC_GENERIC_SERVER',
411  unconstrained_client='async-limited',
412  use_generic_payload=True,
413  secure=secure,
414  client_threads_per_cq=1000000,
415  server_threads_per_cq=1000000,
416  categories=[SWEEP])
417 
418  yield _ping_pong_scenario(
419  'cpp_protobuf_async_streaming_qps_unconstrained_1cq_%s' %
420  secstr,
421  rpc_type='STREAMING',
422  client_type='ASYNC_CLIENT',
423  server_type='ASYNC_SERVER',
424  unconstrained_client='async-limited',
425  secure=secure,
426  client_threads_per_cq=1000000,
427  server_threads_per_cq=1000000,
428  categories=inproc_categories + [SCALABLE])
429 
430  yield _ping_pong_scenario(
431  'cpp_protobuf_async_unary_qps_unconstrained_1cq_%s' % secstr,
432  rpc_type='UNARY',
433  client_type='ASYNC_CLIENT',
434  server_type='ASYNC_SERVER',
435  unconstrained_client='async-limited',
436  secure=secure,
437  client_threads_per_cq=1000000,
438  server_threads_per_cq=1000000,
439  categories=inproc_categories + [SCALABLE])
440 
441  yield _ping_pong_scenario(
442  'cpp_generic_async_streaming_qps_one_server_core_%s' % secstr,
443  rpc_type='STREAMING',
444  client_type='ASYNC_CLIENT',
445  server_type='ASYNC_GENERIC_SERVER',
446  unconstrained_client='async-limited',
447  use_generic_payload=True,
448  async_server_threads=1,
449  minimal_stack=not secure,
450  secure=secure,
451  categories=[SWEEP])
452 
453  yield _ping_pong_scenario(
454  'cpp_protobuf_async_client_sync_server_unary_qps_unconstrained_%s'
455  % (secstr),
456  rpc_type='UNARY',
457  client_type='ASYNC_CLIENT',
458  server_type='SYNC_SERVER',
459  unconstrained_client='async',
460  secure=secure,
461  minimal_stack=not secure,
462  categories=smoketest_categories + inproc_categories +
463  [SCALABLE])
464 
465  yield _ping_pong_scenario(
466  'cpp_protobuf_async_client_unary_1channel_64wide_128Breq_8MBresp_%s'
467  % (secstr),
468  rpc_type='UNARY',
469  client_type='ASYNC_CLIENT',
470  server_type='ASYNC_SERVER',
471  channels=1,
472  outstanding=64,
473  req_size=128,
474  resp_size=8 * 1024 * 1024,
475  secure=secure,
476  minimal_stack=not secure,
477  categories=inproc_categories + [SCALABLE])
478 
479  yield _ping_pong_scenario(
480  'cpp_protobuf_async_client_sync_server_streaming_qps_unconstrained_%s'
481  % secstr,
482  rpc_type='STREAMING',
483  client_type='ASYNC_CLIENT',
484  server_type='SYNC_SERVER',
485  unconstrained_client='async',
486  secure=secure,
487  minimal_stack=not secure,
488  categories=[SWEEP])
489 
490  yield _ping_pong_scenario(
491  'cpp_protobuf_async_unary_ping_pong_%s_1MB' % secstr,
492  rpc_type='UNARY',
493  client_type='ASYNC_CLIENT',
494  server_type='ASYNC_SERVER',
495  req_size=1024 * 1024,
496  resp_size=1024 * 1024,
497  secure=secure,
498  minimal_stack=not secure,
499  categories=smoketest_categories + inproc_categories +
500  [SCALABLE])
501 
502  for rpc_type in [
503  'unary', 'streaming', 'streaming_from_client',
504  'streaming_from_server'
505  ]:
506  for synchronicity in ['sync', 'async']:
507  yield _ping_pong_scenario(
508  'cpp_protobuf_%s_%s_ping_pong_%s' %
509  (synchronicity, rpc_type, secstr),
510  rpc_type=rpc_type.upper(),
511  client_type='%s_CLIENT' % synchronicity.upper(),
512  server_type='%s_SERVER' % synchronicity.upper(),
513  async_server_threads=1,
514  minimal_stack=not secure,
515  secure=secure)
516 
517  for size in geometric_progression(1, 1024 * 1024 * 1024 + 1,
518  8):
519  yield _ping_pong_scenario(
520  'cpp_protobuf_%s_%s_qps_unconstrained_%s_%db' %
521  (synchronicity, rpc_type, secstr, size),
522  rpc_type=rpc_type.upper(),
523  req_size=size,
524  resp_size=size,
525  client_type='%s_CLIENT' % synchronicity.upper(),
526  server_type='%s_SERVER' % synchronicity.upper(),
527  unconstrained_client=synchronicity,
528  secure=secure,
529  minimal_stack=not secure,
530  categories=[SWEEP])
531 
532  maybe_scalable = [SCALABLE]
533  if rpc_type == 'streaming_from_server' and synchronicity == 'async' and secure:
534  # protobuf_async_streaming_from_server_qps_unconstrained_secure is very flaky
535  # and has extremely high variance so running it isn't really useful.
536  # see b/198275705
537  maybe_scalable = [SWEEP]
538 
539  yield _ping_pong_scenario(
540  'cpp_protobuf_%s_%s_qps_unconstrained_%s' %
541  (synchronicity, rpc_type, secstr),
542  rpc_type=rpc_type.upper(),
543  client_type='%s_CLIENT' % synchronicity.upper(),
544  server_type='%s_SERVER' % synchronicity.upper(),
545  unconstrained_client=synchronicity,
546  secure=secure,
547  minimal_stack=not secure,
548  server_threads_per_cq=2,
549  client_threads_per_cq=2,
550  categories=inproc_categories + maybe_scalable)
551 
552  # TODO(vjpai): Re-enable this test. It has a lot of timeouts
553  # and hasn't yet been conclusively identified as a test failure
554  # or race in the library
555  # yield _ping_pong_scenario(
556  # 'cpp_protobuf_%s_%s_qps_unconstrained_%s_500kib_resource_quota' % (synchronicity, rpc_type, secstr),
557  # rpc_type=rpc_type.upper(),
558  # client_type='%s_CLIENT' % synchronicity.upper(),
559  # server_type='%s_SERVER' % synchronicity.upper(),
560  # unconstrained_client=synchronicity,
561  # secure=secure,
562  # categories=smoketest_categories+[SCALABLE],
563  # resource_quota_size=500*1024)
564 
565  if rpc_type == 'streaming':
566  for mps in geometric_progression(10, 20, 10):
567  yield _ping_pong_scenario(
568  'cpp_protobuf_%s_%s_qps_unconstrained_%smps_%s'
569  % (synchronicity, rpc_type, mps, secstr),
570  rpc_type=rpc_type.upper(),
571  client_type='%s_CLIENT' % synchronicity.upper(),
572  server_type='%s_SERVER' % synchronicity.upper(),
573  unconstrained_client=synchronicity,
574  secure=secure,
575  messages_per_stream=mps,
576  minimal_stack=not secure,
577  categories=inproc_categories + [SCALABLE])
578 
579  for mps in geometric_progression(1, 200, math.sqrt(10)):
580  yield _ping_pong_scenario(
581  'cpp_protobuf_%s_%s_qps_unconstrained_%smps_%s'
582  % (synchronicity, rpc_type, mps, secstr),
583  rpc_type=rpc_type.upper(),
584  client_type='%s_CLIENT' % synchronicity.upper(),
585  server_type='%s_SERVER' % synchronicity.upper(),
586  unconstrained_client=synchronicity,
587  secure=secure,
588  messages_per_stream=mps,
589  minimal_stack=not secure,
590  categories=[SWEEP])
591 
592  for channels in geometric_progression(
593  1, 20000, math.sqrt(10)):
594  for outstanding in geometric_progression(
595  1, 200000, math.sqrt(10)):
596  if synchronicity == 'sync' and outstanding > 1200:
597  continue
598  if outstanding < channels:
599  continue
600  yield _ping_pong_scenario(
601  'cpp_protobuf_%s_%s_qps_unconstrained_%s_%d_channels_%d_outstanding'
602  % (synchronicity, rpc_type, secstr, channels,
603  outstanding),
604  rpc_type=rpc_type.upper(),
605  client_type='%s_CLIENT' % synchronicity.upper(),
606  server_type='%s_SERVER' % synchronicity.upper(),
607  unconstrained_client=synchronicity,
608  secure=secure,
609  minimal_stack=not secure,
610  categories=[SWEEP],
611  channels=channels,
612  outstanding=outstanding)
613 
614  def __str__(self):
615  return 'c++'
616 
617 
618 class CSharpLanguage(Language):
619  """The legacy Grpc.Core implementation from grpc/grpc."""
620 
621  def worker_cmdline(self):
622  return ['tools/run_tests/performance/run_worker_csharp.sh']
623 
624  def worker_port_offset(self):
625  return 100
626 
627  def scenarios(self):
628  yield _ping_pong_scenario('csharp_generic_async_streaming_ping_pong',
629  rpc_type='STREAMING',
630  client_type='ASYNC_CLIENT',
631  server_type='ASYNC_GENERIC_SERVER',
632  use_generic_payload=True,
633  categories=[SMOKETEST, SCALABLE])
634 
635  yield _ping_pong_scenario(
636  'csharp_generic_async_streaming_ping_pong_insecure_1MB',
637  rpc_type='STREAMING',
638  client_type='ASYNC_CLIENT',
639  server_type='ASYNC_GENERIC_SERVER',
640  req_size=1024 * 1024,
641  resp_size=1024 * 1024,
642  use_generic_payload=True,
643  secure=False,
644  categories=[SMOKETEST, SCALABLE])
645 
646  yield _ping_pong_scenario(
647  'csharp_generic_async_streaming_qps_unconstrained_insecure',
648  rpc_type='STREAMING',
649  client_type='ASYNC_CLIENT',
650  server_type='ASYNC_GENERIC_SERVER',
651  unconstrained_client='async',
652  use_generic_payload=True,
653  secure=False,
654  categories=[SMOKETEST, SCALABLE])
655 
656  yield _ping_pong_scenario('csharp_protobuf_async_streaming_ping_pong',
657  rpc_type='STREAMING',
658  client_type='ASYNC_CLIENT',
659  server_type='ASYNC_SERVER')
660 
661  yield _ping_pong_scenario('csharp_protobuf_async_unary_ping_pong',
662  rpc_type='UNARY',
663  client_type='ASYNC_CLIENT',
664  server_type='ASYNC_SERVER',
665  categories=[SMOKETEST, SCALABLE])
666 
667  yield _ping_pong_scenario(
668  'csharp_protobuf_sync_to_async_unary_ping_pong',
669  rpc_type='UNARY',
670  client_type='SYNC_CLIENT',
671  server_type='ASYNC_SERVER')
672 
673  yield _ping_pong_scenario(
674  'csharp_protobuf_async_unary_qps_unconstrained',
675  rpc_type='UNARY',
676  client_type='ASYNC_CLIENT',
677  server_type='ASYNC_SERVER',
678  unconstrained_client='async',
679  categories=[SMOKETEST, SCALABLE])
680 
681  yield _ping_pong_scenario(
682  'csharp_protobuf_async_streaming_qps_unconstrained',
683  rpc_type='STREAMING',
684  client_type='ASYNC_CLIENT',
685  server_type='ASYNC_SERVER',
686  unconstrained_client='async',
687  categories=[SCALABLE])
688 
689  yield _ping_pong_scenario('csharp_to_cpp_protobuf_sync_unary_ping_pong',
690  rpc_type='UNARY',
691  client_type='SYNC_CLIENT',
692  server_type='SYNC_SERVER',
693  server_language='c++',
694  async_server_threads=1,
695  categories=[SMOKETEST, SCALABLE])
696 
697  yield _ping_pong_scenario(
698  'csharp_to_cpp_protobuf_async_streaming_ping_pong',
699  rpc_type='STREAMING',
700  client_type='ASYNC_CLIENT',
701  server_type='ASYNC_SERVER',
702  server_language='c++',
703  async_server_threads=1)
704 
705  yield _ping_pong_scenario(
706  'csharp_to_cpp_protobuf_async_unary_qps_unconstrained',
707  rpc_type='UNARY',
708  client_type='ASYNC_CLIENT',
709  server_type='ASYNC_SERVER',
710  unconstrained_client='async',
711  server_language='c++',
712  categories=[SCALABLE])
713 
714  yield _ping_pong_scenario(
715  'csharp_to_cpp_protobuf_sync_to_async_unary_qps_unconstrained',
716  rpc_type='UNARY',
717  client_type='SYNC_CLIENT',
718  server_type='ASYNC_SERVER',
719  unconstrained_client='sync',
720  server_language='c++',
721  categories=[SCALABLE])
722 
723  yield _ping_pong_scenario(
724  'cpp_to_csharp_protobuf_async_unary_qps_unconstrained',
725  rpc_type='UNARY',
726  client_type='ASYNC_CLIENT',
727  server_type='ASYNC_SERVER',
728  unconstrained_client='async',
729  client_language='c++',
730  categories=[SCALABLE])
731 
732  yield _ping_pong_scenario('csharp_protobuf_async_unary_ping_pong_1MB',
733  rpc_type='UNARY',
734  client_type='ASYNC_CLIENT',
735  server_type='ASYNC_SERVER',
736  req_size=1024 * 1024,
737  resp_size=1024 * 1024,
738  categories=[SMOKETEST, SCALABLE])
739 
740  def __str__(self):
741  return 'csharp'
742 
743 
744 class DotnetLanguage(Language):
745  """The pure C# implementation from grpc/grpc-dotnet."""
746 
747  def worker_cmdline(self):
748  # grpc-dotnet worker is only supported by the new GKE based OSS benchmark
749  # framework, and the worker_cmdline() is only used by run_performance_tests.py
750  return ['grpc_dotnet_not_supported_by_legacy_performance_runner.sh']
751 
752  def worker_port_offset(self):
753  return 1100
754 
755  def scenarios(self):
756  yield _ping_pong_scenario('dotnet_generic_async_streaming_ping_pong',
757  rpc_type='STREAMING',
758  client_type='ASYNC_CLIENT',
759  server_type='ASYNC_GENERIC_SERVER',
760  use_generic_payload=True,
761  categories=[SMOKETEST, SCALABLE])
762 
763  yield _ping_pong_scenario(
764  'dotnet_generic_async_streaming_ping_pong_insecure_1MB',
765  rpc_type='STREAMING',
766  client_type='ASYNC_CLIENT',
767  server_type='ASYNC_GENERIC_SERVER',
768  req_size=1024 * 1024,
769  resp_size=1024 * 1024,
770  use_generic_payload=True,
771  secure=False,
772  categories=[SMOKETEST, SCALABLE])
773 
774  yield _ping_pong_scenario(
775  'dotnet_generic_async_streaming_qps_unconstrained_insecure',
776  rpc_type='STREAMING',
777  client_type='ASYNC_CLIENT',
778  server_type='ASYNC_GENERIC_SERVER',
779  unconstrained_client='async',
780  use_generic_payload=True,
781  secure=False,
782  categories=[SMOKETEST, SCALABLE])
783 
784  yield _ping_pong_scenario('dotnet_protobuf_async_streaming_ping_pong',
785  rpc_type='STREAMING',
786  client_type='ASYNC_CLIENT',
787  server_type='ASYNC_SERVER')
788 
789  yield _ping_pong_scenario('dotnet_protobuf_async_unary_ping_pong',
790  rpc_type='UNARY',
791  client_type='ASYNC_CLIENT',
792  server_type='ASYNC_SERVER',
793  categories=[SMOKETEST, SCALABLE])
794 
795  yield _ping_pong_scenario(
796  'dotnet_protobuf_sync_to_async_unary_ping_pong',
797  rpc_type='UNARY',
798  client_type='SYNC_CLIENT',
799  server_type='ASYNC_SERVER')
800 
801  yield _ping_pong_scenario(
802  'dotnet_protobuf_async_unary_qps_unconstrained',
803  rpc_type='UNARY',
804  client_type='ASYNC_CLIENT',
805  server_type='ASYNC_SERVER',
806  unconstrained_client='async',
807  categories=[SMOKETEST, SCALABLE])
808 
809  yield _ping_pong_scenario(
810  'dotnet_protobuf_async_streaming_qps_unconstrained',
811  rpc_type='STREAMING',
812  client_type='ASYNC_CLIENT',
813  server_type='ASYNC_SERVER',
814  unconstrained_client='async',
815  categories=[SCALABLE])
816 
817  yield _ping_pong_scenario('dotnet_to_cpp_protobuf_sync_unary_ping_pong',
818  rpc_type='UNARY',
819  client_type='SYNC_CLIENT',
820  server_type='SYNC_SERVER',
821  server_language='c++',
822  async_server_threads=1,
823  categories=[SMOKETEST, SCALABLE])
824 
825  yield _ping_pong_scenario(
826  'dotnet_to_cpp_protobuf_async_streaming_ping_pong',
827  rpc_type='STREAMING',
828  client_type='ASYNC_CLIENT',
829  server_type='ASYNC_SERVER',
830  server_language='c++',
831  async_server_threads=1)
832 
833  yield _ping_pong_scenario(
834  'dotnet_to_cpp_protobuf_async_unary_qps_unconstrained',
835  rpc_type='UNARY',
836  client_type='ASYNC_CLIENT',
837  server_type='ASYNC_SERVER',
838  unconstrained_client='async',
839  server_language='c++',
840  categories=[SCALABLE])
841 
842  yield _ping_pong_scenario(
843  'dotnet_to_cpp_protobuf_sync_to_async_unary_qps_unconstrained',
844  rpc_type='UNARY',
845  client_type='SYNC_CLIENT',
846  server_type='ASYNC_SERVER',
847  unconstrained_client='sync',
848  server_language='c++',
849  categories=[SCALABLE])
850 
851  yield _ping_pong_scenario(
852  'cpp_to_dotnet_protobuf_async_unary_qps_unconstrained',
853  rpc_type='UNARY',
854  client_type='ASYNC_CLIENT',
855  server_type='ASYNC_SERVER',
856  unconstrained_client='async',
857  client_language='c++',
858  categories=[SCALABLE])
859 
860  yield _ping_pong_scenario('dotnet_protobuf_async_unary_ping_pong_1MB',
861  rpc_type='UNARY',
862  client_type='ASYNC_CLIENT',
863  server_type='ASYNC_SERVER',
864  req_size=1024 * 1024,
865  resp_size=1024 * 1024,
866  categories=[SMOKETEST, SCALABLE])
867 
868  def __str__(self):
869  return 'dotnet'
870 
871 
872 class PythonLanguage(Language):
873 
874  def worker_cmdline(self):
875  return ['tools/run_tests/performance/run_worker_python.sh']
876 
877  def worker_port_offset(self):
878  return 500
879 
880  def scenarios(self):
881  yield _ping_pong_scenario(
882  'python_protobuf_async_unary_5000rpcs_1KB_psm',
883  rpc_type='UNARY',
884  client_type='ASYNC_CLIENT',
885  server_type='ASYNC_SERVER',
886  req_size=1024,
887  resp_size=1024,
888  outstanding=5000,
889  channels=1,
890  num_clients=1,
891  secure=False,
892  async_server_threads=1,
893  categories=[PSM])
894 
895  yield _ping_pong_scenario('python_generic_sync_streaming_ping_pong',
896  rpc_type='STREAMING',
897  client_type='SYNC_CLIENT',
898  server_type='ASYNC_GENERIC_SERVER',
899  use_generic_payload=True,
900  categories=[SMOKETEST, SCALABLE])
901 
902  yield _ping_pong_scenario('python_protobuf_sync_streaming_ping_pong',
903  rpc_type='STREAMING',
904  client_type='SYNC_CLIENT',
905  server_type='ASYNC_SERVER')
906 
907  yield _ping_pong_scenario('python_protobuf_async_unary_ping_pong',
908  rpc_type='UNARY',
909  client_type='ASYNC_CLIENT',
910  server_type='ASYNC_SERVER')
911 
912  yield _ping_pong_scenario('python_protobuf_sync_unary_ping_pong',
913  rpc_type='UNARY',
914  client_type='SYNC_CLIENT',
915  server_type='ASYNC_SERVER',
916  categories=[SMOKETEST, SCALABLE])
917 
918  yield _ping_pong_scenario(
919  'python_protobuf_sync_unary_qps_unconstrained',
920  rpc_type='UNARY',
921  client_type='SYNC_CLIENT',
922  server_type='ASYNC_SERVER',
923  unconstrained_client='sync')
924 
925  yield _ping_pong_scenario(
926  'python_protobuf_sync_streaming_qps_unconstrained',
927  rpc_type='STREAMING',
928  client_type='SYNC_CLIENT',
929  server_type='ASYNC_SERVER',
930  unconstrained_client='sync')
931 
932  yield _ping_pong_scenario('python_to_cpp_protobuf_sync_unary_ping_pong',
933  rpc_type='UNARY',
934  client_type='SYNC_CLIENT',
935  server_type='ASYNC_SERVER',
936  server_language='c++',
937  async_server_threads=0,
938  categories=[SMOKETEST, SCALABLE])
939 
940  yield _ping_pong_scenario(
941  'python_to_cpp_protobuf_sync_streaming_ping_pong',
942  rpc_type='STREAMING',
943  client_type='SYNC_CLIENT',
944  server_type='ASYNC_SERVER',
945  server_language='c++',
946  async_server_threads=1)
947 
948  yield _ping_pong_scenario('python_protobuf_sync_unary_ping_pong_1MB',
949  rpc_type='UNARY',
950  client_type='SYNC_CLIENT',
951  server_type='ASYNC_SERVER',
952  req_size=1024 * 1024,
953  resp_size=1024 * 1024,
954  categories=[SMOKETEST, SCALABLE])
955 
956  def __str__(self):
957  return 'python'
958 
959 
960 class PythonAsyncIOLanguage(Language):
961 
962  def worker_cmdline(self):
963  return ['tools/run_tests/performance/run_worker_python_asyncio.sh']
964 
965  def worker_port_offset(self):
966  return 1200
967 
968  def scenarios(self):
969  yield _ping_pong_scenario(
970  'python_asyncio_protobuf_async_unary_5000rpcs_1KB_psm',
971  rpc_type='UNARY',
972  client_type='ASYNC_CLIENT',
973  server_type='ASYNC_SERVER',
974  req_size=1024,
975  resp_size=1024,
976  outstanding=5000,
977  channels=1,
978  num_clients=1,
979  secure=False,
980  async_server_threads=1,
981  categories=[PSM])
982 
983  for outstanding in [64, 128, 256, 512]:
984  for channels in [1, 4]:
985  yield _ping_pong_scenario(
986  'python_asyncio_protobuf_async_unary_ping_pong_%dx%d_max' %
987  (
988  outstanding,
989  channels,
990  ),
991  rpc_type='UNARY',
992  client_type='ASYNC_CLIENT',
993  server_type='ASYNC_SERVER',
994  outstanding=outstanding * channels,
995  channels=channels,
996  client_processes=0,
997  server_processes=0,
998  unconstrained_client='async',
999  categories=[SCALABLE])
1000 
1001  yield _ping_pong_scenario(
1002  'python_asyncio_protobuf_async_unary_ping_pong_%d_1thread' %
1003  outstanding,
1004  rpc_type='UNARY',
1005  client_type='ASYNC_CLIENT',
1006  server_type='ASYNC_SERVER',
1007  outstanding=outstanding,
1008  channels=1,
1009  client_processes=1,
1010  server_processes=1,
1011  unconstrained_client='async',
1012  categories=[SCALABLE])
1013 
1014  yield _ping_pong_scenario(
1015  'python_asyncio_generic_async_streaming_ping_pong',
1016  rpc_type='STREAMING',
1017  client_type='ASYNC_CLIENT',
1018  server_type='ASYNC_GENERIC_SERVER',
1019  channels=1,
1020  client_processes=1,
1021  server_processes=1,
1022  use_generic_payload=True,
1023  categories=[SMOKETEST, SCALABLE])
1024 
1025  yield _ping_pong_scenario(
1026  'python_asyncio_protobuf_async_streaming_ping_pong',
1027  rpc_type='STREAMING',
1028  client_type='ASYNC_CLIENT',
1029  server_type='ASYNC_SERVER',
1030  channels=1,
1031  client_processes=1,
1032  server_processes=1,
1033  categories=[SMOKETEST, SCALABLE])
1034 
1035  yield _ping_pong_scenario(
1036  'python_asyncio_protobuf_async_unary_ping_pong',
1037  rpc_type='UNARY',
1038  client_type='ASYNC_CLIENT',
1039  server_type='ASYNC_SERVER',
1040  client_processes=1,
1041  server_processes=1,
1042  categories=[SMOKETEST, SCALABLE])
1043 
1044  yield _ping_pong_scenario(
1045  'python_asyncio_protobuf_async_unary_ping_pong',
1046  rpc_type='UNARY',
1047  client_type='ASYNC_CLIENT',
1048  server_type='ASYNC_SERVER',
1049  channels=1,
1050  client_processes=1,
1051  server_processes=1,
1052  categories=[SMOKETEST, SCALABLE])
1053 
1054  yield _ping_pong_scenario(
1055  'python_asyncio_protobuf_async_unary_qps_unconstrained',
1056  rpc_type='UNARY',
1057  client_type='ASYNC_CLIENT',
1058  server_type='ASYNC_SERVER',
1059  channels=1,
1060  unconstrained_client='async')
1061 
1062  yield _ping_pong_scenario(
1063  'python_asyncio_protobuf_async_streaming_qps_unconstrained',
1064  rpc_type='STREAMING',
1065  client_type='ASYNC_CLIENT',
1066  server_type='ASYNC_SERVER',
1067  channels=1,
1068  unconstrained_client='async')
1069 
1070  yield _ping_pong_scenario(
1071  'python_asyncio_to_cpp_protobuf_async_unary_ping_pong_1thread',
1072  rpc_type='UNARY',
1073  client_type='ASYNC_CLIENT',
1074  server_type='ASYNC_SERVER',
1075  server_language='c++',
1076  channels=1,
1077  client_processes=1,
1078  unconstrained_client='async',
1079  categories=[SMOKETEST, SCALABLE])
1080 
1081  yield _ping_pong_scenario(
1082  'python_asyncio_to_cpp_protobuf_async_unary_ping_pong_max',
1083  rpc_type='UNARY',
1084  client_type='ASYNC_CLIENT',
1085  server_type='ASYNC_SERVER',
1086  unconstrained_client='async',
1087  channels=1,
1088  client_processes=0,
1089  server_language='c++',
1090  categories=[SMOKETEST, SCALABLE])
1091 
1092  yield _ping_pong_scenario(
1093  'python_asyncio_to_cpp_protobuf_sync_streaming_ping_pong_1thread',
1094  rpc_type='STREAMING',
1095  client_type='ASYNC_CLIENT',
1096  server_type='ASYNC_SERVER',
1097  channels=1,
1098  client_processes=1,
1099  server_processes=1,
1100  unconstrained_client='async',
1101  server_language='c++')
1102 
1103  yield _ping_pong_scenario(
1104  'python_asyncio_protobuf_async_unary_ping_pong_1MB',
1105  rpc_type='UNARY',
1106  client_type='ASYNC_CLIENT',
1107  server_type='ASYNC_SERVER',
1108  req_size=1024 * 1024,
1109  resp_size=1024 * 1024,
1110  channels=1,
1111  client_processes=1,
1112  server_processes=1,
1113  categories=[SMOKETEST, SCALABLE])
1114 
1115  def __str__(self):
1116  return 'python_asyncio'
1117 
1118 
1119 class RubyLanguage(Language):
1120 
1121  def worker_cmdline(self):
1122  return ['tools/run_tests/performance/run_worker_ruby.sh']
1123 
1124  def worker_port_offset(self):
1125  return 300
1126 
1127  def scenarios(self):
1128  yield _ping_pong_scenario('ruby_protobuf_sync_streaming_ping_pong',
1129  rpc_type='STREAMING',
1130  client_type='SYNC_CLIENT',
1131  server_type='SYNC_SERVER',
1132  categories=[SMOKETEST, SCALABLE])
1133 
1134  yield _ping_pong_scenario('ruby_protobuf_unary_ping_pong',
1135  rpc_type='UNARY',
1136  client_type='SYNC_CLIENT',
1137  server_type='SYNC_SERVER',
1138  categories=[SMOKETEST, SCALABLE])
1139 
1140  yield _ping_pong_scenario('ruby_protobuf_sync_unary_qps_unconstrained',
1141  rpc_type='UNARY',
1142  client_type='SYNC_CLIENT',
1143  server_type='SYNC_SERVER',
1144  unconstrained_client='sync')
1145 
1146  yield _ping_pong_scenario(
1147  'ruby_protobuf_sync_streaming_qps_unconstrained',
1148  rpc_type='STREAMING',
1149  client_type='SYNC_CLIENT',
1150  server_type='SYNC_SERVER',
1151  unconstrained_client='sync')
1152 
1153  yield _ping_pong_scenario('ruby_to_cpp_protobuf_sync_unary_ping_pong',
1154  rpc_type='UNARY',
1155  client_type='SYNC_CLIENT',
1156  server_type='SYNC_SERVER',
1157  server_language='c++',
1158  async_server_threads=1)
1159 
1160  yield _ping_pong_scenario(
1161  'ruby_to_cpp_protobuf_sync_streaming_ping_pong',
1162  rpc_type='STREAMING',
1163  client_type='SYNC_CLIENT',
1164  server_type='SYNC_SERVER',
1165  server_language='c++',
1166  async_server_threads=1)
1167 
1168  yield _ping_pong_scenario('ruby_protobuf_unary_ping_pong_1MB',
1169  rpc_type='UNARY',
1170  client_type='SYNC_CLIENT',
1171  server_type='SYNC_SERVER',
1172  req_size=1024 * 1024,
1173  resp_size=1024 * 1024,
1174  categories=[SMOKETEST, SCALABLE])
1175 
1176  def __str__(self):
1177  return 'ruby'
1178 
1179 
1180 class Php7Language(Language):
1181 
1182  def __init__(self, php7_protobuf_c=False):
1183  super().__init__()
1184  self.php7_protobuf_c = php7_protobuf_c
1185 
1186  def worker_cmdline(self):
1187  if self.php7_protobuf_c:
1188  return [
1189  'tools/run_tests/performance/run_worker_php.sh',
1190  '--use_protobuf_c_extension'
1191  ]
1192  return ['tools/run_tests/performance/run_worker_php.sh']
1193 
1194  def worker_port_offset(self):
1195  if self.php7_protobuf_c:
1196  return 900
1197  return 800
1198 
1199  def scenarios(self):
1200  php7_extension_mode = 'php7_protobuf_php_extension'
1201  if self.php7_protobuf_c:
1202  php7_extension_mode = 'php7_protobuf_c_extension'
1203 
1204  yield _ping_pong_scenario(
1205  '%s_to_cpp_protobuf_async_unary_5000rpcs_1KB_psm' %
1206  php7_extension_mode,
1207  rpc_type='UNARY',
1208  client_type='ASYNC_CLIENT',
1209  server_type='ASYNC_SERVER',
1210  server_language='c++',
1211  req_size=1024,
1212  resp_size=1024,
1213  outstanding=5000,
1214  channels=1,
1215  num_clients=1,
1216  secure=False,
1217  async_server_threads=1,
1218  categories=[PSM])
1219 
1220  yield _ping_pong_scenario('%s_to_cpp_protobuf_sync_unary_ping_pong' %
1221  php7_extension_mode,
1222  rpc_type='UNARY',
1223  client_type='SYNC_CLIENT',
1224  server_type='SYNC_SERVER',
1225  server_language='c++',
1226  async_server_threads=1)
1227 
1228  yield _ping_pong_scenario(
1229  '%s_to_cpp_protobuf_sync_streaming_ping_pong' % php7_extension_mode,
1230  rpc_type='STREAMING',
1231  client_type='SYNC_CLIENT',
1232  server_type='SYNC_SERVER',
1233  server_language='c++',
1234  async_server_threads=1)
1235 
1236  # TODO(ddyihai): Investigate why when async_server_threads=1/CPU usage 340%, the QPS performs
1237  # better than async_server_threads=0/CPU usage 490%.
1238  yield _ping_pong_scenario(
1239  '%s_to_cpp_protobuf_sync_unary_qps_unconstrained' %
1240  php7_extension_mode,
1241  rpc_type='UNARY',
1242  client_type='SYNC_CLIENT',
1243  server_type='ASYNC_SERVER',
1244  server_language='c++',
1245  outstanding=1,
1246  async_server_threads=1,
1247  unconstrained_client='sync')
1248 
1249  yield _ping_pong_scenario(
1250  '%s_to_cpp_protobuf_sync_streaming_qps_unconstrained' %
1251  php7_extension_mode,
1252  rpc_type='STREAMING',
1253  client_type='SYNC_CLIENT',
1254  server_type='ASYNC_SERVER',
1255  server_language='c++',
1256  outstanding=1,
1257  async_server_threads=1,
1258  unconstrained_client='sync')
1259 
1260  def __str__(self):
1261  if self.php7_protobuf_c:
1262  return 'php7_protobuf_c'
1263  return 'php7'
1264 
1265 
1266 class JavaLanguage(Language):
1267 
1268  def worker_cmdline(self):
1269  return ['tools/run_tests/performance/run_worker_java.sh']
1270 
1271  def worker_port_offset(self):
1272  return 400
1273 
1274  def scenarios(self):
1275  yield _ping_pong_scenario('java_protobuf_async_unary_5000rpcs_1KB_psm',
1276  rpc_type='UNARY',
1277  client_type='ASYNC_CLIENT',
1278  server_type='ASYNC_SERVER',
1279  req_size=1024,
1280  resp_size=1024,
1281  outstanding=5000,
1282  channels=1,
1283  num_clients=1,
1284  secure=False,
1285  async_server_threads=1,
1286  warmup_seconds=JAVA_WARMUP_SECONDS,
1287  categories=[PSM])
1288 
1289  for secure in [True, False]:
1290  secstr = 'secure' if secure else 'insecure'
1291  smoketest_categories = ([SMOKETEST] if secure else []) + [SCALABLE]
1292 
1293  yield _ping_pong_scenario(
1294  'java_generic_async_streaming_ping_pong_%s' % secstr,
1295  rpc_type='STREAMING',
1296  client_type='ASYNC_CLIENT',
1297  server_type='ASYNC_GENERIC_SERVER',
1298  use_generic_payload=True,
1299  async_server_threads=1,
1300  secure=secure,
1301  warmup_seconds=JAVA_WARMUP_SECONDS,
1302  categories=smoketest_categories)
1303 
1304  yield _ping_pong_scenario(
1305  'java_protobuf_async_streaming_ping_pong_%s' % secstr,
1306  rpc_type='STREAMING',
1307  client_type='ASYNC_CLIENT',
1308  server_type='ASYNC_SERVER',
1309  async_server_threads=1,
1310  secure=secure,
1311  warmup_seconds=JAVA_WARMUP_SECONDS)
1312 
1313  yield _ping_pong_scenario('java_protobuf_async_unary_ping_pong_%s' %
1314  secstr,
1315  rpc_type='UNARY',
1316  client_type='ASYNC_CLIENT',
1317  server_type='ASYNC_SERVER',
1318  async_server_threads=1,
1319  secure=secure,
1320  warmup_seconds=JAVA_WARMUP_SECONDS,
1321  categories=smoketest_categories)
1322 
1323  yield _ping_pong_scenario('java_protobuf_unary_ping_pong_%s' %
1324  secstr,
1325  rpc_type='UNARY',
1326  client_type='SYNC_CLIENT',
1327  server_type='SYNC_SERVER',
1328  async_server_threads=1,
1329  secure=secure,
1330  warmup_seconds=JAVA_WARMUP_SECONDS)
1331 
1332  yield _ping_pong_scenario(
1333  'java_protobuf_async_unary_qps_unconstrained_%s' % secstr,
1334  rpc_type='UNARY',
1335  client_type='ASYNC_CLIENT',
1336  server_type='ASYNC_SERVER',
1337  unconstrained_client='async',
1338  secure=secure,
1339  warmup_seconds=JAVA_WARMUP_SECONDS,
1340  categories=smoketest_categories + [SCALABLE])
1341 
1342  yield _ping_pong_scenario(
1343  'java_protobuf_async_streaming_qps_unconstrained_%s' % secstr,
1344  rpc_type='STREAMING',
1345  client_type='ASYNC_CLIENT',
1346  server_type='ASYNC_SERVER',
1347  unconstrained_client='async',
1348  secure=secure,
1349  warmup_seconds=JAVA_WARMUP_SECONDS,
1350  categories=[SCALABLE])
1351 
1352  yield _ping_pong_scenario(
1353  'java_generic_async_streaming_qps_unconstrained_%s' % secstr,
1354  rpc_type='STREAMING',
1355  client_type='ASYNC_CLIENT',
1356  server_type='ASYNC_GENERIC_SERVER',
1357  unconstrained_client='async',
1358  use_generic_payload=True,
1359  secure=secure,
1360  warmup_seconds=JAVA_WARMUP_SECONDS,
1361  categories=[SCALABLE])
1362 
1363  yield _ping_pong_scenario(
1364  'java_generic_async_streaming_qps_one_server_core_%s' % secstr,
1365  rpc_type='STREAMING',
1366  client_type='ASYNC_CLIENT',
1367  server_type='ASYNC_GENERIC_SERVER',
1368  unconstrained_client='async-limited',
1369  use_generic_payload=True,
1370  async_server_threads=1,
1371  secure=secure,
1372  warmup_seconds=JAVA_WARMUP_SECONDS)
1373 
1374  # TODO(jtattermusch): add scenarios java vs C++
1375 
1376  def __str__(self):
1377  return 'java'
1378 
1379 
1380 class GoLanguage(Language):
1381 
1382  def worker_cmdline(self):
1383  return ['tools/run_tests/performance/run_worker_go.sh']
1384 
1385  def worker_port_offset(self):
1386  return 600
1387 
1388  def scenarios(self):
1389  yield _ping_pong_scenario('go_protobuf_async_unary_5000rpcs_1KB_psm',
1390  rpc_type='UNARY',
1391  client_type='ASYNC_CLIENT',
1392  server_type='ASYNC_SERVER',
1393  req_size=1024,
1394  resp_size=1024,
1395  outstanding=5000,
1396  channels=1,
1397  num_clients=1,
1398  secure=False,
1399  async_server_threads=1,
1400  categories=[PSM])
1401 
1402  for secure in [True, False]:
1403  secstr = 'secure' if secure else 'insecure'
1404  smoketest_categories = ([SMOKETEST] if secure else []) + [SCALABLE]
1405 
1406  # ASYNC_GENERIC_SERVER for Go actually uses a sync streaming server,
1407  # but that's mostly because of lack of better name of the enum value.
1408  yield _ping_pong_scenario('go_generic_sync_streaming_ping_pong_%s' %
1409  secstr,
1410  rpc_type='STREAMING',
1411  client_type='SYNC_CLIENT',
1412  server_type='ASYNC_GENERIC_SERVER',
1413  use_generic_payload=True,
1414  async_server_threads=1,
1415  secure=secure,
1416  categories=smoketest_categories)
1417 
1418  yield _ping_pong_scenario(
1419  'go_protobuf_sync_streaming_ping_pong_%s' % secstr,
1420  rpc_type='STREAMING',
1421  client_type='SYNC_CLIENT',
1422  server_type='SYNC_SERVER',
1423  async_server_threads=1,
1424  secure=secure)
1425 
1426  yield _ping_pong_scenario('go_protobuf_sync_unary_ping_pong_%s' %
1427  secstr,
1428  rpc_type='UNARY',
1429  client_type='SYNC_CLIENT',
1430  server_type='SYNC_SERVER',
1431  async_server_threads=1,
1432  secure=secure,
1433  categories=smoketest_categories)
1434 
1435  # unconstrained_client='async' is intended (client uses goroutines)
1436  yield _ping_pong_scenario(
1437  'go_protobuf_sync_unary_qps_unconstrained_%s' % secstr,
1438  rpc_type='UNARY',
1439  client_type='SYNC_CLIENT',
1440  server_type='SYNC_SERVER',
1441  unconstrained_client='async',
1442  secure=secure,
1443  categories=smoketest_categories + [SCALABLE])
1444 
1445  # unconstrained_client='async' is intended (client uses goroutines)
1446  yield _ping_pong_scenario(
1447  'go_protobuf_sync_streaming_qps_unconstrained_%s' % secstr,
1448  rpc_type='STREAMING',
1449  client_type='SYNC_CLIENT',
1450  server_type='SYNC_SERVER',
1451  unconstrained_client='async',
1452  secure=secure,
1453  categories=[SCALABLE])
1454 
1455  # unconstrained_client='async' is intended (client uses goroutines)
1456  # ASYNC_GENERIC_SERVER for Go actually uses a sync streaming server,
1457  # but that's mostly because of lack of better name of the enum value.
1458  yield _ping_pong_scenario(
1459  'go_generic_sync_streaming_qps_unconstrained_%s' % secstr,
1460  rpc_type='STREAMING',
1461  client_type='SYNC_CLIENT',
1462  server_type='ASYNC_GENERIC_SERVER',
1463  unconstrained_client='async',
1464  use_generic_payload=True,
1465  secure=secure,
1466  categories=[SCALABLE])
1467 
1468  # TODO(jtattermusch): add scenarios go vs C++
1469 
1470  def __str__(self):
1471  return 'go'
1472 
1473 
1474 class NodeLanguage(Language):
1475 
1476  def __init__(self, node_purejs=False):
1477  super().__init__()
1478  self.node_purejs = node_purejs
1479 
1480  def worker_cmdline(self):
1481  fixture = 'native_js' if self.node_purejs else 'native_native'
1482  return [
1483  'tools/run_tests/performance/run_worker_node.sh', fixture,
1484  '--benchmark_impl=grpc'
1485  ]
1486 
1487  def worker_port_offset(self):
1488  if self.node_purejs:
1489  return 1100
1490  return 1000
1491 
1492  def scenarios(self):
1493  node_implementation = 'node_purejs' if self.node_purejs else 'node'
1494 
1495  yield _ping_pong_scenario(
1496  '%s_to_node_protobuf_async_unary_5000rpcs_1KB_psm' %
1497  (node_implementation),
1498  rpc_type='UNARY',
1499  client_type='ASYNC_CLIENT',
1500  server_type='ASYNC_SERVER',
1501  server_language='node',
1502  req_size=1024,
1503  resp_size=1024,
1504  outstanding=5000,
1505  channels=1,
1506  num_clients=1,
1507  secure=False,
1508  async_server_threads=1,
1509  categories=[PSM])
1510 
1511  for secure in [True, False]:
1512  secstr = 'secure' if secure else 'insecure'
1513  smoketest_categories = ([SMOKETEST] if secure else []) + [SCALABLE]
1514 
1515  yield _ping_pong_scenario(
1516  '%s_to_node_generic_async_streaming_ping_pong_%s' %
1517  (node_implementation, secstr),
1518  rpc_type='STREAMING',
1519  client_type='ASYNC_CLIENT',
1520  server_type='ASYNC_GENERIC_SERVER',
1521  server_language='node',
1522  use_generic_payload=True,
1523  async_server_threads=1,
1524  secure=secure,
1525  categories=smoketest_categories)
1526 
1527  yield _ping_pong_scenario(
1528  '%s_to_node_protobuf_async_streaming_ping_pong_%s' %
1529  (node_implementation, secstr),
1530  rpc_type='STREAMING',
1531  client_type='ASYNC_CLIENT',
1532  server_type='ASYNC_SERVER',
1533  server_language='node',
1534  async_server_threads=1,
1535  secure=secure)
1536 
1537  yield _ping_pong_scenario(
1538  '%s_to_node_protobuf_async_unary_ping_pong_%s' %
1539  (node_implementation, secstr),
1540  rpc_type='UNARY',
1541  client_type='ASYNC_CLIENT',
1542  server_type='ASYNC_SERVER',
1543  server_language='node',
1544  async_server_threads=1,
1545  secure=secure,
1546  categories=smoketest_categories)
1547 
1548  yield _ping_pong_scenario(
1549  '%s_to_node_protobuf_async_unary_qps_unconstrained_%s' %
1550  (node_implementation, secstr),
1551  rpc_type='UNARY',
1552  client_type='ASYNC_CLIENT',
1553  server_type='ASYNC_SERVER',
1554  server_language='node',
1555  unconstrained_client='async',
1556  secure=secure,
1557  categories=smoketest_categories + [SCALABLE])
1558 
1559  yield _ping_pong_scenario(
1560  '%s_to_node_protobuf_async_streaming_qps_unconstrained_%s' %
1561  (node_implementation, secstr),
1562  rpc_type='STREAMING',
1563  client_type='ASYNC_CLIENT',
1564  server_type='ASYNC_SERVER',
1565  server_language='node',
1566  unconstrained_client='async',
1567  secure=secure,
1568  categories=[SCALABLE])
1569 
1570  yield _ping_pong_scenario(
1571  '%s_to_node_generic_async_streaming_qps_unconstrained_%s' %
1572  (node_implementation, secstr),
1573  rpc_type='STREAMING',
1574  client_type='ASYNC_CLIENT',
1575  server_type='ASYNC_GENERIC_SERVER',
1576  server_language='node',
1577  unconstrained_client='async',
1578  use_generic_payload=True,
1579  secure=secure,
1580  categories=[SCALABLE])
1581 
1582  # TODO(murgatroid99): add scenarios node vs C++
1583 
1584  def __str__(self):
1585  if self.node_purejs:
1586  return 'node_purejs'
1587  return 'node'
1588 
1589 
1590 LANGUAGES = {
1591  'c++': CXXLanguage(),
1592  'csharp': CSharpLanguage(),
1593  'dotnet': DotnetLanguage(),
1594  'ruby': RubyLanguage(),
1595  'php7': Php7Language(),
1596  'php7_protobuf_c': Php7Language(php7_protobuf_c=True),
1597  'java': JavaLanguage(),
1598  'python': PythonLanguage(),
1599  'python_asyncio': PythonAsyncIOLanguage(),
1600  'go': GoLanguage(),
1601  'node': NodeLanguage(),
1602  'node_purejs': NodeLanguage(node_purejs=True)
1603 }
xds_interop_client.str
str
Definition: xds_interop_client.py:487
performance.scenario_config._load_params
def _load_params(offered_load)
Definition: scenario_config.py:89
performance.scenario_config.NodeLanguage.worker_cmdline
def worker_cmdline(self)
Definition: scenario_config.py:1480
performance.scenario_config._add_channel_arg
def _add_channel_arg(config, key, value)
Definition: scenario_config.py:100
performance.scenario_config._payload_type
def _payload_type(use_generic_payload, req_size, resp_size)
Definition: scenario_config.py:76
performance.scenario_config.CSharpLanguage.scenarios
def scenarios(self)
Definition: scenario_config.py:627
performance.scenario_config.Php7Language.worker_cmdline
def worker_cmdline(self)
Definition: scenario_config.py:1186
performance.scenario_config.JavaLanguage.scenarios
def scenarios(self)
Definition: scenario_config.py:1274
performance.scenario_config.PythonLanguage.__str__
def __str__(self)
Definition: scenario_config.py:956
performance.scenario_config.NodeLanguage.worker_port_offset
def worker_port_offset(self)
Definition: scenario_config.py:1487
performance.scenario_config.Php7Language
Definition: scenario_config.py:1180
performance.scenario_config.Language
Definition: scenario_config.py:238
performance.scenario_config.Php7Language.scenarios
def scenarios(self)
Definition: scenario_config.py:1199
performance.scenario_config.JavaLanguage
Definition: scenario_config.py:1266
performance.scenario_config.CXXLanguage.worker_cmdline
def worker_cmdline(self)
Definition: scenario_config.py:251
performance.scenario_config.CXXLanguage.scenarios
def scenarios(self)
Definition: scenario_config.py:257
performance.scenario_config.geometric_progression
def geometric_progression(start, stop, step)
Definition: scenario_config.py:69
performance.scenario_config.remove_nonproto_fields
def remove_nonproto_fields(scenario)
Definition: scenario_config.py:56
performance.scenario_config.RubyLanguage.scenarios
def scenarios(self)
Definition: scenario_config.py:1127
performance.scenario_config.DotnetLanguage.__str__
def __str__(self)
Definition: scenario_config.py:868
performance.scenario_config.DotnetLanguage.scenarios
def scenarios(self)
Definition: scenario_config.py:755
xds_interop_client.int
int
Definition: xds_interop_client.py:113
performance.scenario_config.Php7Language.php7_protobuf_c
php7_protobuf_c
Definition: scenario_config.py:1184
performance.scenario_config.CXXLanguage.worker_port_offset
def worker_port_offset(self)
Definition: scenario_config.py:254
max
int max
Definition: bloaty/third_party/zlib/examples/enough.c:170
performance.scenario_config.PythonLanguage
Definition: scenario_config.py:872
performance.scenario_config.NodeLanguage.__init__
def __init__(self, node_purejs=False)
Definition: scenario_config.py:1476
performance.scenario_config.RubyLanguage.worker_cmdline
def worker_cmdline(self)
Definition: scenario_config.py:1121
round
static int round(int n)
Definition: bloaty/third_party/re2/util/benchmark.cc:91
performance.scenario_config.GoLanguage.worker_port_offset
def worker_port_offset(self)
Definition: scenario_config.py:1385
performance.scenario_config.CXXLanguage
Definition: scenario_config.py:245
performance.scenario_config.GoLanguage.scenarios
def scenarios(self)
Definition: scenario_config.py:1388
performance.scenario_config.CSharpLanguage.worker_cmdline
def worker_cmdline(self)
Definition: scenario_config.py:621
performance.scenario_config.NodeLanguage.__str__
def __str__(self)
Definition: scenario_config.py:1584
performance.scenario_config.PythonAsyncIOLanguage.__str__
def __str__(self)
Definition: scenario_config.py:1115
performance.scenario_config.PythonAsyncIOLanguage.worker_cmdline
def worker_cmdline(self)
Definition: scenario_config.py:962
performance.scenario_config.RubyLanguage.worker_port_offset
def worker_port_offset(self)
Definition: scenario_config.py:1124
performance.scenario_config.GoLanguage
Definition: scenario_config.py:1380
performance.scenario_config.JavaLanguage.worker_port_offset
def worker_port_offset(self)
Definition: scenario_config.py:1271
performance.scenario_config.RubyLanguage.__str__
def __str__(self)
Definition: scenario_config.py:1176
performance.scenario_config._ping_pong_scenario
def _ping_pong_scenario(name, rpc_type, client_type, server_type, secure=True, use_generic_payload=False, req_size=0, resp_size=0, unconstrained_client=None, client_language=None, server_language=None, async_server_threads=0, client_processes=0, server_processes=0, server_threads_per_cq=0, client_threads_per_cq=0, warmup_seconds=WARMUP_SECONDS, categories=None, channels=None, outstanding=None, num_clients=None, resource_quota_size=None, messages_per_stream=None, excluded_poll_engines=None, minimal_stack=False, offered_load=None)
Definition: scenario_config.py:114
performance.scenario_config.DotnetLanguage
Definition: scenario_config.py:744
performance.scenario_config.GoLanguage.worker_cmdline
def worker_cmdline(self)
Definition: scenario_config.py:1382
performance.scenario_config.GoLanguage.__str__
def __str__(self)
Definition: scenario_config.py:1470
performance.scenario_config.PythonLanguage.scenarios
def scenarios(self)
Definition: scenario_config.py:880
performance.scenario_config.Php7Language.__str__
def __str__(self)
Definition: scenario_config.py:1260
performance.scenario_config.NodeLanguage.node_purejs
node_purejs
Definition: scenario_config.py:1478
performance.scenario_config.CSharpLanguage
Definition: scenario_config.py:618
performance.scenario_config.PythonAsyncIOLanguage.scenarios
def scenarios(self)
Definition: scenario_config.py:968
performance.scenario_config.DotnetLanguage.worker_cmdline
def worker_cmdline(self)
Definition: scenario_config.py:747
performance.scenario_config.RubyLanguage
Definition: scenario_config.py:1119
performance.scenario_config.PythonLanguage.worker_cmdline
def worker_cmdline(self)
Definition: scenario_config.py:874
performance.scenario_config.JavaLanguage.worker_cmdline
def worker_cmdline(self)
Definition: scenario_config.py:1268
performance.scenario_config.Php7Language.worker_port_offset
def worker_port_offset(self)
Definition: scenario_config.py:1194
performance.scenario_config.NodeLanguage.scenarios
def scenarios(self)
Definition: scenario_config.py:1492
performance.scenario_config.CSharpLanguage.worker_port_offset
def worker_port_offset(self)
Definition: scenario_config.py:624
performance.scenario_config.PythonAsyncIOLanguage.worker_port_offset
def worker_port_offset(self)
Definition: scenario_config.py:965
performance.scenario_config.CXXLanguage.__str__
def __str__(self)
Definition: scenario_config.py:614
performance.scenario_config.CXXLanguage.safename
def safename(self)
Definition: scenario_config.py:248
performance.scenario_config.Language.safename
def safename(self)
Definition: scenario_config.py:241
performance.scenario_config.PythonAsyncIOLanguage
Definition: scenario_config.py:960
performance.scenario_config.CSharpLanguage.__str__
def __str__(self)
Definition: scenario_config.py:740
performance.scenario_config.PythonLanguage.worker_port_offset
def worker_port_offset(self)
Definition: scenario_config.py:877
performance.scenario_config.NodeLanguage
Definition: scenario_config.py:1474
performance.scenario_config.JavaLanguage.__str__
def __str__(self)
Definition: scenario_config.py:1376
performance.scenario_config._get_secargs
def _get_secargs(is_secure)
Definition: scenario_config.py:49
performance.scenario_config.Php7Language.__init__
def __init__(self, php7_protobuf_c=False)
Definition: scenario_config.py:1182
performance.scenario_config.DotnetLanguage.worker_port_offset
def worker_port_offset(self)
Definition: scenario_config.py:752

grpc
Author(s):
autogenerated on Thu Mar 13 2025 03:01:15