test_gil_scoped.py
Go to the documentation of this file.
1 from __future__ import annotations
2 
3 import multiprocessing
4 import sys
5 import threading
6 import time
7 
8 import pytest
9 
10 import env
11 from pybind11_tests import gil_scoped as m
12 
13 
14 class ExtendedVirtClass(m.VirtClass):
15  def virtual_func(self):
16  pass
17 
18  def pure_virtual_func(self):
19  pass
20 
21 
23  m.test_callback_py_obj(lambda: None)
24 
25 
27  m.test_callback_std_func(lambda: None)
28 
29 
31  extended = ExtendedVirtClass()
32  m.test_callback_virtual_func(extended)
33 
34 
36  extended = ExtendedVirtClass()
37  m.test_callback_pure_virtual_func(extended)
38 
39 
41  """Makes sure that the GIL can be acquired by another module from a GIL-released state."""
42  m.test_cross_module_gil_released() # Should not raise a SIGSEGV
43 
44 
46  """Makes sure that the GIL can be acquired by another module from a GIL-acquired state."""
47  m.test_cross_module_gil_acquired() # Should not raise a SIGSEGV
48 
49 
51  """Makes sure that the GIL can be acquired/released by another module
52  from a GIL-released state using custom locking logic."""
53  m.test_cross_module_gil_inner_custom_released()
54 
55 
57  """Makes sure that the GIL can be acquired/acquired by another module
58  from a GIL-acquired state using custom locking logic."""
59  m.test_cross_module_gil_inner_custom_acquired()
60 
61 
63  """Makes sure that the GIL can be acquired/released by another module
64  from a GIL-released state using pybind11 locking logic."""
65  m.test_cross_module_gil_inner_pybind11_released()
66 
67 
69  """Makes sure that the GIL can be acquired/acquired by another module
70  from a GIL-acquired state using pybind11 locking logic."""
71  m.test_cross_module_gil_inner_pybind11_acquired()
72 
73 
75  """Makes sure that the GIL can be nested acquired/released by another module
76  from a GIL-released state using custom locking logic."""
77  m.test_cross_module_gil_nested_custom_released()
78 
79 
81  """Makes sure that the GIL can be nested acquired/acquired by another module
82  from a GIL-acquired state using custom locking logic."""
83  m.test_cross_module_gil_nested_custom_acquired()
84 
85 
87  """Makes sure that the GIL can be nested acquired/released by another module
88  from a GIL-released state using pybind11 locking logic."""
89  m.test_cross_module_gil_nested_pybind11_released()
90 
91 
93  """Makes sure that the GIL can be nested acquired/acquired by another module
94  from a GIL-acquired state using pybind11 locking logic."""
95  m.test_cross_module_gil_nested_pybind11_acquired()
96 
97 
99  assert m.test_release_acquire(0xAB) == "171"
100 
101 
103  assert m.test_nested_acquire(0xAB) == "171"
104 
105 
107  for bits in range(16 * 8):
108  internals_ids = m.test_multi_acquire_release_cross_module(bits)
109  assert len(internals_ids) == 2 if bits % 8 else 1
110 
111 
112 # Intentionally putting human review in the loop here, to guard against accidents.
113 VARS_BEFORE_ALL_BASIC_TESTS = dict(vars()) # Make a copy of the dict (critical).
114 ALL_BASIC_TESTS = (
115  test_callback_py_obj,
116  test_callback_std_func,
117  test_callback_virtual_func,
118  test_callback_pure_virtual_func,
119  test_cross_module_gil_released,
120  test_cross_module_gil_acquired,
121  test_cross_module_gil_inner_custom_released,
122  test_cross_module_gil_inner_custom_acquired,
123  test_cross_module_gil_inner_pybind11_released,
124  test_cross_module_gil_inner_pybind11_acquired,
125  test_cross_module_gil_nested_custom_released,
126  test_cross_module_gil_nested_custom_acquired,
127  test_cross_module_gil_nested_pybind11_released,
128  test_cross_module_gil_nested_pybind11_acquired,
129  test_release_acquire,
130  test_nested_acquire,
131  test_multi_acquire_release_cross_module,
132 )
133 
134 
136  num_found = 0
137  for key, value in VARS_BEFORE_ALL_BASIC_TESTS.items():
138  if not key.startswith("test_"):
139  continue
140  assert value in ALL_BASIC_TESTS
141  num_found += 1
142  assert len(ALL_BASIC_TESTS) == num_found
143 
144 
146  m.intentional_deadlock()
147 
148 
149 ALL_BASIC_TESTS_PLUS_INTENTIONAL_DEADLOCK = ALL_BASIC_TESTS + (_intentional_deadlock,)
150 
151 
152 def _run_in_process(target, *args, **kwargs):
153  test_fn = target if len(args) == 0 else args[0]
154  # Do not need to wait much, 10s should be more than enough.
155  timeout = 0.1 if test_fn is _intentional_deadlock else 10
156  process = multiprocessing.Process(target=target, args=args, kwargs=kwargs)
157  process.daemon = True
158  try:
159  t_start = time.time()
160  process.start()
161  if timeout >= 100: # For debugging.
162  print(
163  "\nprocess.pid STARTED", process.pid, (sys.argv, target, args, kwargs)
164  )
165  print(f"COPY-PASTE-THIS: gdb {sys.argv[0]} -p {process.pid}", flush=True)
166  process.join(timeout=timeout)
167  if timeout >= 100:
168  print("\nprocess.pid JOINED", process.pid, flush=True)
169  t_delta = time.time() - t_start
170  if process.exitcode == 66 and m.defined_THREAD_SANITIZER: # Issue #2754
171  # WOULD-BE-NICE-TO-HAVE: Check that the message below is actually in the output.
172  # Maybe this could work:
173  # https://gist.github.com/alexeygrigorev/01ce847f2e721b513b42ea4a6c96905e
174  pytest.skip(
175  "ThreadSanitizer: starting new threads after multi-threaded fork is not supported."
176  )
177  elif test_fn is _intentional_deadlock:
178  assert process.exitcode is None
179  return 0
180 
181  if process.exitcode is None:
182  assert t_delta > 0.9 * timeout
183  msg = "DEADLOCK, most likely, exactly what this test is meant to detect."
184  if env.PYPY and env.WIN:
185  pytest.skip(msg)
186  raise RuntimeError(msg)
187  return process.exitcode
188  finally:
189  if process.is_alive():
190  process.terminate()
191 
192 
193 def _run_in_threads(test_fn, num_threads, parallel):
194  threads = []
195  for _ in range(num_threads):
196  thread = threading.Thread(target=test_fn)
197  thread.daemon = True
198  thread.start()
199  if parallel:
200  threads.append(thread)
201  else:
202  thread.join()
203  for thread in threads:
204  thread.join()
205 
206 
207 # TODO: FIXME, sometimes returns -11 (segfault) instead of 0 on macOS Python 3.9
208 @pytest.mark.parametrize("test_fn", ALL_BASIC_TESTS_PLUS_INTENTIONAL_DEADLOCK)
210  """Makes sure there is no GIL deadlock when running in a thread.
211 
212  It runs in a separate process to be able to stop and assert if it deadlocks.
213  """
214  assert _run_in_process(_run_in_threads, test_fn, num_threads=1, parallel=False) == 0
215 
216 
217 # TODO: FIXME on macOS Python 3.9
218 @pytest.mark.parametrize("test_fn", ALL_BASIC_TESTS_PLUS_INTENTIONAL_DEADLOCK)
220  """Makes sure there is no GIL deadlock when running in a thread multiple times in parallel.
221 
222  It runs in a separate process to be able to stop and assert if it deadlocks.
223  """
224  assert _run_in_process(_run_in_threads, test_fn, num_threads=8, parallel=True) == 0
225 
226 
227 # TODO: FIXME on macOS Python 3.9
228 @pytest.mark.parametrize("test_fn", ALL_BASIC_TESTS_PLUS_INTENTIONAL_DEADLOCK)
230  """Makes sure there is no GIL deadlock when running in a thread multiple times sequentially.
231 
232  It runs in a separate process to be able to stop and assert if it deadlocks.
233  """
234  assert _run_in_process(_run_in_threads, test_fn, num_threads=8, parallel=False) == 0
235 
236 
237 # TODO: FIXME on macOS Python 3.9
238 @pytest.mark.parametrize("test_fn", ALL_BASIC_TESTS_PLUS_INTENTIONAL_DEADLOCK)
240  """Makes sure there is no GIL deadlock when using processes.
241 
242  This test is for completion, but it was never an issue.
243  """
244  assert _run_in_process(test_fn) == 0
Eigen::internal::print
EIGEN_STRONG_INLINE Packet4f print(const Packet4f &a)
Definition: NEON/PacketMath.h:3115
test_gil_scoped.test_cross_module_gil_released
def test_cross_module_gil_released()
Definition: test_gil_scoped.py:40
test_gil_scoped.test_cross_module_gil_inner_custom_released
def test_cross_module_gil_inner_custom_released()
Definition: test_gil_scoped.py:50
test_gil_scoped.test_cross_module_gil_nested_custom_released
def test_cross_module_gil_nested_custom_released()
Definition: test_gil_scoped.py:74
test_gil_scoped.test_multi_acquire_release_cross_module
def test_multi_acquire_release_cross_module()
Definition: test_gil_scoped.py:106
test_gil_scoped.ExtendedVirtClass.virtual_func
def virtual_func(self)
Definition: test_gil_scoped.py:15
test_gil_scoped.test_release_acquire
def test_release_acquire()
Definition: test_gil_scoped.py:98
test_gil_scoped.test_run_in_process_multiple_threads_parallel
def test_run_in_process_multiple_threads_parallel(test_fn)
Definition: test_gil_scoped.py:219
test_gil_scoped.test_run_in_process_direct
def test_run_in_process_direct(test_fn)
Definition: test_gil_scoped.py:239
test_gil_scoped.test_cross_module_gil_acquired
def test_cross_module_gil_acquired()
Definition: test_gil_scoped.py:45
test_gil_scoped.test_run_in_process_one_thread
def test_run_in_process_one_thread(test_fn)
Definition: test_gil_scoped.py:209
gtsam::range
Double_ range(const Point2_ &p, const Point2_ &q)
Definition: slam/expressions.h:30
dict
Definition: pytypes.h:2107
test_gil_scoped.ExtendedVirtClass.pure_virtual_func
def pure_virtual_func(self)
Definition: test_gil_scoped.py:18
test_gil_scoped.test_cross_module_gil_nested_pybind11_acquired
def test_cross_module_gil_nested_pybind11_acquired()
Definition: test_gil_scoped.py:92
test_gil_scoped.test_cross_module_gil_nested_pybind11_released
def test_cross_module_gil_nested_pybind11_released()
Definition: test_gil_scoped.py:86
test_gil_scoped.test_callback_std_func
def test_callback_std_func()
Definition: test_gil_scoped.py:26
test_gil_scoped.test_callback_py_obj
def test_callback_py_obj()
Definition: test_gil_scoped.py:22
test_gil_scoped.test_run_in_process_multiple_threads_sequential
def test_run_in_process_multiple_threads_sequential(test_fn)
Definition: test_gil_scoped.py:229
test_gil_scoped.ExtendedVirtClass
Definition: test_gil_scoped.py:14
test_gil_scoped.test_cross_module_gil_inner_custom_acquired
def test_cross_module_gil_inner_custom_acquired()
Definition: test_gil_scoped.py:56
test_gil_scoped.test_callback_pure_virtual_func
def test_callback_pure_virtual_func()
Definition: test_gil_scoped.py:35
test_gil_scoped.test_cross_module_gil_inner_pybind11_acquired
def test_cross_module_gil_inner_pybind11_acquired()
Definition: test_gil_scoped.py:68
test_gil_scoped.test_nested_acquire
def test_nested_acquire()
Definition: test_gil_scoped.py:102
test_gil_scoped.test_cross_module_gil_inner_pybind11_released
def test_cross_module_gil_inner_pybind11_released()
Definition: test_gil_scoped.py:62
test_gil_scoped.test_all_basic_tests_completeness
def test_all_basic_tests_completeness()
Definition: test_gil_scoped.py:135
test_gil_scoped._run_in_threads
def _run_in_threads(test_fn, num_threads, parallel)
Definition: test_gil_scoped.py:193
len
size_t len(handle h)
Get the length of a Python object.
Definition: pytypes.h:2446
test_gil_scoped.test_callback_virtual_func
def test_callback_virtual_func()
Definition: test_gil_scoped.py:30
test_gil_scoped._intentional_deadlock
def _intentional_deadlock()
Definition: test_gil_scoped.py:145
test_gil_scoped._run_in_process
def _run_in_process(target, *args, **kwargs)
Definition: test_gil_scoped.py:152
test_gil_scoped.test_cross_module_gil_nested_custom_acquired
def test_cross_module_gil_nested_custom_acquired()
Definition: test_gil_scoped.py:80


gtsam
Author(s):
autogenerated on Sat Nov 16 2024 04:07:05