setup_helpers.py
Go to the documentation of this file.
1 """
2 This module provides helpers for C++11+ projects using pybind11.
3 
4 LICENSE:
5 
6 Copyright (c) 2016 Wenzel Jakob <wenzel.jakob@epfl.ch>, All rights reserved.
7 
8 Redistribution and use in source and binary forms, with or without
9 modification, are permitted provided that the following conditions are met:
10 
11 1. Redistributions of source code must retain the above copyright notice, this
12  list of conditions and the following disclaimer.
13 
14 2. Redistributions in binary form must reproduce the above copyright notice,
15  this list of conditions and the following disclaimer in the documentation
16  and/or other materials provided with the distribution.
17 
18 3. Neither the name of the copyright holder nor the names of its contributors
19  may be used to endorse or promote products derived from this software
20  without specific prior written permission.
21 
22 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
23 ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
24 WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
25 DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
26 FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
27 DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
28 SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
29 CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
30 OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
31 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
32 """
33 
34 # IMPORTANT: If you change this file in the pybind11 repo, also review
35 # setup_helpers.pyi for matching changes.
36 #
37 # If you copy this file in, you don't
38 # need the .pyi file; it's just an interface file for static type checkers.
39 from __future__ import annotations
40 
41 import contextlib
42 import os
43 import platform
44 import shlex
45 import shutil
46 import sys
47 import sysconfig
48 import tempfile
49 import threading
50 import warnings
51 from functools import lru_cache
52 from pathlib import Path
53 from typing import (
54  Any,
55  Callable,
56  Iterable,
57  Iterator,
58  List,
59  Optional,
60  Tuple,
61  TypeVar,
62  Union,
63 )
64 
65 try:
66  from setuptools import Extension as _Extension
67  from setuptools.command.build_ext import build_ext as _build_ext
68 except ImportError:
69  from distutils.command.build_ext import ( # type: ignore[assignment]
70  build_ext as _build_ext,
71  )
72  from distutils.extension import Extension as _Extension # type: ignore[assignment]
73 
74 import distutils.ccompiler
75 import distutils.errors
76 
77 WIN = sys.platform.startswith("win32") and "mingw" not in sysconfig.get_platform()
78 MACOS = sys.platform.startswith("darwin")
79 STD_TMPL = "/std:c++{}" if WIN else "-std=c++{}"
80 
81 
82 # It is recommended to use PEP 518 builds if using this module. However, this
83 # file explicitly supports being copied into a user's project directory
84 # standalone, and pulling pybind11 with the deprecated setup_requires feature.
85 # If you copy the file, remember to add it to your MANIFEST.in, and add the current
86 # directory into your path if it sits beside your setup.py.
87 
88 
89 class Pybind11Extension(_Extension):
90  """
91  Build a C++11+ Extension module with pybind11. This automatically adds the
92  recommended flags when you init the extension and assumes C++ sources - you
93  can further modify the options yourself.
94 
95  The customizations are:
96 
97  * ``/EHsc`` and ``/bigobj`` on Windows
98  * ``stdlib=libc++`` on macOS
99  * ``visibility=hidden`` and ``-g0`` on Unix
100 
101  Finally, you can set ``cxx_std`` via constructor or afterwards to enable
102  flags for C++ std, and a few extra helper flags related to the C++ standard
103  level. It is _highly_ recommended you either set this, or use the provided
104  ``build_ext``, which will search for the highest supported extension for
105  you if the ``cxx_std`` property is not set. Do not set the ``cxx_std``
106  property more than once, as flags are added when you set it. Set the
107  property to None to disable the addition of C++ standard flags.
108 
109  If you want to add pybind11 headers manually, for example for an exact
110  git checkout, then set ``include_pybind11=False``.
111  """
112 
113  # flags are prepended, so that they can be further overridden, e.g. by
114  # ``extra_compile_args=["-g"]``.
115 
116  def _add_cflags(self, flags: list[str]) -> None:
117  self.extra_compile_args[:0] = flags
118 
119  def _add_ldflags(self, flags: list[str]) -> None:
120  self.extra_link_args[:0] = flags
121 
122  def __init__(self, *args: Any, **kwargs: Any) -> None:
123  self._cxx_level = 0
124  cxx_std = kwargs.pop("cxx_std", 0)
125 
126  if "language" not in kwargs:
127  kwargs["language"] = "c++"
128 
129  include_pybind11 = kwargs.pop("include_pybind11", True)
130 
131  super().__init__(*args, **kwargs)
132 
133  # Include the installed package pybind11 headers
134  if include_pybind11:
135  # If using setup_requires, this fails the first time - that's okay
136  try:
137  import pybind11
138 
139  pyinc = pybind11.get_include()
140 
141  if pyinc not in self.include_dirs:
142  self.include_dirs.append(pyinc)
143  except ModuleNotFoundError:
144  pass
145 
146  self.cxx_std = cxx_std
147 
148  cflags = []
149  if WIN:
150  cflags += ["/EHsc", "/bigobj"]
151  else:
152  cflags += ["-fvisibility=hidden"]
153  env_cflags = os.environ.get("CFLAGS", "")
154  env_cppflags = os.environ.get("CPPFLAGS", "")
155  c_cpp_flags = shlex.split(env_cflags) + shlex.split(env_cppflags)
156  if not any(opt.startswith("-g") for opt in c_cpp_flags):
157  cflags += ["-g0"]
158  self._add_cflags(cflags)
159 
160  @property
161  def cxx_std(self) -> int:
162  """
163  The CXX standard level. If set, will add the required flags. If left at
164  0, it will trigger an automatic search when pybind11's build_ext is
165  used. If None, will have no effect. Besides just the flags, this may
166  add a macos-min 10.9 or 10.14 flag if MACOSX_DEPLOYMENT_TARGET is
167  unset.
168  """
169  return self._cxx_level
170 
171  @cxx_std.setter
172  def cxx_std(self, level: int) -> None:
173  if self._cxx_level:
174  warnings.warn(
175  "You cannot safely change the cxx_level after setting it!", stacklevel=2
176  )
177 
178  # MSVC 2015 Update 3 and later only have 14 (and later 17) modes, so
179  # force a valid flag here.
180  if WIN and level == 11:
181  level = 14
182 
183  self._cxx_level = level
184 
185  if not level:
186  return
187 
188  cflags = [STD_TMPL.format(level)]
189  ldflags = []
190 
191  if MACOS and "MACOSX_DEPLOYMENT_TARGET" not in os.environ:
192  # C++17 requires a higher min version of macOS. An earlier version
193  # (10.12 or 10.13) can be set manually via environment variable if
194  # you are careful in your feature usage, but 10.14 is the safest
195  # setting for general use. However, never set higher than the
196  # current macOS version!
197  current_macos = tuple(int(x) for x in platform.mac_ver()[0].split(".")[:2])
198  desired_macos = (10, 9) if level < 17 else (10, 14)
199  macos_string = ".".join(str(x) for x in min(current_macos, desired_macos))
200  macosx_min = f"-mmacosx-version-min={macos_string}"
201  cflags += [macosx_min]
202  ldflags += [macosx_min]
203 
204  self._add_cflags(cflags)
205  self._add_ldflags(ldflags)
206 
207 
208 # Just in case someone clever tries to multithread
209 tmp_chdir_lock = threading.Lock()
210 
211 
212 @contextlib.contextmanager
213 def tmp_chdir() -> Iterator[str]:
214  "Prepare and enter a temporary directory, cleanup when done"
215 
216  # Threadsafe
217  with tmp_chdir_lock:
218  olddir = os.getcwd()
219  try:
220  tmpdir = tempfile.mkdtemp()
221  os.chdir(tmpdir)
222  yield tmpdir
223  finally:
224  os.chdir(olddir)
225  shutil.rmtree(tmpdir)
226 
227 
228 # cf http://bugs.python.org/issue26689
229 def has_flag(compiler: Any, flag: str) -> bool:
230  """
231  Return the flag if a flag name is supported on the
232  specified compiler, otherwise None (can be used as a boolean).
233  If multiple flags are passed, return the first that matches.
234  """
235 
236  with tmp_chdir():
237  fname = Path("flagcheck.cpp")
238  # Don't trigger -Wunused-parameter.
239  fname.write_text("int main (int, char **) { return 0; }", encoding="utf-8")
240 
241  try:
242  compiler.compile([str(fname)], extra_postargs=[flag])
243  except distutils.errors.CompileError:
244  return False
245  return True
246 
247 
248 # Every call will cache the result
249 cpp_flag_cache = None
250 
251 
252 @lru_cache()
253 def auto_cpp_level(compiler: Any) -> str | int:
254  """
255  Return the max supported C++ std level (17, 14, or 11). Returns latest on Windows.
256  """
257 
258  if WIN:
259  return "latest"
260 
261  levels = [17, 14, 11]
262 
263  for level in levels:
264  if has_flag(compiler, STD_TMPL.format(level)):
265  return level
266 
267  msg = "Unsupported compiler -- at least C++11 support is needed!"
268  raise RuntimeError(msg)
269 
270 
271 class build_ext(_build_ext): # noqa: N801
272  """
273  Customized build_ext that allows an auto-search for the highest supported
274  C++ level for Pybind11Extension. This is only needed for the auto-search
275  for now, and is completely optional otherwise.
276  """
277 
278  def build_extensions(self) -> None:
279  """
280  Build extensions, injecting C++ std for Pybind11Extension if needed.
281  """
282 
283  for ext in self.extensions:
284  if hasattr(ext, "_cxx_level") and ext._cxx_level == 0:
285  ext.cxx_std = auto_cpp_level(self.compiler)
286 
287  super().build_extensions()
288 
289 
291  paths: Iterable[str], package_dir: dict[str, str] | None = None
292 ) -> list[Pybind11Extension]:
293  """
294  Generate Pybind11Extensions from source files directly located in a Python
295  source tree.
296 
297  ``package_dir`` behaves as in ``setuptools.setup``. If unset, the Python
298  package root parent is determined as the first parent directory that does
299  not contain an ``__init__.py`` file.
300  """
301  exts = []
302 
303  if package_dir is None:
304  for path in paths:
305  parent, _ = os.path.split(path)
306  while os.path.exists(os.path.join(parent, "__init__.py")):
307  parent, _ = os.path.split(parent)
308  relname, _ = os.path.splitext(os.path.relpath(path, parent))
309  qualified_name = relname.replace(os.path.sep, ".")
310  exts.append(Pybind11Extension(qualified_name, [path]))
311  return exts
312 
313  for path in paths:
314  for prefix, parent in package_dir.items():
315  if path.startswith(parent):
316  relname, _ = os.path.splitext(os.path.relpath(path, parent))
317  qualified_name = relname.replace(os.path.sep, ".")
318  if prefix:
319  qualified_name = prefix + "." + qualified_name
320  exts.append(Pybind11Extension(qualified_name, [path]))
321  break
322  else:
323  msg = (
324  f"path {path} is not a child of any of the directories listed "
325  f"in 'package_dir' ({package_dir})"
326  )
327  raise ValueError(msg)
328 
329  return exts
330 
331 
332 def naive_recompile(obj: str, src: str) -> bool:
333  """
334  This will recompile only if the source file changes. It does not check
335  header files, so a more advanced function or Ccache is better if you have
336  editable header files in your package.
337  """
338  return os.stat(obj).st_mtime < os.stat(src).st_mtime
339 
340 
341 def no_recompile(obg: str, src: str) -> bool: # noqa: ARG001
342  """
343  This is the safest but slowest choice (and is the default) - will always
344  recompile sources.
345  """
346  return True
347 
348 
349 S = TypeVar("S", bound="ParallelCompile")
350 
351 CCompilerMethod = Callable[
352  [
353  distutils.ccompiler.CCompiler,
354  List[str],
355  Optional[str],
356  Optional[List[Union[Tuple[str], Tuple[str, Optional[str]]]]],
357  Optional[List[str]],
358  bool,
359  Optional[List[str]],
360  Optional[List[str]],
361  Optional[List[str]],
362  ],
363  List[str],
364 ]
365 
366 
367 # Optional parallel compile utility
368 # inspired by: http://stackoverflow.com/questions/11013851/speeding-up-build-process-with-distutils
369 # and: https://github.com/tbenthompson/cppimport/blob/stable/cppimport/build_module.py
370 # and NumPy's parallel distutils module:
371 # https://github.com/numpy/numpy/blob/master/numpy/distutils/ccompiler.py
373  """
374  Make a parallel compile function. Inspired by
375  numpy.distutils.ccompiler.CCompiler.compile and cppimport.
376 
377  This takes several arguments that allow you to customize the compile
378  function created:
379 
380  envvar:
381  Set an environment variable to control the compilation threads, like
382  NPY_NUM_BUILD_JOBS
383  default:
384  0 will automatically multithread, or 1 will only multithread if the
385  envvar is set.
386  max:
387  The limit for automatic multithreading if non-zero
388  needs_recompile:
389  A function of (obj, src) that returns True when recompile is needed. No
390  effect in isolated mode; use ccache instead, see
391  https://github.com/matplotlib/matplotlib/issues/1507/
392 
393  To use::
394 
395  ParallelCompile("NPY_NUM_BUILD_JOBS").install()
396 
397  or::
398 
399  with ParallelCompile("NPY_NUM_BUILD_JOBS"):
400  setup(...)
401 
402  By default, this assumes all files need to be recompiled. A smarter
403  function can be provided via needs_recompile. If the output has not yet
404  been generated, the compile will always run, and this function is not
405  called.
406  """
407 
408  __slots__ = ("envvar", "default", "max", "_old", "needs_recompile")
409 
410  def __init__(
411  self,
412  envvar: str | None = None,
413  default: int = 0,
414  max: int = 0, # pylint: disable=redefined-builtin
415  needs_recompile: Callable[[str, str], bool] = no_recompile,
416  ) -> None:
417  self.envvar = envvar
418  self.default = default
419  self.max = max
420  self.needs_recompile = needs_recompile
421  self._old: list[CCompilerMethod] = []
422 
423  def function(self) -> CCompilerMethod:
424  """
425  Builds a function object usable as distutils.ccompiler.CCompiler.compile.
426  """
427 
428  def compile_function(
429  compiler: distutils.ccompiler.CCompiler,
430  sources: list[str],
431  output_dir: str | None = None,
432  macros: list[tuple[str] | tuple[str, str | None]] | None = None,
433  include_dirs: list[str] | None = None,
434  debug: bool = False,
435  extra_preargs: list[str] | None = None,
436  extra_postargs: list[str] | None = None,
437  depends: list[str] | None = None,
438  ) -> Any:
439  # These lines are directly from distutils.ccompiler.CCompiler
440  macros, objects, extra_postargs, pp_opts, build = compiler._setup_compile( # type: ignore[attr-defined]
441  output_dir, macros, include_dirs, sources, depends, extra_postargs
442  )
443  cc_args = compiler._get_cc_args(pp_opts, debug, extra_preargs) # type: ignore[attr-defined]
444 
445  # The number of threads; start with default.
446  threads = self.default
447 
448  # Determine the number of compilation threads, unless set by an environment variable.
449  if self.envvar is not None:
450  threads = int(os.environ.get(self.envvar, self.default))
451 
452  def _single_compile(obj: Any) -> None:
453  try:
454  src, ext = build[obj]
455  except KeyError:
456  return
457 
458  if not os.path.exists(obj) or self.needs_recompile(obj, src):
459  compiler._compile(obj, src, ext, cc_args, extra_postargs, pp_opts) # type: ignore[attr-defined]
460 
461  try:
462  # Importing .synchronize checks for platforms that have some multiprocessing
463  # capabilities but lack semaphores, such as AWS Lambda and Android Termux.
464  import multiprocessing.synchronize
465  from multiprocessing.pool import ThreadPool
466  except ImportError:
467  threads = 1
468 
469  if threads == 0:
470  try:
471  threads = multiprocessing.cpu_count()
472  threads = self.max if self.max and self.max < threads else threads
473  except NotImplementedError:
474  threads = 1
475 
476  if threads > 1:
477  with ThreadPool(threads) as pool:
478  for _ in pool.imap_unordered(_single_compile, objects):
479  pass
480  else:
481  for ob in objects:
482  _single_compile(ob)
483 
484  return objects
485 
486  return compile_function
487 
488  def install(self: S) -> S:
489  """
490  Installs the compile function into distutils.ccompiler.CCompiler.compile.
491  """
492  distutils.ccompiler.CCompiler.compile = self.function() # type: ignore[assignment]
493  return self
494 
495  def __enter__(self: S) -> S:
496  self._old.append(distutils.ccompiler.CCompiler.compile)
497  return self.install()
498 
499  def __exit__(self, *args: Any) -> None:
500  distutils.ccompiler.CCompiler.compile = self._old.pop() # type: ignore[assignment]
gtsam.examples.DogLegOptimizerExample.int
int
Definition: DogLegOptimizerExample.py:111
pybind11.setup_helpers.ParallelCompile.__init__
None __init__(self, str|None envvar=None, int default=0, int max=0, Callable[[str, str], bool] needs_recompile=no_recompile)
Definition: setup_helpers.py:410
pybind11.setup_helpers.auto_cpp_level
str|int auto_cpp_level(Any compiler)
Definition: setup_helpers.py:253
pybind11.setup_helpers.ParallelCompile
Definition: setup_helpers.py:372
pybind11.setup_helpers.ParallelCompile.__exit__
None __exit__(self, *Any args)
Definition: setup_helpers.py:499
pybind11.setup_helpers.ParallelCompile.max
max
Definition: setup_helpers.py:413
hasattr
bool hasattr(handle obj, handle name)
Definition: pytypes.h:870
pybind11.setup_helpers.tmp_chdir
Iterator[str] tmp_chdir()
Definition: setup_helpers.py:213
pybind11.setup_helpers.Pybind11Extension
Definition: setup_helpers.py:89
pybind11.setup_helpers.build_ext.build_extensions
None build_extensions(self)
Definition: setup_helpers.py:278
pybind11.setup_helpers.Pybind11Extension._add_ldflags
None _add_ldflags(self, list[str] flags)
Definition: setup_helpers.py:119
pybind11.setup_helpers.Pybind11Extension.cxx_std
cxx_std
Definition: setup_helpers.py:146
pybind11.setup_helpers.ParallelCompile.default
default
Definition: setup_helpers.py:412
pybind11.setup_helpers.naive_recompile
bool naive_recompile(str obj, str src)
Definition: setup_helpers.py:332
pybind11.setup_helpers.ParallelCompile.__enter__
S __enter__(S self)
Definition: setup_helpers.py:495
pybind11.setup_helpers.intree_extensions
list[Pybind11Extension] intree_extensions(Iterable[str] paths, dict[str, str]|None package_dir=None)
Definition: setup_helpers.py:290
pybind11.setup_helpers.Pybind11Extension.__init__
None __init__(self, *Any args, **Any kwargs)
Definition: setup_helpers.py:122
pybind11.setup_helpers.Pybind11Extension._add_cflags
None _add_cflags(self, list[str] flags)
Definition: setup_helpers.py:116
str
Definition: pytypes.h:1558
pybind11.setup_helpers.ParallelCompile.needs_recompile
needs_recompile
Definition: setup_helpers.py:414
Eigen::ThreadPool
ThreadPoolTempl< StlThreadEnvironment > ThreadPool
Definition: NonBlockingThreadPool.h:482
pybind11.setup_helpers.ParallelCompile.function
CCompilerMethod function(self)
Definition: setup_helpers.py:423
tuple
Definition: pytypes.h:2077
pybind11.setup_helpers.ParallelCompile.install
S install(S self)
Definition: setup_helpers.py:488
pybind11.setup_helpers.no_recompile
bool no_recompile(str obg, str src)
Definition: setup_helpers.py:341
min
#define min(a, b)
Definition: datatypes.h:19
gtsam::split
void split(const G &g, const PredecessorMap< KEY > &tree, G &Ab1, G &Ab2)
Definition: graph-inl.h:245
pybind11.setup_helpers.ParallelCompile.envvar
envvar
Definition: setup_helpers.py:411
pybind11.setup_helpers.has_flag
bool has_flag(Any compiler, str flag)
Definition: setup_helpers.py:229
pybind11.setup_helpers.Pybind11Extension._cxx_level
_cxx_level
Definition: setup_helpers.py:123
pybind11.setup_helpers.build_ext
Definition: setup_helpers.py:271


gtsam
Author(s):
autogenerated on Fri Nov 1 2024 03:35:15