3 GTSAM Copyright 2010-2020, Georgia Tech Research Corporation,
4 Atlanta, Georgia 30332-0415
7 See LICENSE for the license information
9 Code generator for wrapping a C++ module with Pybind11
10 Author: Duy Nguyen Ta, Fan Jiang, Matthew Sklar, Varun Agrawal, and Frank Dellaert
16 from pathlib
import Path
17 from typing
import List
25 Class to generate binding code for Pybind11 specifically.
30 top_module_namespaces='',
31 use_boost_serialization=False,
41 'lambda',
'False',
'def',
'if',
'raise',
'None',
'del',
'import',
42 'return',
'True',
'elif',
'in',
'try',
'and',
'else',
'is',
43 'while',
'as',
'except',
'lambda',
'with',
'assert',
'finally',
44 'nonlocal',
'yield',
'break',
'for',
'not',
'class',
'from',
'or',
45 'continue',
'global',
'pass'
55 "svg",
"png",
"jpeg",
"html",
"javascript",
"markdown",
"latex"
59 """Set the argument names in Pybind11 format."""
63 for arg
in args.list():
64 if arg.default
is not None:
65 default =
' = {arg.default}'.
format(arg=arg)
68 argument =
'py::arg("{name}"){default}'.
format(
69 name=arg.name, default=
'{0}'.
format(default))
70 py_args.append(argument)
71 return ", " +
", ".join(py_args)
76 """Generate the argument types and names as per the method signature."""
77 cpp_types = args.to_cpp()
80 "{} {}".
format(ctype, name)
81 for ctype, name
in zip(cpp_types, names)
84 return ', '.join(types_names)
87 """Wrap the constructors."""
89 for ctor
in my_class.ctors:
90 res += (self.
method_indent +
'.def(py::init<{args_cpp_types}>()'
92 args_cpp_types=
", ".join(ctor.args.to_cpp()),
98 """Helper method to add serialize, deserialize and pickle methods to the wrapped class."""
103 ".def(\"serialize\", []({class_inst} self){{ return gtsam::serialize(*self); }})".
format(class_inst=cpp_class +
'*')
106 '.def("deserialize", []({class_inst} self, string serialized)' \
107 '{{ gtsam::deserialize(serialized, *self); }}, py::arg("serialized"))' \
108 .
format(class_inst=cpp_class +
'*')
112 ".def(py::pickle({indent} [](const {cpp_class} &a){{ /* __getstate__: Returns a string that encodes the state of the object */ return py::make_tuple(gtsam::serialize(a)); }},{indent} [](py::tuple t){{ /* __setstate__ */ {cpp_class} obj; gtsam::deserialize(t[0].cast<std::string>(), obj); return obj; }}))"
114 return serialize_method + deserialize_method + \
115 pickle_method.format(cpp_class=cpp_class, indent=self.
method_indent)
117 def _wrap_print(self, ret: str, method: parser.Method, cpp_class: str,
118 args_names: List[str], args_signature_with_names: str,
119 py_args_names: str, prefix: str, suffix: str):
121 Update the print method to print to the output stream and append a __repr__ method.
124 ret (str): The result of the parser.
125 method (parser.Method): The method to be wrapped.
126 cpp_class (str): The C++ name of the class to which the method belongs.
127 args_names (List[str]): List of argument variable names passed to the method.
128 args_signature_with_names (str): C++ arguments containing their names and type signatures.
129 py_args_names (str): The pybind11 formatted version of the argument list.
130 prefix (str): Prefix to add to the wrapped method when writing to the cpp file.
131 suffix (str): Suffix to add to the wrapped method when writing to the cpp file.
134 str: The wrapped print method.
138 ret = ret.replace(
'self->print',
139 'py::scoped_ostream_redirect output; self->print')
142 ret +=
'''{prefix}.def("__repr__",
143 [](const {cpp_class}& self{opt_comma}{args_signature_with_names}){{
144 gtsam::RedirectCout redirect;
145 self.{method_name}({method_args});
146 return redirect.str();
147 }}{py_args_names}){suffix}'''.
format(
150 opt_comma=
', ' if args_names
else '',
151 args_signature_with_names=args_signature_with_names,
152 method_name=method.name,
153 method_args=
", ".join(args_names)
if args_names
else '',
154 py_args_names=py_args_names,
165 Wrap a Python double-underscore (dunder) method.
167 E.g. __len__() gets wrapped as `.def("__len__", [](gtsam::KeySet* self) {return self->size();})`
169 Supported methods are:
174 py_method = method.name + method_suffix
175 args_names = method.args.names()
179 if method.name ==
'len':
180 function_call =
"return std::distance(self->begin(), self->end());"
181 elif method.name ==
'contains':
182 function_call = f
"return std::find(self->begin(), self->end(), {method.args.args_list[0].name}) != self->end();"
183 elif method.name ==
'iter':
184 function_call =
"return py::make_iterator(self->begin(), self->end());"
186 ret = (
'{prefix}.def("__{py_method}__",'
187 '[]({self}{opt_comma}{args_signature_with_names}){{'
190 '{py_args_names}){suffix}'.
format(
193 self=f
"{cpp_class}* self",
194 opt_comma=
', ' if args_names
else '',
195 args_signature_with_names=args_signature_with_names,
196 function_call=function_call,
197 py_args_names=py_args_names,
210 Wrap the `method` for the class specified by `cpp_class`.
213 method: The method to wrap.
214 cpp_class: The C++ name of the class to which the method belongs.
215 prefix: Prefix to add to the wrapped method when writing to the cpp file.
216 suffix: Suffix to add to the wrapped method when writing to the cpp file.
217 method_suffix: A string to append to the wrapped method name.
219 py_method = method.name + method_suffix
220 cpp_method = method.to_cpp()
222 args_names = method.args.names()
227 if cpp_method
in [
"serialize",
"serializable"]:
237 py_method = f
"_repr_{self._ipython_special_methods[idx]}_"
241 py_method = py_method +
"_"
244 method, (parser.Method, instantiator.InstantiatedMethod))
247 (parser.StaticMethod, instantiator.InstantiatedStaticMethod))
248 return_void = method.return_type.is_void()
250 caller = cpp_class +
"::" if not is_method
else "self->"
251 function_call = (
'{opt_return} {caller}{method_name}'
253 opt_return=
'return' if not return_void
else '',
255 method_name=cpp_method,
256 args_names=
', '.join(args_names),
259 ret = (
'{prefix}.{cdef}("{py_method}",'
260 '[]({opt_self}{opt_comma}{args_signature_with_names}){{'
263 '{py_args_names}){suffix}'.
format(
265 cdef=
"def_static" if is_static
else "def",
267 opt_self=
"{cpp_class}* self".
format(
268 cpp_class=cpp_class)
if is_method
else "",
269 opt_comma=
', ' if is_method
and args_names
else '',
270 args_signature_with_names=args_signature_with_names,
271 function_call=function_call,
272 py_args_names=py_args_names,
278 if method.name ==
'print':
279 ret = self.
_wrap_print(ret, method, cpp_class, args_names,
280 args_signature_with_names, py_args_names,
288 prefix='\n' + ' ' * 8,
291 for method
in methods:
302 prefix='\n' + ' ' * 8,
305 Wrap all the methods in the `cpp_class`.
308 for method
in methods:
311 if method.name ==
'insert' and cpp_class ==
'gtsam::Values':
312 name_list = method.args.names()
313 type_list = method.args.to_cpp()
315 if type_list[0].strip() ==
'size_t':
316 method_suffix =
'_' + name_list[1].strip()
321 method_suffix=method_suffix)
336 prefix='\n' + ' ' * 8):
338 Wrap a variable that's not part of a class (i.e. global)
341 if variable.default
is None:
342 variable_value = variable.name
344 variable_value = variable.default
346 return '{prefix}{module_var}.attr("{variable_name}") = {namespace}{variable_value};'.
format(
348 module_var=module_var,
349 variable_name=variable.name,
351 variable_value=variable_value)
354 """Wrap all the properties in the `cpp_class`."""
356 for prop
in properties:
357 res += (
'{prefix}.def_{property}("{property_name}", '
358 '&{cpp_class}::{property_name})'.
format(
361 if prop.ctype.is_const
else "readwrite",
363 property_name=prop.name,
368 """Wrap all the overloaded operators in the `cpp_class`."""
370 template =
"{prefix}.def({{0}})".
format(prefix=prefix)
372 if op.operator ==
"[]":
373 res +=
"{prefix}.def(\"__getitem__\", &{cpp_class}::operator[])".
format(
374 prefix=prefix, cpp_class=cpp_class)
375 elif op.operator ==
"()":
376 res +=
"{prefix}.def(\"__call__\", &{cpp_class}::operator())".
format(
377 prefix=prefix, cpp_class=cpp_class)
379 res += template.format(
"{0}py::self".
format(op.operator))
381 res += template.format(
"py::self {0} py::self".
format(
385 def wrap_enum(self, enum, class_name='', module=None, prefix=' ' * 4):
390 enum: The parsed enum to wrap.
391 class_name: The class under which the enum is defined.
392 prefix: The amount of indentation.
397 cpp_class = enum.cpp_typename().
to_cpp()
400 cpp_class = class_name +
"::" + cpp_class
402 res =
'{prefix}py::enum_<{cpp_class}>({module}, "{enum.name}", py::arithmetic())'.
format(
403 prefix=prefix, module=module, enum=enum, cpp_class=cpp_class)
404 for enumerator
in enum.enumerators:
405 res +=
'\n{prefix} .value("{enumerator.name}", {cpp_class}::{enumerator.name})'.
format(
406 prefix=prefix, enumerator=enumerator, cpp_class=cpp_class)
410 def wrap_enums(self, enums, instantiated_class, prefix=' ' * 4):
411 """Wrap multiple enums defined in a class."""
412 cpp_class = instantiated_class.to_cpp()
413 module_var = instantiated_class.name.lower()
418 enum, class_name=cpp_class, module=module_var, prefix=prefix)
422 self, instantiated_class: instantiator.InstantiatedClass):
423 """Wrap the class."""
425 cpp_class = instantiated_class.to_cpp()
428 if instantiated_class.parent_class:
429 class_parent =
"{instantiated_class.parent_class}, ".
format(
430 instantiated_class=instantiated_class)
434 if instantiated_class.enums:
436 instance_name = instantiated_class.name.lower()
437 class_declaration = (
438 '\n py::class_<{cpp_class}, {class_parent}'
439 'std::shared_ptr<{cpp_class}>> '
440 '{instance_name}({module_var}, "{class_name}");'
441 '\n {instance_name}').
format(
443 class_name=instantiated_class.name,
444 class_parent=class_parent,
445 instance_name=instance_name,
446 module_var=module_var)
447 module_var = instance_name
450 class_declaration = (
451 '\n py::class_<{cpp_class}, {class_parent}'
452 'std::shared_ptr<{cpp_class}>>({module_var}, "{class_name}")'
453 ).
format(cpp_class=cpp_class,
454 class_name=instantiated_class.name,
455 class_parent=class_parent,
456 module_var=module_var)
458 return (
'{class_declaration}'
461 '{wrapped_static_methods}'
462 '{wrapped_dunder_methods}'
463 '{wrapped_properties}'
464 '{wrapped_operators};\n'.
format(
465 class_declaration=class_declaration,
466 wrapped_ctors=self.
wrap_ctors(instantiated_class),
468 instantiated_class.methods, cpp_class),
470 instantiated_class.static_methods, cpp_class),
472 instantiated_class.dunder_methods, cpp_class),
474 instantiated_class.properties, cpp_class),
476 instantiated_class.operators, cpp_class)))
479 self, instantiated_decl: instantiator.InstantiatedDeclaration):
480 """Wrap the forward declaration."""
482 cpp_class = instantiated_decl.to_cpp()
486 res = (
'\n py::class_<{cpp_class}, '
487 'std::shared_ptr<{cpp_class}>>({module_var}, "{class_name}");'
488 ).
format(cpp_class=cpp_class,
489 class_name=instantiated_decl.name,
490 module_var=module_var)
494 """Wrap STL containers."""
496 cpp_class = stl_class.to_cpp()
500 return (
'\n py::class_<{cpp_class}, {class_parent}'
501 'std::shared_ptr<{cpp_class}>>({module_var}, "{class_name}")'
504 '{wrapped_static_methods}'
505 '{wrapped_properties};\n'.
format(
507 class_name=stl_class.name,
508 class_parent=
str(stl_class.parent_class) +
509 (
', ' if stl_class.parent_class
else ''),
510 module_var=module_var,
515 stl_class.static_methods, cpp_class),
517 stl_class.properties, cpp_class),
523 prefix='\n' + ' ' * 8,
526 Wrap all the global functions.
529 for function
in functions:
531 function_name = function.name
535 if function_name
in python_keywords:
536 function_name = function_name +
"_"
538 cpp_method = function.to_cpp()
540 is_static =
isinstance(function, parser.StaticMethod)
541 return_void = function.return_type.is_void()
542 args_names = function.args.names()
546 caller = namespace +
"::"
547 function_call = (
'{opt_return} {caller}{function_name}'
550 if not return_void
else '',
552 function_name=cpp_method,
553 args_names=
', '.join(args_names),
556 ret = (
'{prefix}.{cdef}("{function_name}",'
557 '[]({args_signature}){{'
560 '{py_args_names}){suffix}'.
format(
562 cdef=
"def_static" if is_static
else "def",
563 function_name=function_name,
564 args_signature=args_signature,
565 function_call=function_call,
566 py_args_names=py_args_names,
575 if namespaces1[i] != namespaces2[i]:
580 """Get the Pybind11 module name from the namespaces."""
583 return "m_{}".
format(
'_'.join(sub_module_namespaces))
588 idx = 1
if not namespaces[0]
else 0
589 return '::'.join(namespaces[idx:] + [name])
594 """Wrap the complete `namespace`."""
598 namespaces = namespace.full_namespaces()
603 for element
in namespace.content:
605 include =
"{}\n".
format(element)
607 include = include.replace(
'<',
'"').replace(
'>',
'"')
615 wrapped += wrapped_namespace
616 includes += includes_namespace
622 ' ' * 4 +
'pybind11::module {module_var} = '
623 '{parent_module_var}.def_submodule("{namespace}", "'
624 '{namespace} submodule");\n'.
format(
625 module_var=module_var,
626 namespace=namespace.name,
632 for element
in namespace.content:
634 include =
"{}\n".
format(element)
636 include = include.replace(
'<',
'"').replace(
'>',
'"')
641 wrapped += wrapped_namespace
642 includes += includes_namespace
644 elif isinstance(element, instantiator.InstantiatedClass):
646 wrapped += self.
wrap_enums(element.enums, element)
648 elif isinstance(element, instantiator.InstantiatedDeclaration):
654 module_var=module_var,
656 prefix=
'\n' +
' ' * 4)
663 func
for func
in namespace.content
665 instantiator.InstantiatedGlobalFunction))
670 prefix=
'\n' +
' ' * 4 + module_var,
674 return wrapped, includes
676 def wrap_file(self, content, module_name=None, submodules=None):
678 Wrap the code in the interface file.
681 content: The contents of the interface file.
682 module_name: The name of the module.
683 submodules: List of other interface file names that should be linked to.
686 module = parser.Module.parseString(content)
688 module = instantiator.instantiate_namespace(module)
693 includes +=
"#include <boost/serialization/export.hpp>"
696 boost_class_export =
""
701 new_name = re.sub(
"[,:<> ]",
"", cpp_class)
702 boost_class_export +=
"typedef {cpp_class} {new_name};\n".
format(
703 cpp_class=cpp_class, new_name=new_name)
705 boost_class_export +=
"BOOST_CLASS_EXPORT({new_name})\n".
format(
708 boost_class_export =
""
715 if submodules
is not None:
716 module_def =
"PYBIND11_MODULE({0}, m_)".
format(module_name)
718 for idx, submodule
in enumerate(submodules):
719 submodules[idx] =
"void {0}(py::module_ &);".
format(submodule)
720 submodules_init.append(
"{0}(m_);".
format(submodule))
723 module_def =
"void {0}(py::module_ &m_)".
format(module_name)
727 module_def=module_def,
728 module_name=module_name,
730 wrapped_namespace=wrapped_namespace,
731 boost_class_export=boost_class_export,
732 submodules=
"\n".join(submodules),
733 submodules_init=
"\n".join(submodules_init),
738 Wrap a list of submodule files, i.e. a set of interface files which are
739 in support of a larger wrapping project.
741 E.g. This is used in GTSAM where we have a main gtsam.i, but various smaller .i files
742 which are the submodules.
743 The benefit of this scheme is that it reduces compute and memory usage during compilation.
746 source: Interface file which forms the submodule.
748 filename = Path(source).name
749 module_name = Path(source).stem
752 with open(source,
"r", encoding=
"UTF-8")
as f:
755 cc_content = self.
wrap_file(content, module_name=module_name)
758 with open(filename.replace(
".i",
".cpp"),
"w", encoding=
"UTF-8")
as f:
761 def wrap(self, sources, main_module_name):
763 Wrap all the main interface file.
766 sources: List of all interface files.
767 The first file should be the main module.
768 main_module_name: The name for the main module.
770 main_module = sources[0]
774 for source
in sources[1:]:
775 module_name = Path(source).stem
776 submodules.append(module_name)
778 with open(main_module,
"r", encoding=
"UTF-8")
as f:
782 submodules=submodules)
785 with open(main_module_name,
"w", encoding=
"UTF-8")
as f: