36 #include <opencv2/imgproc/types_c.h>    38 #ifdef RTABMAP_OPENVINS    39 #include "core/VioManager.h"    40 #include "core/VioManagerOptions.h"    41 #include "core/RosVisualizer.h"    42 #include "utils/dataset_reader.h"    43 #include "utils/parse_ros.h"    44 #include "utils/sensor_data.h"    45 #include "state/State.h"    46 #include "types/Type.h"    53 #ifdef RTABMAP_OPENVINS
    64 #ifdef RTABMAP_OPENVINS    72 #ifdef RTABMAP_OPENVINS    77                 previousPose_.setIdentity();
    78                 previousLocalTransform_.setNull();
    92 #ifdef RTABMAP_OPENVINS    98                 imuBuffer_.insert(std::make_pair(data.
stamp(), data.
imu()));
   104                 if(imuBuffer_.empty())
   106                         UWARN(
"Waiting IMU for initialization...");
   111                         UINFO(
"OpenVINS Initialization");
   114                         ov_msckf::VioManagerOptions 
params;
   130                         params.state_options.num_cameras = 2;
   137                         if( params.state_options.feat_rep_msckf == LandmarkRepresentation::Representation::UNKNOWN ||
   138                                 params.state_options.feat_rep_slam == LandmarkRepresentation::Representation::UNKNOWN ||
   139                                 params.state_options.feat_rep_aruco == LandmarkRepresentation::Representation::UNKNOWN)
   141                                 printf(
RED "VioManager(): invalid feature representation specified:\n" RESET);
   142                                 printf(
RED "\t- GLOBAL_3D\n" RESET);
   143                                 printf(
RED "\t- GLOBAL_FULL_INVERSE_DEPTH\n" RESET);
   144                                 printf(
RED "\t- ANCHORED_3D\n" RESET);
   145                                 printf(
RED "\t- ANCHORED_FULL_INVERSE_DEPTH\n" RESET);
   146                                 printf(
RED "\t- ANCHORED_MSCKF_INVERSE_DEPTH\n" RESET);
   147                                 printf(
RED "\t- ANCHORED_INVERSE_DEPTH_SINGLE\n" RESET);
   148                                 std::exit(EXIT_FAILURE);
   190                         params.use_stereo = 
true;
   192                         params.use_aruco = 
false;
   222                         params.camera_fisheye.insert(std::make_pair(0, fisheye));
   223                         params.camera_fisheye.insert(std::make_pair(1, fisheye));
   225                         Eigen::VectorXd camLeft(8), camRight(8);
   268                         params.camera_intrinsics.insert(std::make_pair(0, camLeft));
   269                         params.camera_intrinsics.insert(std::make_pair(1, camRight));
   271                         const IMU & imu = imuBuffer_.begin()->second;
   288                         Eigen::Matrix4d cam0_eigen = imuCam0.
toEigen4d();
   289                         Eigen::Matrix4d cam1_eigen = imuCam1.toEigen4d();
   290                         Eigen::Matrix<double,7,1> cam_eigen0;
   291                         cam_eigen0.block(0,0,4,1) = rot_2_quat(cam0_eigen.block(0,0,3,3).transpose());
   292                         cam_eigen0.block(4,0,3,1) = -cam0_eigen.block(0,0,3,3).transpose()*cam0_eigen.block(0,3,3,1);
   293                         Eigen::Matrix<double,7,1> cam_eigen1;
   294                         cam_eigen1.block(0,0,4,1) = rot_2_quat(cam1_eigen.block(0,0,3,3).transpose());
   295                         cam_eigen1.block(4,0,3,1) = -cam1_eigen.block(0,0,3,3).transpose()*cam1_eigen.block(0,3,3,1);
   296                         params.camera_extrinsics.insert(std::make_pair(0, cam_eigen0));
   297                         params.camera_extrinsics.insert(std::make_pair(1, cam_eigen1));
   302                         vioManager_ = 
new ov_msckf::VioManager(params);
   307                 if(data.
imageRaw().type() == CV_8UC3)
   309                         cv::cvtColor(data.
imageRaw(), left, CV_BGR2GRAY);
   311                 else if(data.
imageRaw().type() == CV_8UC1)
   317                         UFATAL(
"Not supported color type!");
   319                 if(data.
rightRaw().type() == CV_8UC3)
   321                         cv::cvtColor(data.
rightRaw(), right, CV_BGR2GRAY);
   323                 else if(data.
rightRaw().type() == CV_8UC1)
   329                         UFATAL(
"Not supported color type!");
   333                 ov_core::CameraData message;
   334                 message.timestamp = data.
stamp();
   335                 message.sensor_ids.push_back(0);
   336                 message.sensor_ids.push_back(1);
   337                 message.images.push_back(left);
   338                 message.images.push_back(right);
   339                 message.masks.push_back(cv::Mat::zeros(left.size(), CV_8UC1));
   340                 message.masks.push_back(cv::Mat::zeros(right.size(), CV_8UC1));
   343                 vioManager_->feed_measurement_camera(message);
   346                 double lastIMUstamp = 0.0;
   347                 while(!imuBuffer_.empty())
   349                         std::map<double, IMU>::iterator iter = imuBuffer_.begin();
   352                         ov_core::ImuData message;
   353                         message.timestamp = iter->first;
   354                         message.wm << iter->second.angularVelocity().val[0], iter->second.angularVelocity().val[1], iter->second.angularVelocity().val[2];
   355                         message.am << iter->second.linearAcceleration().val[0], iter->second.linearAcceleration().val[1], iter->second.linearAcceleration().val[2];
   357                         UDEBUG(
"IMU update stamp=%f", message.timestamp);
   360                         vioManager_->feed_measurement_imu(message);
   362                         lastIMUstamp = iter->first;
   364                         imuBuffer_.erase(iter);
   366                         if(lastIMUstamp > data.
stamp())
   372                 if(vioManager_->initialized())
   375                         std::shared_ptr<ov_msckf::State> 
state = vioManager_->get_state();
   377                         if(state->_timestamp != data.
stamp())
   379                                 UWARN(
"OpenVINS: Stamp of the current state %f is not the same "   380                                                 "than last image processed %f (last IMU stamp=%f). There could be "   381                                                 "a synchronization issue between camera and IMU. ",
   388                                         (
float)state->_imu->pos()(0),
   389                                         (
float)state->_imu->pos()(1),
   390                                         (
float)state->_imu->pos()(2),
   391                                         (
float)state->_imu->quat()(0),
   392                                         (
float)state->_imu->quat()(1),
   393                                         (
float)state->_imu->quat()(2),
   394                                         (
float)state->_imu->quat()(3));
   398                         std::vector<std::shared_ptr<ov_type::Type>> statevars;
   399                         statevars.push_back(state->_imu->pose()->p());
   400                         statevars.push_back(state->_imu->pose()->q());
   402                         cv::Mat covariance = cv::Mat::eye(6,6, CV_64FC1);
   409                                 Eigen::Matrix<double,6,6> covariance_posori = ov_msckf::StateHelper::get_marginal_covariance(vioManager_->get_state(),statevars);
   410                                 for(
int r=0; r<6; r++) {
   411                                         for(
int c=0; c<6; c++) {
   412                                                 ((
double *)covariance.data)[6*r+c] = covariance_posori(r,c);
   419                                 p =  p * imuLocalTransform_.inverse();
   427                                 if(previousPose_.isIdentity())
   434                                 t = previousPoseInv*p;
   445                                         for (
auto &it_per_id : vioManager_->get_features_SLAM())
   448                                                 pt3d.
x = it_per_id[0];
   449                                                 pt3d.y = it_per_id[1];
   450                                                 pt3d.z = it_per_id[2];
   466                                 UINFO(
"Odom update time = %fs p=%s", timer.
elapsed(), p.prettyPrint().c_str());
   472                 UERROR(
"OpenVINS doesn't work with RGB-D data, stereo images are required!");
   476                 UERROR(
"OpenVINS requires stereo images!");
   480                 UERROR(
"OpenVINS requires stereo images (only one stereo camera and should be calibrated)!");
   484         UERROR(
"RTAB-Map is not built with OpenVINS support! Select another visual odometry approach.");
 
bool imagesAlreadyRectified() const
cv::Point3f RTABMAP_EXP transformPoint(const cv::Point3f &pt, const Transform &transform)
OdometryOpenVINS(const rtabmap::ParametersMap ¶meters=rtabmap::ParametersMap())
virtual void reset(const Transform &initialPose=Transform::getIdentity())
unsigned int framesProcessed() const
std::map< int, cv::Point3f > localMap
const cv::Mat & depthOrRightRaw() const
const std::vector< StereoCameraModel > & stereoCameraModels() const
std::map< std::string, std::string > ParametersMap
std::vector< int > inliersIDs
virtual Odometry::Type getType()
virtual Transform computeTransform(SensorData &image, const Transform &guess=Transform(), OdometryInfo *info=0)
#define UASSERT(condition)
Wrappers of STL for convenient functions. 
const cv::Mat & imageRaw() const
RecoveryProgressState state
GLM_FUNC_DECL detail::tquat< T, P > rotation(detail::tvec3< T, P > const &orig, detail::tvec3< T, P > const &dest)
const Transform & localTransform() const
GLM_FUNC_DECL bool isIdentity(matType< T, P > const &m, T const &epsilon)
virtual void reset(const Transform &initialPose=Transform::getIdentity())
virtual ~OdometryOpenVINS()
ULogger class and convenient macros. 
std::vector< cv::Point2f > newCorners
const Transform & getPose() const