36 #include <opencv2/imgproc/types_c.h> 38 #ifdef RTABMAP_OPENVINS 39 #include "core/VioManager.h" 40 #include "core/VioManagerOptions.h" 41 #include "core/RosVisualizer.h" 42 #include "utils/dataset_reader.h" 43 #include "utils/parse_ros.h" 44 #include "utils/sensor_data.h" 45 #include "state/State.h" 46 #include "types/Type.h" 53 #ifdef RTABMAP_OPENVINS
64 #ifdef RTABMAP_OPENVINS 72 #ifdef RTABMAP_OPENVINS 77 previousPose_.setIdentity();
78 previousLocalTransform_.setNull();
92 #ifdef RTABMAP_OPENVINS 98 imuBuffer_.insert(std::make_pair(data.
stamp(), data.
imu()));
104 if(imuBuffer_.empty())
106 UWARN(
"Waiting IMU for initialization...");
111 UINFO(
"OpenVINS Initialization");
114 ov_msckf::VioManagerOptions
params;
130 params.state_options.num_cameras = 2;
137 if( params.state_options.feat_rep_msckf == LandmarkRepresentation::Representation::UNKNOWN ||
138 params.state_options.feat_rep_slam == LandmarkRepresentation::Representation::UNKNOWN ||
139 params.state_options.feat_rep_aruco == LandmarkRepresentation::Representation::UNKNOWN)
141 printf(
RED "VioManager(): invalid feature representation specified:\n" RESET);
142 printf(
RED "\t- GLOBAL_3D\n" RESET);
143 printf(
RED "\t- GLOBAL_FULL_INVERSE_DEPTH\n" RESET);
144 printf(
RED "\t- ANCHORED_3D\n" RESET);
145 printf(
RED "\t- ANCHORED_FULL_INVERSE_DEPTH\n" RESET);
146 printf(
RED "\t- ANCHORED_MSCKF_INVERSE_DEPTH\n" RESET);
147 printf(
RED "\t- ANCHORED_INVERSE_DEPTH_SINGLE\n" RESET);
148 std::exit(EXIT_FAILURE);
190 params.use_stereo =
true;
192 params.use_aruco =
false;
222 params.camera_fisheye.insert(std::make_pair(0, fisheye));
223 params.camera_fisheye.insert(std::make_pair(1, fisheye));
225 Eigen::VectorXd camLeft(8), camRight(8);
268 params.camera_intrinsics.insert(std::make_pair(0, camLeft));
269 params.camera_intrinsics.insert(std::make_pair(1, camRight));
271 const IMU & imu = imuBuffer_.begin()->second;
288 Eigen::Matrix4d cam0_eigen = imuCam0.
toEigen4d();
289 Eigen::Matrix4d cam1_eigen = imuCam1.toEigen4d();
290 Eigen::Matrix<double,7,1> cam_eigen0;
291 cam_eigen0.block(0,0,4,1) = rot_2_quat(cam0_eigen.block(0,0,3,3).transpose());
292 cam_eigen0.block(4,0,3,1) = -cam0_eigen.block(0,0,3,3).transpose()*cam0_eigen.block(0,3,3,1);
293 Eigen::Matrix<double,7,1> cam_eigen1;
294 cam_eigen1.block(0,0,4,1) = rot_2_quat(cam1_eigen.block(0,0,3,3).transpose());
295 cam_eigen1.block(4,0,3,1) = -cam1_eigen.block(0,0,3,3).transpose()*cam1_eigen.block(0,3,3,1);
296 params.camera_extrinsics.insert(std::make_pair(0, cam_eigen0));
297 params.camera_extrinsics.insert(std::make_pair(1, cam_eigen1));
302 vioManager_ =
new ov_msckf::VioManager(params);
307 if(data.
imageRaw().type() == CV_8UC3)
309 cv::cvtColor(data.
imageRaw(), left, CV_BGR2GRAY);
311 else if(data.
imageRaw().type() == CV_8UC1)
317 UFATAL(
"Not supported color type!");
319 if(data.
rightRaw().type() == CV_8UC3)
321 cv::cvtColor(data.
rightRaw(), right, CV_BGR2GRAY);
323 else if(data.
rightRaw().type() == CV_8UC1)
329 UFATAL(
"Not supported color type!");
333 ov_core::CameraData message;
334 message.timestamp = data.
stamp();
335 message.sensor_ids.push_back(0);
336 message.sensor_ids.push_back(1);
337 message.images.push_back(left);
338 message.images.push_back(right);
339 message.masks.push_back(cv::Mat::zeros(left.size(), CV_8UC1));
340 message.masks.push_back(cv::Mat::zeros(right.size(), CV_8UC1));
343 vioManager_->feed_measurement_camera(message);
346 double lastIMUstamp = 0.0;
347 while(!imuBuffer_.empty())
349 std::map<double, IMU>::iterator iter = imuBuffer_.begin();
352 ov_core::ImuData message;
353 message.timestamp = iter->first;
354 message.wm << iter->second.angularVelocity().val[0], iter->second.angularVelocity().val[1], iter->second.angularVelocity().val[2];
355 message.am << iter->second.linearAcceleration().val[0], iter->second.linearAcceleration().val[1], iter->second.linearAcceleration().val[2];
357 UDEBUG(
"IMU update stamp=%f", message.timestamp);
360 vioManager_->feed_measurement_imu(message);
362 lastIMUstamp = iter->first;
364 imuBuffer_.erase(iter);
366 if(lastIMUstamp > data.
stamp())
372 if(vioManager_->initialized())
375 std::shared_ptr<ov_msckf::State>
state = vioManager_->get_state();
377 if(state->_timestamp != data.
stamp())
379 UWARN(
"OpenVINS: Stamp of the current state %f is not the same " 380 "than last image processed %f (last IMU stamp=%f). There could be " 381 "a synchronization issue between camera and IMU. ",
388 (
float)state->_imu->pos()(0),
389 (
float)state->_imu->pos()(1),
390 (
float)state->_imu->pos()(2),
391 (
float)state->_imu->quat()(0),
392 (
float)state->_imu->quat()(1),
393 (
float)state->_imu->quat()(2),
394 (
float)state->_imu->quat()(3));
398 std::vector<std::shared_ptr<ov_type::Type>> statevars;
399 statevars.push_back(state->_imu->pose()->p());
400 statevars.push_back(state->_imu->pose()->q());
402 cv::Mat covariance = cv::Mat::eye(6,6, CV_64FC1);
409 Eigen::Matrix<double,6,6> covariance_posori = ov_msckf::StateHelper::get_marginal_covariance(vioManager_->get_state(),statevars);
410 for(
int r=0; r<6; r++) {
411 for(
int c=0; c<6; c++) {
412 ((
double *)covariance.data)[6*r+c] = covariance_posori(r,c);
419 p = p * imuLocalTransform_.inverse();
427 if(previousPose_.isIdentity())
434 t = previousPoseInv*p;
445 for (
auto &it_per_id : vioManager_->get_features_SLAM())
448 pt3d.
x = it_per_id[0];
449 pt3d.y = it_per_id[1];
450 pt3d.z = it_per_id[2];
466 UINFO(
"Odom update time = %fs p=%s", timer.
elapsed(), p.prettyPrint().c_str());
472 UERROR(
"OpenVINS doesn't work with RGB-D data, stereo images are required!");
476 UERROR(
"OpenVINS requires stereo images!");
480 UERROR(
"OpenVINS requires stereo images (only one stereo camera and should be calibrated)!");
484 UERROR(
"RTAB-Map is not built with OpenVINS support! Select another visual odometry approach.");
bool imagesAlreadyRectified() const
cv::Point3f RTABMAP_EXP transformPoint(const cv::Point3f &pt, const Transform &transform)
OdometryOpenVINS(const rtabmap::ParametersMap ¶meters=rtabmap::ParametersMap())
virtual void reset(const Transform &initialPose=Transform::getIdentity())
unsigned int framesProcessed() const
std::map< int, cv::Point3f > localMap
const cv::Mat & depthOrRightRaw() const
const std::vector< StereoCameraModel > & stereoCameraModels() const
std::map< std::string, std::string > ParametersMap
std::vector< int > inliersIDs
virtual Odometry::Type getType()
virtual Transform computeTransform(SensorData &image, const Transform &guess=Transform(), OdometryInfo *info=0)
#define UASSERT(condition)
Wrappers of STL for convenient functions.
const cv::Mat & imageRaw() const
RecoveryProgressState state
GLM_FUNC_DECL detail::tquat< T, P > rotation(detail::tvec3< T, P > const &orig, detail::tvec3< T, P > const &dest)
const Transform & localTransform() const
GLM_FUNC_DECL bool isIdentity(matType< T, P > const &m, T const &epsilon)
virtual void reset(const Transform &initialPose=Transform::getIdentity())
virtual ~OdometryOpenVINS()
ULogger class and convenient macros.
std::vector< cv::Point2f > newCorners
const Transform & getPose() const