36 #include <opencv2/imgproc/types_c.h>    39 #include <estimator/estimator.h>    40 #include <estimator/parameters.h>    41 #include <camodocal/camera_models/PinholeCamera.h>    42 #include <camodocal/camera_models/EquidistantCamera.h>    43 #include <utility/visualization.h>    49 class VinsEstimator: 
public Estimator
    53                         const Transform & imuLocalTransform,
    54                         const StereoCameraModel & 
model,
    55                         bool rectified) : Estimator()
    61                 UASSERT(NUM_OF_CAM >= 1 && NUM_OF_CAM <=2);
    63                 if( (NUM_OF_CAM == 2 && model.left().D_raw().cols == 4 && model.right().D_raw().cols == 4) ||
    64                         (NUM_OF_CAM == 1 && model.left().D_raw().cols == 4))
    66                         UWARN(
"Overwriting VINS camera calibration config with received pinhole model... rectified=%d", rectified?1:0);
    67                         featureTracker.m_camera.clear();
    69                         camodocal::PinholeCameraPtr camera( 
new camodocal::PinholeCamera );
    70                         camodocal::PinholeCamera::Parameters 
params(
    72                                         model.left().imageWidth(), model.left().imageHeight(),
    73                                         rectified?0:model.left().D_raw().at<
double>(0,0),
    74                                         rectified?0:model.left().D_raw().at<
double>(0,1),
    75                                         rectified?0:model.left().D_raw().at<
double>(0,2),
    76                                         rectified?0:model.left().D_raw().at<
double>(0,3),
    77                                         rectified?model.left().fx():model.left().K_raw().at<
double>(0,0),
    78                                         rectified?model.left().fy():model.left().K_raw().at<
double>(1,1),
    79                                         rectified?model.left().cx():model.left().K_raw().at<
double>(0,2),
    80                                         rectified?model.left().cy():model.left().K_raw().at<
double>(1,2));
    81                         camera->setParameters(params);
    82                         featureTracker.m_camera.push_back(camera);
    86                                 camodocal::PinholeCameraPtr camera( 
new camodocal::PinholeCamera );
    87                                 camodocal::PinholeCamera::Parameters 
params(
    89                                                 model.right().imageWidth(), model.right().imageHeight(),
    90                                                 rectified?0:model.right().D_raw().at<
double>(0,0),
    91                                                 rectified?0:model.right().D_raw().at<
double>(0,1),
    92                                                 rectified?0:model.right().D_raw().at<
double>(0,2),
    93                                                 rectified?0:model.right().D_raw().at<
double>(0,3),
    94                                                 rectified?model.right().fx():model.right().K_raw().at<
double>(0,0),
    95                                                 rectified?model.right().fy():model.right().K_raw().at<
double>(1,1),
    96                                                 rectified?model.right().cx():model.right().K_raw().at<
double>(0,2),
    97                                                 rectified?model.right().cy():model.right().K_raw().at<
double>(1,2));
    98                                 camera->setParameters(params);
    99                                 featureTracker.m_camera.push_back(camera);
   104                         UWARN(
"Images are rectified but received calibration cannot be "   105                                         "used, make sure calibration in config file doesn't have "   106                                         "distortion or send raw images to VINS odometry.");
   107                         if(!featureTracker.m_camera.empty())
   109                                 if(featureTracker.m_camera.front()->imageWidth() != model.left().imageWidth() ||
   110                                    featureTracker.m_camera.front()->imageHeight() != model.left().imageHeight())
   112                                         UERROR(
"Received images don't have same size (%dx%d) than in the config file (%dx%d)!",
   113                                                         model.left().imageWidth(),
   114                                                         model.left().imageHeight(),
   115                                                         featureTracker.m_camera.front()->imageWidth(),
   116                                                         featureTracker.m_camera.front()->imageHeight());
   121                 Transform imuCam0 = imuLocalTransform.inverse() * model.localTransform();
   123                 tic[0] = Vector3d(imuCam0.x(), imuCam0.y(), imuCam0.z());
   124                 ric[0] = imuCam0.toEigen4d().block<3,3>(0,0);
   131                                 cam0cam1 = Transform(
   132                                                 1, 0, 0, model.baseline(),
   138                                 cam0cam1 = model.stereoTransform().inverse();
   141                         Transform imuCam1 = imuCam0 * cam0cam1;
   143                         tic[1] = Vector3d(imuCam1.x(), imuCam1.y(), imuCam1.z());
   144                         ric[1] = imuCam1.toEigen4d().block<3,3>(0,0);
   147                 for (
int i = 0; i < NUM_OF_CAM; i++)
   149                         cout << 
" exitrinsic cam " << i << endl  << ric[i] << endl << tic[i].transpose() << endl;
   151                 f_manager.setRic(ric);
   152                 ProjectionTwoFrameOneCamFactor::sqrt_info = FOCAL_LENGTH / 1.5 * Matrix2d::Identity();
   153                 ProjectionTwoFrameTwoCamFactor::sqrt_info = FOCAL_LENGTH / 1.5 * Matrix2d::Identity();
   154                 ProjectionOneFrameTwoCamFactor::sqrt_info = FOCAL_LENGTH / 1.5 * Matrix2d::Identity();
   157                 cout << 
"set g " << g.transpose() << endl;
   161         void inputImage(
double t, 
const cv::Mat &_img, 
const cv::Mat &_img1)
   164             map<int, vector<pair<int, Eigen::Matrix<double, 7, 1>>>> featureFrame;
   165             TicToc featureTrackerTime;
   167                 featureFrame = featureTracker.trackImage(t, _img);
   169                 featureFrame = featureTracker.trackImage(t, _img, _img1);
   184                 featureBuf.push(make_pair(t, featureFrame));
   187                 processMeasurements();
   188                 UDEBUG(
"VINS process time: %f", processTime.toc());
   194         void inputIMU(
double t, 
const Vector3d &linearAcceleration, 
const Vector3d &angularVelocity)
   197                 accBuf.push(make_pair(t, linearAcceleration));
   198                 gyrBuf.push(make_pair(t, angularVelocity));
   202                 fastPredictIMU(t, linearAcceleration, angularVelocity);
   208         void processMeasurements()
   213                         pair<double, map<int, vector<pair<int, Eigen::Matrix<double, 7, 1> > > > > feature;
   214                         vector<pair<double, Eigen::Vector3d>> accVector, gyrVector;
   215                         if(!featureBuf.empty())
   217                                 feature = featureBuf.front();
   218                                 curTime = feature.first + td;
   221                                         if (!((!USE_IMU  || IMUAvailable(feature.first + td))))
   226                                                 printf(
"wait for imu ... \n");
   235                                         getIMUInterval(prevTime, curTime, accVector, gyrVector);
   242                                         if(!initFirstPoseFlag)
   243                                                 initFirstIMUPose(accVector);
   244                                         UDEBUG(
"accVector.size() = %d", accVector.size());
   245                                         for(
size_t i = 0; i < accVector.size(); i++)
   249                                                         dt = accVector[i].first - prevTime;
   250                                                 else if (i == accVector.size() - 1)
   251                                                         dt = curTime - accVector[i - 1].first;
   253                                                         dt = accVector[i].first - accVector[i - 1].first;
   254                                                 processIMU(accVector[i].first, dt, accVector[i].second, gyrVector[i].second);
   258                                 processImage(feature.second, feature.first);
   262                                 printStatistics(*
this, 0);
   297         std::string configFilename;
   298         Parameters::parse(parameters, Parameters::kOdomVINSConfigPath(), configFilename);
   299         if(configFilename.empty())
   301                 UERROR(
"VINS config file is empty (%s=%s)!",
   302                                 Parameters::kOdomVINSConfigPath().c_str(),
   303                                 Parameters::kOdomVINSConfigPath().c_str());
   315         delete vinsEstimator_;
   325                 delete vinsEstimator_;
   327                 previousPose_.setIdentity();
   329                 previousLocalTransform_.setNull();
   331         initGravity_ = 
false;
   345         if(USE_IMU!=0 && !data.
imu().
empty())
   347                 double t = data.
stamp();
   354                 Vector3d acc(dx, dy, dz);
   355                 Vector3d gyr(rx, ry, rz);
   359                 if(vinsEstimator_ != 0)
   361                         vinsEstimator_->inputIMU(t, acc, gyr);
   365                         lastImu_ = data.
imu();
   366                         UWARN(
"Waiting an image for initialization...");
   372                 if(USE_IMU==1 && lastImu_.localTransform().isNull())
   374                         UWARN(
"Waiting IMU for initialization...");
   377                 if(vinsEstimator_ == 0)
   380                         vinsEstimator_ = 
new VinsEstimator(
   389                 if(data.
imageRaw().type() == CV_8UC3)
   391                         cv::cvtColor(data.
imageRaw(), left, CV_BGR2GRAY);
   393                 else if(data.
imageRaw().type() == CV_8UC1)
   399                         UFATAL(
"Not supported color type!");
   401                 if(data.
rightRaw().type() == CV_8UC3)
   403                         cv::cvtColor(data.
rightRaw(), right, CV_BGR2GRAY);
   405                 else if(data.
rightRaw().type() == CV_8UC1)
   411                         UFATAL(
"Not supported color type!");
   414                 vinsEstimator_->inputImage(data.
stamp(), left, right);
   416                 if(vinsEstimator_->solver_flag == Estimator::NON_LINEAR)
   419                         tmp_Q = Quaterniond(vinsEstimator_->Rs[WINDOW_SIZE]);
   421                                         vinsEstimator_->Ps[WINDOW_SIZE].x(),
   422                                         vinsEstimator_->Ps[WINDOW_SIZE].y(),
   423                                         vinsEstimator_->Ps[WINDOW_SIZE].z(),
   431                                 if(!lastImu_.localTransform().isNull())
   433                                         p = p * lastImu_.localTransform().
inverse();
   442                                 if(previousPose_.isIdentity())
   449                                 t = previousPoseInv*p;
   460                                         for (
auto &it_per_id : vinsEstimator_->f_manager.feature)
   463                                                 used_num = it_per_id.feature_per_frame.
size();
   464                                                 if (!(used_num >= 2 && it_per_id.start_frame < WINDOW_SIZE - 2))
   466                                                 if (it_per_id.start_frame > WINDOW_SIZE * 3.0 / 4.0 || it_per_id.solve_flag != 1)
   468                                                 int imu_i = it_per_id.start_frame;
   469                                                 Vector3d pts_i = it_per_id.feature_per_frame[it_per_id.feature_per_frame.size()-1].point * it_per_id.estimated_depth;
   470                                                 Vector3d w_pts_i = vinsEstimator_->Rs[imu_i] * (vinsEstimator_->ric[0] * pts_i + vinsEstimator_->tic[0]) + vinsEstimator_->Ps[imu_i];
   477                                                 info->
localMap.insert(std::make_pair(it_per_id.feature_id, p));
   482                                                         data.
stereoCameraModels()[0].left().reproject(pts_i(0), pts_i(1), pts_i(2), pt.x, pt.y);
   490                                 UINFO(
"Odom update time = %fs p=%s", timer.
elapsed(), p.prettyPrint().c_str());
   495                         UWARN(
"VINS not yet initialized... waiting to get enough IMU messages");
   500                 UERROR(
"VINS-Fusion doesn't work with RGB-D data, stereo images are required!");
   504                 UERROR(
"VINS-Fusion requires stereo images!");
   508                 UERROR(
"VINS-Fusion requires stereo images (and only one stereo camera with valid calibration)!");
   511         UERROR(
"RTAB-Map is not built with VINS support! Select another visual odometry approach.");
 
static std::string homeDir()
static bool parse(const ParametersMap ¶meters, const std::string &key, bool &value)
bool imagesAlreadyRectified() const
cv::Point3f RTABMAP_EXP transformPoint(const cv::Point3f &pt, const Transform &transform)
virtual void reset(const Transform &initialPose=Transform::getIdentity())
unsigned int framesProcessed() const
std::map< int, cv::Point3f > localMap
const cv::Mat & depthOrRightRaw() const
const std::vector< StereoCameraModel > & stereoCameraModels() const
std::map< std::string, std::string > ParametersMap
std::vector< int > inliersIDs
virtual Transform computeTransform(SensorData &image, const Transform &guess=Transform(), OdometryInfo *info=0)
virtual Odometry::Type getType()
virtual void reset(const Transform &initialPose=Transform::getIdentity())
#define UASSERT(condition)
Wrappers of STL for convenient functions. 
const cv::Mat & imageRaw() const
std::string UTILITE_EXP uReplaceChar(const std::string &str, char before, char after)
GLM_FUNC_DECL detail::tquat< T, P > rotation(detail::tvec3< T, P > const &orig, detail::tvec3< T, P > const &dest)
GLM_FUNC_DECL bool isIdentity(matType< T, P > const &m, T const &epsilon)
const cv::Vec3d & angularVelocity() const
ULogger class and convenient macros. 
OdometryVINS(const rtabmap::ParametersMap ¶meters=rtabmap::ParametersMap())
std::vector< cv::Point2f > newCorners
const cv::Vec3d linearAcceleration() const
const Transform & getPose() const