34 #include <opencv2/imgproc/types_c.h>
37 #include <libfovis/fovis.hpp>
55 if(parameters.find(Parameters::kOdomVisKeyFrameThr()) != parameters.end())
57 fovisParameters_.insert(*parameters.find(Parameters::kOdomVisKeyFrameThr()));
118 if(
data.imageRaw().empty() ||
119 data.imageRaw().rows !=
data.depthOrRightRaw().rows ||
120 data.imageRaw().cols !=
data.depthOrRightRaw().cols)
122 UERROR(
"Not supported input!");
126 if(!((
data.cameraModels().size() == 1 &&
data.cameraModels()[0].isValidForReprojection()) ||
127 (
data.stereoCameraModels().size() == 1 &&
data.stereoCameraModels()[0].isValidForProjection())))
129 UERROR(
"Invalid camera model! Mono cameras=%d (reproj=%d), Stereo cameras=%d (reproj=%d)",
130 (
int)
data.cameraModels().size(),
131 data.cameraModels().size() &&
data.cameraModels()[0].isValidForReprojection()?1:0,
132 (
int)
data.stereoCameraModels().size(),
133 data.stereoCameraModels().size() &&
data.stereoCameraModels()[0].isValidForProjection()?1:0);
138 if(
data.imageRaw().type() == CV_8UC3)
140 cv::cvtColor(
data.imageRaw(), gray, CV_BGR2GRAY);
142 else if(
data.imageRaw().type() == CV_8UC1)
144 gray =
data.imageRaw();
148 UFATAL(
"Not supported color type!");
151 fovis::VisualOdometryOptions
options;
152 if(fovis_ == 0 || (
data.cameraModels().size() != 1 && stereoDepth_ == 0))
154 options = fovis::VisualOdometry::getDefaultOptions();
157 options[
"feature-window-size"] =
uValue(
fovisParameters_, Parameters::kOdomFovisFeatureWindowSize(), defaults.at(Parameters::kOdomFovisFeatureWindowSize()));
158 options[
"max-pyramid-level"] =
uValue(
fovisParameters_, Parameters::kOdomFovisMaxPyramidLevel(), defaults.at(Parameters::kOdomFovisMaxPyramidLevel()));
159 options[
"min-pyramid-level"] =
uValue(
fovisParameters_, Parameters::kOdomFovisMinPyramidLevel(), defaults.at(Parameters::kOdomFovisMinPyramidLevel()));
160 options[
"target-pixels-per-feature"] =
uValue(
fovisParameters_, Parameters::kOdomFovisTargetPixelsPerFeature(), defaults.at(Parameters::kOdomFovisTargetPixelsPerFeature()));
161 options[
"fast-threshold"] =
uValue(
fovisParameters_, Parameters::kOdomFovisFastThreshold(), defaults.at(Parameters::kOdomFovisFastThreshold()));
162 options[
"use-adaptive-threshold"] =
uValue(
fovisParameters_, Parameters::kOdomFovisUseAdaptiveThreshold(), defaults.at(Parameters::kOdomFovisUseAdaptiveThreshold()));
163 options[
"fast-threshold-adaptive-gain"] =
uValue(
fovisParameters_, Parameters::kOdomFovisFastThresholdAdaptiveGain(), defaults.at(Parameters::kOdomFovisFastThresholdAdaptiveGain()));
164 options[
"use-homography-initialization"] =
uValue(
fovisParameters_, Parameters::kOdomFovisUseHomographyInitialization(), defaults.at(Parameters::kOdomFovisUseHomographyInitialization()));
168 options[
"use-bucketing"] =
uValue(
fovisParameters_, Parameters::kOdomFovisUseBucketing(), defaults.at(Parameters::kOdomFovisUseBucketing()));
170 options[
"bucket-height"] =
uValue(
fovisParameters_, Parameters::kOdomFovisBucketHeight(), defaults.at(Parameters::kOdomFovisBucketHeight()));
171 options[
"max-keypoints-per-bucket"] =
uValue(
fovisParameters_, Parameters::kOdomFovisMaxKeypointsPerBucket(), defaults.at(Parameters::kOdomFovisMaxKeypointsPerBucket()));
172 options[
"use-image-normalization"] =
uValue(
fovisParameters_, Parameters::kOdomFovisUseImageNormalization(), defaults.at(Parameters::kOdomFovisUseImageNormalization()));
175 options[
"inlier-max-reprojection-error"] =
uValue(
fovisParameters_, Parameters::kOdomFovisInlierMaxReprojectionError(), defaults.at(Parameters::kOdomFovisInlierMaxReprojectionError()));
176 options[
"clique-inlier-threshold"] =
uValue(
fovisParameters_, Parameters::kOdomFovisCliqueInlierThreshold(), defaults.at(Parameters::kOdomFovisCliqueInlierThreshold()));
177 options[
"min-features-for-estimate"] =
uValue(
fovisParameters_, Parameters::kOdomFovisMinFeaturesForEstimate(), defaults.at(Parameters::kOdomFovisMinFeaturesForEstimate()));
178 options[
"max-mean-reprojection-error"] =
uValue(
fovisParameters_, Parameters::kOdomFovisMaxMeanReprojectionError(), defaults.at(Parameters::kOdomFovisMaxMeanReprojectionError()));
179 options[
"use-subpixel-refinement"] =
uValue(
fovisParameters_, Parameters::kOdomFovisUseSubpixelRefinement(), defaults.at(Parameters::kOdomFovisUseSubpixelRefinement()));
180 options[
"feature-search-window"] =
uValue(
fovisParameters_, Parameters::kOdomFovisFeatureSearchWindow(), defaults.at(Parameters::kOdomFovisFeatureSearchWindow()));
181 options[
"update-target-features-with-refined"] =
uValue(
fovisParameters_, Parameters::kOdomFovisUpdateTargetFeaturesWithRefined(), defaults.at(Parameters::kOdomFovisUpdateTargetFeaturesWithRefined()));
184 options[
"stereo-require-mutual-match"] =
uValue(
fovisParameters_, Parameters::kOdomFovisStereoRequireMutualMatch(), defaults.at(Parameters::kOdomFovisStereoRequireMutualMatch()));
185 options[
"stereo-max-dist-epipolar-line"] =
uValue(
fovisParameters_, Parameters::kOdomFovisStereoMaxDistEpipolarLine(), defaults.at(Parameters::kOdomFovisStereoMaxDistEpipolarLine()));
186 options[
"stereo-max-refinement-displacement"] =
uValue(
fovisParameters_, Parameters::kOdomFovisStereoMaxRefinementDisplacement(), defaults.at(Parameters::kOdomFovisStereoMaxRefinementDisplacement()));
187 options[
"stereo-max-disparity"] =
uValue(
fovisParameters_, Parameters::kOdomFovisStereoMaxDisparity(), defaults.at(Parameters::kOdomFovisStereoMaxDisparity()));
190 fovis::DepthSource * depthSource = 0;
194 if(
data.cameraModels().size() == 1)
197 fovis::CameraIntrinsicsParameters rgb_params;
198 memset(&rgb_params, 0,
sizeof(fovis::CameraIntrinsicsParameters));
199 rgb_params.width =
data.cameraModels()[0].imageWidth();
200 rgb_params.height =
data.cameraModels()[0].imageHeight();
202 rgb_params.fx =
data.cameraModels()[0].fx();
203 rgb_params.fy =
data.cameraModels()[0].fy();
204 rgb_params.cx =
data.cameraModels()[0].cx()==0.0?double(rgb_params.width) / 2.0:
data.cameraModels()[0].cx();
205 rgb_params.cy =
data.cameraModels()[0].cy()==0.0?double(rgb_params.height) / 2.0:
data.cameraModels()[0].cy();
206 localTransform =
data.cameraModels()[0].localTransform();
210 UINFO(
"Init rgbd fovis: %dx%d fx=%f fy=%f cx=%f cy=%f", rgb_params.width, rgb_params.height, rgb_params.fx, rgb_params.fy, rgb_params.cx, rgb_params.cy);
211 rect_ =
new fovis::Rectification(rgb_params);
216 depthImage_ =
new fovis::DepthImage(rgb_params, rgb_params.width, rgb_params.height);
219 if(
data.depthRaw().type() == CV_32FC1)
221 depth =
data.depthRaw();
222 for(
int i=0;
i<depth.rows; ++
i)
224 for(
int j=0;
j<depth.cols; ++
j)
226 float &
d = depth.at<
float>(
i,
j);
234 else if(
data.depthRaw().type() == CV_16UC1)
236 depth = cv::Mat(
data.depthRaw().size(), CV_32FC1);
241 float d =
float(
data.depthRaw().at<
unsigned short>(
i,
j))/1000.0f;
242 depth.at<
float>(
i,
j) =
d==0.0
f?NAN:
d;
248 UFATAL(
"Unknown depth format!");
250 depthImage_->setDepthImage((
float*)depth.data);
251 depthSource = depthImage_;
253 else if(
data.stereoCameraModels().size() == 1)
257 fovis::CameraIntrinsicsParameters left_parameters;
258 left_parameters.width =
data.stereoCameraModels()[0].left().imageWidth();
259 left_parameters.height =
data.stereoCameraModels()[0].left().imageHeight();
260 left_parameters.fx =
data.stereoCameraModels()[0].left().fx();
261 left_parameters.fy =
data.stereoCameraModels()[0].left().fy();
262 left_parameters.cx =
data.stereoCameraModels()[0].left().cx()==0.0?double(left_parameters.width) / 2.0:
data.stereoCameraModels()[0].left().cx();
263 left_parameters.cy =
data.stereoCameraModels()[0].left().cy()==0.0?double(left_parameters.height) / 2.0:
data.stereoCameraModels()[0].left().cy();
264 localTransform =
data.stereoCameraModels()[0].localTransform();
268 UINFO(
"Init stereo fovis: %dx%d fx=%f fy=%f cx=%f cy=%f", left_parameters.width, left_parameters.height, left_parameters.fx, left_parameters.fy, left_parameters.cx, left_parameters.cy);
269 rect_ =
new fovis::Rectification(left_parameters);
272 if(stereoCalib_ == 0)
275 fovis::CameraIntrinsicsParameters right_parameters;
276 right_parameters.width =
data.stereoCameraModels()[0].right().imageWidth();
277 right_parameters.height =
data.stereoCameraModels()[0].right().imageHeight();
278 right_parameters.fx =
data.stereoCameraModels()[0].right().fx();
279 right_parameters.fy =
data.stereoCameraModels()[0].right().fy();
280 right_parameters.cx =
data.stereoCameraModels()[0].right().cx()==0.0?double(right_parameters.width) / 2.0:
data.stereoCameraModels()[0].right().cx();
281 right_parameters.cy =
data.stereoCameraModels()[0].right().cy()==0.0?double(right_parameters.height) / 2.0:
data.stereoCameraModels()[0].right().cy();
285 fovis::StereoCalibrationParameters stereo_parameters;
286 stereo_parameters.left_parameters = left_parameters;
287 stereo_parameters.right_parameters = right_parameters;
288 stereo_parameters.right_to_left_rotation[0] = 1.0;
289 stereo_parameters.right_to_left_rotation[1] = 0.0;
290 stereo_parameters.right_to_left_rotation[2] = 0.0;
291 stereo_parameters.right_to_left_rotation[3] = 0.0;
292 stereo_parameters.right_to_left_translation[0] = -
data.stereoCameraModels()[0].baseline();
293 stereo_parameters.right_to_left_translation[1] = 0.0;
294 stereo_parameters.right_to_left_translation[2] = 0.0;
296 stereoCalib_ =
new fovis::StereoCalibration(stereo_parameters);
299 if(stereoDepth_ == 0)
301 stereoDepth_ =
new fovis::StereoDepth(stereoCalib_,
options);
303 if(
data.rightRaw().type() == CV_8UC3)
305 cv::cvtColor(
data.rightRaw(), right, CV_BGR2GRAY);
307 else if(
data.rightRaw().type() == CV_8UC1)
309 right =
data.rightRaw();
313 UFATAL(
"Not supported color type!");
315 stereoDepth_->setRightImage(right.data);
316 depthSource = stereoDepth_;
321 fovis_ =
new fovis::VisualOdometry(rect_,
options);
325 fovis_->processFrame(gray.data, depthSource);
331 fovis::MotionEstimateStatusCode statusCode = fovis_->getMotionEstimator()->getMotionEstimateStatus();
332 if(statusCode > fovis::SUCCESS)
334 UWARN(
"Fovis error status: %s", fovis::MotionEstimateStatusCodeStrings[statusCode]);
337 covariance = cv::Mat::eye(6,6, CV_64FC1)*9999.0;
344 covariance = cv::Mat::eye(6,6, CV_64FC1)*9999.0;
349 const Eigen::MatrixXd& cov = fovis_->getMotionEstimator()->getMotionEstimateCov();
350 if(cov.cols() == 6 && cov.rows() == 6 && cov(0,0) > 0.0)
352 covariance = cv::Mat::eye(6,6, CV_64FC1);
353 memcpy(covariance.data, cov.data(), 36*
sizeof(
double));
358 if(!
t.isNull() && !
t.isIdentity() && !localTransform.
isIdentity() && !localTransform.
isNull())
367 t = localTransform *
t * localTransform.
inverse();
375 info->keyFrameAdded = fovis_->getChangeReferenceFrames();
376 info->features = fovis_->getTargetFrame()->getNumDetectedKeypoints();
377 info->reg.matches = fovis_->getMotionEstimator()->getNumMatches();
378 info->reg.inliers = fovis_->getMotionEstimator()->getNumInliers();
379 info->reg.covariance = covariance;
383 const fovis::FeatureMatch *
matches = fovis_->getMotionEstimator()->getMatches();
384 int numMatches = fovis_->getMotionEstimator()->getNumMatches();
387 info->refCorners.resize(numMatches);
388 info->newCorners.resize(numMatches);
389 info->cornerInliers.resize(numMatches);
391 for (
int i = 0;
i < numMatches; ++
i)
395 info->newCorners[
i].x =
matches[
i].target_keypoint->base_uv[0];
396 info->newCorners[
i].y =
matches[
i].target_keypoint->base_uv[1];
397 info->cornerInliers[oi++] =
i;
399 info->cornerInliers.resize(oi);
404 UINFO(
"Odom update time = %fs status=%s",
timer.
elapsed(), fovis::MotionEstimateStatusCodeStrings[statusCode]);
407 UERROR(
"RTAB-Map is not built with FOVIS support! Select another visual odometry approach.");