37 #include <QApplication> 45 "rtabmap-report [\"Statistic/Id\"] [--latex] [--kitti] [--scale] [--poses] path\n" 46 " path Directory containing rtabmap databases or path of a database.\n" 47 " --latex Print table formatted in LaTeX with results.\n" 48 " --kitti Compute error based on KITTI benchmark.\n" 49 " --scale Find the best scale for the map against the ground truth\n" 50 " and compute error based on the scaled path.\n" 51 " --poses Export poses to [path]_poses.txt, ground truth to [path]_gt.txt\n" 52 " and valid ground truth indices to [path]_indices.txt \n\n");
56 int main(
int argc,
char * argv[])
66 QApplication
app(argc, argv);
68 bool outputLatex =
false;
69 bool outputScaled =
false;
70 bool outputPoses =
false;
71 bool outputKittiError =
false;
72 std::map<std::string, UPlot*> figures;
73 for(
int i=1; i<argc-1; ++i)
75 if(strcmp(argv[i],
"--latex") == 0)
79 else if(strcmp(argv[i],
"--kitti") == 0)
81 outputKittiError =
true;
83 else if(strcmp(argv[i],
"--scale") == 0)
87 else if(strcmp(argv[i],
"--poses") == 0)
93 std::string figureTitle = argv[i];
94 printf(
"Plot %s\n", figureTitle.c_str());
96 fig->setTitle(figureTitle.c_str());
97 fig->setXLabel(
"Time (s)");
98 figures.insert(std::make_pair(figureTitle, fig));
102 std::string path = argv[argc-1];
105 std::string fileName;
106 std::list<std::string> paths;
107 paths.push_back(path);
108 std::vector<std::map<std::string, std::vector<float> > > outputLatexStatistics;
109 std::map<std::string, std::vector<float> > outputLatexStatisticsMap;
110 bool odomRAMSet =
false;
111 std::set<std::string> topDirs;
114 std::string currentPath = paths.front();
117 bool currentPathIsDatabase =
false;
122 currentPathIsDatabase=
true;
123 printf(
"Database: %s\n", currentPath.c_str());
130 std::list<std::string> subDirs;
131 if(!currentPathIsDatabase)
133 printf(
"Directory: %s\n", currentPath.c_str());
134 std::list<std::string> fileNames = currentDir.
getFileNames();
137 for(std::list<std::string>::iterator iter = fileNames.begin(); iter!=fileNames.end(); ++iter)
139 topDirs.insert(currentPath+
"/"+*iter);
144 if(topDirs.find(currentPath) != topDirs.end())
146 if(outputLatexStatisticsMap.size())
148 outputLatexStatistics.push_back(outputLatexStatisticsMap);
149 outputLatexStatisticsMap.clear();
155 while(currentPathIsDatabase || !(fileName = currentDir.
getNextFileName()).empty())
159 std::string filePath;
160 if(currentPathIsDatabase)
162 filePath = currentPath;
178 std::map<int, std::pair<std::map<std::string, float>,
double> > stats = driver->
getAllStatistics();
179 std::map<int, Transform> odomPoses, gtPoses;
180 std::vector<float> cameraTime;
181 cameraTime.reserve(ids.size());
182 std::vector<float> odomTime;
183 odomTime.reserve(ids.size());
184 std::vector<float> slamTime;
185 slamTime.reserve(ids.size());
188 float maxOdomRAM = -1;
189 float maxMapRAM = -1;
190 std::map<std::string, UPlotCurve*> curves;
191 std::map<std::string, double> firstStamps;
192 for(std::map<std::string, UPlot*>::iterator iter=figures.begin(); iter!=figures.end(); ++iter)
194 curves.insert(std::make_pair(iter->first, iter->second->addCurve(filePath.c_str())));
197 for(std::set<int>::iterator iter=ids.begin(); iter!=ids.end(); ++iter)
204 std::vector<float> v;
205 if(driver->
getNodeInfo(*iter, p, m, w, l, s, gt, v, gps))
207 odomPoses.insert(std::make_pair(*iter, p));
210 gtPoses.insert(std::make_pair(*iter, gt));
214 const std::map<std::string, float> & stat = stats.at(*iter).first;
215 if(
uContains(stat, Statistics::kGtTranslational_rmse()))
217 rmse = stat.at(Statistics::kGtTranslational_rmse());
218 if(maxRMSE==-1 || maxRMSE < rmse)
223 if(
uContains(stat, std::string(
"Camera/TotalTime/ms")))
225 cameraTime.push_back(stat.at(std::string(
"Camera/TotalTime/ms")));
227 if(
uContains(stat, std::string(
"Odometry/TotalTime/ms")))
229 odomTime.push_back(stat.at(std::string(
"Odometry/TotalTime/ms")));
232 if(
uContains(stat, std::string(
"RtabmapROS/TotalTime/ms")))
234 if(w>=0 || stat.at(
"RtabmapROS/TotalTime/ms") > 10.0f)
236 slamTime.push_back(stat.at(
"RtabmapROS/TotalTime/ms"));
239 else if(
uContains(stat, Statistics::kTimingTotal()))
241 if(w>=0 || stat.at(Statistics::kTimingTotal()) > 10.0f)
243 slamTime.push_back(stat.at(Statistics::kTimingTotal()));
247 if(
uContains(stat, std::string(Statistics::kMemoryRAM_usage())))
249 float ram = stat.at(Statistics::kMemoryRAM_usage());
250 if(maxMapRAM==-1 || maxMapRAM < ram)
255 if(
uContains(stat, std::string(
"Odometry/RAM_usage/MB")))
257 float ram = stat.at(
"Odometry/RAM_usage/MB");
258 if(maxOdomRAM==-1 || maxOdomRAM < ram)
264 for(std::map<std::string, UPlotCurve*>::iterator jter=curves.begin(); jter!=curves.end(); ++jter)
270 firstStamps.insert(std::make_pair(jter->first, s));
272 float x = s - firstStamps.at(jter->first);
273 float y = stat.at(jter->first);
274 jter->second->addValue(x,y);
281 std::multimap<int, Link> links;
283 std::multimap<int, Link> loopClosureLinks;
284 for(std::multimap<int, Link>::iterator jter=links.begin(); jter!=links.end(); ++jter)
287 graph::findLink(loopClosureLinks, jter->second.from(), jter->second.to()) == loopClosureLinks.end())
289 loopClosureLinks.insert(*jter);
293 float bestScale = 1.0f;
295 float bestRMSEAng = -1;
296 float bestVoRMSE = -1;
298 float kitti_t_err = 0.0f;
299 float kitti_r_err = 0.0f;
302 std::map<int, Transform> posesOut;
303 std::multimap<int, Link> linksOut;
304 int firstId = *ids.begin();
308 std::map<int, Transform> poses = optimizer->
optimize(firstId, posesOut, linksOut);
312 UWARN(
"Optimization failed! Try incremental optimization...");
316 UERROR(
"Incremental optimization also failed! Only original RMSE will be shown.");
321 UWARN(
"Incremental optimization succeeded!");
327 std::map<int, Transform> groundTruth;
328 for(std::map<int, Transform>::const_iterator iter=poses.begin(); iter!=poses.end(); ++iter)
330 if(gtPoses.find(iter->first) != gtPoses.end())
332 groundTruth.insert(*gtPoses.find(iter->first));
336 outputScaled = outputScaled && groundTruth.size();
339 std::map<int, Transform> scaledPoses;
340 std::map<int, Transform> scaledOdomPoses;
342 for(std::map<int, Transform>::iterator iter=poses.begin(); iter!=poses.end(); ++iter)
348 scaledPoses.insert(std::make_pair(iter->first, t));
350 UASSERT(posesOut.find(iter->first)!=posesOut.end());
351 t = posesOut.at(iter->first).
clone();
355 scaledOdomPoses.insert(std::make_pair(iter->first, t));
358 float translational_rmse = 0.0f;
359 float translational_mean = 0.0f;
360 float translational_median = 0.0f;
361 float translational_std = 0.0f;
362 float translational_min = 0.0f;
363 float translational_max = 0.0f;
364 float rotational_rmse = 0.0f;
365 float rotational_mean = 0.0f;
366 float rotational_median = 0.0f;
367 float rotational_std = 0.0f;
368 float rotational_min = 0.0f;
369 float rotational_max = 0.0f;
376 translational_median,
386 float translational_rmse_vo = translational_rmse;
393 translational_median,
404 if(bestRMSE!=-1 && translational_rmse > bestRMSE)
408 bestRMSE = translational_rmse;
409 bestVoRMSE = translational_rmse_vo;
410 bestRMSEAng = rotational_rmse;
412 bestGtToMap = gtToMap;
420 for(std::map<int, Transform>::iterator iter=poses.begin(); iter!=poses.end(); ++iter)
422 iter->second.
x()*=bestScale;
423 iter->second.y()*=bestScale;
424 iter->second.z()*=bestScale;
425 iter->second = bestGtToMap * iter->second;
430 if(groundTruth.size() == poses.size())
437 printf(
"Cannot compute KITTI statistics as optimized poses and ground truth don't have the same size (%d vs %d).\n",
438 (
int)poses.size(), (int)groundTruth.size());
446 dbName = dbName.substr(0, dbName.size()-3);
450 printf(
"Could not export the poses to \"%s\"!?!\n", path.c_str());
452 if(groundTruth.size())
455 std::vector<int> validIndices(poses.size(), 1);
457 for(std::map<int, Transform>::iterator iter=poses.begin(); iter!=poses.end(); ++iter, ++i)
459 if(groundTruth.find(iter->first) == groundTruth.end())
461 groundTruth.insert(std::make_pair(iter->first,
Transform()));
468 printf(
"Could not export the ground truth to \"%s\"!?!\n", path.c_str());
476 fopen_s(&file, path.c_str(),
"w");
478 file = fopen(path.c_str(),
"w");
483 for(
unsigned int k=0; k<validIndices.size(); ++k)
485 fprintf(file,
"%d\n", validIndices[k]);
494 printf(
" %s (%d, s=%.3f):\terror lin=%.3fm (max=%.3fm, odom=%.3fm) ang=%.1fdeg%s, slam: avg=%dms (max=%dms) loops=%d, odom: avg=%dms (max=%dms), camera: avg=%dms, %smap=%dMB\n",
502 !outputKittiError?
"":
uFormat(
", KITTI: t_err=%.2f%% r_err=%.2f deg/100m", kitti_t_err, kitti_r_err*100).c_str(),
503 (int)
uMean(slamTime), (int)
uMax(slamTime),
504 (int)loopClosureLinks.size(),
505 (int)
uMean(odomTime), (int)
uMax(odomTime),
506 (int)
uMean(cameraTime),
507 maxOdomRAM!=-1.0f?
uFormat(
"RAM odom=%dMB ", (
int)maxOdomRAM).c_str():
"",
512 std::vector<float> stats;
513 stats.push_back(ids.size());
514 stats.push_back(bestRMSE);
515 stats.push_back(maxRMSE);
516 stats.push_back(bestRMSEAng);
517 stats.push_back(
uMean(odomTime));
518 stats.push_back(
uMean(slamTime));
519 stats.push_back(
uMax(slamTime));
520 stats.push_back(maxOdomRAM);
521 stats.push_back(maxMapRAM);
522 outputLatexStatisticsMap.insert(std::make_pair(filePath, stats));
524 if(maxOdomRAM != -1.0
f)
533 else if(
uSplit(fileName,
'.').size() == 1)
538 currentPathIsDatabase =
false;
541 for(std::list<std::string>::iterator iter=subDirs.begin(); iter!=subDirs.end(); ++iter)
543 paths.push_front(*iter);
546 if(outputLatexStatisticsMap.size() && paths.empty())
548 outputLatexStatistics.push_back(outputLatexStatisticsMap);
549 outputLatexStatisticsMap.clear();
553 if(outputLatex && outputLatexStatistics.size())
555 printf(
"\nLaTeX output:\n----------------\n");
556 printf(
"\\begin{table*}[!t]\n");
557 printf(
"\\caption{$t_{end}$ is the absolute translational RMSE value at the end " 558 "of the experiment as $ATE_{max}$ is the maximum during the experiment. " 559 "$r_{end}$ is rotational RMSE value at the end of the experiment. " 560 "$o_{avg}$ and $m_{avg}$ are the average computational time " 561 "for odometry (front-end) and map update (back-end). " 562 "$m_{avg}$ is the maximum computational time for map update. " 563 "$O_{end}$ and $M_{end}$ are the RAM usage at the end of the experiment " 564 "for odometry and map management respectively.}\n");
565 printf(
"\\label{}\n");
566 printf(
"\\centering\n");
569 printf(
"\\begin{tabular}{l|c|c|c|c|c|c|c|c|c}\n");
570 printf(
"\\cline{2-10}\n");
571 printf(
" & Size & $t_{end}$ & $t_{max}$ & $r_{end}$ & $o_{avg}$ & $m_{avg}$ & $m_{max}$ & $O_{end}$ & $M_{end}$ \\\\\n");
572 printf(
" & (nodes) & (m) & (m) & (deg) & (ms) & (ms) & (ms) & (MB) & (MB) \\\\\n");
576 printf(
"\\begin{tabular}{l|c|c|c|c|c|c|c|c}\n");
577 printf(
"\\cline{2-9}\n");
578 printf(
" & Size & $t_{end}$ & $t_{max}$ & $r_{end}$ & $o_{avg}$ & $m_{avg}$ & $m_{max}$ & $M_{end}$ \\\\\n");
579 printf(
" & (nodes) & (m) & (m) & (deg) & (ms) & (ms) & (ms) & (MB) \\\\\n");
584 for(
unsigned int j=0; j<outputLatexStatistics.size(); ++j)
586 if(outputLatexStatistics[j].size())
588 std::vector<int> lowestIndex;
589 if(outputLatexStatistics[j].size() > 1)
591 std::vector<float> lowestValue(outputLatexStatistics[j].begin()->second.size(),-1);
592 lowestIndex = std::vector<int>(lowestValue.size(),0);
594 for(std::map<std::string, std::vector<float> >::iterator iter=outputLatexStatistics[j].begin(); iter!=outputLatexStatistics[j].end(); ++iter)
596 UASSERT(lowestValue.size() == iter->second.size());
597 for(
unsigned int i=0; i<iter->second.size(); ++i)
599 if(lowestValue[i] == -1 || (iter->second[i]>0.0f && lowestValue[i]>iter->second[i]))
601 lowestValue[i] = iter->second[i];
602 lowestIndex[i] = index;
610 for(std::map<std::string, std::vector<float> >::iterator iter=outputLatexStatistics[j].begin(); iter!=outputLatexStatistics[j].end(); ++iter)
612 UASSERT(iter->second.size() == 9);
613 printf(
"%s & ",
uReplaceChar(iter->first.c_str(),
'_',
'-').c_str());
614 printf(
"%d & ", (
int)iter->second[0]);
615 printf(
"%s%.3f%s & ", lowestIndex.size()&&lowestIndex[1]==index?
"\\textbf{":
"", iter->second[1], lowestIndex.size()&&lowestIndex[1]==index?
"}":
"");
616 printf(
"%s%.3f%s & ", lowestIndex.size()&&lowestIndex[2]==index?
"\\textbf{":
"", iter->second[2], lowestIndex.size()&&lowestIndex[2]==index?
"}":
"");
617 printf(
"%s%.2f%s & ", lowestIndex.size()&&lowestIndex[3]==index?
"\\textbf{":
"", iter->second[3], lowestIndex.size()&&lowestIndex[3]==index?
"}":
"");
618 printf(
"%s%d%s & ", lowestIndex.size()&&lowestIndex[4]==index?
"\\textbf{":
"", (int)iter->second[4], lowestIndex.size()&&lowestIndex[4]==index?
"}":
"");
619 printf(
"%s%d%s & ", lowestIndex.size()&&lowestIndex[5]==index?
"\\textbf{":
"", (int)iter->second[5], lowestIndex.size()&&lowestIndex[5]==index?
"}":
"");
620 printf(
"%s%d%s & ", lowestIndex.size()&&lowestIndex[6]==index?
"\\textbf{":
"", (int)iter->second[6], lowestIndex.size()&&lowestIndex[6]==index?
"}":
"");
623 printf(
"%s%d%s & ", lowestIndex.size()&&lowestIndex[7]==index?
"\\textbf{":
"", (int)iter->second[7], lowestIndex.size()&&lowestIndex[7]==index?
"}":
"");
625 printf(
"%s%d%s ", lowestIndex.size()&&lowestIndex[8]==index?
"\\textbf{":
"", (int)iter->second[8], lowestIndex.size()&&lowestIndex[8]==index?
"}":
"");
633 printf(
"\\end{tabular}\n");
634 printf(
"\\end{table*}\n----------------\n");
639 for(std::map<std::string, UPlot*>::iterator iter=figures.begin(); iter!=figures.end(); ++iter)
641 iter->second->show();
static std::string homeDir()
bool RTABMAP_EXP exportPoses(const std::string &filePath, int format, const std::map< int, Transform > &poses, const std::multimap< int, Link > &constraints=std::multimap< int, Link >(), const std::map< int, double > &stamps=std::map< int, double >(), bool g2oRobust=false)
T uMean(const T *v, unsigned int size)
const std::list< std::string > & getFileNames() const
void getAllNodeIds(std::set< int > &ids, bool ignoreChildren=false, bool ignoreBadSignatures=false) const
static std::string getDir(const std::string &filePath)
static std::string separator()
void getAllLinks(std::multimap< int, Link > &links, bool ignoreNullLinks=true) const
std::map< std::string, std::string > ParametersMap
Basic mathematics functions.
bool getNodeInfo(int signatureId, Transform &pose, int &mapId, int &weight, std::string &label, double &stamp, Transform &groundTruthPose, std::vector< float > &velocity, GPS &gps) const
std::string getExtension()
static void setLevel(ULogger::Level level)
std::multimap< int, Link > RTABMAP_EXP filterDuplicateLinks(const std::multimap< int, Link > &links)
std::list< std::string > uSplit(const std::string &str, char separator= ' ')
bool openConnection(const std::string &url, bool overwritten=false)
#define UASSERT(condition)
Wrappers of STL for convenient functions.
std::map< int, Transform > optimizeIncremental(int rootId, const std::map< int, Transform > &poses, const std::multimap< int, Link > &constraints, std::list< std::map< int, Transform > > *intermediateGraphes=0, double *finalError=0, int *iterationsDone=0)
void closeConnection(bool save=true, const std::string &outputUrl="")
std::map< int, Transform > optimize(int rootId, const std::map< int, Transform > &poses, const std::multimap< int, Link > &constraints, std::list< std::map< int, Transform > > *intermediateGraphes=0, double *finalError=0, int *iterationsDone=0)
std::map< int, std::pair< std::map< std::string, float >, double > > getAllStatistics() const
Transform RTABMAP_EXP calcRMSE(const std::map< int, Transform > &groundTruth, const std::map< int, Transform > &poses, float &translational_rmse, float &translational_mean, float &translational_median, float &translational_std, float &translational_min, float &translational_max, float &rotational_rmse, float &rotational_mean, float &rotational_median, float &rotational_std, float &rotational_min, float &rotational_max)
static void setType(Type type, const std::string &fileName=kDefaultLogFileName, bool append=true)
std::string UTILITE_EXP uReplaceChar(const std::string &str, char before, char after)
bool uContains(const std::list< V > &list, const V &value)
static DBDriver * create(const ParametersMap ¶meters=ParametersMap())
T uMax(const T *v, unsigned int size, unsigned int &index)
std::vector< V > uValues(const std::multimap< K, V > &mm)
std::string getNextFileName()
std::multimap< int, Link >::iterator RTABMAP_EXP findLink(std::multimap< int, Link > &links, int from, int to, bool checkBothWays=true)
ParametersMap getLastParameters() const
ULogger class and convenient macros.
int main(int argc, char *argv[])
static void getConnectedGraph(int fromId, const std::map< int, Transform > &posesIn, const std::multimap< int, Link > &linksIn, std::map< int, Transform > &posesOut, std::multimap< int, Link > &linksOut, int depth=0)
std::string UTILITE_EXP uFormat(const char *fmt,...)
static Optimizer * create(const ParametersMap ¶meters)
void RTABMAP_EXP calcKittiSequenceErrors(const std::vector< Transform > &poses_gt, const std::vector< Transform > &poses_result, float &t_err, float &r_err)