example/main.cpp
Go to the documentation of this file.
1 /*
2 Copyright (c) 2011-2014, Mathieu Labbe - IntRoLab - Universite de Sherbrooke
3 All rights reserved.
4 
5 Redistribution and use in source and binary forms, with or without
6 modification, are permitted provided that the following conditions are met:
7  * Redistributions of source code must retain the above copyright
8  notice, this list of conditions and the following disclaimer.
9  * Redistributions in binary form must reproduce the above copyright
10  notice, this list of conditions and the following disclaimer in the
11  documentation and/or other materials provided with the distribution.
12  * Neither the name of the Universite de Sherbrooke nor the
13  names of its contributors may be used to endorse or promote products
14  derived from this software without specific prior written permission.
15 
16 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
17 ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
18 WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
19 DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY
20 DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
21 (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
22 LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
23 ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
24 (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
25 SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
26 */
27 
28 #include <stdio.h>
29 
30 // Qt stuff
31 #include <QtCore/QTime>
32 #include <QtCore/QTimer>
33 #include <QApplication>
34 #include <QGraphicsRectItem>
35 #include <QPen>
36 #include <QColor>
37 
38 // OpenCV stuff
39 #include <opencv2/core/core.hpp>
40 #include <opencv2/highgui/highgui.hpp>
41 #include <opencv2/features2d/features2d.hpp>
42 #include <opencv2/calib3d/calib3d.hpp> // for homography
43 
44 #include <opencv2/opencv_modules.hpp>
45 
46 #ifdef HAVE_OPENCV_NONFREE
47  #if CV_MAJOR_VERSION == 2 && CV_MINOR_VERSION >=4
48  #include <opencv2/nonfree/gpu.hpp>
49  #include <opencv2/nonfree/features2d.hpp>
50  #endif
51 #endif
52 #ifdef HAVE_OPENCV_XFEATURES2D
53  #include <opencv2/xfeatures2d.hpp>
54  #include <opencv2/xfeatures2d/cuda.hpp>
55 #endif
56 
57 
58 // From this project
59 #include "find_object/ObjWidget.h"
60 #include "find_object/QtOpenCV.h"
61 
62 using namespace find_object;
63 
64 void showUsage()
65 {
66  printf("\n");
67  printf("Usage :\n");
68  printf(" ./example object.png scene.png\n");
69  exit(1);
70 }
71 
72 int main(int argc, char * argv[])
73 {
74  if(argc<3)
75  {
76  showUsage();
77  }
78  QTime time;
79 
80  // GUI stuff
81  QApplication app(argc, argv);
82 
83  time.start();
84  //Load as grayscale
85  cv::Mat objectImg = cv::imread(argv[1], cv::IMREAD_GRAYSCALE);
86  cv::Mat sceneImg = cv::imread(argv[2], cv::IMREAD_GRAYSCALE);
87 
88  if(!objectImg.empty() && !sceneImg.empty())
89  {
90  printf("Loading images: %d ms\n", time.restart());
91  std::vector<cv::KeyPoint> objectKeypoints;
92  std::vector<cv::KeyPoint> sceneKeypoints;
93  cv::Mat objectDescriptors;
94  cv::Mat sceneDescriptors;
95 
97  // EXTRACT KEYPOINTS
99  cv::Ptr<cv::FeatureDetector> detector;
100  // The detector can be any of (see OpenCV features2d.hpp):
101 #if CV_MAJOR_VERSION == 2
102  // detector = cv::Ptr(new cv::DenseFeatureDetector());
103  // detector = cv::Ptr(new cv::FastFeatureDetector());
104  // detector = cv::Ptr(new cv::GFTTDetector());
105  // detector = cv::Ptr(new cv::MSER());
106  // detector = cv::Ptr(new cv::ORB());
107  detector = cv::Ptr<cv::FeatureDetector>(new cv::SIFT());
108  // detector = cv::Ptr(new cv::StarFeatureDetector());
109  // detector = cv::Ptr(new cv::SURF(600.0));
110  // detector = cv::Ptr(new cv::BRISK());
111 #else
112  detector = cv::xfeatures2d::SIFT::create();
113 #endif
114  detector->detect(objectImg, objectKeypoints);
115  printf("Object: %d keypoints detected in %d ms\n", (int)objectKeypoints.size(), time.restart());
116  detector->detect(sceneImg, sceneKeypoints);
117  printf("Scene: %d keypoints detected in %d ms\n", (int)sceneKeypoints.size(), time.restart());
118 
120  // EXTRACT DESCRIPTORS
122  cv::Ptr<cv::DescriptorExtractor> extractor;
123 #if CV_MAJOR_VERSION == 2
124  // The extractor can be any of (see OpenCV features2d.hpp):
125  // extractor = cv::Ptr(new cv::BriefDescriptorExtractor());
126  // extractor = cv::Ptr(new cv::ORB());
127  extractor = cv::Ptr<cv::DescriptorExtractor>(new cv::SIFT());
128  // extractor = cv::Ptr(new cv::SURF(600.0));
129  // extractor = cv::Ptr(new cv::BRISK());
130  // extractor = cv::Ptr(new cv::FREAK());
131 #else
132  extractor = cv::xfeatures2d::SIFT::create();
133 #endif
134  extractor->compute(objectImg, objectKeypoints, objectDescriptors);
135  printf("Object: %d descriptors extracted in %d ms\n", objectDescriptors.rows, time.restart());
136  extractor->compute(sceneImg, sceneKeypoints, sceneDescriptors);
137  printf("Scene: %d descriptors extracted in %d ms\n", sceneDescriptors.rows, time.restart());
138 
140  // NEAREST NEIGHBOR MATCHING USING FLANN LIBRARY (included in OpenCV)
142  cv::Mat results;
143  cv::Mat dists;
144  std::vector<std::vector<cv::DMatch> > matches;
145  int k=2; // find the 2 nearest neighbors
146  bool useBFMatcher = false; // SET TO TRUE TO USE BRUTE FORCE MATCHER
147  if(objectDescriptors.type()==CV_8U)
148  {
149  // Binary descriptors detected (from ORB, Brief, BRISK, FREAK)
150  printf("Binary descriptors detected...\n");
151  if(useBFMatcher)
152  {
153  cv::BFMatcher matcher(cv::NORM_HAMMING); // use cv::NORM_HAMMING2 for ORB descriptor with WTA_K == 3 or 4 (see ORB constructor)
154  matcher.knnMatch(objectDescriptors, sceneDescriptors, matches, k);
155  }
156  else
157  {
158  // Create Flann LSH index
159  cv::flann::Index flannIndex(sceneDescriptors, cv::flann::LshIndexParams(12, 20, 2), cvflann::FLANN_DIST_HAMMING);
160  printf("Time creating FLANN LSH index = %d ms\n", time.restart());
161 
162  // search (nearest neighbor)
163  flannIndex.knnSearch(objectDescriptors, results, dists, k, cv::flann::SearchParams() );
164  }
165  }
166  else
167  {
168  // assume it is CV_32F
169  printf("Float descriptors detected...\n");
170  if(useBFMatcher)
171  {
172  cv::BFMatcher matcher(cv::NORM_L2);
173  matcher.knnMatch(objectDescriptors, sceneDescriptors, matches, k);
174  }
175  else
176  {
177  // Create Flann KDTree index
178  cv::flann::Index flannIndex(sceneDescriptors, cv::flann::KDTreeIndexParams(), cvflann::FLANN_DIST_EUCLIDEAN);
179  printf("Time creating FLANN KDTree index = %d ms\n", time.restart());
180 
181  // search (nearest neighbor)
182  flannIndex.knnSearch(objectDescriptors, results, dists, k, cv::flann::SearchParams() );
183  }
184  }
185  printf("Time nearest neighbor search = %d ms\n", time.restart());
186 
187  // Conversion to CV_32F if needed
188  if(dists.type() == CV_32S)
189  {
190  cv::Mat temp;
191  dists.convertTo(temp, CV_32F);
192  dists = temp;
193  }
194 
195 
197  // PROCESS NEAREST NEIGHBOR RESULTS
199  // Set gui data
200  ObjWidget objWidget(0, objectKeypoints, QMultiMap<int,int>(), cvtCvMat2QImage(objectImg));
201  ObjWidget sceneWidget(0, sceneKeypoints, QMultiMap<int,int>(), cvtCvMat2QImage(sceneImg));
202 
203  // Find correspondences by NNDR (Nearest Neighbor Distance Ratio)
204  float nndrRatio = 0.8f;
205  std::vector<cv::Point2f> mpts_1, mpts_2; // Used for homography
206  std::vector<int> indexes_1, indexes_2; // Used for homography
207  std::vector<uchar> outlier_mask; // Used for homography
208  // Check if this descriptor matches with those of the objects
209  if(!useBFMatcher)
210  {
211  for(int i=0; i<objectDescriptors.rows; ++i)
212  {
213  // Apply NNDR
214  //printf("q=%d dist1=%f dist2=%f\n", i, dists.at<float>(i,0), dists.at<float>(i,1));
215  if(results.at<int>(i,0) >= 0 && results.at<int>(i,1) >= 0 &&
216  dists.at<float>(i,0) <= nndrRatio * dists.at<float>(i,1))
217  {
218  mpts_1.push_back(objectKeypoints.at(i).pt);
219  indexes_1.push_back(i);
220 
221  mpts_2.push_back(sceneKeypoints.at(results.at<int>(i,0)).pt);
222  indexes_2.push_back(results.at<int>(i,0));
223  }
224  }
225  }
226  else
227  {
228  for(unsigned int i=0; i<matches.size(); ++i)
229  {
230  // Apply NNDR
231  //printf("q=%d dist1=%f dist2=%f\n", matches.at(i).at(0).queryIdx, matches.at(i).at(0).distance, matches.at(i).at(1).distance);
232  if(matches.at(i).size() == 2 &&
233  matches.at(i).at(0).distance <= nndrRatio * matches.at(i).at(1).distance)
234  {
235  mpts_1.push_back(objectKeypoints.at(matches.at(i).at(0).queryIdx).pt);
236  indexes_1.push_back(matches.at(i).at(0).queryIdx);
237 
238  mpts_2.push_back(sceneKeypoints.at(matches.at(i).at(0).trainIdx).pt);
239  indexes_2.push_back(matches.at(i).at(0).trainIdx);
240  }
241  }
242  }
243 
244  // FIND HOMOGRAPHY
245  unsigned int minInliers = 8;
246  if(mpts_1.size() >= minInliers)
247  {
248  time.start();
249  cv::Mat H = findHomography(mpts_1,
250  mpts_2,
251  cv::RANSAC,
252  1.0,
253  outlier_mask);
254  printf("Time finding homography = %d ms\n", time.restart());
255  int inliers=0, outliers=0;
256  for(unsigned int k=0; k<mpts_1.size();++k)
257  {
258  if(outlier_mask.at(k))
259  {
260  ++inliers;
261  }
262  else
263  {
264  ++outliers;
265  }
266  }
267  QTransform hTransform(
268  H.at<double>(0,0), H.at<double>(1,0), H.at<double>(2,0),
269  H.at<double>(0,1), H.at<double>(1,1), H.at<double>(2,1),
270  H.at<double>(0,2), H.at<double>(1,2), H.at<double>(2,2));
271 
272  // GUI : Change color and add homography rectangle
273  QColor color(Qt::green);
274  int alpha = 130;
275  color.setAlpha(alpha);
276  for(unsigned int k=0; k<mpts_1.size();++k)
277  {
278  if(outlier_mask.at(k))
279  {
280  objWidget.setKptColor(indexes_1.at(k), color);
281  sceneWidget.setKptColor(indexes_2.at(k), color);
282  }
283  else
284  {
285  objWidget.setKptColor(indexes_1.at(k), QColor(255,0,0,alpha));
286  sceneWidget.setKptColor(indexes_2.at(k), QColor(255,0,0,alpha));
287  }
288  }
289  QPen rectPen(color);
290  rectPen.setWidth(4);
291  QGraphicsRectItem * rectItem = new QGraphicsRectItem(objWidget.pixmap().rect());
292  rectItem->setPen(rectPen);
293  rectItem->setTransform(hTransform);
294  sceneWidget.addRect(rectItem);
295  printf("Inliers=%d Outliers=%d\n", inliers, outliers);
296  }
297  else
298  {
299  printf("Not enough matches (%d) for homography...\n", (int)mpts_1.size());
300  }
301 
302  // Wait for gui
303  objWidget.setGraphicsViewMode(false);
304  objWidget.setWindowTitle("Object");
305  if(objWidget.pixmap().width() <= 800)
306  {
307  objWidget.setMinimumSize(objWidget.pixmap().width(), objWidget.pixmap().height());
308  }
309  else
310  {
311  objWidget.setMinimumSize(800, 600);
312  objWidget.setAutoScale(false);
313  }
314 
315  sceneWidget.setGraphicsViewMode(false);
316  sceneWidget.setWindowTitle("Scene");
317  if(sceneWidget.pixmap().width() <= 800)
318  {
319  sceneWidget.setMinimumSize(sceneWidget.pixmap().width(), sceneWidget.pixmap().height());
320  }
321  else
322  {
323  sceneWidget.setMinimumSize(800, 600);
324  sceneWidget.setAutoScale(false);
325  }
326 
327  sceneWidget.show();
328  objWidget.show();
329 
330  int r = app.exec();
331  printf("Closing...\n");
332 
333  return r;
334  }
335  else
336  {
337  printf("Images are not valid!\n");
338  showUsage();
339  }
340 
341  return 1;
342 }
app
void setKptColor(int index, const QColor &color)
Definition: ObjWidget.cpp:341
void setAutoScale(bool autoScale)
Definition: ObjWidget.cpp:179
void addRect(QGraphicsRectItem *rect)
Definition: ObjWidget.cpp:372
void showUsage()
void setGraphicsViewMode(bool on)
Definition: ObjWidget.cpp:146
int main(int argc, char *argv[])
const QPixmap & pixmap() const
Definition: ObjWidget.h:83
FINDOBJECT_EXP QImage cvtCvMat2QImage(const cv::Mat &image, bool isBgr=true)
Definition: QtOpenCV.cpp:34


find_object_2d
Author(s): Mathieu Labbe
autogenerated on Mon Jun 10 2019 13:21:31