frame.h

Go to the documentation of this file.

/*********************************************************************
* Software License Agreement (BSD License)
*
*  Copyright (c) 2009, Willow Garage, Inc.
*  All rights reserved.
*
*  Redistribution and use in source and binary forms, with or without
*  modification, are permitted provided that the following conditions
*  are met:
*
*   * Redistributions of source code must retain the above copyright
*     notice, this list of conditions and the following disclaimer.
*   * Redistributions in binary form must reproduce the above
*     copyright notice, this list of conditions and the following
*     disclaimer in the documentation and/or other materials provided
*     with the distribution.
*   * Neither the name of the Willow Garage nor the names of its
*     contributors may be used to endorse or promote products derived
*     from this software without specific prior written permission.
*
*  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
*  "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
*  LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
*  FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
*  COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
*  INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
*  BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
*  LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
*  CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
*  LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
*  ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
*  POSSIBILITY OF SUCH DAMAGE.
*********************************************************************/

//
// Visual Odometry classes and functions
//

#ifndef _FRAME_H_
#define _FRAME_H_

#ifndef EIGEN_USE_NEW_STDVECTOR
#define EIGEN_USE_NEW_STDVECTOR
#endif // EIGEN_USE_NEW_STDVECTOR

#include <stdio.h>
#include <iostream>
#include <Eigen/Core>
#include <Eigen/StdVector>
#include <vector>
#include <boost/shared_ptr.hpp>
#include <boost/make_shared.hpp>
#include <frame_common/stereo.h>
#include <frame_common/camparams.h>

// PCL headers
#include <pcl/common/common.h>
#include <pcl/point_types.h>
#include <pcl/point_cloud.h>
#include <pcl/features/normal_3d.h>
#include <pcl/kdtree/kdtree_flann.h>
#include <pcl/filters/passthrough.h>
#include <pcl/filters/voxel_grid.h>
#include <pcl/filters/filter.h>
#include <pcl/filters/extract_indices.h>
#include <pcl/common/transforms.h>
#include <pcl/registration/transforms.h>
#include <pcl/registration/icp.h>
#include <pcl/registration/icp_nl.h>

#include <pcl/io/pcd_io.h>


// Forward declaration.
namespace frame_common
{

  class Frame
  {
  public:    
    int frameId; 

    // image section
    bool isStereo;              
    cv::Mat img;                
    CamParams cam;              
    Eigen::Matrix3d iproj;     

    // feature section: keypoints and corresponding descriptors
    std::vector<cv::KeyPoint> kpts; 
    std::vector<cv::KeyPoint> tkpts; 
    cv::Mat dtors;              

    // stereo
    cv::Mat imgRight;           
    Eigen::Matrix4d disp_to_cart; 
    Eigen::Matrix4d cart_to_disp; 
    void setCamParams(const CamParams &c); 

    // these are for stereo; do we need monocular ones?
    Eigen::Vector3d cam2pix(const Eigen::Vector3d &cam_coord) const;
    Eigen::Vector3d pix2cam(const Eigen::Vector3d &pix_coord) const;
    Eigen::Vector3d pix2cam(const cv::KeyPoint &pix_coord, double disp) const;

    void setTKpts(Eigen::Vector4d trans, Eigen::Quaterniond rot);

    std::vector<Eigen::Vector4d, Eigen::aligned_allocator<Eigen::Vector4d> > pts;
    std::vector<char> goodPts;  
    std::vector<double> disps;  
    std::vector<int> ipts;      

    // these are for point-to-plane matches
    pcl::PointCloud<pcl::PointXYZRGBNormal> pointcloud;
    
    pcl::PointCloud<pcl::PointXYZRGB> dense_pointcloud;
    
    std::vector<Eigen::Vector3d, Eigen::aligned_allocator<Eigen::Vector3d> > pl_kpts;
    std::vector<Eigen::Vector4d, Eigen::aligned_allocator<Eigen::Vector4d> > pl_pts;
    std::vector<Eigen::Vector4d, Eigen::aligned_allocator<Eigen::Vector4d> > pl_normals;
    std::vector<int> pl_ipts;  
     
    EIGEN_MAKE_ALIGNED_OPERATOR_NEW // needed for 16B alignment
  };


  class FrameProc
  {
  public:
    FrameProc(int v = 25);

    cv::Ptr<cv::FeatureDetector> detector;
    cv::Ptr<cv::DescriptorExtractor> extractor;

    void setFrameDetector(const cv::Ptr<cv::FeatureDetector>& detector);
    void setFrameDescriptor(const cv::Ptr<cv::DescriptorExtractor>& extractor);

    void setMonoFrame(Frame& frame, const cv::Mat &img, const cv::Mat& mask = cv::Mat() );

    void setStereoPoints(Frame &frame, int nfrac = 0, bool setPointCloud=false);
    int ndisp;                  
    bool doSparse;              

    void setStereoFrame(Frame &frame, const cv::Mat &img, const cv::Mat &imgr, int nfrac = 0, 
                        bool setPointCloud=false);

    void setStereoFrame(Frame &frame, const cv::Mat &img, const cv::Mat &imgr, const cv::Mat &left_mask, int nfrac = 0,
                        bool setPointCloud=false);

  };
  
  class PointcloudProc
  {
    public:
      // TODO: Have an initializer with some parameters here.
      
      void setPointcloud(Frame &frame, const pcl::PointCloud<pcl::PointXYZRGB>& input_cloud) const;
      
      void match(const Frame& frame0, const Frame& frame1, 
                  const Eigen::Vector3d& trans, const Eigen::Quaterniond& rot, 
                  std::vector<cv::DMatch>& matches) const;
      
    private:
      void reduceCloud(const pcl::PointCloud<pcl::PointXYZRGB>& input, 
                        pcl::PointCloud<pcl::PointXYZRGBNormal>& output) const;

      Eigen::Vector3d projectPoint(Eigen::Vector4d& point, CamParams cam) const;
      
      void getMatchingIndices(const pcl::PointCloud<pcl::PointXYZRGBNormal>& input, 
          const pcl::PointCloud<pcl::PointXYZRGBNormal>& output, 
          std::vector<int>& input_indices, std::vector<int>& output_indices) const;
  };

  void drawVOtracks(const cv::Mat &image,
                    const std::vector<Frame, Eigen::aligned_allocator<Frame> > &frames,
                    cv::Mat &display);

} // end of namespace frame_common

/*namespace pe
{
  struct Match
  {
    int index1;      //!< The index of the descriptor from the first set.
    int index2;      //!< The index of the descriptor from the second set.
    float distance;

    Match(int index1, int index2, float distance = 0.0f)
      : index1(index1), index2(index2), distance(distance)
    {
    }
  };
} // namespace pe
*/
#endif // _FRAME_H_