face_normalizer.h

Go to the documentation of this file.

#include <opencv2/opencv.hpp>

#include <iostream>
#include <boost/lexical_cast.hpp>

using namespace cv;

namespace FACE
{
enum FEATURE_TYPE
{
        LEFTEYE, RIGHTEYE, NOSE, MOUTH,
};

template<class T>
class FaceFeatures
{
public:

        FaceFeatures<T>()
        {
        }
        ;

        ~FaceFeatures<T>()
        {
        }
        ;

        T lefteye; 
        T righteye; 
        T nose; 
        T mouth; 

        void sub_offset(int& ox, int& oy)
        {
                this->lefteye.x -= ox;
                this->lefteye.y -= oy;
                this->righteye.x -= ox;
                this->righteye.y -= oy;
                this->mouth.x -= ox;
                this->mouth.y -= oy;
                this->nose.x -= ox;
                this->nose.y -= oy;
        }

        void add_offset(int& ox, int& oy)
        {
                this->lefteye.x += ox;
                this->lefteye.y += oy;
                this->righteye.x += ox;
                this->righteye.y += oy;
                this->mouth.x += ox;
                this->mouth.y += oy;
                this->nose.x += ox;
                this->nose.y += oy;
        }

        void scale(double scale)
        {
                lefteye *= scale;
                righteye *= scale;
                nose *= scale;
                mouth *= scale;
        }

        // @return Vector containing lefteye,righteye,nose,mouth
        std::vector<T> as_vector()
        {
                std::vector<T> vec;
                vec.push_back(lefteye);
                vec.push_back(righteye);
                vec.push_back(nose);
                vec.push_back(mouth);
                return vec;
        }
        ;

        bool valid()
        {
                if (std::isnan(lefteye.x) || std::isnan(lefteye.y))
                        return false;
                if (std::isnan(righteye.x) || std::isnan(righteye.y))
                        return false;
                if (std::isnan(nose.x) || std::isnan(nose.y))
                        return false;
                //if(std::isnan(mouth.x)|| std::isnan(mouth.y)) return false;
                else
                        return true;
        }
};

}
;

class FaceNormalizer
{

public:
        struct FNConfig
        {
                bool eq_ill; 
                bool align; 
                bool resize; 
                bool cvt2gray; 
                bool extreme_illumination_condtions; 
        };

        FaceNormalizer()
        {
        }
        ;

        ~FaceNormalizer();

        void init();

        void init(FNConfig& i_config);

        void init(std::string i_classifier_directory, FNConfig& i_config);

        void init(std::string i_classifier_directory, std::string i_storage_directory, FNConfig& i_config, int i_epoch_ctr, bool i_debug, bool i_record_scene);

        bool normalizeFace(cv::Mat & RGB, cv::Size& norm_size);

        bool normalizeFace(cv::Mat& RGB, cv::Mat& XYZ, cv::Size& norm_size, cv::Mat& DM);

        bool normalizeFace(cv::Mat & RGB, cv::Mat& XYZ, cv::Size& norm_size);

        //TODO documentation
        bool synthFace(cv::Mat &RGB, cv::Mat& XYZ, cv::Size& norm_size, std::vector<cv::Mat>& synth_images);
        bool synth_head_poses(cv::Mat& img, cv::Mat& depth, std::vector<cv::Mat>& synth_images);
        bool synth_head_poses_relative(cv::Mat& img, cv::Mat& depth, std::vector<cv::Mat>& synth_images);
        bool eliminate_background(cv::Mat& RGB, cv::Mat& XYZ, float background_thresh);
        bool isolateFace(cv::Mat& RGB, cv::Mat& XYZ);
        bool interpolate_head(cv::Mat& RGB, cv::Mat& XYZ);
        bool recordFace(cv::Mat&RGB, cv::Mat& XYZ);

        bool save_scene(cv::Mat& RGB, cv::Mat& XYZ, std::string path);

        bool read_scene(cv::Mat& RGB, cv::Mat& XYZ, std::string path);

        FNConfig config_; 
protected:

        bool normalize_geometry_depth(cv::Mat& img, cv::Mat& depth);

        bool rotate_head(cv::Mat& img, cv::Mat& depth);

        bool features_from_color(cv::Mat& img);

        bool features_from_depth(cv::Mat& depth);

        bool detect_feature(cv::Mat& img, cv::Point2f& coords, FACE::FEATURE_TYPE type);

        bool projectPointCloud(cv::Mat& RGB, cv::Mat& XYZ, cv::Mat& img_res, cv::Mat& depth_res);

        bool projectPoint(cv::Point3f& xyz, cv::Point2f& uv);

        void create_DM(cv::Mat& XYZ, cv::Mat& DM);

        bool normalize_radiometry(cv::Mat& img);

        void extractVChannel(cv::Mat& img, cv::Mat& V);

        void subVChannel(cv::Mat& img, cv::Mat& V);

        void GammaDCT(cv::Mat& img);

        void GammaDoG(cv::Mat& img);

        bool normalize_img_type(cv::Mat& in, cv::Mat& out);

        void dump_features(cv::Mat& img);

        void dump_img(cv::Mat& data, std::string name);

        bool initialized_; 
        int epoch_ctr_; 
        bool debug_; 
        bool record_scene_; 

        std::string storage_directory_; 
        std::string classifier_directory_; 

        //intrinsics
        cv::Mat cam_mat_; 
        cv::Mat dist_coeffs_; 

        CvHaarClassifierCascade* nose_cascade_; 
        CvMemStorage* nose_storage_; 

        CvHaarClassifierCascade* eye_l_cascade_; 
        CvMemStorage* eye_l_storage_; 

        CvHaarClassifierCascade* eye_r_cascade_; 
        CvMemStorage* eye_r_storage_; 

        FACE::FaceFeatures<cv::Point2f> f_det_img_; 
        FACE::FaceFeatures<cv::Point3f> f_det_xyz_; 

        cv::Size norm_size_; 
        cv::Size input_size_; 


        template<class T>
        void despeckle(cv::Mat& src, cv::Mat& dst)
        {
                if (src.channels() == 1)
                {
                        T* lptr = src.ptr<T>(1, 0);
                        T* rptr = src.ptr<T>(1, 2);
                        T* mptr = src.ptr<T>(1, 1);
                        T* uptr = src.ptr<T>(0, 1);
                        T* dptr = src.ptr<T>(2, 1);

                        int normalizer = 4;

                        for (int px = 2 * src.cols + 2; px < (dst.rows * src.cols); ++px)
                        {
                                if (*mptr == 0)
                                {
                                        normalizer = 4;
                                        if (*lptr == 0)
                                                normalizer -= 1;
                                        if (*rptr == 0)
                                                normalizer -= 1;
                                        if (*uptr == 0)
                                                normalizer -= 1;
                                        if (*dptr == 0)
                                                normalizer -= 1;
                                        if (normalizer > 1)
                                                *mptr = (*lptr + *rptr + *uptr + *dptr) / normalizer;
                                        else
                                                *mptr = 75;
                                }
                                ++lptr;
                                ++rptr;
                                ++mptr;
                                ++uptr;
                                ++dptr;
                        }
                }

                if (src.channels() == 3)
                {
                        cv::Vec<T, 3>* lptr = src.ptr<cv::Vec<T, 3> >(1, 0);
                        cv::Vec<T, 3>* rptr = src.ptr<cv::Vec<T, 3> >(1, 2);
                        cv::Vec<T, 3>* mptr = src.ptr<cv::Vec<T, 3> >(1, 1);
                        cv::Vec<T, 3>* uptr = src.ptr<cv::Vec<T, 3> >(0, 1);
                        cv::Vec<T, 3>* dptr = src.ptr<cv::Vec<T, 3> >(2, 1);

                        int normalizer = 4;
                        for (int px = 2 * src.cols + 2; px < (dst.rows * src.cols); ++px)
                        {
                                if ((*mptr)[0] == 0)
                                {
                                        normalizer = 4;
                                        if ((*lptr)[0] == 0)
                                                normalizer -= 1;
                                        if ((*rptr)[0] == 0)
                                                normalizer -= 1;
                                        if ((*uptr)[0] == 0)
                                                normalizer -= 1;
                                        if ((*dptr)[0] == 0)
                                                normalizer -= 1;
                                        if (normalizer > 1)
                                                cv::divide((*lptr + *rptr + *uptr + *dptr), normalizer, *mptr);
                                        else
                                                *mptr = 75;
                                }
                                ++lptr;
                                ++rptr;
                                ++mptr;
                                ++uptr;
                                ++dptr;
                        }
                }
        }
};