face.h

Go to the documentation of this file.

// Copyright 2021 PAL Robotics S.L.
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are met:
//
//    * Redistributions of source code must retain the above copyright
//      notice, this list of conditions and the following disclaimer.
//
//    * Redistributions in binary form must reproduce the above copyright
//      notice, this list of conditions and the following disclaimer in the
//      documentation and/or other materials provided with the distribution.
//
//    * Neither the name of the PAL Robotics S.L. nor the names of its
//      contributors may be used to endorse or promote products derived from
//      this software without specific prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
// POSSIBILITY OF SUCH DAMAGE.


#ifndef HRI_FACE_H
#define HRI_FACE_H

#include <hri_msgs/NormalizedPointOfInterest2D.h>
#include <hri_msgs/FacialLandmarks.h>
#include <hri_msgs/SoftBiometrics.h>
#include <sensor_msgs/RegionOfInterest.h>
#include <sensor_msgs/Image.h>
#include <memory>
#include <boost/optional.hpp>

#include "base.h"
#include "ros/subscriber.h"

#include <opencv2/core.hpp>

#include "tf2_ros/transform_listener.h"

namespace hri
{
struct IntensityConfidence
{
  float intensity;
  float confidence;
};

enum Gender
{
  FEMALE = 1,
  MALE = 2,
  OTHER = 3
};

// the tf prefixes follow REP-155
const static std::string FACE_TF_PREFIX("face_");
const static std::string GAZE_TF_PREFIX("gaze_");
const static ros::Duration FACE_TF_TIMEOUT(0.01);

class Face : public FeatureTracker
{
public:
  Face(ID id, ros::NodeHandle& nh, tf2_ros::Buffer* tf_buffer_ptr,
       const std::string& reference_frame);

  virtual ~Face();

  std::string frame() const
  {
    return FACE_TF_PREFIX + id_;
  }

  std::string gazeFrame() const
  {
    return GAZE_TF_PREFIX + id_;
  }

  cv::Rect roi() const;


  cv::Mat cropped() const;

  cv::Mat aligned() const;

  boost::optional<std::array<hri_msgs::NormalizedPointOfInterest2D, 70>> facialLandmarks() const
  {
    return facial_landmarks_;
  }

  boost::optional<std::array<IntensityConfidence, 99>> facialActionUnits() const
  {
    return facial_action_units_;
  }

  /* \brief estimated age of this face, if available (eg, the
   * '/softbiometrics' is published)
   */
  boost::optional<float> age() const;

  /* \brief estimated gender of this face, if available (eg, the
   * '/softbiometrics' is published)
   */
  boost::optional<Gender> gender() const;

  boost::optional<geometry_msgs::TransformStamped> transform() const;

  boost::optional<geometry_msgs::TransformStamped> gazeTransform() const;

  void init() override;

private:
  size_t nb_roi;

  ros::Subscriber roi_subscriber_;
  void onRoI(sensor_msgs::RegionOfInterestConstPtr roi);
  cv::Rect roi_;

  ros::Subscriber cropped_subscriber_;
  void onCropped(sensor_msgs::ImageConstPtr roi);
  cv::Mat cropped_;

  ros::Subscriber aligned_subscriber_;
  void onAligned(sensor_msgs::ImageConstPtr roi);
  cv::Mat aligned_;

  ros::Subscriber landmarks_subscriber_;
  void onLandmarks(hri_msgs::FacialLandmarksConstPtr landmarks);
  std::array<hri_msgs::NormalizedPointOfInterest2D, 70> facial_landmarks_;

  ros::Subscriber softbiometrics_subscriber_;
  void onSoftBiometrics(hri_msgs::SoftBiometricsConstPtr biometrics);
  hri_msgs::SoftBiometricsConstPtr softbiometrics_;


  std::array<IntensityConfidence, 99> facial_action_units_;

  std::string _reference_frame;
  tf2_ros::Buffer* _tf_buffer_ptr;
};

typedef std::shared_ptr<Face> FacePtr;
typedef std::shared_ptr<const Face> FaceConstPtr;
typedef std::weak_ptr<Face> FaceWeakPtr;
typedef std::weak_ptr<const Face> FaceWeakConstPtr;

}  // namespace hri
#endif