depth_self_filter_nodelet.cpp

Go to the documentation of this file.

/*********************************************************************
 * Software License Agreement (BSD License)
 *
 *  Copyright (c) 2013, Willow Garage, Inc.
 *  All rights reserved.
 *
 *  Redistribution and use in source and binary forms, with or without
 *  modification, are permitted provided that the following conditions
 *  are met:
 *
 *   * Redistributions of source code must retain the above copyright
 *     notice, this list of conditions and the following disclaimer.
 *   * Redistributions in binary form must reproduce the above
 *     copyright notice, this list of conditions and the following
 *     disclaimer in the documentation and/or other materials provided
 *     with the distribution.
 *   * Neither the name of Willow Garage nor the names of its
 *     contributors may be used to endorse or promote products derived
 *     from this software without specific prior written permission.
 *
 *  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
 *  "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
 *  LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
 *  FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
 *  COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
 *  INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
 *  BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
 *  LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
 *  CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
 *  LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
 *  ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
 *  POSSIBILITY OF SUCH DAMAGE.
 *********************************************************************/

/* Author: Suat Gedikli */

#include <moveit/mesh_filter/depth_self_filter_nodelet.h>
#include <moveit/mesh_filter/stereo_camera_model.h>
#include <moveit/mesh_filter/mesh_filter.h>
#include <ros/ros.h>
#include <image_transport/subscriber_filter.h>
#include <sensor_msgs/image_encodings.h>
#include <moveit/robot_model_loader/robot_model_loader.h>
#include <moveit/robot_model/robot_model.h>
#include <eigen3/Eigen/Eigen>
#include <cv_bridge/cv_bridge.h>

namespace enc = sensor_msgs::image_encodings;
using namespace std;
using namespace boost;

mesh_filter::DepthSelfFiltering::~DepthSelfFiltering()
{
}

void mesh_filter::DepthSelfFiltering::onInit()
{
  ros::NodeHandle& nh = getNodeHandle();
  ros::NodeHandle& private_nh = getPrivateNodeHandle();
  input_depth_transport_.reset(new image_transport::ImageTransport(nh));
  filtered_depth_transport_.reset(new image_transport::ImageTransport(nh));
  filtered_label_transport_.reset(new image_transport::ImageTransport(nh));
  model_depth_transport_.reset(new image_transport::ImageTransport(nh));
  model_label_transport_.reset(new image_transport::ImageTransport(nh));

  // Read parameters
  private_nh.param("queue_size", queue_size_, 1);
  private_nh.param("near_clipping_plane_distance", near_clipping_plane_distance_, 0.4);
  private_nh.param("far_clipping_plane_distance", far_clipping_plane_distance_, 5.0);
  ;
  private_nh.param("shadow_threshold", shadow_threshold_, 0.3);
  private_nh.param("padding_scale", padding_scale_, 1.0);
  private_nh.param("padding_offset", padding_offset_, 0.005);
  double tf_update_rate = 30;
  private_nh.param("tf_update_rate", tf_update_rate, 30.0);
  transform_provider_.setUpdateInterval(long(1000000.0 / tf_update_rate));

  image_transport::SubscriberStatusCallback itssc = bind(&DepthSelfFiltering::connectCb, this);
  ros::SubscriberStatusCallback rssc = bind(&DepthSelfFiltering::connectCb, this);

  lock_guard<mutex> lock(connect_mutex_);
  pub_filtered_depth_image_ =
      filtered_depth_transport_->advertiseCamera("/filtered/depth", queue_size_, itssc, itssc, rssc, rssc);
  pub_filtered_label_image_ =
      filtered_label_transport_->advertiseCamera("/filtered/labels", queue_size_, itssc, itssc, rssc, rssc);
  pub_model_depth_image_ =
      model_depth_transport_->advertiseCamera("/model/depth", queue_size_, itssc, itssc, rssc, rssc);
  pub_model_label_image_ =
      model_depth_transport_->advertiseCamera("/model/label", queue_size_, itssc, itssc, rssc, rssc);

  filtered_depth_ptr_.reset(new cv_bridge::CvImage);
  filtered_label_ptr_.reset(new cv_bridge::CvImage);
  model_depth_ptr_.reset(new cv_bridge::CvImage);
  model_label_ptr_.reset(new cv_bridge::CvImage);

  mesh_filter_.reset(
      new MeshFilter<StereoCameraModel>(bind(&TransformProvider::getTransform, &transform_provider_, _1, _2),
                                        mesh_filter::StereoCameraModel::RegisteredPSDKParams));
  mesh_filter_->parameters().setDepthRange(near_clipping_plane_distance_, far_clipping_plane_distance_);
  mesh_filter_->setShadowThreshold(shadow_threshold_);
  mesh_filter_->setPaddingOffset(padding_offset_);
  mesh_filter_->setPaddingScale(padding_scale_);
  // add meshesfla
  addMeshes(*mesh_filter_);
  transform_provider_.start();
}

void mesh_filter::DepthSelfFiltering::filter(const sensor_msgs::ImageConstPtr& depth_msg,
                                             const sensor_msgs::CameraInfoConstPtr& info_msg)
{
  transform_provider_.setFrame(depth_msg->header.frame_id);
  // do filtering here
  mesh_filter::StereoCameraModel::Parameters& params = mesh_filter_->parameters();
  params.setCameraParameters(info_msg->K[0], info_msg->K[4], info_msg->K[2], info_msg->K[5]);
  params.setImageSize(depth_msg->width, depth_msg->height);

  const float* src = (const float*)&depth_msg->data[0];
  //*
  static unsigned dataSize = 0;
  static unsigned short* data = 0;
  if (dataSize < depth_msg->width * depth_msg->height)
    data = new unsigned short[depth_msg->width * depth_msg->height];
  for (unsigned idx = 0; idx < depth_msg->width * depth_msg->height; ++idx)
    data[idx] = (unsigned short)(src[idx] * 1000.0);

  mesh_filter_->filter(data, GL_UNSIGNED_SHORT);
  // delete[] data;
  /*/
  mesh_filter_->filter ((void*) &depth_msg->data[0], GL_FLOAT);
  //*/
  if (pub_filtered_depth_image_.getNumSubscribers() > 0)
  {
    filtered_depth_ptr_->encoding = sensor_msgs::image_encodings::TYPE_32FC1;
    filtered_depth_ptr_->header = depth_msg->header;

    if (filtered_depth_ptr_->image.cols != depth_msg->width || filtered_depth_ptr_->image.rows != depth_msg->height)
      filtered_depth_ptr_->image = cv::Mat(depth_msg->height, depth_msg->width, CV_32FC1);
    mesh_filter_->getFilteredDepth((float*)filtered_depth_ptr_->image.data);
    pub_filtered_depth_image_.publish(filtered_depth_ptr_->toImageMsg(), info_msg);
  }

  // this is from rendering of the model
  if (pub_model_depth_image_.getNumSubscribers() > 0)
  {
    model_depth_ptr_->encoding = sensor_msgs::image_encodings::TYPE_32FC1;
    model_depth_ptr_->header = depth_msg->header;

    if (model_depth_ptr_->image.cols != depth_msg->width || model_depth_ptr_->image.rows != depth_msg->height)
      model_depth_ptr_->image = cv::Mat(depth_msg->height, depth_msg->width, CV_32FC1);
    mesh_filter_->getModelDepth((float*)model_depth_ptr_->image.data);
    pub_model_depth_image_.publish(model_depth_ptr_->toImageMsg(), info_msg);
  }

  if (pub_filtered_label_image_.getNumSubscribers() > 0)
  {
    filtered_label_ptr_->encoding = sensor_msgs::image_encodings::RGBA8;
    filtered_label_ptr_->header = depth_msg->header;

    if (filtered_label_ptr_->image.cols != depth_msg->width || filtered_label_ptr_->image.rows != depth_msg->height)
      filtered_label_ptr_->image = cv::Mat(depth_msg->height, depth_msg->width, CV_8UC4);
    mesh_filter_->getFilteredLabels((unsigned int*)filtered_label_ptr_->image.data);
    pub_filtered_label_image_.publish(filtered_label_ptr_->toImageMsg(), info_msg);
  }

  if (pub_model_label_image_.getNumSubscribers() > 0)
  {
    model_label_ptr_->encoding = sensor_msgs::image_encodings::RGBA8;
    model_label_ptr_->header = depth_msg->header;
    if (model_label_ptr_->image.cols != depth_msg->width || model_label_ptr_->image.rows != depth_msg->height)
      model_label_ptr_->image = cv::Mat(depth_msg->height, depth_msg->width, CV_8UC4);
    mesh_filter_->getModelLabels((unsigned int*)model_label_ptr_->image.data);
    pub_model_label_image_.publish(model_label_ptr_->toImageMsg(), info_msg);
  }
}

void mesh_filter::DepthSelfFiltering::addMeshes(MeshFilter<StereoCameraModel>& mesh_filter)
{
  robot_model_loader::RobotModelLoader robotModelLoader("robot_description");
  robot_model::RobotModelConstPtr robotModel = robotModelLoader.getModel();
  const vector<robot_model::LinkModel*>& links = robotModel->getLinkModelsWithCollisionGeometry();
  for (size_t i = 0; i < links.size(); ++i)
  {
    shapes::ShapeConstPtr shape = links[i]->getShape();
    if (shape->type == shapes::MESH)
    {
      const shapes::Mesh& m = static_cast<const shapes::Mesh&>(*shape);
      MeshHandle mesh_handle = mesh_filter.addMesh(m);
      transform_provider_.addHandle(mesh_handle, links[i]->getName());
    }
  }
}

// Handles (un)subscribing when clients (un)subscribe
void mesh_filter::DepthSelfFiltering::connectCb()
{
  lock_guard<mutex> lock(connect_mutex_);
  if (pub_filtered_depth_image_.getNumSubscribers() == 0 && pub_filtered_label_image_.getNumSubscribers() == 0 &&
      pub_model_depth_image_.getNumSubscribers() == 0 && pub_model_label_image_.getNumSubscribers() == 0)
  {
    sub_depth_image_.shutdown();
  }
  else if (!sub_depth_image_)
  {
    image_transport::TransportHints hints("raw", ros::TransportHints(), getPrivateNodeHandle());
    sub_depth_image_ =
        input_depth_transport_->subscribeCamera("depth", queue_size_, &DepthSelfFiltering::depthCb, this, hints);
  }
}

void mesh_filter::DepthSelfFiltering::depthCb(const sensor_msgs::ImageConstPtr& depth_msg,
                                              const sensor_msgs::CameraInfoConstPtr& info_msg)
{
  filter(depth_msg, info_msg);
}

#include <pluginlib/class_list_macros.hpp>
PLUGINLIB_EXPORT_CLASS(mesh_filter::DepthSelfFiltering, nodelet::Nodelet)