00001
00002
00003
00004
00005
00006
00007
00008
00009
00010
00011
00012
00013
00014
00015
00016
00017
00018
00019
00020
00021
00022
00023
00024
00025
00026
00027
00028
00029
00030
00031
00032
00033
00034
00035
00036 #include "jsk_pcl_ros/color_histogram_matcher.h"
00037 #include <pluginlib/class_list_macros.h>
00038 #include <pcl/filters/extract_indices.h>
00039 #include <pcl/point_types_conversion.h>
00040 #include <pcl/common/centroid.h>
00041 #include <geometry_msgs/PoseStamped.h>
00042
00043 namespace jsk_pcl_ros
00044 {
00045 void ColorHistogramMatcher::onInit()
00046 {
00047 ConnectionBasedNodelet::onInit();
00048 srv_ = boost::make_shared <dynamic_reconfigure::Server<Config> > (*pnh_);
00049 dynamic_reconfigure::Server<Config>::CallbackType f =
00050 boost::bind (&ColorHistogramMatcher::configCallback, this, _1, _2);
00051 srv_->setCallback (f);
00052
00053 policy_ = USE_HUE_AND_SATURATION;
00054 reference_set_ = false;
00055
00056 all_histogram_pub_
00057 = advertise<jsk_recognition_msgs::ColorHistogramArray>(
00058 *pnh_, "output_histograms", 1);
00059 best_pub_
00060 = advertise<geometry_msgs::PoseStamped>(*pnh_, "best_match", 1);
00061 reference_histogram_pub_
00062 = advertise<jsk_recognition_msgs::ColorHistogram>(*pnh_, "output_reference", 1);
00063 result_pub_
00064 = advertise<jsk_recognition_msgs::ClusterPointIndices>(*pnh_, "output", 1);
00065 coefficient_points_pub_
00066 = advertise<sensor_msgs::PointCloud2>(*pnh_, "coefficient_points", 1);
00067 }
00068
00069 void ColorHistogramMatcher::subscribe()
00070 {
00071 reference_sub_ = pnh_->subscribe("input_reference_cloud", 1,
00072 &ColorHistogramMatcher::reference,
00073 this);
00074 reference_histogram_sub_ = pnh_->subscribe(
00075 "input_reference", 1,
00076 &ColorHistogramMatcher::referenceHistogram,
00077 this);
00078 sub_input_.subscribe(*pnh_, "input", 1);
00079 sub_indices_.subscribe(*pnh_, "input_indices", 1);
00080 sync_ = boost::make_shared<message_filters::Synchronizer<SyncPolicy> >(100);
00081 sync_->connectInput(sub_input_, sub_indices_);
00082 sync_->registerCallback(boost::bind(&ColorHistogramMatcher::feature,
00083 this, _1, _2));
00084 }
00085
00086 void ColorHistogramMatcher::unsubscribe()
00087 {
00088 reference_sub_.shutdown();
00089 reference_histogram_sub_.shutdown();
00090 sub_input_.unsubscribe();
00091 sub_indices_.unsubscribe();
00092 }
00093
00094 void ColorHistogramMatcher::configCallback(Config &config, uint32_t level)
00095 {
00096 boost::mutex::scoped_lock lock(mutex_);
00097 coefficient_thr_ = config.coefficient_thr;
00098 bin_size_ = config.bin_size;
00099 publish_colored_cloud_ = config.publish_colored_cloud;
00100 power_ = config.power;
00101 color_min_coefficient_ = config.color_min_coefficient;
00102 color_max_coefficient_ = config.color_max_coefficient;
00103 show_method_ = config.show_method;
00104 ComparePolicy new_histogram;
00105 if (config.histogram_method == 0) {
00106 new_histogram = USE_HUE;
00107 }
00108 else if (config.histogram_method == 1) {
00109 new_histogram = USE_SATURATION;
00110 }
00111 else if (config.histogram_method == 2) {
00112 new_histogram = USE_VALUE;
00113 }
00114 else if (config.histogram_method == 3) {
00115 new_histogram = USE_HUE_AND_SATURATION;
00116 }
00117 else {
00118 JSK_ROS_WARN("unknown histogram method");
00119 return;
00120 }
00121 if (new_histogram != policy_) {
00122 policy_ = new_histogram;
00123 reference_set_ = false;
00124 JSK_ROS_WARN("histogram method is reset, please specify histogram again");
00125 }
00126 }
00127
00128
00129 void ColorHistogramMatcher::feature(
00130 const sensor_msgs::PointCloud2::ConstPtr& input_cloud,
00131 const jsk_recognition_msgs::ClusterPointIndices::ConstPtr& input_indices)
00132 {
00133 boost::mutex::scoped_lock lock(mutex_);
00134 if (!reference_set_) {
00135 JSK_NODELET_WARN("reference histogram is not available yet");
00136 return;
00137 }
00138 pcl::PointCloud<pcl::PointXYZRGB>::Ptr pcl_cloud (new pcl::PointCloud<pcl::PointXYZRGB>);
00139 pcl::fromROSMsg(*input_cloud, *pcl_cloud);
00140 pcl::PointCloud<pcl::PointXYZHSV>::Ptr hsv_cloud (new pcl::PointCloud<pcl::PointXYZHSV>);
00141 pcl::PointCloudXYZRGBtoXYZHSV(*pcl_cloud, *hsv_cloud);
00142 for (size_t i = 0; i < pcl_cloud->points.size(); i++) {
00143 hsv_cloud->points[i].x = pcl_cloud->points[i].x;
00144 hsv_cloud->points[i].y = pcl_cloud->points[i].y;
00145 hsv_cloud->points[i].z = pcl_cloud->points[i].z;
00146 }
00147
00148 std::vector<std::vector<float> > histograms;
00149 histograms.resize(input_indices->cluster_indices.size());
00150 unsigned long point_all_size=0;
00151 pcl::ExtractIndices<pcl::PointXYZHSV> extract;
00152 extract.setInputCloud(hsv_cloud);
00153
00154 jsk_recognition_msgs::ColorHistogramArray histogram_array;
00155 histogram_array.header = input_cloud->header;
00156 std::vector<pcl::PointCloud<pcl::PointXYZHSV>::Ptr > segmented_clouds;
00157 for (size_t i = 0; i < input_indices->cluster_indices.size(); i++) {
00158 pcl::IndicesPtr indices (new std::vector<int>(input_indices->cluster_indices[i].indices));
00159 point_all_size+=indices->size();
00160 extract.setIndices(indices);
00161 pcl::PointCloud<pcl::PointXYZHSV> segmented_cloud;
00162 extract.filter(segmented_cloud);
00163 segmented_clouds.push_back(segmented_cloud.makeShared());
00164 std::vector<float> histogram;
00165 computeHistogram(segmented_cloud, histogram, policy_);
00166 histograms[i] = histogram;
00167 jsk_recognition_msgs::ColorHistogram ros_histogram;
00168 ros_histogram.header = input_cloud->header;
00169 ros_histogram.histogram = histogram;
00170 histogram_array.histograms.push_back(ros_histogram);
00171 }
00172 all_histogram_pub_.publish(histogram_array);
00173
00174
00175 jsk_recognition_msgs::ClusterPointIndices result;
00176 result.header = input_indices->header;
00177 double best_coefficient = - DBL_MAX;
00178 int best_index = -1;
00179 pcl::PointCloud<pcl::PointXYZRGB>::Ptr output_cloud (new pcl::PointCloud<pcl::PointXYZRGB>);
00180 if(publish_colored_cloud_){
00181 output_cloud->width=point_all_size;
00182 output_cloud->height=1;
00183 output_cloud->resize(point_all_size);
00184 }
00185 unsigned long count_points=0;
00186 for (size_t i = 0; i < input_indices->cluster_indices.size(); i++) {
00187 const double coefficient = bhattacharyyaCoefficient(histograms[i], reference_histogram_);
00188 JSK_NODELET_DEBUG_STREAM("coefficient: " << i << "::" << coefficient);
00189 if(publish_colored_cloud_){
00190 int tmp_point_size = input_indices->cluster_indices[i].indices.size();
00191 double color_standard;
00192 if(color_min_coefficient_ > coefficient){
00193 color_standard = 0;
00194 } else if(color_max_coefficient_ < coefficient){
00195 color_standard = 1;
00196 } else{
00197 color_standard = (coefficient - color_min_coefficient_) / (color_max_coefficient_ - color_min_coefficient_);
00198 }
00199 double color_standard_powered = 1;
00200 for (int k=0; k<power_; k++){
00201 color_standard_powered *= color_standard;
00202 }
00203 unsigned char color_r, color_g, color_b;
00204 switch(show_method_){
00205 case 0:
00206 color_r=(int)(255*color_standard_powered);
00207 color_g=0;
00208 color_b=(int)(255*(1-color_standard_powered));
00209 break;
00210 case 1:
00211
00212 int color_index = (int)(color_standard_powered*1280);
00213 switch(color_index/256){
00214 case 0:
00215 color_r=0; color_g=0; color_b=color_index;
00216 break;
00217 case 1:
00218 color_r=color_index-256; color_g=0; color_b=255;
00219 break;
00220 case 2:
00221 color_r=255; color_g=0; color_b=255-(color_index-256*2);
00222 break;
00223 case 3:
00224 color_r=255; color_g=color_index-256*3; color_b=0;
00225 break;
00226 case 4:
00227 color_r=255; color_g=255; color_b=color_index-256*4;
00228 break;
00229 case 5:
00230 color_r=255; color_g=255; color_b=255;
00231 break;
00232 }
00233 break;
00234 }
00235 for(int j=0; j<tmp_point_size; j++){
00236 output_cloud->points[j+count_points].x=segmented_clouds[i]->points[j].x;
00237 output_cloud->points[j+count_points].y=segmented_clouds[i]->points[j].y;
00238 output_cloud->points[j+count_points].z=segmented_clouds[i]->points[j].z;
00239 output_cloud->points[j+count_points].r=color_r;
00240 output_cloud->points[j+count_points].g=color_g;
00241 output_cloud->points[j+count_points].b=color_b;
00242 }
00243 count_points+=tmp_point_size;
00244 }
00245 if (coefficient > coefficient_thr_) {
00246 result.cluster_indices.push_back(input_indices->cluster_indices[i]);
00247 if (best_coefficient < coefficient) {
00248 best_coefficient = coefficient;
00249 best_index = i;
00250 }
00251 }
00252 }
00253 JSK_NODELET_DEBUG("best coefficients: %f, %d", best_coefficient, best_index);
00254
00255 sensor_msgs::PointCloud2 p_msg;
00256 if(publish_colored_cloud_){
00257 pcl::toROSMsg(*output_cloud, p_msg);
00258 p_msg.header=input_cloud->header;
00259 coefficient_points_pub_.publish(p_msg);
00260 }
00261 result_pub_.publish(result);
00262 if (best_index != -1) {
00263 pcl::PointCloud<pcl::PointXYZHSV>::Ptr best_cloud
00264 = segmented_clouds[best_index];
00265 Eigen::Vector4f center;
00266 pcl::compute3DCentroid(*best_cloud, center);
00267 geometry_msgs::PoseStamped best_pose;
00268 best_pose.header = input_cloud->header;
00269 best_pose.pose.position.x = center[0];
00270 best_pose.pose.position.y = center[1];
00271 best_pose.pose.position.z = center[2];
00272 best_pose.pose.orientation.w = 1.0;
00273 best_pub_.publish(best_pose);
00274 }
00275 }
00276
00277 double ColorHistogramMatcher::bhattacharyyaCoefficient(const std::vector<float>& a, const std::vector<float>& b)
00278 {
00279 if (a.size() != b.size()) {
00280 JSK_NODELET_ERROR("the bin size of histograms do not match");
00281 return 0.0;
00282 }
00283 double sum = 0.0;
00284 for (size_t i = 0; i < a.size(); i++) {
00285 sum += sqrt(a[i] * b[i]);
00286 }
00287 return sum;
00288 }
00289
00290 void ColorHistogramMatcher::computeHistogram(
00291 const pcl::PointCloud<pcl::PointXYZHSV>& cloud,
00292 std::vector<float>& output,
00293 const ComparePolicy policy)
00294 {
00295 if (policy == USE_HUE_AND_SATURATION) {
00296 std::vector<float> hue, saturation;
00297 computeHistogram(cloud, hue, USE_HUE);
00298 computeHistogram(cloud, saturation, USE_SATURATION);
00299
00300 output.resize(hue.size() + saturation.size());
00301 for (size_t i = 0; i < hue.size(); i++) {
00302 output[i] = hue[i];
00303 }
00304 for (size_t j = hue.size(); j < hue.size() + saturation.size(); j++) {
00305 output[j] = saturation[j - hue.size()];
00306 }
00307 }
00308 else {
00309 double val_max, val_min;
00310 if (policy == USE_HUE) {
00311 val_max = 360.0;
00312 val_min = 0.0;
00313 }
00314 else {
00315 val_max = 1.0;
00316 val_min = 0.0;
00317 }
00318 output.resize(bin_size_, 0);
00319 for (size_t i = 0; i < cloud.points.size(); i++) {
00320 pcl::PointXYZHSV output_point = cloud.points[i];
00321
00322 double val;
00323 if (policy == USE_HUE) {
00324 val = output_point.h;
00325 }
00326 else if (policy == USE_SATURATION) {
00327 val = output_point.s;
00328 }
00329 else if (policy == USE_VALUE) {
00330 val = output_point.v;
00331 }
00332 int index = int((val - val_min) / (val_max - val_min) * bin_size_);
00333 if (index >= bin_size_) {
00334 index = bin_size_ - 1;
00335 }
00336 output[index] += 1.0;
00337 }
00338 }
00339
00340 double sum = 0;
00341 for (size_t i = 0; i < output.size(); i++) {
00342 sum += output[i];
00343 }
00344 for (size_t i = 0; i < output.size(); i++) {
00345 if (sum != 0.0) {
00346 output[i] /= sum;
00347 }
00348 else {
00349 output[i] = 0.0;
00350 }
00351 }
00352 }
00353
00354 void ColorHistogramMatcher::reference(
00355 const sensor_msgs::PointCloud2::ConstPtr& input_cloud)
00356 {
00357 boost::mutex::scoped_lock lock(mutex_);
00358 std::vector<float> hist;
00359 pcl::PointCloud<pcl::PointXYZRGB> pcl_cloud;
00360 pcl::fromROSMsg(*input_cloud, pcl_cloud);
00361 pcl::PointCloud<pcl::PointXYZHSV> hsv_cloud;
00362 pcl::PointCloudXYZRGBtoXYZHSV(pcl_cloud, hsv_cloud);
00363 computeHistogram(hsv_cloud, hist, policy_);
00364 reference_histogram_ = hist;
00365 JSK_NODELET_INFO("update reference");
00366 reference_set_ = true;
00367 jsk_recognition_msgs::ColorHistogram ros_histogram;
00368 ros_histogram.header = input_cloud->header;
00369 ros_histogram.histogram = reference_histogram_;
00370 reference_histogram_pub_.publish(ros_histogram);
00371 }
00372
00373 void ColorHistogramMatcher::referenceHistogram(
00374 const jsk_recognition_msgs::ColorHistogram::ConstPtr& input_histogram)
00375 {
00376 boost::mutex::scoped_lock lock(mutex_);
00377 JSK_NODELET_INFO("update reference");
00378 reference_histogram_ = input_histogram->histogram;
00379 reference_histogram_pub_.publish(input_histogram);
00380 reference_set_ = true;
00381 }
00382
00383 }
00384
00385 PLUGINLIB_EXPORT_CLASS (jsk_pcl_ros::ColorHistogramMatcher,
00386 nodelet::Nodelet);