auto_selector.py

Go to the documentation of this file.

#!/usr/bin/env python
import roslib
roslib.load_manifest('face_contour_detector')
import rospy
import face_contour_detector.image_information.sql_database as sql_database
import k_nearest_neighbour
from face_contour_detector.srv import *
from face_contour_detector.msg import *
import face_contour_detector.image_information.image_information as image_information
import face_contour_detector.image_information.face_areas as face_areas
import cv_bridge
import cv
import face_contour_detector.gui.image_settings_frame as image_settings_frame
from portrait_robot_msgs.srv import *

## @package face_contour_detector.learing.auto_selector
#  This file contains the AutoSelector class which can be used to create filter setting proposals form a knowledge-base.
#  When this file is executed as main script it initializes a rosservice called "experience_autoselector". See service file srv/experience_request.srv
#  @author Fabian Wenzelmann

## @brief This class is used to generate proposals for filter settings.
#  @author Fabian Wenzelmann
#
#  A  face_contour_detector.image_information.sql_database.SQLImageDatabase is used to store information about some images. From this values proposals are calculated using the users raiting.
class AutoSelector:
        ## @brief constructor method
        #  @param self the object pointe
        #  @param bridge a cv_bridge. If you already have a bridge this bridge will be used here. Otherwise a new cv_bridge is created
    def __init__(self, bridge=None):
                self.images = []
                if bridge is None:
                        bridge = cv_bridge.CvBridge()
                self.bridge = bridge
        
    # field documentation
    
    ## @var images
    #a list of images - type is similar to the return value of face_contour_detector.image_information.sql_database.SQLImageDatabase.get_images()
    
    ## @var bridge
    #  cv_bridge used to convert images
    
    ## @brief Load the image information database form it's default path.
    #  @brief self the object pointer
    #
    #  Try to load the database from its default path. The <code>images</code> field is set to the value returned by face_contour_detector.image_information.sql_database.SQLImageDatabase.get_images().
    #  If opening the database fails images is set to an empty list.
    def load_file(self):
        sql_db = sql_database.SQLImageDatabase(bridge=self.bridge)
        try:
            sql_db.load_file()
            self.images = sql_db.get_images()
        except:
            self.images = []
        msg_srv = rospy.ServiceProxy("/alubsc/status_msg", alubsc_status_msg)
        try:
            msg_srv(2, ("opened database containing information about " + str(len(self.images)) + " face images").encode("ascii"))
        except:
            pass
      
    ## @brief Choose the parameters where self.images is used as information database.
    #  @param self the object pointer
    #  @param img sensor_msgs/Image the image to choose the parameter for. The image must be an colored face image (RGB) where all pixels that are not on the mask are set to white.
    #  @param mask sensor_msgs/Image the mask to calculate the face areas from.
    #
    #  The database stores the average brightness of images and the standard deviation of the brightness. We assume that these values correlate with the filter settings. We think that these values are the most important for the parameter settings.
    #  We first just find images in the database that are "similar" to img. Brightness and standard deviation of the brightness are used as distance measure.
    #  We have a fixed set of possible value combinations (returned by the function face_contour_detector.learing.auto_selector._get_parameter_combinations()). Now we find for each of the face areas the best under those combinations.
    #  To do this we check all combinations for this area: Now we find under the similar images those that have "similar" filter settings for this area, according to the curren combination. We then calculate the average raiting of that images. After everything is done we choose the combination with the best average raiting.
    #  This image gives an overview:
    #  @image html auto_selector.png
    #  If the database is empty an exception is thrown.
    def choose_parameters(self, img, mask):
        cv_img = self.bridge.imgmsg_to_cv(img)
        cv_mask = self.bridge.imgmsg_to_cv(mask, "mono8")
        extractor = image_information.InformationExtractor()
        extractor.information_extractors = [ image_information.AverageValueExtractor() ]
        information = extractor.to_dict(cv_img)
        new_img = {}
        new_img["AveragePixelBrightness"], new_img["StandardBrightnessDeviation"] = information["AveragePixelBrightness"][0], information["StandardBrightnessDeviation"][0]
        
        # now calculate the 10 most similar images
        similar_images = k_nearest_neighbour.k_nearest_neighbours(new_img, self.images, 20, key=self._entry_key_func, key2=self._entry_key_func)
        combinations = _get_parameter_combinations()
        # now for each area calculate it's parameters
        area_resp = face_areas.find_areas(self.bridge.imgmsg_to_cv(mask, "mono8"))
        new_areas = [ area_resp.left_eye, area_resp.right_eye, area_resp.nose, area_resp.mouth, area_resp.complete_face ]
        selected_areas = []
        for area in new_areas:
            area_best_raiting = None
            best_combination = None
            for combination in combinations:
                # for this area find under the similar images those that are near to this image occording the current combination
                parameter_similar = k_nearest_neighbour.k_nearest_neighbours(combination, similar_images, 10, key=lambda params : self._parameter_key(params), key2=self._img_parameter_key(area.name))
                # calculate average raiting
                raiting_sum = 0
                for best_image in parameter_similar:
                    raiting_sum += best_image["Raiting"]
                average_raiting = float(raiting_sum) / len(parameter_similar)
                if area_best_raiting is None:
                    area_best_raiting = average_raiting
                    best_combination = combination
                elif average_raiting < area_best_raiting:
                    area_best_raiting = average_raiting
                    best_combination = combination
            blur_width_pair = name_value_pair("blurwidth".encode("ascii"), str( best_combination["blurwidth"]).encode("ascii"), "int".encode("ascii"))
            blur_height_pair = name_value_pair("blurheight".encode("ascii"), str( best_combination["blurheight"] ).encode("ascii"), "int".encode("ascii"))
            
            threshold1_pair = name_value_pair( "threshold1".encode("ascii"), str(best_combination["threshold1"]).encode("ascii"), "float".encode("ascii"))
            threshold2_pair = name_value_pair("threshold2".encode("ascii"), str(best_combination["threshold2"]).encode("ascii"), "float".encode("ascii"))
            l2gradientent_pair = name_value_pair("l2gradientent".encode("ascii"), "true".encode("ascii"), "bool".encode("ascii"))
            
            blur_setup = filter_setup("GaussianBlur".encode("ascii"), [blur_width_pair, blur_height_pair])
            canny_setup = filter_setup("Canny".encode("ascii"), [threshold1_pair, threshold2_pair, l2gradientent_pair])
            
            min_length_pair = name_value_pair("searchRadius".encode("ascii"), "5".encode("ascii"), "int".encode("ascii"))
            edge_setup = filter_setup("EdgeConnector".encode("ascii"), [min_length_pair])
            
            min_num_pair = name_value_pair("minNumPixels".encode("ascii"), "7".encode("ascii"), "int".encode("ascii"))
            delete_setup = filter_setup("DeleteShortLines".encode("ascii"), [min_num_pair])
            
            filters = [blur_setup, canny_setup, edge_setup, delete_setup]
            
            fa_setup = filter_area_setup(area, filters)
            
            selected_areas.append(fa_setup)
            area_order = ["Complete Face", "Nose", "LeftEye", "RightEye", "Mouth"]
            area_sorted = [None, None, None, None, None]
            for area in selected_areas:
                if area.area.name == "Complete Face":
                    area_sorted[0] = area
                elif area.area.name == "Nose":
                    area_sorted[1] = area
                elif area.area.name == "LeftEye":
                    area_sorted[2] = area
                elif area.area.name == "RightEye":
                    area_sorted[3] = area
                else:
                    area_sorted[4] = area
            
        return autoselect_result(img, area_sorted)
    
    ## @brief Internal method that is not really important to understand
    def _img_parameter_key(self, area_name):
        return lambda img : self._parameter_key(img["Areas"][area_name])
    
    ## @brief Internal method that is not really important to understand
    def _parameter_key(self, params):
        blur_width = normalize(params["blurwidth"], image_settings_frame.BLUR_WIDTH_MIN, image_settings_frame.BLUR_WIDTH_MAX, 0, 1)
        blur_height = normalize(params["blurheight"], image_settings_frame.BLUR_HEIGHT_MIN, image_settings_frame.BLUR_HEIGHT_MAX, 0, 1)
        threshold1 = normalize(params["threshold1"], image_settings_frame.CANNY_THRESHOLD1_MIN, image_settings_frame.CANNY_THRESHOLD1_MAX, 0, 1)
        threshold2 = normalize(params["threshold1"], image_settings_frame.CANNY_THRESHOLD2_MIN, image_settings_frame.CANNY_THRESHOLD2_MAX, 0, 1)
        return blur_width, blur_height, threshold1, threshold2            
    
    ## @brief Internal method that is not really important to understand 
    def _entry_key_func(self, entry):
        return normalize(entry["AveragePixelBrightness"], 0.0, 255.0, 0.0, 1.0), normalize(entry["StandardBrightnessDeviation"], 0.0, 127.5, 0.0, 1.0)
        
## @brief Normalize a value, that is in the finite domain between \f$min_{value}\f$ and \f$max_{value}\f$, to a new value between \f$norm_{min}\f$ and \f$norm_{max}\f$
#  @param value the value to normalize
#  @param min_value minimal value of the old domain
#  @param max_value maximal value of the old domain
#  @param norm_min minimal value of the new domain
#  @param norm_max maximal value of the new domain
def normalize(value, min_value, max_value, norm_min, norm_max):
    return (value - min_value) * ( (norm_max - norm_min) / (max_value - min_value) ) + norm_min

## @brief Returns a finite set of parameter combinations that can be used as filter settings for an image area.   
def _get_parameter_combinations():
    blur_width_params = [15, 16, 18, 21, 23]
    blur_height_params = [15, 16, 18, 21, 23]
    threshold1_params = [2500, 3500, 5500, 7500, 8500, 10000, 15000, 20000]
    threshold2_params = [2500, 3500, 5500, 7500, 8500, 10000, 15000, 20000]
    
    combinations = []
    
    for blur_width in blur_width_params:
        for blur_height in blur_height_params:
            for threshold1 in threshold1_params:
                for threshold2 in threshold2_params:
                    combinations.append( { "blurwidth" : blur_width, "blurheight" : blur_height, "threshold1" : threshold1, "threshold2" : threshold2 } )
    return combinations

## @brief Handler for service requests
#  @param req the request, see experience_request
def auto_select_handler(req):
    try:
        selector = AutoSelector()
        selector.load_file()
        return experience_requestResponse([selector.choose_parameters(req.image, req.mask)])
    except Exception as e:
        print(e)
        return experience_requestResponse([])

## @brief Init a service node with name experience_autoselector and type experience_request
def init_service():
    rospy.init_node("face_contour_detector_auto_selector_server")
    s1 = rospy.Service("experience_autoselector", experience_request, auto_select_handler)
    rospy.spin()
    
if __name__ == "__main__":
    init_service()