render_target.cpp

Go to the documentation of this file.

/*
 * Copyright (c) 2010, Willow Garage, Inc.
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions are met:
 *
 *     * Redistributions of source code must retain the above copyright
 *       notice, this list of conditions and the following disclaimer.
 *     * Redistributions in binary form must reproduce the above copyright
 *       notice, this list of conditions and the following disclaimer in the
 *       documentation and/or other materials provided with the distribution.
 *     * Neither the name of the Willow Garage, Inc. nor the names of its
 *       contributors may be used to endorse or promote products derived from
 *       this software without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
 * POSSIBILITY OF SUCH DAMAGE.
 */

#include "rve_render_server/render_target.h"
#include "rve_render_server/camera.h"
#include "rve_render_server/renderer.h"
#include "rve_render_server/init.h"
#include "rve_render_server/server.h"
#include "rve_render_server/pickable.h"
#include "rve_render_server/screen_rect.h"

#include <OGRE/OgreRenderTarget.h>
#include <OGRE/OgreTextureManager.h>
#include <OGRE/OgreMaterialManager.h>
#include <OGRE/OgreTechnique.h>
#include <OGRE/OgrePass.h>
#include <OGRE/OgreTextureUnitState.h>
#include <OGRE/OgreRoot.h>
#include <OGRE/OgreRenderSystem.h>
#include <OGRE/OgreRenderTarget.h>
#include <OGRE/OgreRenderTexture.h>
#include <OGRE/OgreHardwarePixelBuffer.h>
#include <OGRE/OgreRectangle2D.h>
#include <OGRE/OgreSceneManager.h>
#include <OGRE/OgreSceneNode.h>
#include <OGRE/OgreSceneQuery.h>

#include <ros/assert.h>
#include <ros/ros.h>
#include <sensor_msgs/Image.h>
#include <sensor_msgs/image_encodings.h>

#include <boost/foreach.hpp>


using namespace rve_common;

namespace rve_render_server
{

std::string dashToUnderscore(const std::string& str)
{
  std::string out = str;
  for (size_t i = 0; i < out.size(); ++i)
  {
    if (out[i] == '-')
    {
      out[i] = '_';
    }
  }

  return out;
}

class RenderTargetToROSImage
{
public:
  RenderTargetToROSImage(Ogre::RenderTarget* target, const std::string& name)
  {
    setTarget(target);
    pub_ = getServer()->getNodeHandle().advertise<sensor_msgs::Image>("render_targets/" + name, 1);
  }

  void setTarget(Ogre::RenderTarget* target)
  {
    target_ = target;
  }

  ~RenderTargetToROSImage()
  {
  }

  void update()
  {
    if (pub_.getNumSubscribers() == 0)
    {
      return;
    }

    uint32_t width = target_->getWidth();
    uint32_t height = target_->getHeight();

    sensor_msgs::ImagePtr image(new sensor_msgs::Image);
    image->encoding = sensor_msgs::image_encodings::RGBA8;
    image->width = width;
    image->height = height;
    image->step = width * 4;
    image->data.resize(image->step * height);

    Ogre::PixelBox pbox(width, height, 1, Ogre::PF_BYTE_RGBA, &image->data.front());
    target_->copyContentsToMemory(pbox, Ogre::RenderTarget::FB_FRONT);
    pub_.publish(image);
  }

private:

  Ogre::RenderTarget* target_;
  ros::Publisher pub_;
};



RenderTarget::RenderTarget(const rve_common::UUID& id, Renderer* rend, uint32_t width, uint32_t height)
: id_(id)
, final_target_(0)
, renderer_(rend)
, cam_(0)
, width_(width)
, height_(height)
, recreate_resources_(false)
, screen_rect_(0)
, picked_(false)
, pick_target_updated_this_frame_(false)
, pick_buffer_valid_(false)
, pick_scene_query_(0)
, render_needed_(false)
, render_in_progress_(false)
{
  screen_rect_ = new Ogre::Rectangle2D(true);
  screen_rect_->setCorners(-1, 1, 1, -1, false);
  screen_rect_->setBoundingBox(Ogre::AxisAlignedBox::BOX_INFINITE);

  std::stringstream ss;
  ss << getID() << "RectMaterial";
  rect_material_ = Ogre::MaterialManager::getSingleton().getByName("DeferredShading/DeferredWithWeightedFinal", ROS_PACKAGE_NAME);
  rect_material_ = rect_material_->clone(ss.str());
  screen_rect_->setMaterial(rect_material_->getName());

  createResources();

  stats_.id = getID();
}

RenderTarget::~RenderTarget()
{
  if (cam_)
  {
    cam_->removeRenderTarget(this);
  }

  destroyResources();

  {
    V_ScreenRect::iterator it = screen_rects_.begin();
    V_ScreenRect::iterator end = screen_rects_.end();
    for (; it != end; ++it)
    {
      delete *it;
    }
    screen_rects_.clear();
  }
}

void RenderTarget::setTarget(Ogre::RenderTarget* target)
{
  if (!final_target_)
  {
    final_target_out_.reset(new RenderTargetToROSImage(final_target_, dashToUnderscore(getID().toString())));
  }

  final_target_ = target;
  final_target_out_->setTarget(final_target_);
}

void RenderTarget::destroyMRT(MRT& mrt)
{
  uint32_t count = mrt.textures.size() - 1;

  Ogre::Root::getSingleton().getRenderSystem()->destroyRenderTarget(mrt.mrt->getName());

  V_OgreTexture::reverse_iterator it = mrt.textures.rbegin();
  V_OgreTexture::reverse_iterator end = mrt.textures.rend();
  for (; it != end; ++it, --count)
  {
    Ogre::TexturePtr& tex = *it;
    Ogre::TextureManager::getSingleton().remove(tex->getName());
  }

  mrt.textures.clear();
  mrt.rtts.clear();
  mrt.mrt = 0;
}

void RenderTarget::destroyResources()
{
  destroyMRT(gbuffer_target_);
  destroyMRT(weighted_average_alpha_target_);

  // Only destroy the picking MRT if it exists
  if (picked_)
  {
    destroyMRT(picking_target_);
  }
}

void RenderTarget::createMRT(MRT& mrt, const std::string& name, const std::string& scheme, uint32_t format)
{
  std::stringstream ss;
  ss << getID() << "_" << name;
  mrt.mrt = Ogre::Root::getSingleton().getRenderSystem()->createMultiRenderTarget(ss.str());
  mrt.mrt->setAutoUpdated(false);
  mrt.mrt->setActive(true);

  mrt.ros_output.resize(2);
  for (uint32_t i = 0; i < 2; ++i)
  {
    std::stringstream ss2;
    ss2 << ss.str() << "rt" << i;
    mrt.textures.push_back(Ogre::TextureManager::getSingleton().createManual(ss2.str(), ROS_PACKAGE_NAME, Ogre::TEX_TYPE_2D, width_, height_, 0, (Ogre::PixelFormat)format, Ogre::TU_RENDERTARGET));
    mrt.rtts.push_back(mrt.textures[i]->getBuffer()->getRenderTarget());
    mrt.mrt->bindSurface(i, mrt.rtts[i]);

    if (!mrt.ros_output[i])
    {
      mrt.ros_output[i].reset(new RenderTargetToROSImage(mrt.rtts[i], dashToUnderscore(ss2.str())));
    }
    else
    {
      mrt.ros_output[i]->setTarget(mrt.rtts[i]);
    }
  }

  setupRT(mrt.mrt);
  if (mrt.mrt->getNumViewports())
  {
    mrt.mrt->getViewport(0)->setMaterialScheme(scheme);
  }
}

void RenderTarget::createResources()
{
  createMRT(gbuffer_target_, "gbuffer", "GBuffer", Ogre::PF_FLOAT16_RGBA);
  createMRT(weighted_average_alpha_target_, "waa", "WeightedAverageAlpha", Ogre::PF_FLOAT16_RGBA);

  // Only create the picking MRT and buffer if we've been asked to pick before
  if (picked_)
  {
    createMRT(picking_target_, "pick", "Pick", Ogre::PF_BYTE_RGBA);
    pick_buffer_.reset(new uint32_t[width_ * height_ * 2]);
  }

  rect_material_->getTechnique(0)->getPass(0)->getTextureUnitState(0)->setTextureName(gbuffer_target_.textures[0]->getName());
  rect_material_->getTechnique(0)->getPass(0)->getTextureUnitState(1)->setTextureName(gbuffer_target_.textures[1]->getName());
  rect_material_->getTechnique(0)->getPass(0)->getTextureUnitState(2)->setTextureName(weighted_average_alpha_target_.textures[0]->getName());
  rect_material_->getTechnique(0)->getPass(0)->getTextureUnitState(3)->setTextureName(weighted_average_alpha_target_.textures[1]->getName());
}

void RenderTarget::setupRT(Ogre::RenderTarget* rt)
{
  rt->removeAllViewports();

  if (cam_)
  {
    Ogre::Viewport* vp = rt->addViewport(cam_->getOgreCamera());
    ROS_INFO("RenderTarget::setupRT() ogre_viewport size: %d x %d",
             vp->getActualWidth(), vp->getActualHeight());
    vp->setBackgroundColour(Ogre::ColourValue(0.0, 0.0, 0.0, 0.0));
    vp->setClearEveryFrame(false);
    vp->setOverlaysEnabled(false);
  }
}

const rve_common::UUID& RenderTarget::getID()
{
  return id_;
}

void RenderTarget::resize(uint32_t width, uint32_t height)
{
  width_ = width;
  height_ = height;

  recreate_resources_ = true;
}

void RenderTarget::attachCamera(Camera* cam)
{
  if (cam_ && pick_scene_query_)
  {
    cam_->getOgreCamera()->getSceneManager()->destroyQuery(pick_scene_query_);
  }

  if (cam_ && !cam)
  {
    destroyResources();
    cam_ = 0;
    createResources();
    return;
  }

  cam_ = cam;

  setupRT(gbuffer_target_.mrt);
  setupRT(weighted_average_alpha_target_.mrt);
  setupRT(final_target_);

  if (picked_)
  {
    setupRT(picking_target_.mrt);
    pick_scene_query_ = cam_->getOgreCamera()->getSceneManager()->createPlaneBoundedVolumeQuery(Ogre::PlaneBoundedVolumeList());
  }

  gbuffer_target_.mrt->getViewport(0)->setMaterialScheme("GBuffer");
  weighted_average_alpha_target_.mrt->getViewport(0)->setMaterialScheme("WeightedAverageAlpha");
}

struct Timer
{
  Timer()
  : start_(ros::WallTime::now())
  {}

  double elapsed() { return (ros::WallTime::now() - start_).toSec(); }

  ros::WallTime start_;
};

bool rectSort(ScreenRect* lhs, ScreenRect* rhs)
{
  return lhs->getZOrder() > rhs->getZOrder();
}

void RenderTarget::requestRender()
{
  render_needed_ = true;
}

void RenderTarget::beginRender()
{
  ROS_ASSERT(!render_in_progress_);

  if (recreate_resources_)
  {
    destroyResources();
    createResources();
    recreate_resources_ = false;
    render_needed_ = true;
  }

  if (render_needed_)
  {
    render_in_progress_ = true;
    render_needed_ = false;
  }
  else
  {
    return;
  }

  if (cam_)
  {
    // Work around OGRE bug #340: http://www.ogre3d.org/mantis/view.php?id=340
    Ogre::Root::getSingleton().getRenderSystem()->_setViewport(gbuffer_target_.mrt->getViewport(0));
    gbuffer_target_.mrt->getViewport(0)->clear(Ogre::FBT_COLOUR|Ogre::FBT_DEPTH|Ogre::FBT_STENCIL, Ogre::ColourValue(0.0, 0.0, 0.0, 0.0));
    weighted_average_alpha_target_.mrt->getViewport(0)->clear(Ogre::FBT_COLOUR|Ogre::FBT_DEPTH|Ogre::FBT_STENCIL, Ogre::ColourValue(0.0, 0.0, 0.0, 0.0));
  }

  {
    Timer t;
    gbuffer_target_.mrt->update(false);
    gbuffer_target_.ros_output[0]->update();
    gbuffer_target_.ros_output[1]->update();
    stats_.gbuffer_update = t.elapsed();
  }

  {
    Timer t;
    weighted_average_alpha_target_.mrt->update(false);
    weighted_average_alpha_target_.ros_output[0]->update();
    weighted_average_alpha_target_.ros_output[1]->update();
    t.elapsed(); // TODO
  }

  {
    Timer t;
    if (cam_)
    {
      final_target_->getViewport(0)->clear(Ogre::FBT_COLOUR|Ogre::FBT_DEPTH|Ogre::FBT_STENCIL);
#if 01
      cam_->getOgreCamera()->getSceneManager()->manualRender(screen_rect_, rect_material_->getTechnique(0)->getPass(0),
                                                             final_target_->getViewport(0), cam_->getOgreCamera()->getViewMatrix(),
                                                             cam_->getOgreCamera()->getProjectionMatrix(), true, false, false, 0);
#else
      final_target_->update(false);
#endif

      {
        // Now render the screen elements
        std::sort(screen_rects_.begin(), screen_rects_.end(), rectSort);
        V_ScreenRect::iterator it = screen_rects_.begin();
        V_ScreenRect::iterator end = screen_rects_.end();
        for (; it != end; ++it)
        {
          ScreenRect* rect = *it;
          Ogre::Renderable* rend = rect->getRenderable();
          cam_->getOgreCamera()->getSceneManager()->manualRender(rend, rend->getMaterial()->getTechnique(0)->getPass(0),
                                                                 final_target_->getViewport(0), cam_->getOgreCamera()->getViewMatrix(),
                                                                 cam_->getOgreCamera()->getProjectionMatrix(), true, false, false, 0);
        }
      }

      final_target_out_->update();
    }

    stats_.blit_to_main = t.elapsed();
  }

  pick_buffer_valid_ = false;
  pick_target_updated_this_frame_ = false;
  pick_requests_.swap(new_pick_requests_);

  if (!pick_requests_.empty())
  {
    if (cam_)
    {
      // Work around OGRE bug #340: http://www.ogre3d.org/mantis/view.php?id=340
      Ogre::Root::getSingleton().getRenderSystem()->_setViewport(picking_target_.mrt->getViewport(0));
      picking_target_.mrt->getViewport(0)->clear(Ogre::FBT_COLOUR|Ogre::FBT_DEPTH|Ogre::FBT_STENCIL, Ogre::ColourValue(0.0, 0.0, 0.0, 0.0));
    }

    // Figure out the min/max x/y
    uint32_t x0 = 9999999;
    uint32_t y0 = 9999999;
    uint32_t x1 = 0;
    uint32_t y1 = 0;
    V_PickRequest::iterator it = pick_requests_.begin();
    V_PickRequest::iterator end = pick_requests_.end();
    for (; it != end; ++it)
    {
      const PickRequest& r = *it;
      x0 = std::min(x0, std::min(r.x0, r.x1));
      x1 = std::max(x1, std::max(r.x0, r.x1));
      y0 = std::min(y0, std::min(r.y0, r.y1));
      y1 = std::max(y1, std::max(r.y0, r.y1));
    }

    x1 += 1;
    y1 += 1;
    x0 = std::min(x0, width_);
    x1 = std::min(x1, width_);
    y0 = std::min(y0, height_);
    y1 = std::min(y1, height_);
    float left = (float)x0 / width_;
    float right = (float)x1 / width_;
    float top = (float)y0 / height_;
    float bottom = (float)y1 / height_;

    // ROS_INFO("right,bottom = %f,%f", right, bottom);

    Ogre::Viewport* vp = picking_target_.mrt->getViewport(0);
    vp->setVisibilityMask(0x10000000);

    Ogre::Camera* cam = cam_->getOgreCamera();

    Ogre::PlaneBoundedVolumeList pbvl;
    pbvl.push_back(cam->getCameraToViewportBoxVolume(left, top, right, bottom, false));

    pick_scene_query_->setVolumes(pbvl);
    pick_scene_query_->clearResults();
    Ogre::SceneQueryResult& results = pick_scene_query_->execute();
    //ROS_INFO("****************************************");
    BOOST_FOREACH(Ogre::MovableObject* obj, results.movables)
    {
      obj->addVisibilityFlags(0x10000000);
      //ROS_INFO("%p", obj);
    }

    Timer t;
    picking_target_.mrt->update(false);
    picking_target_.ros_output[0]->update();
    picking_target_.ros_output[1]->update();
    //ROS_INFO_STREAM(t.elapsed()); // TODO

    pick_target_updated_this_frame_ = true;

    BOOST_FOREACH(Ogre::MovableObject* obj, results.movables)
    {
      obj->removeVisibilityFlags(0x10000000);
    }
  }
}

void RenderTarget::finishRender()
{
  if (!render_in_progress_)
  {
    return;
  }

  {
    Timer t;
    gbuffer_target_.mrt->swapBuffers(false);
    stats_.gbuffer_swap = t.elapsed();
  }

  {
    Timer t;
    weighted_average_alpha_target_.mrt->swapBuffers(false);
    t.elapsed(); // TODO
  }

  {
    Timer t;
    final_target_->swapBuffers(false);
    stats_.main_swap = t.elapsed();
  }

  if (pick_target_updated_this_frame_)
  {
    Timer t;
    updatePickBuffer();
    processPickRequests();
    t.elapsed(); // TODO
  }

  render_in_progress_ = false;
}

void RenderTarget::pick(const PickRequest& pr)
{
  ROS_ASSERT(pick_buffer_valid_);

  uint32_t x0 = std::min(std::min(pr.x0, pr.x1), width_);
  uint32_t x1 = std::min(std::max(pr.x0, pr.x1), width_);
  uint32_t y0 = std::min(std::min(pr.y0, pr.y1), height_);
  uint32_t y1 = std::min(std::max(pr.y0, pr.y1), height_);
  uint32_t pick_width = x1 - x0 + 1;
  uint32_t pick_height = y1 - y0 + 1;
  uint32_t width = width_;
  uint32_t height = height_;
  uint32_t target_array_size = width * height;

  if (x0 >= width_ - 1 || y0 >= height_ - 1)
  {
    V_PickRenderValues vals;
    pr.cb(vals);
    return;
  }

  uint32_t* buf = pick_buffer_.get();

  V_PickRenderValues out_picked;
  for (uint32_t y = y0; y < y0 + pick_height; ++y)
  {
    for (uint32_t x = x0; x < x0 + pick_width; ++x)
    {
      uint32_t offset = y * width + x;
      uint32_t obj = buf[offset];
      uint32_t extra = buf[target_array_size + offset];
      out_picked.push_back(PickRenderValues(obj, extra));
    }
  }

  pr.cb(out_picked);
}

void RenderTarget::updatePickBuffer()
{
  //Ogre::Viewport* vp = picking_target_.mrt->getViewport(0);
  //vp->setDimensions((double)x / width_, (double)y / height_, 1, 1);

  ROS_ASSERT(pick_target_updated_this_frame_);

  picking_target_.mrt->swapBuffers(false);

  uint32_t width = width_;
  uint32_t height = height_;
  uint32_t target_array_size = width * height;
  Ogre::PixelBox pbox(width, height, 1, Ogre::PF_BYTE_RGBA, pick_buffer_.get());
  picking_target_.rtts[0]->copyContentsToMemory(pbox, Ogre::RenderTarget::FB_AUTO);
  pbox = Ogre::PixelBox(width, height, 1, Ogre::PF_BYTE_RGBA, pick_buffer_.get() + target_array_size);
  picking_target_.rtts[1]->copyContentsToMemory(pbox, Ogre::RenderTarget::FB_AUTO);

  //vp->setDimensions(0, 0, 1.0, 1.0);

  pick_buffer_valid_ = true;
}

void RenderTarget::pick(uint32_t x0, uint32_t y0, uint32_t x1, uint32_t y1, const PickCallback& cb)
{
  if (!cam_)
  {
    V_PickRenderValues empty;
    cb(empty);
    return;
  }

  if (!picked_)
  {
    picked_ = true;
    pick_buffer_.reset(new uint32_t[width_ * height_ * 2]);
    createMRT(picking_target_, "pick", "Pick", Ogre::PF_BYTE_RGBA);
    pick_scene_query_ = cam_->getOgreCamera()->getSceneManager()->createPlaneBoundedVolumeQuery(Ogre::PlaneBoundedVolumeList());
  }

  PickRequest pr(x0, y0, x1, y1, cb);
#if 0
  if (pick_buffer_valid_)
  {
    pick(pr);
  }
  else
#endif
  {
    new_pick_requests_.push_back(pr);
  }

  render_needed_ = true;
}

void RenderTarget::processPickRequests()
{
  V_PickRequest::iterator it = pick_requests_.begin();
  V_PickRequest::iterator end = pick_requests_.end();
  for (; it != end; ++it)
  {
    pick(*it);
  }
  pick_requests_.clear();
}

ScreenRect* RenderTarget::createScreenRect(const rve_common::UUID& id, uint32_t zorder, float x0, float y0, float x1, float y1)
{
  ScreenRect* rect = new ScreenRect(id);
  rect->setZOrder(zorder);
  rect->setCorners(x0, y0, x1, y1);
  screen_rects_.push_back(rect);
  return rect;
}

bool rectMatches(ScreenRect* rect, const rve_common::UUID& id)
{
  return rect->getID() == id;
}

ScreenRect* RenderTarget::getScreenRect(const rve_common::UUID& id)
{
  V_ScreenRect::iterator it = std::find_if(screen_rects_.begin(), screen_rects_.end(), boost::bind(rectMatches, _1, boost::ref(id)));
  if (it == screen_rects_.end())
  {
    throw std::runtime_error("No ScreenRect with id [" + id.toString() + "]");
  }

  return *it;
}

void RenderTarget::destroyScreenRect(const rve_common::UUID& id)
{
  V_ScreenRect::iterator it = std::find_if(screen_rects_.begin(), screen_rects_.end(), boost::bind(rectMatches, _1, boost::ref(id)));
  if (it == screen_rects_.end())
  {
    throw std::runtime_error("No ScreenRect with id [" + id.toString() + "]");
  }

  ScreenRect* rect = *it;
  screen_rects_.erase(it);
  delete rect;
}

} // namespace rve_render_server