colorizer-gl.cpp
Go to the documentation of this file.
1 // License: Apache 2.0. See LICENSE file in root directory.
2 // Copyright(c) 2019 Intel Corporation. All Rights Reserved.
3 
7 
8 #include "../proc/synthetic-stream.h"
9 #include "synthetic-stream-gl.h"
10 #include "../proc/disparity-transform.h"
11 #include "colorizer-gl.h"
12 #include "option.h"
13 
14 #ifndef NOMINMAX
15 #define NOMINMAX
16 #endif // NOMINMAX
17 
18 #include <glad/glad.h>
19 
20 #include <iostream>
21 #include <chrono>
22 
23 static const char* fragment_shader_text =
24 "#version 110\n"
25 "varying vec2 textCoords;\n"
26 "uniform sampler2D textureSampler;\n"
27 "uniform sampler2D cmSampler;\n"
28 "uniform sampler2D histSampler;\n"
29 "uniform float opacity;\n"
30 "uniform float depth_units;\n"
31 "uniform float min_depth;\n"
32 "uniform float max_depth;\n"
33 "uniform float max_disparity;\n"
34 "uniform float equalize;\n"
35 "uniform float disparity;\n"
36 "void main(void) {\n"
37 " vec2 tex = vec2(textCoords.x, 1.0 - textCoords.y);\n"
38 " vec4 depth = texture2D(textureSampler, tex);\n"
39 " float dx = depth.x;\n"
40 " float dy = depth.y;\n"
41 " float nd = dx + dy * 256.0;\n"
42 " float d = 0.0;\n"
43 " if (disparity > 0.0) {;\n"
44 " d = dx;\n"
45 " } else {\n"
46 " d = nd * 256.0;\n"
47 " }\n"
48 " if (d > 0.0){\n"
49 " float f = 0.0;\n"
50 " if (equalize > 0.0){\n"
51 " float x;\n"
52 " float y;\n"
53 " vec4 hist;\n"
54 " if (disparity > 0.0) {;\n"
55 " hist = texture2D(histSampler, vec2(d / max_disparity, 0.0));\n"
56 " } else {\n"
57 " x = dx * 0.99;\n"
58 " y = dy + (1.0 / 256.0);\n"
59 " hist = texture2D(histSampler, vec2(x, y));\n"
60 " }\n"
61 " f = hist.x;\n"
62 " } else {\n"
63 " if (disparity > 0.0) {\n"
64 " f = ((d - min_depth) / (max_depth - min_depth));\n"
65 " } else {\n"
66 " f = (d * depth_units - min_depth) / (max_depth - min_depth);\n"
67 " }\n"
68 " }\n"
69 " f = clamp(f, 0.0, 0.99);\n"
70 " vec4 color = texture2D(cmSampler, vec2(f, 0.0));\n"
71 " gl_FragColor = vec4(color.x / 256.0, color.y / 256.0, color.z / 256.0, opacity);\n"
72 " } else {\n"
73 " gl_FragColor = vec4(0.0, 0.0, 0.0, opacity);\n"
74 " }\n"
75 "}";
76 
77 using namespace rs2;
78 using namespace librealsense::gl;
79 
81 {
82 public:
85  texture_2d_shader::default_vertex_shader(),
86  fragment_shader_text, "position", "textureCoords"))
87  {
88  _depth_units_location = _shader->get_uniform_location("depth_units");
89  _min_depth_location = _shader->get_uniform_location("min_depth");
90  _max_depth_location = _shader->get_uniform_location("max_depth");
91  _max_disparity_location = _shader->get_uniform_location("max_disparity");
92  _equalize_location = _shader->get_uniform_location("equalize");
93  _is_disparity_location = _shader->get_uniform_location("disparity");
94 
95  auto texture0_sampler_location = _shader->get_uniform_location("textureSampler");
96  auto texture1_sampler_location = _shader->get_uniform_location("cmSampler");
97  auto texture2_sampler_location = _shader->get_uniform_location("histSampler");
98 
99  _shader->begin();
100  _shader->load_uniform(texture0_sampler_location, texture_slot());
101  _shader->load_uniform(texture1_sampler_location, color_map_slot());
102  _shader->load_uniform(texture2_sampler_location, histogram_slot());
103  _shader->end();
104  }
105 
106  int texture_slot() const { return 0; }
107  int color_map_slot() const { return 1; }
108  int histogram_slot() const { return 2; }
109 
110  void set_params(float units, float min, float max, float max_disparity, bool equalize, bool disparity)
111  {
112  _shader->load_uniform(_depth_units_location, units);
113  _shader->load_uniform(_min_depth_location, min);
114  _shader->load_uniform(_max_depth_location, max);
115  _shader->load_uniform(_max_disparity_location, max_disparity);
116  _shader->load_uniform(_equalize_location, equalize ? 1.f : 0.f);
117  _shader->load_uniform(_is_disparity_location, disparity ? 1.f : 0.f);
118  }
119 
120 private:
127 };
128 
129 using namespace rs2;
130 
131 namespace librealsense
132 {
133  namespace gl
134  {
135  void colorizer::cleanup_gpu_resources()
136  {
137  _viz.reset();
138  _fbo.reset();
139 
140  if (_cm_texture) glDeleteTextures(1, &_cm_texture);
141 
142  _enabled = 0;
143  }
144 
145  void colorizer::create_gpu_resources()
146  {
147  _viz = std::make_shared<visualizer_2d>(std::make_shared<colorize_shader>());
148  _fbo = std::make_shared<fbo>(_width, _height);
149 
150  glGenTextures(1, &_cm_texture);
151  auto& curr_map = _maps[_map_index]->get_cache();
152  _last_selected_cm = _map_index;
153  auto size = static_cast<GLsizei>(curr_map.size());
154  glBindTexture(GL_TEXTURE_2D, _cm_texture);
155  glTexImage2D(GL_TEXTURE_2D, 0, GL_RGB16F, size, 1, 0, GL_RGB, GL_FLOAT, curr_map.data());
158 
159  _enabled = glsl_enabled() ? 1 : 0;
160  }
161 
163  : librealsense::colorizer("Depth Visualization (GLSL)"), _cm_texture(0)
164  {
165  _fhist = std::vector<float>(MAX_DEPTH, 0);
166  _fhist_data = _fhist.data();
168 
169  auto opt = std::make_shared<librealsense::ptr_option<int>>(
170  0, 1, 0, 1, &_enabled, "GLSL enabled");
172 
173  initialize();
174  }
175 
177  {
178  try
179  {
180  perform_gl_action( [&]()
181  {
183  }, [] {} );
184  }
185  catch(...)
186  {
187  LOG_DEBUG( "Error while performing cleaning up gpu resources" );
188  }
189  }
190 
192  {
193  float total = float(hist[MAX_DEPTH-1]);
194  for (int i = 0; i < MAX_DEPTH; i++)
195  f[i] = hist[i] / total;
196  }
197 
199  {
200  if(f.as<rs2::depth_frame>())
201  _depth_units = ((depth_frame*)f.get())->get_units();
202  if (f.get_profile().get() != _source_stream_profile.get())
203  {
204  _source_stream_profile = f.get_profile();
210  _width = vp.width(); _height = vp.height();
211 
213  _d2d_convert_factor = info.d2d_convert_factor;
214 
215  perform_gl_action([&]()
216  {
217  _fbo = std::make_shared<fbo>(_width, _height);
218  }, [this] {
219  _enabled = false;
220  });
221  }
222 
223  rs2::frame res = f;
224 
225  perform_gl_action([&]()
226  {
227  //scoped_timer t("colorizer.gl");
228  auto& curr_map = _maps[_map_index]->get_cache();
229 
231  {
233  auto size = static_cast<GLsizei>(curr_map.size());
234  glTexImage2D(GL_TEXTURE_2D, 0, GL_RGB16F, size, 1, 0, GL_RGB, GL_FLOAT, curr_map.data());
237 
239  }
240 
241  res = src.allocate_video_frame(_target_stream_profile, f, 3, _width, _height, _width * 3, RS2_EXTENSION_VIDEO_FRAME_GL);
242 
243  if (!res) return;
244 
245  auto fi = (frame_interface*)f.get();
246  auto df = dynamic_cast<librealsense::depth_frame*>(fi);
247  if (!df)
248  throw std::runtime_error("Frame interface is not depth frame");
249 
250  _depth_units = df->get_units();
251  bool disparity = f.get_profile().format() == RS2_FORMAT_DISPARITY32 ? true : false;
252 
253  uint32_t depth_texture;
254  uint32_t hist_texture = _cm_texture;
255 
256  if (auto input_frame = f.as<rs2::gl::gpu_frame>())
257  {
258  depth_texture = input_frame.get_texture_id(0);
259  hist_texture = input_frame.get_texture_id(1);
260  }
261  else
262  {
263  glGenTextures(1, &depth_texture);
264  glBindTexture(GL_TEXTURE_2D, depth_texture);
265 
266  if (disparity)
267  {
269  }
270  else
271  {
273  }
274 
277 
278  if (_equalize)
279  {
280  glGenTextures(1, &hist_texture);
281  glBindTexture(GL_TEXTURE_2D, hist_texture);
282 
283  if (disparity)
284  {
285  update_histogram(_hist_data, reinterpret_cast<const float*>(f.get_data()), _width, _height);
288  }
289  else
290  {
291  update_histogram(_hist_data, reinterpret_cast<const uint16_t*>(f.get_data()), _width, _height);
294  }
295 
298  }
299  }
300 
301  if( auto gf = dynamic_cast< gpu_addon_interface * >( (frame_interface *)res.get() ) )
302  {
303  uint32_t output_rgb = 0;
304  gf->get_gpu_section().output_texture( 0, &output_rgb, TEXTYPE_RGB );
305  glBindTexture( GL_TEXTURE_2D, output_rgb );
309 
310  gf->get_gpu_section().set_size( _width, _height );
311 
314 
315  glBindTexture(GL_TEXTURE_2D, output_rgb);
316  _fbo->createTextureAttachment(output_rgb);
317 
318  _fbo->bind();
319  glClearColor(1, 0, 0, 1);
321 
322  auto& shader = (colorize_shader&)_viz->get_shader();
323  shader.begin();
324  float max = _max;;
325  float min = _min;;
326  if (disparity)
327  {
328  auto __min = _min;
329  if (__min < 1e-6f) { __min = 1e-6f; } // Min value set to prevent zero division. only when _min is zero.
330  max = (_d2d_convert_factor / (__min)) * _depth_units + .5f;
331  min = (_d2d_convert_factor / (_max)) * _depth_units + .5f;
332  }
333  shader.set_params(_depth_units, min, max, MAX_DISPARITY, _equalize, disparity);
334  shader.end();
335 
336  glActiveTexture(GL_TEXTURE0 + shader.histogram_slot());
337  glBindTexture(GL_TEXTURE_2D, hist_texture);
338 
339  glActiveTexture(GL_TEXTURE0 + shader.color_map_slot());
341 
342  glActiveTexture(GL_TEXTURE0 + shader.texture_slot());
343  glBindTexture(GL_TEXTURE_2D, depth_texture);
344 
345  _viz->draw_texture(depth_texture);
346 
347  glActiveTexture(GL_TEXTURE0 + shader.texture_slot());
348 
349  _fbo->unbind();
350 
352 
353  if (!f.is<rs2::gl::gpu_frame>())
354  {
355  if (_equalize)
356  {
357  glDeleteTextures(1, &hist_texture);
358  }
359  glDeleteTextures(1, &depth_texture);
360  }
361  }
362 
363  },
364  [this]{
365  _enabled = false;
366  });
367 
368  return res;
369  }
370  }
371 }
librealsense::gl::TEXTYPE_RGB
@ TEXTYPE_RGB
Definition: synthetic-stream-gl.h:33
librealsense::gl::gpu_addon_interface
Definition: synthetic-stream-gl.h:380
librealsense
Definition: algo.h:18
librealsense::gl::colorizer::~colorizer
~colorizer() override
Definition: colorizer-gl.cpp:176
GL_COLOR_BUFFER_BIT
#define GL_COLOR_BUFFER_BIT
Definition: glad/glad/glad.h:144
rs2::gl::gpu_frame
Definition: rs_processing_gl.hpp:93
librealsense::gl::colorizer::_viz
std::shared_ptr< rs2::visualizer_2d > _viz
Definition: colorizer-gl.h:53
glActiveTexture
#define glActiveTexture
Definition: glad/glad/glad.h:2413
min
int min(int a, int b)
Definition: lz4s.c:73
rs2::frame
Definition: rs_frame.hpp:345
rs2::frame_source
Definition: rs_processing.hpp:18
librealsense::gl::colorizer::_enabled
int _enabled
Definition: colorizer-gl.h:43
colorize_shader::texture_slot
int texture_slot() const
Definition: colorizer-gl.cpp:106
GL_RG8
#define GL_RG8
Definition: glad/glad/glad.h:1229
rs2::stream_profile::as
T as() const
Definition: rs_frame.hpp:103
uint16_t
unsigned short uint16_t
Definition: stdint.h:79
librealsense::gl::gpu_processing_object::perform_gl_action
void perform_gl_action(T action, S fallback)
Definition: synthetic-stream-gl.h:320
colorize_shader::set_params
void set_params(float units, float min, float max, float max_disparity, bool equalize, bool disparity)
Definition: colorizer-gl.cpp:110
librealsense::gl::colorizer::_fhist
std::vector< float > _fhist
Definition: colorizer-gl.h:50
glDeleteTextures
#define glDeleteTextures
Definition: glad/glad/glad.h:2336
librealsense::gl::colorizer::_height
int _height
Definition: colorizer-gl.h:45
synthetic-stream-gl.h
units
GLfloat units
Definition: glad/glad/glad.h:2310
librealsense::colorizer::MAX_DISPARITY
static const int MAX_DISPARITY
Definition: colorizer.h:119
librealsense::colorizer::_maps
std::vector< color_map * > _maps
Definition: colorizer.h:160
librealsense::gl::gpu_video_frame
Definition: synthetic-stream-gl.h:421
librealsense::colorizer::_d2d_convert_factor
float _d2d_convert_factor
Definition: colorizer.h:171
librealsense::colorizer::_depth_units
float _depth_units
Definition: colorizer.h:170
GL_R32F
#define GL_R32F
Definition: glad/glad/glad.h:1232
RS2_FORMAT_RGB8
@ RS2_FORMAT_RGB8
Definition: rs_sensor.h:68
librealsense::disparity_info::update_info_from_frame
static info update_info_from_frame(const rs2::frame &f)
Definition: disparity-transform.h:78
librealsense::gl::gpu_processing_object::initialize
void initialize()
Definition: synthetic-stream-gl.h:307
colorize_shader::colorize_shader
colorize_shader()
Definition: colorizer-gl.cpp:83
rs2::texture_2d_shader::begin
void begin()
Definition: opengl3.cpp:346
librealsense::options_container::register_option
void register_option(rs2_option id, std::shared_ptr< option > option)
Definition: options-container.h:50
rs2::stream_profile::stream_type
rs2_stream stream_type() const
Definition: rs_frame.hpp:39
rs2::colorizer::colorizer
colorizer()
Definition: rs_processing.hpp:759
colorize_shader::_depth_units_location
uint32_t _depth_units_location
Definition: colorizer-gl.cpp:121
glTexImage2D
#define glTexImage2D
Definition: glad/glad/glad.h:1417
rs2::stream_profile::clone
stream_profile clone(rs2_stream type, int index, rs2_format format) const
Definition: rs_frame.hpp:63
LOG_DEBUG
#define LOG_DEBUG(...)
Definition: easyloggingpp.h:70
rs2::texture_2d_shader
Definition: opengl3.h:160
GL_RGB16F
#define GL_RGB16F
Definition: glad/glad/glad.h:1034
glTexParameteri
#define glTexParameteri
Definition: glad/glad/glad.h:1408
librealsense::gl::colorizer::_cm_texture
uint32_t _cm_texture
Definition: colorizer-gl.h:47
librealsense::frame_interface
Definition: frame-interface.h:20
GL_NEAREST
#define GL_NEAREST
Definition: glad/glad/glad.h:304
librealsense::processing_block::_source
frame_source _source
Definition: synthetic-stream.h:65
colorize_shader::_equalize_location
uint32_t _equalize_location
Definition: colorizer-gl.cpp:125
GL_FLOAT
#define GL_FLOAT
Definition: glad/glad/glad.h:262
size
GLsizeiptr size
Definition: glad/glad/glad.h:2734
glDrawBuffer
#define glDrawBuffer
Definition: glad/glad/glad.h:1420
librealsense::gl::colorizer::_fhist_data
float * _fhist_data
Definition: colorizer-gl.h:51
GL_TEXTURE_2D
#define GL_TEXTURE_2D
Definition: glad/glad/glad.h:249
librealsense::gl::colorizer::populate_floating_histogram
static void populate_floating_histogram(float *f, int *hist)
Definition: colorizer-gl.cpp:191
uint32_t
unsigned int uint32_t
Definition: stdint.h:80
rs2::video_stream_profile
Definition: rs_frame.hpp:201
librealsense::colorizer::_source_stream_profile
rs2::stream_profile _source_stream_profile
Definition: colorizer.h:168
librealsense::gl::colorizer::process_frame
rs2::frame process_frame(const rs2::frame_source &source, const rs2::frame &f) override
Definition: colorizer-gl.cpp:198
GL_COLOR_ATTACHMENT0
#define GL_COLOR_ATTACHMENT0
Definition: glad/glad/glad.h:1159
rs2::frame::get
rs2_frame * get() const
Definition: rs_frame.hpp:592
GL_LINEAR
#define GL_LINEAR
Definition: glad/glad/glad.h:305
rs_sensor.hpp
GL_RGB
#define GL_RGB
Definition: glad/glad/glad.h:291
GL_FRAMEBUFFER
#define GL_FRAMEBUFFER
Definition: glad/glad/glad.h:1193
f
GLdouble f
Definition: glad/glad/glad.h:1517
i
int i
Definition: rs-pcl-color.cpp:54
librealsense::frame_source::add_extension
void add_extension(rs2_extension ex)
Definition: source.h:50
colorize_shader::_max_depth_location
uint32_t _max_depth_location
Definition: colorizer-gl.cpp:123
rs2
Definition: animated.h:9
test-units.df
df
Definition: test-units.py:21
rs2::shader_program
Definition: opengl3.h:62
unit-test-config.src
string src
Definition: unit-test-config.py:88
glClearColor
#define glClearColor
Definition: glad/glad/glad.h:1426
librealsense::depth_frame
Definition: depth-frame.h:17
RS2_EXTENSION_VIDEO_FRAME_GL
#define RS2_EXTENSION_VIDEO_FRAME_GL
Definition: synthetic-stream-gl.h:23
librealsense::colorizer::update_histogram
static void update_histogram(int *hist, const T *depth_data, int w, int h)
Definition: colorizer.h:105
rs_processing_gl.hpp
rs2::stream_profile::get
const rs2_stream_profile * get() const
Definition: rs_frame.hpp:137
glBindTexture
#define glBindTexture
Definition: glad/glad/glad.h:2333
colorize_shader::_is_disparity_location
uint32_t _is_disparity_location
Definition: colorizer-gl.cpp:126
rs_processing.hpp
glBindFramebuffer
#define glBindFramebuffer
Definition: glad/glad/glad.h:3261
fps.info
info
Definition: fps.py:50
colorize_shader::_min_depth_location
uint32_t _min_depth_location
Definition: colorizer-gl.cpp:122
librealsense::colorizer::_target_stream_profile
rs2::stream_profile _target_stream_profile
Definition: colorizer.h:167
GLsizei
int GLsizei
Definition: glad/glad/glad.h:107
librealsense::gl::colorizer::_width
int _width
Definition: colorizer-gl.h:45
rs2::depth_frame
Definition: rs_frame.hpp:813
GL_TEXTURE0
#define GL_TEXTURE0
Definition: glad/glad/glad.h:711
rs2::shader
Definition: opengl3.h:50
librealsense::colorizer::MAX_DEPTH
static const int MAX_DEPTH
Definition: colorizer.h:118
glGenTextures
#define glGenTextures
Definition: glad/glad/glad.h:2339
RS2_OPTION_COUNT
@ RS2_OPTION_COUNT
Definition: rs_option.h:127
colorize_shader
Definition: colorizer-gl.cpp:80
GL_TEXTURE_MIN_FILTER
#define GL_TEXTURE_MIN_FILTER
Definition: glad/glad/glad.h:311
glClear
#define glClear
Definition: glad/glad/glad.h:1423
librealsense::colorizer::_min
float _min
Definition: colorizer.h:157
rmse.e
e
Definition: rmse.py:177
librealsense::colorizer::_equalize
bool _equalize
Definition: colorizer.h:158
colorizer-gl.h
GL_UNSIGNED_BYTE
#define GL_UNSIGNED_BYTE
Definition: glad/glad/glad.h:257
option.h
librealsense::colorizer::_max
float _max
Definition: colorizer.h:157
GL_RED
#define GL_RED
Definition: glad/glad/glad.h:287
rs2::stream_profile::stream_index
int stream_index() const
Definition: rs_frame.hpp:34
librealsense::gl::colorizer::_fbo
std::shared_ptr< rs2::fbo > _fbo
Definition: colorizer-gl.h:54
librealsense::gl::colorizer::_last_selected_cm
int _last_selected_cm
Definition: colorizer-gl.h:48
librealsense::gl::colorizer::cleanup_gpu_resources
void cleanup_gpu_resources() override
Definition: colorizer-gl.cpp:135
RS2_FORMAT_DISPARITY32
@ RS2_FORMAT_DISPARITY32
Definition: rs_sensor.h:82
librealsense::gl::colorizer
Definition: colorizer-gl.h:29
librealsense::colorizer::_hist_data
int * _hist_data
Definition: colorizer.h:164
colorize_shader::_max_disparity_location
uint32_t _max_disparity_location
Definition: colorizer-gl.cpp:124
GL_RG
#define GL_RG
Definition: glad/glad/glad.h:1225
colorize_shader::color_map_slot
int color_map_slot() const
Definition: colorizer-gl.cpp:107
GL_TEXTURE_MAG_FILTER
#define GL_TEXTURE_MAG_FILTER
Definition: glad/glad/glad.h:310
fragment_shader_text
static const char * fragment_shader_text
Definition: colorizer-gl.cpp:23
librealsense::gl
Definition: align-gl.h:26
colorize_shader::histogram_slot
int histogram_slot() const
Definition: colorizer-gl.cpp:108
librealsense::colorizer::_map_index
int _map_index
Definition: colorizer.h:161


librealsense2
Author(s): LibRealSense ROS Team
autogenerated on Fri Aug 2 2024 08:30:01