From fdb51c9933396c25062c5e03be86a4fa4390a4e9 Mon Sep 17 00:00:00 2001
From: Nicolas Pope <nwpope@utu.fi>
Date: Tue, 8 Oct 2019 09:26:47 +0300
Subject: [PATCH] WIP: Refactor of render code
---
applications/gui/src/camera.cpp | 1 +
applications/gui/src/media_panel.cpp | 9 +
.../cpp/include/ftl/render/splat_render.hpp | 8 +-
components/renderers/cpp/src/splat_render.cpp | 222 ++++++++----------
.../include/ftl/rgbd/channels.hpp | 2 +
.../rgbd-sources/include/ftl/rgbd/frame.hpp | 1 +
6 files changed, 120 insertions(+), 123 deletions(-)
diff --git a/applications/gui/src/camera.cpp b/applications/gui/src/camera.cpp
index fe9967739..c05580482 100644
--- a/applications/gui/src/camera.cpp
+++ b/applications/gui/src/camera.cpp
@@ -381,6 +381,7 @@ const GLTexture &ftl::gui::Camera::captureFrame() {
visualizeEnergy(depth_, tmp, 1.0);
texture_.update(tmp);
break;
+ case Channel::Density:
case Channel::Energy:
if (depth_.rows == 0) { break; }
visualizeEnergy(depth_, tmp, 10.0);
diff --git a/applications/gui/src/media_panel.cpp b/applications/gui/src/media_panel.cpp
index cb44400bb..9aa81932d 100644
--- a/applications/gui/src/media_panel.cpp
+++ b/applications/gui/src/media_panel.cpp
@@ -191,6 +191,15 @@ MediaPanel::MediaPanel(ftl::gui::Screen *screen) : nanogui::Window(screen, ""),
}
});
+ button = new Button(popup, "Density");
+ button->setFlags(Button::RadioButton);
+ button->setCallback([this]() {
+ ftl::gui::Camera *cam = screen_->activeCamera();
+ if (cam) {
+ cam->setChannel(Channel::Density);
+ }
+ });
+
}
MediaPanel::~MediaPanel() {
diff --git a/components/renderers/cpp/include/ftl/render/splat_render.hpp b/components/renderers/cpp/include/ftl/render/splat_render.hpp
index af6affb10..4de9ec491 100644
--- a/components/renderers/cpp/include/ftl/render/splat_render.hpp
+++ b/components/renderers/cpp/include/ftl/render/splat_render.hpp
@@ -26,7 +26,7 @@ class Splatter : public ftl::render::Renderer {
//void setOutputDevice(int);
protected:
- void renderChannel(ftl::render::SplatParams ¶ms, ftl::rgbd::Frame &out, const ftl::rgbd::Channel &channel, cudaStream_t stream);
+ void _renderChannel(ftl::rgbd::Frame &out, const ftl::rgbd::Channel &channel, cudaStream_t stream);
private:
int device_;
@@ -50,6 +50,12 @@ class Splatter : public ftl::render::Renderer {
float3 light_dir_;
uchar4 light_diffuse_;
uchar4 light_ambient_;
+ ftl::render::SplatParams params_;
+
+ template <typename T>
+ void __blendChannel(ftl::rgbd::Frame &, ftl::rgbd::Channel in, ftl::rgbd::Channel out, cudaStream_t);
+ void _blendChannel(ftl::rgbd::Frame &, ftl::rgbd::Channel in, ftl::rgbd::Channel out, cudaStream_t);
+ void _dibr(cudaStream_t);
};
}
diff --git a/components/renderers/cpp/src/splat_render.cpp b/components/renderers/cpp/src/splat_render.cpp
index 553bd3ab6..6aaaebb75 100644
--- a/components/renderers/cpp/src/splat_render.cpp
+++ b/components/renderers/cpp/src/splat_render.cpp
@@ -126,23 +126,73 @@ Splatter::~Splatter() {
}
-void Splatter::renderChannel(
- ftl::render::SplatParams ¶ms, ftl::rgbd::Frame &out,
- const Channel &channel, cudaStream_t stream)
-{
- if (channel == Channel::None) return;
+template <typename T>
+struct AccumSelector {
+ typedef float4 type;
+ static constexpr Channel channel = Channel::Colour;
+ //static constexpr cv::Scalar value = cv::Scalar(0.0f,0.0f,0.0f,0.0f);
+};
+
+template <>
+struct AccumSelector<float> {
+ typedef float type;
+ static constexpr Channel channel = Channel::Colour2;
+ //static constexpr cv::Scalar value = cv::Scalar(0.0f);
+};
+
+template <typename T>
+void Splatter::__blendChannel(ftl::rgbd::Frame &output, ftl::rgbd::Channel in, ftl::rgbd::Channel out, cudaStream_t stream) {
cv::cuda::Stream cvstream = cv::cuda::StreamAccessor::wrapStream(stream);
- temp_.get<GpuMat>(Channel::Depth).setTo(cv::Scalar(0x7FFFFFFF), cvstream);
- temp_.get<GpuMat>(Channel::Depth2).setTo(cv::Scalar(0x7FFFFFFF), cvstream);
- temp_.get<GpuMat>(Channel::Colour).setTo(cv::Scalar(0.0f,0.0f,0.0f,0.0f), cvstream);
+ temp_.create<GpuMat>(
+ AccumSelector<T>::channel,
+ Format<typename AccumSelector<T>::type>(params_.camera.width, params_.camera.height)
+ ).setTo(cv::Scalar(0.0f), cvstream);
temp_.get<GpuMat>(Channel::Contribution).setTo(cv::Scalar(0.0f), cvstream);
- if (scene_->frames.size() < 1) return;
- bool is_float = out.get<GpuMat>(channel).type() == CV_32F; //ftl::rgbd::isFloatChannel(channel);
- bool is_4chan = out.get<GpuMat>(channel).type() == CV_32FC4;
+ temp_.createTexture<float>(Channel::Contribution);
+
+ for (auto &f : scene_->frames) {
+ if (f.get<GpuMat>(in).type() == CV_8UC3) {
+ // Convert to 4 channel colour
+ auto &col = f.get<GpuMat>(in);
+ GpuMat tmp(col.size(), CV_8UC4);
+ cv::cuda::swap(col, tmp);
+ cv::cuda::cvtColor(tmp,col, cv::COLOR_BGR2BGRA);
+ }
+
+ ftl::cuda::dibr_attribute(
+ f.createTexture<T>(in),
+ f.createTexture<float4>(Channel::Points),
+ temp_.getTexture<int>(Channel::Depth2),
+ temp_.createTexture<typename AccumSelector<T>::type>(AccumSelector<T>::channel),
+ temp_.getTexture<float>(Channel::Contribution),
+ params_, stream
+ );
+ }
+
+ ftl::cuda::dibr_normalise(
+ temp_.getTexture<typename AccumSelector<T>::type>(AccumSelector<T>::channel),
+ output.createTexture<T>(out),
+ temp_.getTexture<float>(Channel::Contribution),
+ stream
+ );
+}
+
+void Splatter::_blendChannel(ftl::rgbd::Frame &output, ftl::rgbd::Channel in, ftl::rgbd::Channel out, cudaStream_t stream) {
+ int type = output.get<GpuMat>(out).type(); // == CV_32F; //ftl::rgbd::isFloatChannel(channel);
- // Render each camera into virtual view
- // TODO: Move out of renderChannel, this is a common step to all channels
+ switch (type) {
+ case CV_32F : __blendChannel<float>(output, in, out, stream); break;
+ case CV_32FC4 : __blendChannel<float4>(output, in, out, stream); break;
+ case CV_8UC4 : __blendChannel<uchar4>(output, in, out, stream); break;
+ default : LOG(ERROR) << "Invalid output channel format";
+ }
+}
+
+void Splatter::_dibr(cudaStream_t stream) {
+ cv::cuda::Stream cvstream = cv::cuda::StreamAccessor::wrapStream(stream);
+ temp_.get<GpuMat>(Channel::Depth2).setTo(cv::Scalar(0x7FFFFFFF), cvstream);
+
for (size_t i=0; i < scene_->frames.size(); ++i) {
auto &f = scene_->frames[i];
auto *s = scene_->sources[i];
@@ -156,129 +206,54 @@ void Splatter::renderChannel(
f.createTexture<float4>(Channel::Points),
f.createTexture<float4>(Channel::Normals),
temp_.createTexture<int>(Channel::Depth2),
- params, backcull_, stream
+ params_, backcull_, stream
);
//LOG(INFO) << "DIBR DONE";
}
+}
- temp_.createTexture<float4>(Channel::Colour);
- temp_.createTexture<float>(Channel::Contribution);
- out.create<GpuMat>(Channel::Normals, Format<float4>(params.camera.width, params.camera.height));
- out.get<GpuMat>(Channel::Normals).setTo(cv::Scalar(0.0f,0.0f,0.0f,0.0f), cvstream);
-
- // Create normals first
- for (auto &f : scene_->frames) {
+void Splatter::_renderChannel(
+ ftl::rgbd::Frame &out,
+ const Channel &channel, cudaStream_t stream)
+{
+ if (channel == Channel::None) return;
+ cv::cuda::Stream cvstream = cv::cuda::StreamAccessor::wrapStream(stream);
- ftl::cuda::dibr_attribute(
- f.createTexture<float4>(Channel::Normals),
- f.createTexture<float4>(Channel::Points),
- temp_.getTexture<int>(Channel::Depth2),
- out.createTexture<float4>(Channel::Normals),
- temp_.getTexture<float>(Channel::Contribution),
- params, stream
- );
- }
+ if (scene_->frames.size() < 1) return;
+ bool is_float = out.get<GpuMat>(channel).type() == CV_32F; //ftl::rgbd::isFloatChannel(channel);
+ bool is_4chan = out.get<GpuMat>(channel).type() == CV_32FC4;
- ftl::cuda::dibr_normalise(
- out.getTexture<float4>(Channel::Normals),
- out.getTexture<float4>(Channel::Normals),
- temp_.getTexture<float>(Channel::Contribution),
- stream
- );
- //auto &t = out.createTexture<float4>(Channel::Points, Format<float4>(params.camera.width, params.camera.height));
- //ftl::cuda::point_cloud(t, temp_.getTexture<int>(Channel::Depth2), params.camera, params.m_viewMatrixInverse, 0, stream);
- /*ftl::cuda::normals(out.createTexture<float4>(Channel::Normals),
- temp_.getTexture<float4>(Channel::Normals),
- temp_.getTexture<int>(Channel::Depth2),
- 9, 0.04f,
- params.camera, params.m_viewMatrixInverse.getFloat3x3(), params.m_viewMatrix.getFloat3x3(), stream);*/
+ temp_.createTexture<float4>(Channel::Colour);
+ temp_.createTexture<float>(Channel::Contribution);
- //if (norm_filter_ > -0.1f) {
- // ftl::cuda::normal_filter(f.getTexture<float4>(Channel::Normals), f.getTexture<float4>(Channel::Points), s->parameters(), pose, norm_filter_, stream);
- //}
+ // Generate initial normals for the splats
+ out.create<GpuMat>(Channel::Normals, Format<float4>(params_.camera.width, params_.camera.height));
+ _blendChannel(out, Channel::Normals, Channel::Normals, stream);
- temp_.get<GpuMat>(Channel::Colour).setTo(cv::Scalar(0.0f,0.0f,0.0f,0.0f), cvstream);
- temp_.get<GpuMat>(Channel::Contribution).setTo(cv::Scalar(0.0f), cvstream);
+ // Estimate point density
+ out.create<GpuMat>(Channel::Density, Format<float>(params_.camera.width, params_.camera.height));
+ _blendChannel(out, Channel::Depth, Channel::Density, stream);
+ // FIXME: Using colour 2 in this way seems broken since it is already used
if (is_4chan) {
- temp_.create<GpuMat>(Channel::Colour2, Format<float4>(params.camera.width, params.camera.height));
+ temp_.create<GpuMat>(Channel::Colour2, Format<float4>(params_.camera.width, params_.camera.height));
temp_.get<GpuMat>(Channel::Colour2).setTo(cv::Scalar(0.0f,0.0f,0.0f,0.0f), cvstream);
} else if (is_float) {
- temp_.create<GpuMat>(Channel::Colour2, Format<float>(params.camera.width, params.camera.height));
+ temp_.create<GpuMat>(Channel::Colour2, Format<float>(params_.camera.width, params_.camera.height));
temp_.get<GpuMat>(Channel::Colour2).setTo(cv::Scalar(0.0f), cvstream);
} else {
- temp_.create<GpuMat>(Channel::Colour2, Format<uchar4>(params.camera.width, params.camera.height));
+ temp_.create<GpuMat>(Channel::Colour2, Format<uchar4>(params_.camera.width, params_.camera.height));
temp_.get<GpuMat>(Channel::Colour2).setTo(cv::Scalar(0,0,0,0), cvstream);
}
- // Create attribute first
- for (auto &f : scene_->frames) {
- // Convert colour from BGR to BGRA if needed
- if (f.get<GpuMat>(channel).type() == CV_8UC3) {
- // Convert to 4 channel colour
- auto &col = f.get<GpuMat>(Channel::Colour);
- GpuMat tmp(col.size(), CV_8UC4);
- cv::cuda::swap(col, tmp);
- cv::cuda::cvtColor(tmp,col, cv::COLOR_BGR2BGRA);
- }
-
- if (is_4chan) {
- ftl::cuda::dibr_attribute(
- f.createTexture<float4>(channel),
- f.createTexture<float4>(Channel::Points),
- temp_.getTexture<int>(Channel::Depth2),
- temp_.getTexture<float4>(Channel::Colour),
- temp_.getTexture<float>(Channel::Contribution),
- params, stream
- );
- } else if (is_float) {
- ftl::cuda::dibr_attribute(
- f.createTexture<float>(channel),
- f.createTexture<float4>(Channel::Points),
- temp_.getTexture<int>(Channel::Depth2),
- temp_.createTexture<float>(Channel::Colour2),
- temp_.getTexture<float>(Channel::Contribution),
- params, stream
- );
- } else {
- ftl::cuda::dibr_attribute(
- f.createTexture<uchar4>(channel),
- f.createTexture<float4>(Channel::Points),
- temp_.getTexture<int>(Channel::Depth2),
- temp_.createTexture<float4>(Channel::Colour),
- temp_.getTexture<float>(Channel::Contribution),
- params, stream
- );
- }
- }
-
- if (is_4chan) {
- ftl::cuda::dibr_normalise(
- temp_.getTexture<float4>(Channel::Colour),
- (splat_) ? temp_.createTexture<float4>(Channel::Colour2) : out.createTexture<float4>(channel),
- temp_.getTexture<float>(Channel::Contribution),
- stream
- );
- } else if (is_float) {
- ftl::cuda::dibr_normalise(
- temp_.createTexture<float>(Channel::Colour2),
- (splat_) ? temp_.createTexture<float>(Channel::Colour2) : out.createTexture<float>(channel),
- temp_.getTexture<float>(Channel::Contribution),
- stream
- );
+ if (splat_) {
+ _blendChannel(temp_, channel, Channel::Colour2, stream);
} else {
- ftl::cuda::dibr_normalise(
- temp_.getTexture<float4>(Channel::Colour),
- (splat_) ? temp_.createTexture<uchar4>(Channel::Colour2) : out.createTexture<uchar4>(channel),
- temp_.getTexture<float>(Channel::Contribution),
- stream
- );
+ _blendChannel(out, channel, channel, stream);
}
- //out.get<GpuMat>(Channel::Left).setTo(cv::Scalar(0,0,0,0), cvstream);
-
// Now splat the points
if (splat_) {
if (is_4chan) {
@@ -288,7 +263,7 @@ void Splatter::renderChannel(
temp_.getTexture<int>(Channel::Depth2),
out.createTexture<float>(Channel::Depth),
out.createTexture<float4>(channel),
- params, stream
+ params_, stream
);
} else if (is_float) {
ftl::cuda::splat(
@@ -297,7 +272,7 @@ void Splatter::renderChannel(
temp_.getTexture<int>(Channel::Depth2),
out.createTexture<float>(Channel::Depth),
out.createTexture<float>(channel),
- params, stream
+ params_, stream
);
} else {
ftl::cuda::splat(
@@ -306,7 +281,7 @@ void Splatter::renderChannel(
temp_.getTexture<int>(Channel::Depth2),
out.createTexture<float>(Channel::Depth),
out.createTexture<uchar4>(channel),
- params, stream
+ params_, stream
);
}
}
@@ -334,7 +309,7 @@ bool Splatter::render(ftl::rgbd::VirtualSource *src, ftl::rgbd::Frame &out, cuda
cv::cuda::Stream cvstream = cv::cuda::StreamAccessor::wrapStream(stream);
// Parameters object to pass to CUDA describing the camera
- SplatParams params;
+ SplatParams ¶ms = params_;
params.m_flags = 0;
if (value("show_discontinuity_mask", false)) params.m_flags |= ftl::render::kShowDisconMask;
if (value("normal_weight_colours", true)) params.m_flags |= ftl::render::kNormalWeightColours;
@@ -391,7 +366,8 @@ bool Splatter::render(ftl::rgbd::VirtualSource *src, ftl::rgbd::Frame &out, cuda
}
}
- renderChannel(params, out, Channel::Colour, stream);
+ _dibr(stream);
+ _renderChannel(out, Channel::Colour, stream);
Channel chan = src->getChannel();
if (chan == Channel::Depth)
@@ -401,7 +377,7 @@ bool Splatter::render(ftl::rgbd::VirtualSource *src, ftl::rgbd::Frame &out, cuda
out.create<GpuMat>(Channel::Normals, Format<float4>(camera.width, camera.height));
// Render normal attribute
- renderChannel(params, out, Channel::Normals, stream);
+ _renderChannel(out, Channel::Normals, stream);
// Convert normal to single float value
temp_.create<GpuMat>(Channel::Colour, Format<uchar4>(camera.width, camera.height));
@@ -427,7 +403,9 @@ bool Splatter::render(ftl::rgbd::VirtualSource *src, ftl::rgbd::Frame &out, cuda
out.create<GpuMat>(Channel::Right, Format<uchar4>(camera.width, camera.height));
out.get<GpuMat>(Channel::Right).setTo(background_, cvstream);
- renderChannel(params, out, Channel::Right, stream);
+
+ _dibr(stream); // Need to re-dibr due to pose change
+ _renderChannel(out, Channel::Right, stream);
} else if (chan != Channel::None) {
if (ftl::rgbd::isFloatChannel(chan)) {
out.create<GpuMat>(chan, Format<float>(camera.width, camera.height));
@@ -436,7 +414,7 @@ bool Splatter::render(ftl::rgbd::VirtualSource *src, ftl::rgbd::Frame &out, cuda
out.create<GpuMat>(chan, Format<uchar4>(camera.width, camera.height));
out.get<GpuMat>(chan).setTo(background_, cvstream);
}
- renderChannel(params, out, chan, stream);
+ _renderChannel(out, chan, stream);
}
return true;
diff --git a/components/rgbd-sources/include/ftl/rgbd/channels.hpp b/components/rgbd-sources/include/ftl/rgbd/channels.hpp
index 467df0b1e..e00647271 100644
--- a/components/rgbd-sources/include/ftl/rgbd/channels.hpp
+++ b/components/rgbd-sources/include/ftl/rgbd/channels.hpp
@@ -25,6 +25,7 @@ enum struct Channel : int {
Flow, // 32F
Energy, // 32F
Mask, // 32U
+ Density, // 32F
LeftGray,
RightGray,
Overlay1
@@ -106,6 +107,7 @@ inline bool isFloatChannel(ftl::rgbd::Channel chan) {
case Channel::Depth :
//case Channel::Normals :
case Channel::Confidence:
+ case Channel::Density:
case Channel::Energy : return true;
default : return false;
}
diff --git a/components/rgbd-sources/include/ftl/rgbd/frame.hpp b/components/rgbd-sources/include/ftl/rgbd/frame.hpp
index 338674745..e9036fd6c 100644
--- a/components/rgbd-sources/include/ftl/rgbd/frame.hpp
+++ b/components/rgbd-sources/include/ftl/rgbd/frame.hpp
@@ -187,6 +187,7 @@ ftl::cuda::TextureObject<T> &Frame::getTexture(ftl::rgbd::Channel c) {
auto &m = _get(c);
if (m.tex.cvType() != ftl::traits::OpenCVType<T>::value || m.tex.width() != m.gpu.cols || m.tex.height() != m.gpu.rows || m.gpu.type() != m.tex.cvType()) {
+ LOG(ERROR) << "Texture has not been created for channel = " << (int)c;
throw ftl::exception("Texture has not been created properly for this channel");
}
--
GitLab