From e845501decc73075713a3aaddf75b09cf280ba07 Mon Sep 17 00:00:00 2001
From: Nicolas Pope <nicolas.pope@utu.fi>
Date: Thu, 3 Oct 2019 22:04:51 +0300
Subject: [PATCH] Implements #188 normal shading
---
applications/gui/src/camera.cpp | 2 +-
.../cpp/include/ftl/cuda/normals.hpp | 4 +-
.../cpp/include/ftl/render/splat_render.hpp | 3 ++
components/renderers/cpp/src/normals.cu | 25 +++++-----
components/renderers/cpp/src/splat_render.cpp | 47 ++++++++++++++++---
.../include/ftl/rgbd/channels.hpp | 2 +-
6 files changed, 60 insertions(+), 23 deletions(-)
diff --git a/applications/gui/src/camera.cpp b/applications/gui/src/camera.cpp
index abb7fdb80..32d78a4a0 100644
--- a/applications/gui/src/camera.cpp
+++ b/applications/gui/src/camera.cpp
@@ -376,7 +376,6 @@ const GLTexture &ftl::gui::Camera::captureFrame() {
cv::Mat tmp;
switch(channel_) {
- case Channel::Normals:
case Channel::Energy:
if (depth_.rows == 0) { break; }
visualizeEnergy(depth_, tmp, 10.0);
@@ -400,6 +399,7 @@ const GLTexture &ftl::gui::Camera::captureFrame() {
case Channel::Flow:
case Channel::Confidence:
+ case Channel::Normals:
case Channel::Right:
if (depth_.rows == 0 || depth_.type() != CV_8UC3) { break; }
texture_.update(depth_);
diff --git a/components/renderers/cpp/include/ftl/cuda/normals.hpp b/components/renderers/cpp/include/ftl/cuda/normals.hpp
index 5481b4662..b4d2ced19 100644
--- a/components/renderers/cpp/include/ftl/cuda/normals.hpp
+++ b/components/renderers/cpp/include/ftl/cuda/normals.hpp
@@ -13,8 +13,8 @@ void normals(ftl::cuda::TextureObject<float4> &output,
ftl::cuda::TextureObject<float4> &input, cudaStream_t stream);
void normal_visualise(ftl::cuda::TextureObject<float4> &norm,
- ftl::cuda::TextureObject<float> &output,
- const ftl::rgbd::Camera &camera, const float4x4 &pose,
+ ftl::cuda::TextureObject<uchar4> &output,
+ const float3 &light, const uchar4 &diffuse, const uchar4 &ambient,
cudaStream_t stream);
void normal_filter(ftl::cuda::TextureObject<float4> &norm,
diff --git a/components/renderers/cpp/include/ftl/render/splat_render.hpp b/components/renderers/cpp/include/ftl/render/splat_render.hpp
index 5f8a8ba49..af6affb10 100644
--- a/components/renderers/cpp/include/ftl/render/splat_render.hpp
+++ b/components/renderers/cpp/include/ftl/render/splat_render.hpp
@@ -47,6 +47,9 @@ class Splatter : public ftl::render::Renderer {
bool backcull_;
cv::Scalar background_;
bool splat_;
+ float3 light_dir_;
+ uchar4 light_diffuse_;
+ uchar4 light_ambient_;
};
}
diff --git a/components/renderers/cpp/src/normals.cu b/components/renderers/cpp/src/normals.cu
index eec540ccd..cbbd0e967 100644
--- a/components/renderers/cpp/src/normals.cu
+++ b/components/renderers/cpp/src/normals.cu
@@ -89,38 +89,37 @@ void ftl::cuda::normals(ftl::cuda::TextureObject<float4> &output,
//==============================================================================
__global__ void vis_normals_kernel(ftl::cuda::TextureObject<float4> norm,
- ftl::cuda::TextureObject<float> output,
- ftl::rgbd::Camera camera, float4x4 pose) {
+ ftl::cuda::TextureObject<uchar4> output,
+ float3 direction, uchar4 diffuse, uchar4 ambient) {
const unsigned int x = blockIdx.x*blockDim.x + threadIdx.x;
const unsigned int y = blockIdx.y*blockDim.y + threadIdx.y;
if(x >= norm.width() || y >= norm.height()) return;
- output(x,y) = 0.0f;
- float3 ray = pose.getFloat3x3() * camera.screenToCam(x,y,1.0f);
+ output(x,y) = make_uchar4(0,0,0,0);
+ float3 ray = direction;
ray = ray / length(ray);
float3 n = make_float3(norm.tex2D((int)x,(int)y));
float l = length(n);
if (l == 0) return;
n /= l;
- const float d = dot(ray, n);
- output(x,y) = (1.0f + d)*3.5f; // FIXME: Do not hard code these value scalings
-
- //if (d > 0.2f) {
- // output(x,y) = d * 7.0f;
- //}
+ const float d = max(dot(ray, n), 0.0f);
+ output(x,y) = make_uchar4(
+ min(255.0f, diffuse.x*d + ambient.x),
+ min(255.0f, diffuse.y*d + ambient.y),
+ min(255.0f, diffuse.z*d + ambient.z), 255);
}
void ftl::cuda::normal_visualise(ftl::cuda::TextureObject<float4> &norm,
- ftl::cuda::TextureObject<float> &output,
- const ftl::rgbd::Camera &camera, const float4x4 &pose,
+ ftl::cuda::TextureObject<uchar4> &output,
+ const float3 &light, const uchar4 &diffuse, const uchar4 &ambient,
cudaStream_t stream) {
const dim3 gridSize((norm.width() + T_PER_BLOCK - 1)/T_PER_BLOCK, (norm.height() + T_PER_BLOCK - 1)/T_PER_BLOCK);
const dim3 blockSize(T_PER_BLOCK, T_PER_BLOCK);
- vis_normals_kernel<<<gridSize, blockSize, 0, stream>>>(norm, output, camera, pose);
+ vis_normals_kernel<<<gridSize, blockSize, 0, stream>>>(norm, output, light, diffuse, ambient);
cudaSafeCall( cudaGetLastError() );
#ifdef _DEBUG
diff --git a/components/renderers/cpp/src/splat_render.cpp b/components/renderers/cpp/src/splat_render.cpp
index 26f8e893e..d993e1458 100644
--- a/components/renderers/cpp/src/splat_render.cpp
+++ b/components/renderers/cpp/src/splat_render.cpp
@@ -25,7 +25,10 @@ static Eigen::Affine3d create_rotation_matrix(float ax, float ay, float az) {
return rz * rx * ry;
}
-static cv::Scalar parseColour(const std::string &colour) {
+/*
+ * Parse a CSS style colour string into a scalar.
+ */
+static cv::Scalar parseCVColour(const std::string &colour) {
std::string c = colour;
if (c[0] == '#') {
c.erase(0, 1);
@@ -41,6 +44,25 @@ static cv::Scalar parseColour(const std::string &colour) {
return cv::Scalar(0,0,0,0);
}
+/*
+ * Parse a CSS style colour string into a scalar.
+ */
+static uchar4 parseCUDAColour(const std::string &colour) {
+ std::string c = colour;
+ if (c[0] == '#') {
+ c.erase(0, 1);
+ unsigned long value = stoul(c.c_str(), nullptr, 16);
+ return make_uchar4(
+ (value >> 0) & 0xff,
+ (value >> 8) & 0xff,
+ (value >> 16) & 0xff,
+ (value >> 24) & 0xff
+ );
+ }
+
+ return make_uchar4(0,0,0,0);
+}
+
Splatter::Splatter(nlohmann::json &config, ftl::rgbd::FrameSet *fs) : ftl::render::Renderer(config), scene_(fs) {
if (config["clipping"].is_object()) {
auto &c = config["clipping"];
@@ -84,9 +106,19 @@ Splatter::Splatter(nlohmann::json &config, ftl::rgbd::FrameSet *fs) : ftl::rende
splat_ = value("splatting", true);
});
- background_ = parseColour(value("background", std::string("#e0e0e0")));
+ background_ = parseCVColour(value("background", std::string("#4c4c4c")));
on("background", [this](const ftl::config::Event &e) {
- background_ = parseColour(value("background", std::string("#e0e0e0")));
+ background_ = parseCVColour(value("background", std::string("#4c4c4c")));
+ });
+
+ light_diffuse_ = parseCUDAColour(value("diffuse", std::string("#e0e0e0")));
+ on("diffuse", [this](const ftl::config::Event &e) {
+ light_diffuse_ = parseCUDAColour(value("diffuse", std::string("#e0e0e0")));
+ });
+
+ light_ambient_ = parseCUDAColour(value("ambient", std::string("#0e0e0e")));
+ on("ambient", [this](const ftl::config::Event &e) {
+ light_ambient_ = parseCUDAColour(value("ambient", std::string("#0e0e0e")));
});
}
@@ -322,11 +354,14 @@ bool Splatter::render(ftl::rgbd::VirtualSource *src, ftl::rgbd::Frame &out, cuda
renderChannel(params, out, Channel::Normals, stream);
// Convert normal to single float value
- temp_.create<GpuMat>(Channel::Contribution, Format<float>(camera.width, camera.height));
- ftl::cuda::normal_visualise(out.getTexture<float4>(Channel::Normals), temp_.createTexture<float>(Channel::Contribution), camera, params.m_viewMatrixInverse, stream);
+ temp_.create<GpuMat>(Channel::Colour, Format<uchar4>(camera.width, camera.height));
+ ftl::cuda::normal_visualise(out.getTexture<float4>(Channel::Normals), temp_.createTexture<uchar4>(Channel::Colour),
+ make_float3(-0.3f, 0.2f, 1.0f),
+ light_diffuse_,
+ light_ambient_, stream);
// Put in output as single float
- cv::cuda::swap(temp_.get<GpuMat>(Channel::Contribution), out.create<GpuMat>(Channel::Normals));
+ cv::cuda::swap(temp_.get<GpuMat>(Channel::Colour), out.create<GpuMat>(Channel::Normals));
out.resetTexture(Channel::Normals);
}
else if (chan == Channel::Contribution)
diff --git a/components/rgbd-sources/include/ftl/rgbd/channels.hpp b/components/rgbd-sources/include/ftl/rgbd/channels.hpp
index d1d8dcdbf..e89a42ce3 100644
--- a/components/rgbd-sources/include/ftl/rgbd/channels.hpp
+++ b/components/rgbd-sources/include/ftl/rgbd/channels.hpp
@@ -103,7 +103,7 @@ static const Channels kAllChannels(0xFFFFFFFFu);
inline bool isFloatChannel(ftl::rgbd::Channel chan) {
switch (chan) {
case Channel::Depth :
- case Channel::Normals :
+ //case Channel::Normals :
case Channel::Energy : return true;
default : return false;
}
--
GitLab