diff --git a/components/renderers/cpp/src/reprojection.cu b/components/renderers/cpp/src/reprojection.cu
index ed5580ac5ad99c0588b0cf3c5b7f210f7bf46682..c157f6fe6905b036b73b72668deaa3a4224ae5de 100644
--- a/components/renderers/cpp/src/reprojection.cu
+++ b/components/renderers/cpp/src/reprojection.cu
@@ -61,7 +61,8 @@ __device__ inline void accumulateOutput(TextureObject<float4> &out, TextureObjec
__global__ void reprojection_kernel(
TextureObject<A> in, // Attribute input
TextureObject<float> depth_src,
- TextureObject<int> depth_in, // Virtual depth map
+ TextureObject<int> depth_in, // Virtual depth map
+ TextureObject<float4> normals,
TextureObject<B> out, // Accumulated output
TextureObject<float> contrib,
SplatParams params,
@@ -84,11 +85,24 @@ __global__ void reprojection_kernel(
// Not on screen so stop now...
if (screenPos.x >= depth_src.width() || screenPos.y >= depth_src.height()) return;
- // Is this point near the actual surface and therefore a contributor?
+ // Calculate the dot product of surface normal and camera ray
+ const float3 n = poseInv.getFloat3x3() * make_float3(normals.tex2D((int)x, (int)y));
+ float3 ray = camera.screenToCam(screenPos.x, screenPos.y, 1.0f);
+ ray = ray / length(ray);
+ const float dotproduct = max(dot(ray,n),0.0f);
const float d2 = depth_src.tex2D((int)screenPos.x, (int)screenPos.y);
const A input = in.tex2D((int)screenPos.x, (int)screenPos.y); //generateInput(in.tex2D((int)screenPos.x, (int)screenPos.y), params, worldPos);
- const float weight = ftl::cuda::weighting(fabs(camPos.z - d2), 0.02f);
+ float weight = ftl::cuda::weighting(fabs(camPos.z - d2), 0.02f);
+
+ /* Buehler C. et al. 2001. Unstructured Lumigraph Rendering. */
+ /* Orts-Escolano S. et al. 2016. Holoportation: Virtual 3D teleportation in real-time. */
+ // This is the simple naive colour weighting. It might be good
+ // enough for our purposes if the alignment step prevents ghosting
+ // TODO: Use depth and perhaps the neighbourhood consistency in:
+ // Kuster C. et al. 2011. FreeCam: A hybrid camera system for interactive free-viewpoint video
+ if (params.m_flags & ftl::render::kNormalWeightColours) weight *= dotproduct;
+
const B weighted = make<B>(input) * weight; //weightInput(input, weight);
if (weight > 0.0f) {
@@ -102,7 +116,8 @@ template <typename A, typename B>
void ftl::cuda::reproject(
TextureObject<A> &in,
TextureObject<float> &depth_src, // Original 3D points
- TextureObject<int> &depth_in, // Virtual depth map
+ TextureObject<int> &depth_in, // Virtual depth map
+ TextureObject<float4> &normals,
TextureObject<B> &out, // Accumulated output
TextureObject<float> &contrib,
const SplatParams ¶ms,
@@ -113,7 +128,8 @@ void ftl::cuda::reproject(
reprojection_kernel<<<gridSize, blockSize, 0, stream>>>(
in,
depth_src,
- depth_in,
+ depth_in,
+ normals,
out,
contrib,
params,
@@ -127,6 +143,7 @@ template void ftl::cuda::reproject(
ftl::cuda::TextureObject<uchar4> &in, // Original colour image
ftl::cuda::TextureObject<float> &depth_src, // Original 3D points
ftl::cuda::TextureObject<int> &depth_in, // Virtual depth map
+ ftl::cuda::TextureObject<float4> &normals,
ftl::cuda::TextureObject<float4> &out, // Accumulated output
ftl::cuda::TextureObject<float> &contrib,
const ftl::render::SplatParams ¶ms,
@@ -137,6 +154,7 @@ template void ftl::cuda::reproject(
ftl::cuda::TextureObject<float> &in, // Original colour image
ftl::cuda::TextureObject<float> &depth_src, // Original 3D points
ftl::cuda::TextureObject<int> &depth_in, // Virtual depth map
+ ftl::cuda::TextureObject<float4> &normals,
ftl::cuda::TextureObject<float> &out, // Accumulated output
ftl::cuda::TextureObject<float> &contrib,
const ftl::render::SplatParams ¶ms,
@@ -147,6 +165,7 @@ template void ftl::cuda::reproject(
ftl::cuda::TextureObject<float4> &in, // Original colour image
ftl::cuda::TextureObject<float> &depth_src, // Original 3D points
ftl::cuda::TextureObject<int> &depth_in, // Virtual depth map
+ ftl::cuda::TextureObject<float4> &normals,
ftl::cuda::TextureObject<float4> &out, // Accumulated output
ftl::cuda::TextureObject<float> &contrib,
const ftl::render::SplatParams ¶ms,
diff --git a/components/renderers/cpp/src/splat_render.cpp b/components/renderers/cpp/src/splat_render.cpp
index 9de1f00b8725f30a1dbc9660587e3d02148213df..5b1ea0409f74a59502bdc60eabf4a796b68cb0de 100644
--- a/components/renderers/cpp/src/splat_render.cpp
+++ b/components/renderers/cpp/src/splat_render.cpp
@@ -216,6 +216,7 @@ void Splatter::__reprojectChannel(ftl::rgbd::Frame &output, ftl::codecs::Channel
f.createTexture<T>(in),
f.createTexture<float>(Channel::Depth), // TODO: Use depth?
temp_.getTexture<int>(Channel::Depth2),
+ accum_.getTexture<float4>(Channel::Normals),
temp_.createTexture<typename AccumSelector<T>::type>(AccumSelector<T>::channel),
temp_.getTexture<float>(Channel::Contribution),
params_,
@@ -309,16 +310,6 @@ void Splatter::_renderChannel(
temp_.createTexture<float4>(Channel::Colour);
temp_.createTexture<float>(Channel::Contribution);
- // Generate initial normals for the splats
- //accum_.create<GpuMat>(Channel::Normals, Format<float4>(params_.camera.width, params_.camera.height));
- //_blendChannel(accum_, Channel::Normals, Channel::Normals, stream);
- // Put normals in camera space here...
- //ftl::cuda::transform_normals(accum_.getTexture<float4>(Channel::Normals), params_.m_viewMatrix.getFloat3x3(), stream);
-
- // Estimate point density
- //accum_.create<GpuMat>(Channel::Density, Format<float>(params_.camera.width, params_.camera.height));
- //_blendChannel(accum_, Channel::Depth, Channel::Density, stream);
-
// FIXME: Using colour 2 in this way seems broken since it is already used
if (is_4chan) {
accum_.create<GpuMat>(channel_out, Format<float4>(params_.camera.width, params_.camera.height));
@@ -332,53 +323,6 @@ void Splatter::_renderChannel(
}
_reprojectChannel(out, channel_in, channel_out, stream);
-
- //if (splat_) {
- //_blendChannel(accum_, channel_in, channel_out, stream);
- //} else {
- // _blendChannel(out, channel, channel, stream);
- //}
-
- // Now splat the points
- if (splat_) {
- /*if (is_4chan) {
- ftl::cuda::splat(
- accum_.getTexture<float4>(Channel::Normals),
- accum_.getTexture<float>(Channel::Density),
- accum_.getTexture<float4>(channel_out),
- temp_.getTexture<int>(Channel::Depth2),
- out.createTexture<float>(Channel::Depth),
- out.createTexture<float4>(channel_out),
- params_, stream
- );
- } else if (is_float) {
- ftl::cuda::splat(
- accum_.getTexture<float4>(Channel::Normals),
- accum_.getTexture<float>(Channel::Density),
- accum_.getTexture<float>(channel_out),
- temp_.getTexture<int>(Channel::Depth2),
- out.createTexture<float>(Channel::Depth),
- out.createTexture<float>(channel_out),
- params_, stream
- );
- } else {
- ftl::cuda::splat(
- accum_.getTexture<float4>(Channel::Normals),
- accum_.getTexture<float>(Channel::Density),
- accum_.getTexture<uchar4>(channel_out),
- temp_.getTexture<int>(Channel::Depth2),
- out.createTexture<float>(Channel::Depth),
- out.createTexture<uchar4>(channel_out),
- params_, stream
- );
- }*/
- //temp_.get<GpuMat>(Channel::Depth2).convertTo(out.get<GpuMat>(Channel::Depth), CV_32F, 1.0f / 1000.0f);
- //accum_.swapTo(Channels(channel_out), out);
- } else {
- // Swap accum frames directly to output.
- //accum_.swapTo(Channels(channel_out), out);
- //temp_.get<GpuMat>(Channel::Depth2).convertTo(out.get<GpuMat>(Channel::Depth), CV_32F, 1.0f / 1000.0f);
- }
}
bool Splatter::render(ftl::rgbd::VirtualSource *src, ftl::rgbd::Frame &out) {
diff --git a/components/renderers/cpp/src/splatter_cuda.hpp b/components/renderers/cpp/src/splatter_cuda.hpp
index 5e70225296866111164f7d1197b3650b726495fe..14fb2dc64fc37351fd2fbd8d8ac36110505026d6 100644
--- a/components/renderers/cpp/src/splatter_cuda.hpp
+++ b/components/renderers/cpp/src/splatter_cuda.hpp
@@ -54,6 +54,7 @@ namespace cuda {
ftl::cuda::TextureObject<A> &in, // Original colour image
ftl::cuda::TextureObject<float> &depth_src, // Original 3D points
ftl::cuda::TextureObject<int> &depth_in, // Virtual depth map
+ ftl::cuda::TextureObject<float4> &normals,
ftl::cuda::TextureObject<B> &out, // Accumulated output
ftl::cuda::TextureObject<float> &contrib,
const ftl::render::SplatParams ¶ms,