Skip to content
Snippets Groups Projects
Commit efd6c29e authored by Nicolas Pope's avatar Nicolas Pope
Browse files

Re implement normal based colour weighting

parent 29c2f568
No related branches found
No related tags found
1 merge request!151Implements #216 triangle renderer
Pipeline #16007 passed
Stage: all
Hide whitespace changes
Inline Side-by-side
components/renderers/cpp/src/reprojection.cu
+ 24
− 5
View file @ efd6c29e
......@@ -61,7 +61,8 @@ __device__ inline void accumulateOutput(TextureObject<float4> &out, TextureObjec
__global__ void reprojection_kernel(
TextureObject<A> in, // Attribute input
TextureObject<float> depth_src,
TextureObject<int> depth_in, // Virtual depth map
TextureObject<int> depth_in, // Virtual depth map
TextureObject<float4> normals,
TextureObject<B> out, // Accumulated output
TextureObject<float> contrib,
SplatParams params,
......@@ -84,11 +85,24 @@ __global__ void reprojection_kernel(
// Not on screen so stop now...
if (screenPos.x >= depth_src.width() || screenPos.y >= depth_src.height()) return;
// Is this point near the actual surface and therefore a contributor?
// Calculate the dot product of surface normal and camera ray
const float3 n = poseInv.getFloat3x3() * make_float3(normals.tex2D((int)x, (int)y));
float3 ray = camera.screenToCam(screenPos.x, screenPos.y, 1.0f);
ray = ray / length(ray);
const float dotproduct = max(dot(ray,n),0.0f);
const float d2 = depth_src.tex2D((int)screenPos.x, (int)screenPos.y);
const A input = in.tex2D((int)screenPos.x, (int)screenPos.y); //generateInput(in.tex2D((int)screenPos.x, (int)screenPos.y), params, worldPos);
const float weight = ftl::cuda::weighting(fabs(camPos.z - d2), 0.02f);
float weight = ftl::cuda::weighting(fabs(camPos.z - d2), 0.02f);
/* Buehler C. et al. 2001. Unstructured Lumigraph Rendering. */
/* Orts-Escolano S. et al. 2016. Holoportation: Virtual 3D teleportation in real-time. */
// This is the simple naive colour weighting. It might be good
// enough for our purposes if the alignment step prevents ghosting
// TODO: Use depth and perhaps the neighbourhood consistency in:
// Kuster C. et al. 2011. FreeCam: A hybrid camera system for interactive free-viewpoint video
if (params.m_flags & ftl::render::kNormalWeightColours) weight *= dotproduct;
const B weighted = make<B>(input) * weight; //weightInput(input, weight);
if (weight > 0.0f) {
......@@ -102,7 +116,8 @@ template <typename A, typename B>
void ftl::cuda::reproject(
TextureObject<A> &in,
TextureObject<float> &depth_src, // Original 3D points
TextureObject<int> &depth_in, // Virtual depth map
TextureObject<int> &depth_in, // Virtual depth map
TextureObject<float4> &normals,
TextureObject<B> &out, // Accumulated output
TextureObject<float> &contrib,
const SplatParams &params,
......@@ -113,7 +128,8 @@ void ftl::cuda::reproject(
reprojection_kernel<<<gridSize, blockSize, 0, stream>>>(
in,
depth_src,
depth_in,
depth_in,
normals,
out,
contrib,
params,
......@@ -127,6 +143,7 @@ template void ftl::cuda::reproject(
ftl::cuda::TextureObject<uchar4> &in, // Original colour image
ftl::cuda::TextureObject<float> &depth_src, // Original 3D points
ftl::cuda::TextureObject<int> &depth_in, // Virtual depth map
ftl::cuda::TextureObject<float4> &normals,
ftl::cuda::TextureObject<float4> &out, // Accumulated output
ftl::cuda::TextureObject<float> &contrib,
const ftl::render::SplatParams &params,
......@@ -137,6 +154,7 @@ template void ftl::cuda::reproject(
ftl::cuda::TextureObject<float> &in, // Original colour image
ftl::cuda::TextureObject<float> &depth_src, // Original 3D points
ftl::cuda::TextureObject<int> &depth_in, // Virtual depth map
ftl::cuda::TextureObject<float4> &normals,
ftl::cuda::TextureObject<float> &out, // Accumulated output
ftl::cuda::TextureObject<float> &contrib,
const ftl::render::SplatParams &params,
......@@ -147,6 +165,7 @@ template void ftl::cuda::reproject(
ftl::cuda::TextureObject<float4> &in, // Original colour image
ftl::cuda::TextureObject<float> &depth_src, // Original 3D points
ftl::cuda::TextureObject<int> &depth_in, // Virtual depth map
ftl::cuda::TextureObject<float4> &normals,
ftl::cuda::TextureObject<float4> &out, // Accumulated output
ftl::cuda::TextureObject<float> &contrib,
const ftl::render::SplatParams &params,
......
components/renderers/cpp/src/splat_render.cpp
+ 1
− 57
View file @ efd6c29e
......@@ -216,6 +216,7 @@ void Splatter::__reprojectChannel(ftl::rgbd::Frame &output, ftl::codecs::Channel
f.createTexture<T>(in),
f.createTexture<float>(Channel::Depth), // TODO: Use depth?
temp_.getTexture<int>(Channel::Depth2),
accum_.getTexture<float4>(Channel::Normals),
temp_.createTexture<typename AccumSelector<T>::type>(AccumSelector<T>::channel),
temp_.getTexture<float>(Channel::Contribution),
params_,
......@@ -309,16 +310,6 @@ void Splatter::_renderChannel(
temp_.createTexture<float4>(Channel::Colour);
temp_.createTexture<float>(Channel::Contribution);
// Generate initial normals for the splats
//accum_.create<GpuMat>(Channel::Normals, Format<float4>(params_.camera.width, params_.camera.height));
//_blendChannel(accum_, Channel::Normals, Channel::Normals, stream);
// Put normals in camera space here...
//ftl::cuda::transform_normals(accum_.getTexture<float4>(Channel::Normals), params_.m_viewMatrix.getFloat3x3(), stream);
// Estimate point density
//accum_.create<GpuMat>(Channel::Density, Format<float>(params_.camera.width, params_.camera.height));
//_blendChannel(accum_, Channel::Depth, Channel::Density, stream);
// FIXME: Using colour 2 in this way seems broken since it is already used
if (is_4chan) {
accum_.create<GpuMat>(channel_out, Format<float4>(params_.camera.width, params_.camera.height));
......@@ -332,53 +323,6 @@ void Splatter::_renderChannel(
}
_reprojectChannel(out, channel_in, channel_out, stream);
//if (splat_) {
//_blendChannel(accum_, channel_in, channel_out, stream);
//} else {
// _blendChannel(out, channel, channel, stream);
//}
// Now splat the points
if (splat_) {
/*if (is_4chan) {
ftl::cuda::splat(
accum_.getTexture<float4>(Channel::Normals),
accum_.getTexture<float>(Channel::Density),
accum_.getTexture<float4>(channel_out),
temp_.getTexture<int>(Channel::Depth2),
out.createTexture<float>(Channel::Depth),
out.createTexture<float4>(channel_out),
params_, stream
);
} else if (is_float) {
ftl::cuda::splat(
accum_.getTexture<float4>(Channel::Normals),
accum_.getTexture<float>(Channel::Density),
accum_.getTexture<float>(channel_out),
temp_.getTexture<int>(Channel::Depth2),
out.createTexture<float>(Channel::Depth),
out.createTexture<float>(channel_out),
params_, stream
);
} else {
ftl::cuda::splat(
accum_.getTexture<float4>(Channel::Normals),
accum_.getTexture<float>(Channel::Density),
accum_.getTexture<uchar4>(channel_out),
temp_.getTexture<int>(Channel::Depth2),
out.createTexture<float>(Channel::Depth),
out.createTexture<uchar4>(channel_out),
params_, stream
);
}*/
//temp_.get<GpuMat>(Channel::Depth2).convertTo(out.get<GpuMat>(Channel::Depth), CV_32F, 1.0f / 1000.0f);
//accum_.swapTo(Channels(channel_out), out);
} else {
// Swap accum frames directly to output.
//accum_.swapTo(Channels(channel_out), out);
//temp_.get<GpuMat>(Channel::Depth2).convertTo(out.get<GpuMat>(Channel::Depth), CV_32F, 1.0f / 1000.0f);
}
}
bool Splatter::render(ftl::rgbd::VirtualSource *src, ftl::rgbd::Frame &out) {
......
components/renderers/cpp/src/splatter_cuda.hpp
+ 1
− 0
View file @ efd6c29e
......@@ -54,6 +54,7 @@ namespace cuda {
ftl::cuda::TextureObject<A> &in, // Original colour image
ftl::cuda::TextureObject<float> &depth_src, // Original 3D points
ftl::cuda::TextureObject<int> &depth_in, // Virtual depth map
ftl::cuda::TextureObject<float4> &normals,
ftl::cuda::TextureObject<B> &out, // Accumulated output
ftl::cuda::TextureObject<float> &contrib,
const ftl::render::SplatParams &params,
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment