From f4ac078abc340a81eca59bcab08c09973b4faaf1 Mon Sep 17 00:00:00 2001
From: Nicolas Pope <nicolas.pope@utu.fi>
Date: Tue, 10 Dec 2019 15:13:36 +0200
Subject: [PATCH] Tweaks to improve reconstruct
---
applications/gui/src/camera.cpp | 6 +-
components/operators/src/correspondence.cu | 58 ++++++++++++--------
components/operators/src/mvmls.cpp | 2 +-
components/renderers/cpp/src/reprojection.cu | 4 +-
components/renderers/cpp/src/splatter.cu | 4 +-
components/renderers/cpp/src/tri_render.cpp | 2 +-
6 files changed, 44 insertions(+), 32 deletions(-)
diff --git a/applications/gui/src/camera.cpp b/applications/gui/src/camera.cpp
index 0a9d04c04..64a100a59 100644
--- a/applications/gui/src/camera.cpp
+++ b/applications/gui/src/camera.cpp
@@ -340,10 +340,10 @@ static void visualizeEnergy( const cv::Mat &depth, cv::Mat &out,
depth.convertTo(out, CV_8U, 255.0f / max_depth);
//out = 255 - out;
- cv::Mat mask = (depth >= 39.0f); // TODO (mask for invalid pixels)
+ //cv::Mat mask = (depth >= 39.0f); // TODO (mask for invalid pixels)
applyColorMap(out, out, cv::COLORMAP_JET);
- out.setTo(cv::Scalar(255, 255, 255), mask);
+ //out.setTo(cv::Scalar(255, 255, 255), mask);
}
static void drawEdges( const cv::Mat &in, cv::Mat &out,
@@ -447,7 +447,7 @@ const GLTexture &ftl::gui::Camera::captureFrame() {
case Channel::Smoothing:
case Channel::Confidence:
if (im2_.rows == 0) { break; }
- visualizeEnergy(im2_, tmp, 1.0);
+ visualizeEnergy(im2_, tmp, screen_->root()->value("float_image_max", 1.0f));
texture2_.update(tmp);
break;
diff --git a/components/operators/src/correspondence.cu b/components/operators/src/correspondence.cu
index 5af30cc99..1611a8483 100644
--- a/components/operators/src/correspondence.cu
+++ b/components/operators/src/correspondence.cu
@@ -119,13 +119,13 @@ __global__ void corresponding_point_kernel(
//float bestdepth = 0.0f;
short2 bestScreen = make_short2(-1,-1);
- float bestdepth = 0.0f;
+ //float bestdepth = 0.0f;
//float bestdepth2 = 0.0f;
float bestweight = 0.0f;
float bestcolour = 0.0f;
//float bestdweight = 0.0f;
float totalcolour = 0.0f;
- int count = 0;
+ //int count = 0;
//float contrib = 0.0f;
const float3 camPosOrigin = pose * cam1.screenToCam(x,y,depth1);
@@ -138,8 +138,11 @@ __global__ void corresponding_point_kernel(
float2 linePos;
linePos.x = lineOrigin.x - ((COR_STEPS/2));
linePos.y = lineOrigin.y - (((COR_STEPS/2)) * lineM);
- float depthPos = depth1 - (float((COR_STEPS/2)) * depthM);
- float depthPos2 = camPosOrigin.z - (float((COR_STEPS/2)) * depthM2);
+ //float depthPos = depth1 - (float((COR_STEPS/2)) * depthM);
+ float depthPos2 = camPosOrigin.z - (float((COR_STEPS/2)) * depthM2);
+
+ uint badMask = 0;
+ int bestStep = 0;
// Project to p2 using cam2
@@ -150,51 +153,58 @@ __global__ void corresponding_point_kernel(
// Generate a colour correspondence value
const auto colour2 = c2.tex2D(linePos.x, linePos.y);
+
+ // TODO: Check if other colour dissimilarities are better...
const float cweight = ftl::cuda::colourWeighting(colour1, colour2, params.colour_smooth);
// Generate a depth correspondence value
- const float depth2 = d2.tex2D(int(linePos.x+0.5f), int(linePos.y+0.5f));
+ const float depth2 = d2.tex2D(int(linePos.x+0.5f), int(linePos.y+0.5f));
+
+ // Record which correspondences are invalid
+ badMask |= (depth2 <= cam2.minDepth || depth2 >= cam2.maxDepth) ? 1 << i : 0;
- if (FUNCTION == 1) {
+ //if (FUNCTION == 1) {
weight *= ftl::cuda::weighting(fabs(depth2 - depthPos2), cweight*params.spatial_smooth);
- } else {
- const float dweight = ftl::cuda::weighting(fabs(depth2 - depthPos2), params.spatial_smooth);
- weight *= weightFunction<FUNCTION>(params, dweight, cweight);
- }
+ //} else {
+ // const float dweight = ftl::cuda::weighting(fabs(depth2 - depthPos2), params.spatial_smooth);
+ // weight *= weightFunction<FUNCTION>(params, dweight, cweight);
+ //}
//const float dweight = ftl::cuda::weighting(fabs(depth_adjust), 10.0f*params.range);
//weight *= weightFunction<FUNCTION>(params, dweight, cweight);
- ++count;
- //contrib += weight;
+ //++count;
+
bestcolour = max(cweight, bestcolour);
- //bestdweight = max(dweight, bestdweight);
totalcolour += cweight;
- bestdepth = (weight > bestweight) ? depthPos : bestdepth;
- //bestdepth2 = (weight > bestweight) ? camPos.z : bestdepth2;
- //bestScreen = (weight > bestweight) ? make_short2(screen.x+0.5f, screen.y+0.5f) : bestScreen;
+
+ //bestdepth = (weight > bestweight) ? depthPos : bestdepth;
+ bestStep = (weight > bestweight) ? i : bestStep;
+
bestweight = max(bestweight, weight);
- //bestweight = weight;
- //bestdepth = depth_adjust;
- //bestScreen = make_short2(screen.x+0.5f, screen.y+0.5f);
- //}
+
- depthPos += depthM;
+ //depthPos += depthM;
depthPos2 += depthM2;
linePos.x += 1.0f;
linePos.y += lineM;
}
- const float avgcolour = totalcolour/(float)count;
+ //const float avgcolour = totalcolour/(float)count;
const float confidence = bestcolour / totalcolour; //bestcolour - avgcolour;
-
+ const float bestadjust = float(bestStep-(COR_STEPS/2))*depthM;
+
+ // Detect matches to boundaries, and discard those
+ uint stepMask = 1 << bestStep;
+ if ((stepMask & (badMask << 1)) || (stepMask & (badMask >> 1))) bestweight = 0.0f;
+
//Mask m(mask.tex2D(x,y));
//if (bestweight > 0.0f) {
float old = conf.tex2D(x,y);
if (bestweight * confidence > old) {
- d1(x,y) = (0.4f*(bestdepth-depth1)) + depth1;
+ d1(x,y) = (0.4f*bestadjust) + depth1;
//d2(bestScreen.x, bestScreen.y) = bestdepth2;
//screenOut(x,y) = bestScreen;
conf(x,y) = bestweight * confidence;
diff --git a/components/operators/src/mvmls.cpp b/components/operators/src/mvmls.cpp
index 822b6e09b..4a61e6322 100644
--- a/components/operators/src/mvmls.cpp
+++ b/components/operators/src/mvmls.cpp
@@ -249,7 +249,7 @@ bool MultiViewMLS::apply(ftl::rgbd::FrameSet &in, ftl::rgbd::FrameSet &out, cuda
for (size_t i=0; i<in.frames.size(); ++i) {
auto &f1 = in.frames[i];
//f1.get<GpuMat>(Channel::Depth2).setTo(cv::Scalar(0.0f), cvstream);
- f1.get<GpuMat>(Channel::Confidence).setTo(cv::Scalar(0.0f), cvstream);
+ //f1.get<GpuMat>(Channel::Confidence).setTo(cv::Scalar(0.0f), cvstream);
Eigen::Vector4d d1(0.0, 0.0, 1.0, 0.0);
d1 = in.sources[i]->getPose() * d1;
diff --git a/components/renderers/cpp/src/reprojection.cu b/components/renderers/cpp/src/reprojection.cu
index e6490c47e..b23ad81e9 100644
--- a/components/renderers/cpp/src/reprojection.cu
+++ b/components/renderers/cpp/src/reprojection.cu
@@ -89,7 +89,9 @@ __global__ void reprojection_kernel(
const float3 n = transformR * make_float3(normals.tex2D((int)x, (int)y));
float3 ray = camera.screenToCam(screenPos.x, screenPos.y, 1.0f);
ray = ray / length(ray);
- const float dotproduct = max(dot(ray,n),0.0f);
+
+ // Allow slightly beyond 90 degrees due to normal estimation errors
+ const float dotproduct = (max(dot(ray,n),-0.1f)+0.1) / 1.1f;
const float d2 = depth_src.tex2D(int(screenPos.x+0.5f), int(screenPos.y+0.5f));
const auto input = in.tex2D(screenPos.x, screenPos.y); //generateInput(in.tex2D((int)screenPos.x, (int)screenPos.y), params, worldPos);
diff --git a/components/renderers/cpp/src/splatter.cu b/components/renderers/cpp/src/splatter.cu
index 2986234bb..55706b085 100644
--- a/components/renderers/cpp/src/splatter.cu
+++ b/components/renderers/cpp/src/splatter.cu
@@ -131,7 +131,7 @@ using ftl::cuda::warpSum;
const uint2 screenPos = params.camera.camToScreen<uint2>(camPos);
const unsigned int cx = screenPos.x;
const unsigned int cy = screenPos.y;
- if (d > params.camera.minDepth && d < params.camera.maxDepth && cx < depth.width() && cy < depth.height()) {
+ if (d > params.camera.minDepth && d < params.camera.maxDepth && cx < depth_out.width() && cy < depth_out.height()) {
// Transform estimated point to virtual cam space and output z
atomicMin(&depth_out(cx,cy), d * 100000.0f);
}
@@ -155,7 +155,7 @@ void ftl::cuda::dibr_merge(TextureObject<float4> &points, TextureObject<int> &de
}
void ftl::cuda::dibr_merge(TextureObject<float> &depth, TextureObject<int> &depth_out, const float4x4 &transform, const ftl::rgbd::Camera &cam, SplatParams params, cudaStream_t stream) {
- const dim3 gridSize((depth.width() + T_PER_BLOCK - 1)/T_PER_BLOCK, (depth.height() + T_PER_BLOCK - 1)/T_PER_BLOCK);
+ const dim3 gridSize((depth_out.width() + T_PER_BLOCK - 1)/T_PER_BLOCK, (depth_out.height() + T_PER_BLOCK - 1)/T_PER_BLOCK);
const dim3 blockSize(T_PER_BLOCK, T_PER_BLOCK);
dibr_merge_kernel<<<gridSize, blockSize, 0, stream>>>(depth, depth_out, transform, cam, params);
diff --git a/components/renderers/cpp/src/tri_render.cpp b/components/renderers/cpp/src/tri_render.cpp
index 567a9d139..06d4fe262 100644
--- a/components/renderers/cpp/src/tri_render.cpp
+++ b/components/renderers/cpp/src/tri_render.cpp
@@ -621,7 +621,7 @@ bool Triangular::render(ftl::rgbd::VirtualSource *src, ftl::rgbd::Frame &out, co
);
}
- if (value("show_bad_colour", false)) {
+ if (value("show_bad_colour", true)) {
ftl::cuda::show_missing_colour(
out.getTexture<float>(Channel::Depth),
out.getTexture<uchar4>(Channel::Colour),
--
GitLab