Merge branch 'feature/tweak_recon' into 'master'

Tweaks to improve reconstruct

See merge request nicolas.pope/ftl!193

applications/gui/src/camera.cpp

@@ -340,10 +340,10 @@ static void visualizeEnergy(	const cv::Mat &depth, cv::Mat &out,
 
 	depth.convertTo(out, CV_8U, 255.0f / max_depth);
 	//out = 255 - out;
-	cv::Mat mask = (depth >= 39.0f); // TODO (mask for invalid pixels)
+	//cv::Mat mask = (depth >= 39.0f); // TODO (mask for invalid pixels)
 	
 	applyColorMap(out, out, cv::COLORMAP_JET);
-	out.setTo(cv::Scalar(255, 255, 255), mask);
+	//out.setTo(cv::Scalar(255, 255, 255), mask);
 }
 
 static void drawEdges(	const cv::Mat &in, cv::Mat &out,
@@ -447,7 +447,7 @@ const GLTexture &ftl::gui::Camera::captureFrame() {
 			case Channel::Smoothing:
 			case Channel::Confidence:
 				if (im2_.rows == 0) { break; }
-				visualizeEnergy(im2_, tmp, 1.0);
+				visualizeEnergy(im2_, tmp, screen_->root()->value("float_image_max", 1.0f));
 				texture2_.update(tmp);
 				break;
 			

components/operators/src/correspondence.cu

@@ -119,13 +119,13 @@ __global__ void corresponding_point_kernel(
 
         //float bestdepth = 0.0f;
         short2 bestScreen = make_short2(-1,-1);
-		float bestdepth = 0.0f;
+		//float bestdepth = 0.0f;
 		//float bestdepth2 = 0.0f;
         float bestweight = 0.0f;
         float bestcolour = 0.0f;
         //float bestdweight = 0.0f;
         float totalcolour = 0.0f;
-        int count = 0;
+        //int count = 0;
         //float contrib = 0.0f;
 		
 		const float3 camPosOrigin = pose * cam1.screenToCam(x,y,depth1);
@@ -138,8 +138,11 @@ __global__ void corresponding_point_kernel(
         float2 linePos;
         linePos.x = lineOrigin.x - ((COR_STEPS/2));
         linePos.y = lineOrigin.y - (((COR_STEPS/2)) * lineM);
-		float depthPos = depth1 - (float((COR_STEPS/2)) * depthM);
-		float depthPos2 = camPosOrigin.z - (float((COR_STEPS/2)) * depthM2);
+		//float depthPos = depth1 - (float((COR_STEPS/2)) * depthM);
+        float depthPos2 = camPosOrigin.z - (float((COR_STEPS/2)) * depthM2);
+        
+        uint badMask = 0;
+        int bestStep = 0;
 
 
         // Project to p2 using cam2
@@ -150,51 +153,58 @@ __global__ void corresponding_point_kernel(
 
 			// Generate a colour correspondence value
             const auto colour2 = c2.tex2D(linePos.x, linePos.y);
+
+            // TODO: Check if other colour dissimilarities are better...
             const float cweight = ftl::cuda::colourWeighting(colour1, colour2, params.colour_smooth);
 
             // Generate a depth correspondence value
-			const float depth2 = d2.tex2D(int(linePos.x+0.5f), int(linePos.y+0.5f));
+            const float depth2 = d2.tex2D(int(linePos.x+0.5f), int(linePos.y+0.5f));
+            
+            // Record which correspondences are invalid
+            badMask |= (depth2 <= cam2.minDepth || depth2 >= cam2.maxDepth) ? 1 << i : 0;
 			
-			if (FUNCTION == 1) {
+			//if (FUNCTION == 1) {
 				weight *= ftl::cuda::weighting(fabs(depth2 - depthPos2), cweight*params.spatial_smooth);
-			} else {
-				const float dweight = ftl::cuda::weighting(fabs(depth2 - depthPos2), params.spatial_smooth);
-            	weight *= weightFunction<FUNCTION>(params, dweight, cweight);
-			}
+			//} else {
+			//	const float dweight = ftl::cuda::weighting(fabs(depth2 - depthPos2), params.spatial_smooth);
+            //	weight *= weightFunction<FUNCTION>(params, dweight, cweight);
+			//}
             //const float dweight = ftl::cuda::weighting(fabs(depth_adjust), 10.0f*params.range);
 
             //weight *= weightFunction<FUNCTION>(params, dweight, cweight);
 
-            ++count;
-            //contrib += weight;
+            //++count;
+
             bestcolour = max(cweight, bestcolour);
-            //bestdweight = max(dweight, bestdweight);
             totalcolour += cweight;
-			bestdepth = (weight > bestweight) ? depthPos : bestdepth;
-			//bestdepth2 = (weight > bestweight) ? camPos.z : bestdepth2;
-			//bestScreen = (weight > bestweight) ? make_short2(screen.x+0.5f, screen.y+0.5f) : bestScreen;
+
+            //bestdepth = (weight > bestweight) ? depthPos : bestdepth;
+            bestStep = (weight > bestweight) ? i : bestStep;
+			
 			bestweight = max(bestweight, weight);
-                //bestweight = weight;
-                //bestdepth = depth_adjust;
-                //bestScreen = make_short2(screen.x+0.5f, screen.y+0.5f);
-			//}
+                
 			
-			depthPos += depthM;
+			//depthPos += depthM;
 			depthPos2 += depthM2;
             linePos.x += 1.0f;
             linePos.y += lineM;
         }
 
-        const float avgcolour = totalcolour/(float)count;
+        //const float avgcolour = totalcolour/(float)count;
         const float confidence = bestcolour / totalcolour; //bestcolour - avgcolour;
-        
+        const float bestadjust = float(bestStep-(COR_STEPS/2))*depthM;
+
+        // Detect matches to boundaries, and discard those
+        uint stepMask = 1 << bestStep;
+        if ((stepMask & (badMask << 1)) || (stepMask & (badMask >> 1))) bestweight = 0.0f;
+
         //Mask m(mask.tex2D(x,y));
 
         //if (bestweight > 0.0f) {
             float old = conf.tex2D(x,y);
 
             if (bestweight * confidence > old) {
-				d1(x,y) = (0.4f*(bestdepth-depth1)) + depth1;
+				d1(x,y) = (0.4f*bestadjust) + depth1;
 				//d2(bestScreen.x, bestScreen.y) = bestdepth2;
                 //screenOut(x,y) = bestScreen;
                 conf(x,y) = bestweight * confidence;

components/operators/src/mvmls.cpp

@@ -249,7 +249,7 @@ bool MultiViewMLS::apply(ftl::rgbd::FrameSet &in, ftl::rgbd::FrameSet &out, cuda
             for (size_t i=0; i<in.frames.size(); ++i) {
                 auto &f1 = in.frames[i];
                 //f1.get<GpuMat>(Channel::Depth2).setTo(cv::Scalar(0.0f), cvstream);
-                f1.get<GpuMat>(Channel::Confidence).setTo(cv::Scalar(0.0f), cvstream);
+                //f1.get<GpuMat>(Channel::Confidence).setTo(cv::Scalar(0.0f), cvstream);
 
                 Eigen::Vector4d d1(0.0, 0.0, 1.0, 0.0);
                 d1 = in.sources[i]->getPose() * d1;

components/renderers/cpp/src/reprojection.cu

@@ -89,7 +89,9 @@ __global__ void reprojection_kernel(
 	const float3 n = transformR * make_float3(normals.tex2D((int)x, (int)y));
 	float3 ray = camera.screenToCam(screenPos.x, screenPos.y, 1.0f);
 	ray = ray / length(ray);
-	const float dotproduct = max(dot(ray,n),0.0f);
+
+	// Allow slightly beyond 90 degrees due to normal estimation errors
+	const float dotproduct = (max(dot(ray,n),-0.1f)+0.1) / 1.1f;
     
 	const float d2 = depth_src.tex2D(int(screenPos.x+0.5f), int(screenPos.y+0.5f));
 	const auto input = in.tex2D(screenPos.x, screenPos.y); //generateInput(in.tex2D((int)screenPos.x, (int)screenPos.y), params, worldPos);

components/renderers/cpp/src/splatter.cu

@@ -131,7 +131,7 @@ using ftl::cuda::warpSum;
 	const uint2 screenPos = params.camera.camToScreen<uint2>(camPos);
 	const unsigned int cx = screenPos.x;
 	const unsigned int cy = screenPos.y;
-	if (d > params.camera.minDepth && d < params.camera.maxDepth && cx < depth.width() && cy < depth.height()) {
+	if (d > params.camera.minDepth && d < params.camera.maxDepth && cx < depth_out.width() && cy < depth_out.height()) {
 		// Transform estimated point to virtual cam space and output z
 		atomicMin(&depth_out(cx,cy), d * 100000.0f);
 	}
@@ -155,7 +155,7 @@ void ftl::cuda::dibr_merge(TextureObject<float4> &points, TextureObject<int> &de
 }
 
 void ftl::cuda::dibr_merge(TextureObject<float> &depth, TextureObject<int> &depth_out, const float4x4 &transform, const ftl::rgbd::Camera &cam, SplatParams params, cudaStream_t stream) {
-    const dim3 gridSize((depth.width() + T_PER_BLOCK - 1)/T_PER_BLOCK, (depth.height() + T_PER_BLOCK - 1)/T_PER_BLOCK);
+    const dim3 gridSize((depth_out.width() + T_PER_BLOCK - 1)/T_PER_BLOCK, (depth_out.height() + T_PER_BLOCK - 1)/T_PER_BLOCK);
     const dim3 blockSize(T_PER_BLOCK, T_PER_BLOCK);
 
 	dibr_merge_kernel<<<gridSize, blockSize, 0, stream>>>(depth, depth_out, transform, cam, params);

components/renderers/cpp/src/tri_render.cpp

@@ -621,7 +621,7 @@ bool Triangular::render(ftl::rgbd::VirtualSource *src, ftl::rgbd::Frame &out, co
 		);
 	}
 
-	if (value("show_bad_colour", false)) {
+	if (value("show_bad_colour", true)) {
 		ftl::cuda::show_missing_colour(
 			out.getTexture<float>(Channel::Depth),
 			out.getTexture<uchar4>(Channel::Colour),