From e9b6b8eb657c21f55c5bba3432e8865165977a59 Mon Sep 17 00:00:00 2001
From: Nicolas Pope <nicolas.pope@utu.fi>
Date: Sat, 29 Feb 2020 15:24:42 +0200
Subject: [PATCH] Resolves #334 window effect bugs
---
applications/gui/src/camera.cpp | 202 +++++++++---------
applications/gui/src/camera.hpp | 4 +
.../cpp/include/ftl/render/render_params.hpp | 13 +-
components/renderers/cpp/src/overlay.cpp | 2 +
components/renderers/cpp/src/reprojection.cu | 39 ++--
components/renderers/cpp/src/screen.cu | 19 +-
.../rgbd-sources/include/ftl/rgbd/camera.hpp | 24 +++
7 files changed, 177 insertions(+), 126 deletions(-)
diff --git a/applications/gui/src/camera.cpp b/applications/gui/src/camera.cpp
index 5a0964ab6..4076c1ee6 100644
--- a/applications/gui/src/camera.cpp
+++ b/applications/gui/src/camera.cpp
@@ -13,8 +13,6 @@
#include <ftl/render/colouriser.hpp>
#include <ftl/cuda/transform.hpp>
-#include <ftl/codecs/faces.hpp>
-
#include <ftl/render/overlay.hpp>
#include "statsimage.hpp"
@@ -161,116 +159,114 @@ void ftl::gui::Camera::draw(std::vector<ftl::rgbd::FrameSet*> &fss) {
//auto &fs = *fss[fsid_];
+ _applyPoseEffects(fss);
+
UNIQUE_LOCK(mutex_, lk2);
//state_.getLeft().fx = intrinsics_->value("focal", 700.0f);
//state_.getLeft().fy = state_.getLeft().fx;
_draw(fss);
+}
- if (renderer_->value("window_effect", false)) {
- for (auto *fset : fss) {
- for (const auto &f : fset->frames) {
- if (f.hasChannel(Channel::Faces)) {
- std::vector<ftl::codecs::Face> data;
- f.get(Channel::Faces, data);
-
- if (data.size() > 0) {
- auto &d = *data.rbegin();
-
- cv::Mat over_depth;
- over_depth.create(overlay_.size(), CV_32F);
-
- auto cam = ftl::rgbd::Camera::from(intrinsics_);
-
- float screenWidth = intrinsics_->value("screen_size", 0.6f); // In meters
- float screenHeight = (9.0f/16.0f) * screenWidth;
-
- float screenDistance = (d.depth > cam.minDepth && d.depth < cam.maxDepth) ? d.depth : intrinsics_->value("screen_dist_default", 0.5f); // Face distance from screen in meters
-
- auto pos = f.getLeft().screenToCam(d.box.x+(d.box.width/2), d.box.y+(d.box.height/2), screenDistance);
- Eigen::Vector3f eye;
- eye[0] = -pos.x;
- eye[1] = pos.y;
- eye[2] = -pos.z;
- //eye[3] = 0;
-
- Eigen::Translation3f trans(eye);
- Eigen::Affine3f t(trans);
- Eigen::Matrix4f viewPose = t.matrix();
-
- // Calculate where the screen is within current camera space
- Eigen::Vector4f p1 = viewPose.cast<float>() * (Eigen::Vector4f(screenWidth/2.0, screenHeight/2.0, 0, 1));
- Eigen::Vector4f p2 = viewPose.cast<float>() * (Eigen::Vector4f(screenWidth/2.0, -screenHeight/2.0, 0, 1));
- Eigen::Vector4f p3 = viewPose.cast<float>() * (Eigen::Vector4f(-screenWidth/2.0, screenHeight/2.0, 0, 1));
- Eigen::Vector4f p4 = viewPose.cast<float>() * (Eigen::Vector4f(-screenWidth/2.0, -screenHeight/2.0, 0, 1));
- p1 = p1 / p1[3];
- p2 = p2 / p2[3];
- p3 = p3 / p3[3];
- p4 = p4 / p4[3];
- float2 p1screen = cam.camToScreen<float2>(make_float3(p1[0],p1[1],p1[2]));
- float2 p2screen = cam.camToScreen<float2>(make_float3(p2[0],p2[1],p2[2]));
- float2 p3screen = cam.camToScreen<float2>(make_float3(p3[0],p3[1],p3[2]));
- float2 p4screen = cam.camToScreen<float2>(make_float3(p4[0],p4[1],p4[2]));
-
- //cv::line(im1_, cv::Point2f(p1screen.x, p1screen.y), cv::Point2f(p2screen.x, p2screen.y), cv::Scalar(255,0,255,0));
- //cv::line(im1_, cv::Point2f(p4screen.x, p4screen.y), cv::Point2f(p2screen.x, p2screen.y), cv::Scalar(255,0,255,0));
- //cv::line(im1_, cv::Point2f(p1screen.x, p1screen.y), cv::Point2f(p3screen.x, p3screen.y), cv::Scalar(255,0,255,0));
- //cv::line(im1_, cv::Point2f(p3screen.x, p3screen.y), cv::Point2f(p4screen.x, p4screen.y), cv::Scalar(255,0,255,0));
- //LOG(INFO) << "DRAW LINE: " << p1screen.x << "," << p1screen.y << " -> " << p2screen.x << "," << p2screen.y;
-
- std::vector<cv::Point2f> quad_pts;
- std::vector<cv::Point2f> squre_pts;
- quad_pts.push_back(cv::Point2f(p1screen.x,p1screen.y));
- quad_pts.push_back(cv::Point2f(p2screen.x,p2screen.y));
- quad_pts.push_back(cv::Point2f(p3screen.x,p3screen.y));
- quad_pts.push_back(cv::Point2f(p4screen.x,p4screen.y));
- squre_pts.push_back(cv::Point2f(0,0));
- squre_pts.push_back(cv::Point2f(0,cam.height));
- squre_pts.push_back(cv::Point2f(cam.width,0));
- squre_pts.push_back(cv::Point2f(cam.width,cam.height));
-
- cv::Mat transmtx = cv::getPerspectiveTransform(quad_pts,squre_pts);
- cv::Mat transformed = cv::Mat::zeros(overlay_.rows, overlay_.cols, CV_8UC4);
- //cv::warpPerspective(im1_, im1_, transmtx, im1_.size());
-
- ftl::render::ViewPort vp;
- vp.x = std::min(p4screen.x, std::min(p3screen.x, std::min(p1screen.x,p2screen.x)));
- vp.y = std::min(p4screen.y, std::min(p3screen.y, std::min(p1screen.y,p2screen.y)));
- vp.width = std::max(p4screen.x, std::max(p3screen.x, std::max(p1screen.x,p2screen.x))) - vp.x;
- vp.height = std::max(p4screen.y, std::max(p3screen.y, std::max(p1screen.y,p2screen.y))) - vp.y;
- renderer_->setViewPort(ftl::render::ViewPortMode::Warping, vp);
-
- //Eigen::Matrix4d windowPose;
- //windowPose.setIdentity();
-
- //Eigen::Matrix4d fakepose = (viewPose.cast<double>()).inverse() * state_.getPose();
- //ftl::rgbd::Camera fakecam = cam;
-
- //eye[1] = -eye[1];
- //eye[2] = -eye[2];
- //Eigen::Translation3f trans2(eye);
- //Eigen::Affine3f t2(trans2);
- //Eigen::Matrix4f viewPose2 = t2.matrix();
-
- // Use face tracked window pose for virtual camera
- //state_.getLeft() = fakecam;
- transform_ix_ = fset->id; // Disable keyboard/mouse pose setting
-
- state_.setPose(transforms_[transform_ix_] * viewPose.cast<double>());
-
- //Eigen::Vector4d pt1 = state_.getPose().inverse() * Eigen::Vector4d(eye[0],eye[1],eye[2],1);
- //pt1 /= pt1[3];
- //Eigen::Vector4d pt2 = state_.getPose().inverse() * Eigen::Vector4d(0,0,0,1);
- //pt2 /= pt2[3];
-
-
- //ftl::overlay::draw3DLine(state_.getLeft(), im1_, over_depth, pt1, pt2, cv::Scalar(0,0,255,0));
- //ftl::overlay::drawRectangle(state_.getLeft(), im1_, over_depth, windowPose.inverse() * state_.getPose(), cv::Scalar(255,0,0,0), screenWidth, screenHeight);
- //ftl::overlay::drawCamera(state_.getLeft(), im1_, over_depth, fakecam, fakepose, cv::Scalar(255,0,255,255), 1.0,screen_->root()->value("show_frustrum", false));
- }
+std::pair<const ftl::rgbd::Frame *, const ftl::codecs::Face *> ftl::gui::Camera::_selectFace(std::vector<ftl::rgbd::FrameSet*> &fss) {
+ for (auto *fset : fss) {
+ for (const auto &f : fset->frames) {
+ if (f.hasChannel(Channel::Faces)) {
+ std::vector<ftl::codecs::Face> data;
+ f.get(Channel::Faces, data);
+
+ if (data.size() > 0) {
+ return {&f,&(*data.rbegin())};
}
}
}
}
+ return {nullptr, nullptr};
+}
+
+void ftl::gui::Camera::_generateWindow(const ftl::rgbd::Frame &f, const ftl::codecs::Face &face, Eigen::Matrix4d &pose_adjust, ftl::render::ViewPort &vp) {
+ auto cam = ftl::rgbd::Camera::from(intrinsics_);
+ auto d = face;
+
+ float screenWidth = intrinsics_->value("screen_size", 0.6f); // In meters
+ float screenHeight = (9.0f/16.0f) * screenWidth;
+
+ float screenDistance = (d.depth > cam.minDepth && d.depth < cam.maxDepth) ? d.depth : intrinsics_->value("screen_dist_default", 0.5f); // Face distance from screen in meters
+
+ auto pos = f.getLeft().screenToCam(float(d.box.x+(d.box.width/2)), float(d.box.y+(d.box.height/2)), screenDistance);
+ Eigen::Vector3f eye;
+ eye[0] = -pos.x;
+ eye[1] = pos.y;
+ eye[2] = -pos.z;
+ //eye[3] = 0;
+
+ Eigen::Translation3f trans(eye);
+ Eigen::Affine3f t(trans);
+ Eigen::Matrix4f viewPose = t.matrix();
+
+ // Calculate where the screen is within current camera space
+ Eigen::Vector4f p1 = viewPose.cast<float>() * (Eigen::Vector4f(screenWidth/2.0, screenHeight/2.0, 0, 1));
+ Eigen::Vector4f p2 = viewPose.cast<float>() * (Eigen::Vector4f(screenWidth/2.0, -screenHeight/2.0, 0, 1));
+ Eigen::Vector4f p3 = viewPose.cast<float>() * (Eigen::Vector4f(-screenWidth/2.0, screenHeight/2.0, 0, 1));
+ Eigen::Vector4f p4 = viewPose.cast<float>() * (Eigen::Vector4f(-screenWidth/2.0, -screenHeight/2.0, 0, 1));
+ p1 = p1 / p1[3];
+ p2 = p2 / p2[3];
+ p3 = p3 / p3[3];
+ p4 = p4 / p4[3];
+ float2 p1screen = cam.camToScreen<float2>(make_float3(p1[0],p1[1],p1[2]));
+ float2 p2screen = cam.camToScreen<float2>(make_float3(p2[0],p2[1],p2[2]));
+ float2 p3screen = cam.camToScreen<float2>(make_float3(p3[0],p3[1],p3[2]));
+ float2 p4screen = cam.camToScreen<float2>(make_float3(p4[0],p4[1],p4[2]));
+
+ std::vector<cv::Point2f> quad_pts;
+ std::vector<cv::Point2f> squre_pts;
+ quad_pts.push_back(cv::Point2f(p1screen.x,p1screen.y));
+ quad_pts.push_back(cv::Point2f(p2screen.x,p2screen.y));
+ quad_pts.push_back(cv::Point2f(p3screen.x,p3screen.y));
+ quad_pts.push_back(cv::Point2f(p4screen.x,p4screen.y));
+ squre_pts.push_back(cv::Point2f(0,0));
+ squre_pts.push_back(cv::Point2f(0,cam.height));
+ squre_pts.push_back(cv::Point2f(cam.width,0));
+ squre_pts.push_back(cv::Point2f(cam.width,cam.height));
+
+ cv::Mat transmtx = cv::getPerspectiveTransform(quad_pts,squre_pts);
+ //cv::Mat transformed = cv::Mat::zeros(overlay_.rows, overlay_.cols, CV_8UC4);
+ //cv::warpPerspective(im1_, im1_, transmtx, im1_.size());
+
+ // TODO: Use the transmtx above for perspective distortion..
+
+ //ftl::render::ViewPort vp;
+ vp.x = std::min(p4screen.x, std::min(p3screen.x, std::min(p1screen.x,p2screen.x)));
+ vp.y = std::min(p4screen.y, std::min(p3screen.y, std::min(p1screen.y,p2screen.y)));
+ vp.width = std::max(p4screen.x, std::max(p3screen.x, std::max(p1screen.x,p2screen.x))) - vp.x;
+ vp.height = std::max(p4screen.y, std::max(p3screen.y, std::max(p1screen.y,p2screen.y))) - vp.y;
+ /*vp.warpMatrix.entries[0] = transmtx.at<float>(0,0);
+ vp.warpMatrix.entries[1] = transmtx.at<float>(1,0);
+ vp.warpMatrix.entries[2] = transmtx.at<float>(2,0);
+ vp.warpMatrix.entries[3] = transmtx.at<float>(0,1);
+ vp.warpMatrix.entries[4] = transmtx.at<float>(1,1);
+ vp.warpMatrix.entries[5] = transmtx.at<float>(2,1);
+ vp.warpMatrix.entries[6] = transmtx.at<float>(0,2);
+ vp.warpMatrix.entries[7] = transmtx.at<float>(1,2);
+ vp.warpMatrix.entries[8] = transmtx.at<float>(2,2);
+ vp.warpMatrix = vp.warpMatrix.getInverse(); //.getInverse();*/
+ //renderer_->setViewPort(ftl::render::ViewPortMode::Warping, vp);
+
+ pose_adjust = viewPose.cast<double>();
+}
+
+void ftl::gui::Camera::_applyPoseEffects(std::vector<ftl::rgbd::FrameSet*> &fss) {
+ if (renderer_->value("window_effect", false)) {
+ auto [frame,face] = _selectFace(fss);
+ if (face) {
+ Eigen::Matrix4d windowPose;
+ ftl::render::ViewPort windowViewPort;
+ _generateWindow(*frame, *face, windowPose, windowViewPort);
+
+ // Apply the window effect
+ renderer_->setViewPort(ftl::render::ViewPortMode::Stretch, windowViewPort);
+ state_.getPose() = windowPose * state_.getPose();
+ }
+ }
}
void ftl::gui::Camera::setStereo(bool v) {
diff --git a/applications/gui/src/camera.hpp b/applications/gui/src/camera.hpp
index 2072d60f7..99affdae2 100644
--- a/applications/gui/src/camera.hpp
+++ b/applications/gui/src/camera.hpp
@@ -9,6 +9,7 @@
#include <ftl/streams/filestream.hpp>
#include <ftl/streams/sender.hpp>
+#include <ftl/codecs/faces.hpp>
#include <string>
#include <array>
@@ -173,6 +174,9 @@ class Camera {
#endif
void _draw(std::vector<ftl::rgbd::FrameSet*> &fss);
+ void _applyPoseEffects(std::vector<ftl::rgbd::FrameSet*> &fss);
+ std::pair<const ftl::rgbd::Frame *, const ftl::codecs::Face *> _selectFace(std::vector<ftl::rgbd::FrameSet*> &fss);
+ void _generateWindow(const ftl::rgbd::Frame &, const ftl::codecs::Face &face, Eigen::Matrix4d &pose_adjust, ftl::render::ViewPort &vp);
};
}
diff --git a/components/renderers/cpp/include/ftl/render/render_params.hpp b/components/renderers/cpp/include/ftl/render/render_params.hpp
index c109d7c1e..b1bcce98b 100644
--- a/components/renderers/cpp/include/ftl/render/render_params.hpp
+++ b/components/renderers/cpp/include/ftl/render/render_params.hpp
@@ -25,19 +25,21 @@ struct ViewPort {
return px >= x && px <= x2() && py >= y && py <= y2();
}
- __device__ __host__ inline uint2 map(const ftl::rgbd::Camera &cam, const float2 &pt) const {
- return make_uint2(
+ __device__ __host__ inline float2 map(const ftl::rgbd::Camera &cam, const float2 &pt) const {
+ return make_float2(
(pt.x - static_cast<float>(x)) * (static_cast<float>(cam.width) / static_cast<float>(width)),
(pt.y - static_cast<float>(y)) * (static_cast<float>(cam.height) / static_cast<float>(height))
);
}
- __device__ __host__ inline uint2 reverseMap(const ftl::rgbd::Camera &cam, const float2 &pt) const {
- return make_uint2(
+ __device__ __host__ inline float2 reverseMap(const ftl::rgbd::Camera &cam, const float2 &pt) const {
+ return make_float2(
(pt.x * (static_cast<float>(width) / static_cast<float>(cam.width))) + static_cast<float>(x),
(pt.y * (static_cast<float>(height) / static_cast<float>(cam.height))) + static_cast<float>(y)
);
}
+
+ //float3x3 warpMatrix;
};
/**
@@ -46,7 +48,8 @@ struct ViewPort {
enum class ViewPortMode : uint8_t {
Disabled = 0, // Do not use the viewport data
Clipping = 1, // Clip the rendering to within the viewport for stencil like effect
- Warping = 2 // Stretch viewport region over entire frame
+ Warping = 2, // Stretch and perspective warp the viewport (requires warp matrix)
+ Stretch = 3 // Stretch viewport region over entire frame
};
enum class AccumulationFunction : uint8_t {
diff --git a/components/renderers/cpp/src/overlay.cpp b/components/renderers/cpp/src/overlay.cpp
index 78415f563..9b3328b33 100644
--- a/components/renderers/cpp/src/overlay.cpp
+++ b/components/renderers/cpp/src/overlay.cpp
@@ -237,6 +237,8 @@ void Overlay::_drawAxis(const Eigen::Matrix4d &pose, const Eigen::Vector3f &scal
}
void Overlay::draw(ftl::rgbd::FrameSet &fs, ftl::rgbd::FrameState &state, const Eigen::Vector2f &screenSize) {
+ if (!value("enabled", false)) return;
+
double zfar = 8.0f;
auto intrin = state.getLeft();
intrin = intrin.scaled(screenSize[0], screenSize[1]);
diff --git a/components/renderers/cpp/src/reprojection.cu b/components/renderers/cpp/src/reprojection.cu
index ce0745003..f34759fcf 100644
--- a/components/renderers/cpp/src/reprojection.cu
+++ b/components/renderers/cpp/src/reprojection.cu
@@ -114,12 +114,21 @@ __device__ inline void accumulateOutput(TextureObject<T> &out, TextureObject<int
}
template <ViewPortMode VPMODE>
-__device__ inline uint2 convertScreen(const Parameters ¶ms, int x, int y) {
- return make_uint2(x,y);
+__device__ inline float2 convertScreen(const Parameters ¶ms, int x, int y) {
+ return make_float2(x,y);
}
+/*template <>
+__device__ inline float2 convertScreen<ViewPortMode::Warping>(const Parameters ¶ms, int x, int y) {
+ const float coeff = 1.0f / (params.viewport.warpMatrix.entries[6] * x + params.viewport.warpMatrix.entries[7] * y + params.viewport.warpMatrix.entries[8]);
+ const float xcoo = coeff * (params.viewport.warpMatrix.entries[0] * x + params.viewport.warpMatrix.entries[1] * y + params.viewport.warpMatrix.entries[2]);
+ const float ycoo = coeff * (params.viewport.warpMatrix.entries[3] * x + params.viewport.warpMatrix.entries[4] * y + params.viewport.warpMatrix.entries[5]);
+ float2 pt = params.viewport.reverseMap(params.camera, make_float2(xcoo,ycoo));
+ return pt;
+}*/
+
template <>
-__device__ inline uint2 convertScreen<ViewPortMode::Warping>(const Parameters ¶ms, int x, int y) {
+__device__ inline float2 convertScreen<ViewPortMode::Stretch>(const Parameters ¶ms, int x, int y) {
return params.viewport.reverseMap(params.camera, make_float2(x,y));
}
@@ -167,7 +176,7 @@ __global__ void reprojection_kernel(
const float d = depth_in.tex2D((int)x, (int)y);
if (d > params.camera.minDepth && d < params.camera.maxDepth) {
- const uint2 rpt = convertScreen<VPMODE>(params, x, y);
+ const float2 rpt = convertScreen<VPMODE>(params, x, y);
const float3 camPos = transform * params.camera.screenToCam(rpt.x, rpt.y, d);
if (camPos.z > camera.minDepth && camPos.z < camera.maxDepth) {
const float2 screenPos = camera.camToScreen<float2>(camPos);
@@ -227,7 +236,7 @@ __global__ void reprojection_kernel(
const float d = depth_in.tex2D((int)x, (int)y);
if (d > params.camera.minDepth && d < params.camera.maxDepth) {
- const uint2 rpt = convertScreen<VPMODE>(params, x, y);
+ const float2 rpt = convertScreen<VPMODE>(params, x, y);
const float3 camPos = transform * params.camera.screenToCam(rpt.x, rpt.y, d);
if (camPos.z > camera.minDepth && camPos.z < camera.maxDepth) {
const float2 screenPos = camera.camToScreen<float2>(camPos);
@@ -272,31 +281,31 @@ void ftl::cuda::reproject(
switch (params.viewPortMode) {
case ViewPortMode::Disabled: reprojection_kernel<A,B,ViewPortMode::Disabled,AccumulationFunction::CloseWeights><<<gridSize, blockSize, 0, stream>>>(in, depth_src, depth_in, weights, *normals, out, contrib, params, camera, transform, transformR); break;
case ViewPortMode::Clipping: reprojection_kernel<A,B,ViewPortMode::Clipping,AccumulationFunction::CloseWeights><<<gridSize, blockSize, 0, stream>>>(in, depth_src, depth_in, weights, *normals, out, contrib, params, camera, transform, transformR); break;
- case ViewPortMode::Warping: reprojection_kernel<A,B,ViewPortMode::Warping,AccumulationFunction::CloseWeights><<<gridSize, blockSize, 0, stream>>>(in, depth_src, depth_in, weights, *normals, out, contrib, params, camera, transform, transformR); break;
+ case ViewPortMode::Stretch: reprojection_kernel<A,B,ViewPortMode::Stretch,AccumulationFunction::CloseWeights><<<gridSize, blockSize, 0, stream>>>(in, depth_src, depth_in, weights, *normals, out, contrib, params, camera, transform, transformR); break;
}
} else if (params.accumulationMode == AccumulationFunction::BestWeight) {
switch (params.viewPortMode) {
case ViewPortMode::Disabled: reprojection_kernel<A,B,ViewPortMode::Disabled,AccumulationFunction::BestWeight><<<gridSize, blockSize, 0, stream>>>(in, depth_src, depth_in, weights, *normals, out, contrib, params, camera, transform, transformR); break;
case ViewPortMode::Clipping: reprojection_kernel<A,B,ViewPortMode::Clipping,AccumulationFunction::BestWeight><<<gridSize, blockSize, 0, stream>>>(in, depth_src, depth_in, weights, *normals, out, contrib, params, camera, transform, transformR); break;
- case ViewPortMode::Warping: reprojection_kernel<A,B,ViewPortMode::Warping,AccumulationFunction::BestWeight><<<gridSize, blockSize, 0, stream>>>(in, depth_src, depth_in, weights, *normals, out, contrib, params, camera, transform, transformR); break;
+ case ViewPortMode::Stretch: reprojection_kernel<A,B,ViewPortMode::Stretch,AccumulationFunction::BestWeight><<<gridSize, blockSize, 0, stream>>>(in, depth_src, depth_in, weights, *normals, out, contrib, params, camera, transform, transformR); break;
}
} else if (params.accumulationMode == AccumulationFunction::Simple) {
switch (params.viewPortMode) {
case ViewPortMode::Disabled: reprojection_kernel<A,B,ViewPortMode::Disabled,AccumulationFunction::Simple><<<gridSize, blockSize, 0, stream>>>(in, depth_src, depth_in, weights, *normals, out, contrib, params, camera, transform, transformR); break;
case ViewPortMode::Clipping: reprojection_kernel<A,B,ViewPortMode::Clipping,AccumulationFunction::Simple><<<gridSize, blockSize, 0, stream>>>(in, depth_src, depth_in, weights, *normals, out, contrib, params, camera, transform, transformR); break;
- case ViewPortMode::Warping: reprojection_kernel<A,B,ViewPortMode::Warping,AccumulationFunction::Simple><<<gridSize, blockSize, 0, stream>>>(in, depth_src, depth_in, weights, *normals, out, contrib, params, camera, transform, transformR); break;
+ case ViewPortMode::Stretch: reprojection_kernel<A,B,ViewPortMode::Stretch,AccumulationFunction::Simple><<<gridSize, blockSize, 0, stream>>>(in, depth_src, depth_in, weights, *normals, out, contrib, params, camera, transform, transformR); break;
}
} else if (params.accumulationMode == AccumulationFunction::ColourDiscard) {
switch (params.viewPortMode) {
case ViewPortMode::Disabled: reprojection_kernel<A,B,ViewPortMode::Disabled,AccumulationFunction::ColourDiscard><<<gridSize, blockSize, 0, stream>>>(in, depth_src, depth_in, weights, *normals, out, contrib, params, camera, transform, transformR); break;
case ViewPortMode::Clipping: reprojection_kernel<A,B,ViewPortMode::Clipping,AccumulationFunction::ColourDiscard><<<gridSize, blockSize, 0, stream>>>(in, depth_src, depth_in, weights, *normals, out, contrib, params, camera, transform, transformR); break;
- case ViewPortMode::Warping: reprojection_kernel<A,B,ViewPortMode::Warping,AccumulationFunction::ColourDiscard><<<gridSize, blockSize, 0, stream>>>(in, depth_src, depth_in, weights, *normals, out, contrib, params, camera, transform, transformR); break;
+ case ViewPortMode::Stretch: reprojection_kernel<A,B,ViewPortMode::Stretch,AccumulationFunction::ColourDiscard><<<gridSize, blockSize, 0, stream>>>(in, depth_src, depth_in, weights, *normals, out, contrib, params, camera, transform, transformR); break;
}
} else if (params.accumulationMode == AccumulationFunction::ColourDiscardSmooth) {
switch (params.viewPortMode) {
case ViewPortMode::Disabled: reprojection_kernel<A,B,ViewPortMode::Disabled,AccumulationFunction::ColourDiscardSmooth><<<gridSize, blockSize, 0, stream>>>(in, depth_src, depth_in, weights, *normals, out, contrib, params, camera, transform, transformR); break;
case ViewPortMode::Clipping: reprojection_kernel<A,B,ViewPortMode::Clipping,AccumulationFunction::ColourDiscardSmooth><<<gridSize, blockSize, 0, stream>>>(in, depth_src, depth_in, weights, *normals, out, contrib, params, camera, transform, transformR); break;
- case ViewPortMode::Warping: reprojection_kernel<A,B,ViewPortMode::Warping,AccumulationFunction::ColourDiscardSmooth><<<gridSize, blockSize, 0, stream>>>(in, depth_src, depth_in, weights, *normals, out, contrib, params, camera, transform, transformR); break;
+ case ViewPortMode::Stretch: reprojection_kernel<A,B,ViewPortMode::Stretch,AccumulationFunction::ColourDiscardSmooth><<<gridSize, blockSize, 0, stream>>>(in, depth_src, depth_in, weights, *normals, out, contrib, params, camera, transform, transformR); break;
}
}
} else {
@@ -304,31 +313,31 @@ void ftl::cuda::reproject(
switch (params.viewPortMode) {
case ViewPortMode::Disabled: reprojection_kernel<A,B,ViewPortMode::Disabled,AccumulationFunction::CloseWeights><<<gridSize, blockSize, 0, stream>>>(in, depth_src, depth_in, weights, out, contrib, params, camera, transform, transformR); break;
case ViewPortMode::Clipping: reprojection_kernel<A,B,ViewPortMode::Clipping,AccumulationFunction::CloseWeights><<<gridSize, blockSize, 0, stream>>>(in, depth_src, depth_in, weights, out, contrib, params, camera, transform, transformR); break;
- case ViewPortMode::Warping: reprojection_kernel<A,B,ViewPortMode::Warping,AccumulationFunction::CloseWeights><<<gridSize, blockSize, 0, stream>>>(in, depth_src, depth_in, weights, out, contrib, params, camera, transform, transformR); break;
+ case ViewPortMode::Stretch: reprojection_kernel<A,B,ViewPortMode::Stretch,AccumulationFunction::CloseWeights><<<gridSize, blockSize, 0, stream>>>(in, depth_src, depth_in, weights, out, contrib, params, camera, transform, transformR); break;
}
} else if (params.accumulationMode == AccumulationFunction::BestWeight) {
switch (params.viewPortMode) {
case ViewPortMode::Disabled: reprojection_kernel<A,B,ViewPortMode::Disabled,AccumulationFunction::BestWeight><<<gridSize, blockSize, 0, stream>>>(in, depth_src, depth_in, weights, out, contrib, params, camera, transform, transformR); break;
case ViewPortMode::Clipping: reprojection_kernel<A,B,ViewPortMode::Clipping,AccumulationFunction::BestWeight><<<gridSize, blockSize, 0, stream>>>(in, depth_src, depth_in, weights, out, contrib, params, camera, transform, transformR); break;
- case ViewPortMode::Warping: reprojection_kernel<A,B,ViewPortMode::Warping,AccumulationFunction::BestWeight><<<gridSize, blockSize, 0, stream>>>(in, depth_src, depth_in, weights, out, contrib, params, camera, transform, transformR); break;
+ case ViewPortMode::Stretch: reprojection_kernel<A,B,ViewPortMode::Stretch,AccumulationFunction::BestWeight><<<gridSize, blockSize, 0, stream>>>(in, depth_src, depth_in, weights, out, contrib, params, camera, transform, transformR); break;
}
} else if (params.accumulationMode == AccumulationFunction::Simple) {
switch (params.viewPortMode) {
case ViewPortMode::Disabled: reprojection_kernel<A,B,ViewPortMode::Disabled,AccumulationFunction::Simple><<<gridSize, blockSize, 0, stream>>>(in, depth_src, depth_in, weights, out, contrib, params, camera, transform, transformR); break;
case ViewPortMode::Clipping: reprojection_kernel<A,B,ViewPortMode::Clipping,AccumulationFunction::Simple><<<gridSize, blockSize, 0, stream>>>(in, depth_src, depth_in, weights, out, contrib, params, camera, transform, transformR); break;
- case ViewPortMode::Warping: reprojection_kernel<A,B,ViewPortMode::Warping,AccumulationFunction::Simple><<<gridSize, blockSize, 0, stream>>>(in, depth_src, depth_in, weights, out, contrib, params, camera, transform, transformR); break;
+ case ViewPortMode::Stretch: reprojection_kernel<A,B,ViewPortMode::Stretch,AccumulationFunction::Simple><<<gridSize, blockSize, 0, stream>>>(in, depth_src, depth_in, weights, out, contrib, params, camera, transform, transformR); break;
}
} else if (params.accumulationMode == AccumulationFunction::ColourDiscard) {
switch (params.viewPortMode) {
case ViewPortMode::Disabled: reprojection_kernel<A,B,ViewPortMode::Disabled,AccumulationFunction::ColourDiscard><<<gridSize, blockSize, 0, stream>>>(in, depth_src, depth_in, weights, out, contrib, params, camera, transform, transformR); break;
case ViewPortMode::Clipping: reprojection_kernel<A,B,ViewPortMode::Clipping,AccumulationFunction::ColourDiscard><<<gridSize, blockSize, 0, stream>>>(in, depth_src, depth_in, weights, out, contrib, params, camera, transform, transformR); break;
- case ViewPortMode::Warping: reprojection_kernel<A,B,ViewPortMode::Warping,AccumulationFunction::ColourDiscard><<<gridSize, blockSize, 0, stream>>>(in, depth_src, depth_in, weights, out, contrib, params, camera, transform, transformR); break;
+ case ViewPortMode::Stretch: reprojection_kernel<A,B,ViewPortMode::Stretch,AccumulationFunction::ColourDiscard><<<gridSize, blockSize, 0, stream>>>(in, depth_src, depth_in, weights, out, contrib, params, camera, transform, transformR); break;
}
} else if (params.accumulationMode == AccumulationFunction::ColourDiscardSmooth) {
switch (params.viewPortMode) {
case ViewPortMode::Disabled: reprojection_kernel<A,B,ViewPortMode::Disabled,AccumulationFunction::ColourDiscardSmooth><<<gridSize, blockSize, 0, stream>>>(in, depth_src, depth_in, weights, out, contrib, params, camera, transform, transformR); break;
case ViewPortMode::Clipping: reprojection_kernel<A,B,ViewPortMode::Clipping,AccumulationFunction::ColourDiscardSmooth><<<gridSize, blockSize, 0, stream>>>(in, depth_src, depth_in, weights, out, contrib, params, camera, transform, transformR); break;
- case ViewPortMode::Warping: reprojection_kernel<A,B,ViewPortMode::Warping,AccumulationFunction::ColourDiscardSmooth><<<gridSize, blockSize, 0, stream>>>(in, depth_src, depth_in, weights, out, contrib, params, camera, transform, transformR); break;
+ case ViewPortMode::Stretch: reprojection_kernel<A,B,ViewPortMode::Stretch,AccumulationFunction::ColourDiscardSmooth><<<gridSize, blockSize, 0, stream>>>(in, depth_src, depth_in, weights, out, contrib, params, camera, transform, transformR); break;
}
}
}
diff --git a/components/renderers/cpp/src/screen.cu b/components/renderers/cpp/src/screen.cu
index 1197f875f..cb041ac0b 100644
--- a/components/renderers/cpp/src/screen.cu
+++ b/components/renderers/cpp/src/screen.cu
@@ -10,6 +10,10 @@ using ftl::render::ViewPortMode;
#define T_PER_BLOCK 8
+__device__ inline uint2 make_uint2(const float2 &f) {
+ return {static_cast<uint>(f.x), static_cast<uint>(f.y)};
+}
+
template <ViewPortMode VPMODE>
__device__ inline uint2 convertToScreen(const Parameters ¶ms, const float3 &camPos);
@@ -25,10 +29,19 @@ __device__ inline uint2 convertToScreen<ViewPortMode::Clipping>(const Parameters
}
template <>
-__device__ inline uint2 convertToScreen<ViewPortMode::Warping>(const Parameters ¶ms, const float3 &camPos) {
- return params.viewport.map(params.camera, params.camera.camToScreen<float2>(camPos));
+__device__ inline uint2 convertToScreen<ViewPortMode::Stretch>(const Parameters ¶ms, const float3 &camPos) {
+ return make_uint2(params.viewport.map(params.camera, params.camera.camToScreen<float2>(camPos)));
}
+/*template <>
+__device__ inline uint2 convertToScreen<ViewPortMode::Warping>(const Parameters ¶ms, const float3 &camPos) {
+ float2 pt = params.camera.camToScreen<float2>(camPos); //params.viewport.map(params.camera, params.camera.camToScreen<float2>(camPos));
+ const float coeff = 1.0f / (params.viewport.warpMatrix.entries[6] * pt.x + params.viewport.warpMatrix.entries[7] * pt.y + params.viewport.warpMatrix.entries[8]);
+ const float xcoo = coeff * (params.viewport.warpMatrix.entries[0] * pt.x + params.viewport.warpMatrix.entries[1] * pt.y + params.viewport.warpMatrix.entries[2]);
+ const float ycoo = coeff * (params.viewport.warpMatrix.entries[3] * pt.x + params.viewport.warpMatrix.entries[4] * pt.y + params.viewport.warpMatrix.entries[5]);
+ return make_uint2(xcoo, ycoo);
+}*/
+
/*
* Convert source screen position to output screen coordinates.
*/
@@ -68,7 +81,7 @@ void ftl::cuda::screen_coord(TextureObject<float> &depth, TextureObject<float> &
switch (params.viewPortMode) {
case ViewPortMode::Disabled: screen_coord_kernel<ViewPortMode::Disabled><<<gridSize, blockSize, 0, stream>>>(depth, depth_out, screen_out, params, pose, camera); break;
case ViewPortMode::Clipping: screen_coord_kernel<ViewPortMode::Clipping><<<gridSize, blockSize, 0, stream>>>(depth, depth_out, screen_out, params, pose, camera); break;
- case ViewPortMode::Warping: screen_coord_kernel<ViewPortMode::Warping><<<gridSize, blockSize, 0, stream>>>(depth, depth_out, screen_out, params, pose, camera); break;
+ case ViewPortMode::Stretch: screen_coord_kernel<ViewPortMode::Stretch><<<gridSize, blockSize, 0, stream>>>(depth, depth_out, screen_out, params, pose, camera); break;
}
cudaSafeCall( cudaGetLastError() );
}
diff --git a/components/rgbd-sources/include/ftl/rgbd/camera.hpp b/components/rgbd-sources/include/ftl/rgbd/camera.hpp
index 54f41579d..f6ae4e1b6 100644
--- a/components/rgbd-sources/include/ftl/rgbd/camera.hpp
+++ b/components/rgbd-sources/include/ftl/rgbd/camera.hpp
@@ -37,11 +37,21 @@ struct __align__(16) Camera {
*/
template <typename T> __device__ T camToScreen(const float3 &pos) const;
+ /**
+ * Convert screen plus depth into camera coordinates.
+ */
+ __device__ float3 screenToCam(int ux, int uy, float depth) const;
+
/**
* Convert screen plus depth into camera coordinates.
*/
__device__ float3 screenToCam(uint ux, uint uy, float depth) const;
+ /**
+ * Convert screen plus depth into camera coordinates.
+ */
+ __device__ float3 screenToCam(float ux, float uy, float depth) const;
+
#ifndef __CUDACC__
MSGPACK_DEFINE(fx,fy,cx,cy,width,height,minDepth,maxDepth,baseline,doffs);
@@ -87,4 +97,18 @@ inline float3 ftl::rgbd::Camera::screenToCam(uint ux, uint uy, float depth) cons
return make_float3(depth*x, depth*y, depth);
}
+__device__
+inline float3 ftl::rgbd::Camera::screenToCam(int ux, int uy, float depth) const {
+ const float x = static_cast<float>(((float)ux+cx) / fx);
+ const float y = static_cast<float>(((float)uy+cy) / fy);
+ return make_float3(depth*x, depth*y, depth);
+}
+
+__device__
+inline float3 ftl::rgbd::Camera::screenToCam(float ux, float uy, float depth) const {
+ const float x = static_cast<float>((ux+cx) / fx);
+ const float y = static_cast<float>((uy+cy) / fy);
+ return make_float3(depth*x, depth*y, depth);
+}
+
#endif
--
GitLab