diff --git a/applications/gui/src/camera.cpp b/applications/gui/src/camera.cpp
index eedf63ffd5292edf71301bc2cbca63cbf1484d5c..f1682a06d90bf6dedf74631708b1e927262a8616 100644
--- a/applications/gui/src/camera.cpp
+++ b/applications/gui/src/camera.cpp
@@ -222,6 +222,7 @@ void ftl::gui::Camera::showSettings() {
void ftl::gui::Camera::setChannel(ftl::rgbd::channel_t c) {
channel_ = c;
switch (c) {
+ case ftl::rgbd::kChanEnergy:
case ftl::rgbd::kChanFlow:
case ftl::rgbd::kChanConfidence:
case ftl::rgbd::kChanNormals:
@@ -255,6 +256,19 @@ static void visualizeDepthMap( const cv::Mat &depth, cv::Mat &out,
out.setTo(cv::Scalar(255, 255, 255), mask);
}
+static void visualizeEnergy( const cv::Mat &depth, cv::Mat &out,
+ const float max_depth)
+{
+ DCHECK(max_depth > 0.0);
+
+ depth.convertTo(out, CV_8U, 255.0f / max_depth);
+ //out = 255 - out;
+ cv::Mat mask = (depth >= 39.0f); // TODO (mask for invalid pixels)
+
+ applyColorMap(out, out, cv::COLORMAP_JET);
+ out.setTo(cv::Scalar(255, 255, 255), mask);
+}
+
static void drawEdges( const cv::Mat &in, cv::Mat &out,
const int ksize = 3, double weight = -1.0, const int threshold = 32,
const int threshold_type = cv::THRESH_TOZERO)
@@ -303,6 +317,11 @@ const GLTexture &ftl::gui::Camera::captureFrame() {
cv::Mat tmp;
switch(channel_) {
+ case ftl::rgbd::kChanEnergy:
+ if (depth.rows == 0) { break; }
+ visualizeEnergy(depth, tmp, 10.0);
+ texture_.update(tmp);
+ break;
case ftl::rgbd::kChanDepth:
if (depth.rows == 0) { break; }
visualizeDepthMap(depth, tmp, 7.0);
diff --git a/applications/gui/src/media_panel.cpp b/applications/gui/src/media_panel.cpp
index 6fb6d5085cc02efae38fcdfce0819ee0d2ae7110..44bb92ec099b5d961142ec4fc1fed95e200c7bd2 100644
--- a/applications/gui/src/media_panel.cpp
+++ b/applications/gui/src/media_panel.cpp
@@ -184,6 +184,15 @@ MediaPanel::MediaPanel(ftl::gui::Screen *screen) : nanogui::Window(screen, ""),
}
});
+ button = new Button(popup, "Energy");
+ button->setFlags(Button::RadioButton);
+ button->setCallback([this]() {
+ ftl::gui::Camera *cam = screen_->activeCamera();
+ if (cam) {
+ cam->setChannel(ftl::rgbd::kChanEnergy);
+ }
+ });
+
}
MediaPanel::~MediaPanel() {
diff --git a/applications/reconstruct/src/dibr.cu b/applications/reconstruct/src/dibr.cu
index 8d21279f1c20a30e392749e14cc157adf6f182a3..a7ba0e9df91637c687eda13d1048f323f72a30f4 100644
--- a/applications/reconstruct/src/dibr.cu
+++ b/applications/reconstruct/src/dibr.cu
@@ -400,6 +400,9 @@ __device__ inline float warpMin(float e) {
}
#define ENERGY_THRESHOLD 0.1f
+#define SMOOTHING_MULTIPLIER_A 10.0f // For surface search
+#define SMOOTHING_MULTIPLIER_B 4.0f // For z contribution
+#define SMOOTHING_MULTIPLIER_C 4.0f // For colour contribution
/*
@@ -449,7 +452,7 @@ __device__ inline float warpMin(float e) {
// 1) Depth from original source
// 2) Colour contrast in underlying RGB
// 3) Estimated noise levels in depth values
- if (point.z > clusterBase && point.z < params.camera.m_sensorDepthWorldMax && length(point - camPos) <= 0.04f) {
+ if (point.z > clusterBase && point.z < params.camera.m_sensorDepthWorldMax && length(point - camPos) <= SMOOTHING_MULTIPLIER_A*(point.z / params.camera.fx)) {
atomicMin(&minimum[warp], point.z*1000.0f);
}
}
@@ -471,7 +474,7 @@ __device__ inline float warpMin(float e) {
const float3 point = params.camera.kinectDepthToSkeleton(x+u, y+v, float(point_in.tex2D(x+u, y+v)) / 1000.0f);
// If it is close enough...
- if (point.z > params.camera.m_sensorDepthWorldMin && point.z < params.camera.m_sensorDepthWorldMax && length(point - minPos) <= 0.04f) {
+ if (point.z > params.camera.m_sensorDepthWorldMin && point.z < params.camera.m_sensorDepthWorldMax && length(point - minPos) <= SMOOTHING_MULTIPLIER_A*(point.z / params.camera.fx)) {
// Append to neighbour list
//unsigned int idx = atomicInc(&nidx[warp], MAX_NEIGHBORS_2-1);
unsigned int idx = atomicAdd(&nidx[warp], 1);
@@ -511,7 +514,7 @@ __device__ inline float warpMin(float e) {
// Search for best or threshold energy
for (int k=lane; k<MAX_ITERATIONS; k+=WARP_SIZE) {
const float3 nearest = params.camera.kinectDepthToSkeleton(x,y,minDepth+float(k)*interval);
- const float myenergy = ftl::cuda::mls_point_energy<MAX_NEIGHBORS_2>(neighborhood_cache[warp], nearest, min(nidx[warp], MAX_NEIGHBORS_2), SPATIAL_SMOOTHING);
+ const float myenergy = ftl::cuda::mls_point_energy<MAX_NEIGHBORS_2>(neighborhood_cache[warp], nearest, min(nidx[warp], MAX_NEIGHBORS_2), SMOOTHING_MULTIPLIER_B*(nearest.z/params.camera.fx));
const float newenergy = warpMax(max(myenergy, maxenergy));
bestdepth = (myenergy == newenergy) ? nearest.z : (newenergy > maxenergy) ? 0.0f : bestdepth;
maxenergy = newenergy;
@@ -548,7 +551,7 @@ __device__ inline float warpMin(float e) {
if (maxenergy >= ENERGY_THRESHOLD) return;
// Move to next possible surface...
- clusterBase = minDepth + 0.04f;
+ clusterBase = minDepth + SMOOTHING_MULTIPLIER_B*(minDepth / params.camera.fx);
};
}
@@ -586,7 +589,7 @@ __global__ void dibr_attribute_contrib_kernel(
if (camPos.z > params.camera.m_sensorDepthWorldMax) return;
const uint2 screenPos = params.camera.cameraToKinectScreen(camPos);
- const int upsample = min(UPSAMPLE_MAX, int((5.0f*r) * params.camera.fx / camPos.z));
+ const int upsample = 8; //min(UPSAMPLE_MAX, int((5.0f*r) * params.camera.fx / camPos.z));
// Not on screen so stop now...
if (screenPos.x < 0 || screenPos.y < 0 ||
@@ -611,7 +614,7 @@ __global__ void dibr_attribute_contrib_kernel(
const float3 nearest = params.camera.kinectDepthToSkeleton((int)(screenPos.x+u),(int)(screenPos.y+v),d);
// What is contribution of our current point at this pixel?
- const float weight = ftl::cuda::spatialWeighting(length(nearest - camPos), SPATIAL_SMOOTHING);
+ const float weight = ftl::cuda::spatialWeighting(length(nearest - camPos), SMOOTHING_MULTIPLIER_C*(nearest.z/params.camera.fx));
if (screenPos.x+u < colour_out.width() && screenPos.y+v < colour_out.height() && weight > 0.0f) { // TODO: Use confidence threshold here
const float4 wcolour = colour * weight;
const float4 wnormal = normal * weight;
diff --git a/applications/reconstruct/src/splat_render.cpp b/applications/reconstruct/src/splat_render.cpp
index aad8fb818215f1925ff0f159fc85d2b70b1d127d..df9d99a9f3b48dd59c2c7dd0b508a6a32aaded53 100644
--- a/applications/reconstruct/src/splat_render.cpp
+++ b/applications/reconstruct/src/splat_render.cpp
@@ -95,6 +95,16 @@ void Splatter::render(ftl::rgbd::Source *src, cudaStream_t stream) {
ftl::cuda::int_to_float(depth1_, depth2_, 1.0f / 1000.0f, stream);
src->writeFrames(colour1_, depth2_, stream);
}
+ } else if (src->getChannel() == ftl::rgbd::kChanEnergy) {
+ //ftl::cuda::int_to_float(depth1_, depth2_, 1.0f / 1000.0f, stream);
+ //if (src->value("splatting", false)) {
+ //ftl::cuda::splat_points(depth1_, colour1_, normal1_, depth2_, colour2_, params, stream);
+ //ftl::cuda::int_to_float(depth1_, depth2_, 1.0f / 1000.0f, stream);
+ src->writeFrames(colour1_, depth2_, stream);
+ //} else {
+ //ftl::cuda::int_to_float(depth1_, depth2_, 1.0f / 1000.0f, stream);
+ // src->writeFrames(colour1_, depth2_, stream);
+ //}
} else if (src->getChannel() == ftl::rgbd::kChanRight) {
// Adjust pose to right eye position
Eigen::Affine3f transform(Eigen::Translation3f(camera.baseline,0.0f,0.0f));
diff --git a/components/rgbd-sources/include/ftl/rgbd/detail/source.hpp b/components/rgbd-sources/include/ftl/rgbd/detail/source.hpp
index b76dd27fac1a7f09a07612629c9d6aae8ef47eb3..dcde8047ee902990dad4e03c550cd0f48a8480aa 100644
--- a/components/rgbd-sources/include/ftl/rgbd/detail/source.hpp
+++ b/components/rgbd-sources/include/ftl/rgbd/detail/source.hpp
@@ -21,11 +21,12 @@ static const channel_t kChanDeviation = 0x0010;
static const channel_t kChanNormals = 0x0020;
static const channel_t kChanConfidence = 0x0040;
static const channel_t kChanFlow = 0x0080;
+static const channel_t kChanEnergy = 0x0100;
static const channel_t kChanOverlay1 = 0x1000;
inline bool isFloatChannel(ftl::rgbd::channel_t chan) {
- return (chan == ftl::rgbd::kChanDepth);
+ return (chan == ftl::rgbd::kChanDepth || chan == ftl::rgbd::kChanEnergy);
}
diff --git a/components/rgbd-sources/src/streamer.cpp b/components/rgbd-sources/src/streamer.cpp
index 21821c908964b5f0889330fa931888c9bc10bcf1..e52e20e394bca414e7b2a08e8c628481cf2fe1a0 100644
--- a/components/rgbd-sources/src/streamer.cpp
+++ b/components/rgbd-sources/src/streamer.cpp
@@ -101,6 +101,8 @@ Streamer::Streamer(nlohmann::json &config, Universe *net)
net->bind("set_channel", [this](const string &uri, unsigned int chan) {
SHARED_LOCK(mutex_,slk);
+ LOG(INFO) << "SET CHANNEL " << chan;
+
if (sources_.find(uri) != sources_.end()) {
sources_[uri]->src->setChannel((ftl::rgbd::channel_t)chan);
}