From 5080fc4788da1454d52649e74e26828c738ec5f7 Mon Sep 17 00:00:00 2001
From: Sebastian Hahta <joseha@utu.fi>
Date: Fri, 8 Nov 2019 15:10:16 +0200
Subject: [PATCH 1/7] mls code from reconstruct
---
.../src/sources/stereovideo/stereovideo.cpp | 10 ++++++++++
1 file changed, 10 insertions(+)
diff --git a/components/rgbd-sources/src/sources/stereovideo/stereovideo.cpp b/components/rgbd-sources/src/sources/stereovideo/stereovideo.cpp
index 0b0cd15a1..ce45f436d 100644
--- a/components/rgbd-sources/src/sources/stereovideo/stereovideo.cpp
+++ b/components/rgbd-sources/src/sources/stereovideo/stereovideo.cpp
@@ -8,6 +8,12 @@
#include "ftl/operators/opticalflow.hpp"
#endif
+
+#include "ftl/operators/smoothing.hpp"
+#include "ftl/operators/colours.hpp"
+#include "ftl/operators/normals.hpp"
+#include "ftl/operators/filling.hpp"
+#include "ftl/operators/segmentation.hpp"
#include "ftl/operators/disparity.hpp"
#include "ftl/threads.hpp"
@@ -129,6 +135,10 @@ void StereoVideoSource::init(const string &file) {
pipeline_depth_->append<ftl::operators::OpticalFlowTemporalSmoothing>("optflow_filter");
pipeline_depth_->append<ftl::operators::DisparityBilateralFilter>("bilateral_filter");
pipeline_depth_->append<ftl::operators::DisparityToDepth>("calculate_depth");
+ pipeline_depth_->append<ftl::operators::ColourChannels>("colour"); // Convert BGR to BGRA
+ pipeline_depth_->append<ftl::operators::Normals>("normals"); // Estimate surface normals
+ pipeline_depth_->append<ftl::operators::CrossSupport>("cross")->set("enabled", false);
+ pipeline_depth_->append<ftl::operators::ColourMLS>("mls"); // Perform MLS (using smoothing channel)
LOG(INFO) << "StereoVideo source ready...";
ready_ = true;
--
GitLab
From 19527faf75dccce05006c2d6937b31e72c83c6b4 Mon Sep 17 00:00:00 2001
From: Sebastian Hahta <joseha@utu.fi>
Date: Fri, 8 Nov 2019 15:33:46 +0200
Subject: [PATCH 2/7] config proxy
---
applications/reconstruct/src/main.cpp | 2 ++
1 file changed, 2 insertions(+)
diff --git a/applications/reconstruct/src/main.cpp b/applications/reconstruct/src/main.cpp
index 4deb6256e..ce6e05c81 100644
--- a/applications/reconstruct/src/main.cpp
+++ b/applications/reconstruct/src/main.cpp
@@ -172,6 +172,8 @@ static void run(ftl::Configurable *root) {
configproxy->add(disparity, "source/disparity/algorithm", "algorithm");
configproxy->add(disparity, "source/disparity/bilateral_filter", "bilateral_filter");
configproxy->add(disparity, "source/disparity/optflow_filter", "optflow_filter");
+ configproxy->add(disparity, "source/disparity/mls", "mls");
+ configproxy->add(disparity, "source/disparity/cross", "cross");
}
// Create scene transform, intended for axis aligning the walls and floor
--
GitLab
From e8483607913c07d9a22706ffbaff396ed3e0bdd1 Mon Sep 17 00:00:00 2001
From: Sebastian Hahta <joseha@utu.fi>
Date: Thu, 14 Nov 2019 10:25:35 +0200
Subject: [PATCH 3/7] AggreMLS
---
components/rgbd-sources/src/sources/stereovideo/stereovideo.cpp | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/components/rgbd-sources/src/sources/stereovideo/stereovideo.cpp b/components/rgbd-sources/src/sources/stereovideo/stereovideo.cpp
index fb2fd9694..fe3c40e3e 100644
--- a/components/rgbd-sources/src/sources/stereovideo/stereovideo.cpp
+++ b/components/rgbd-sources/src/sources/stereovideo/stereovideo.cpp
@@ -141,7 +141,7 @@ void StereoVideoSource::init(const string &file) {
pipeline_depth_->append<ftl::operators::ColourChannels>("colour"); // Convert BGR to BGRA
pipeline_depth_->append<ftl::operators::Normals>("normals"); // Estimate surface normals
pipeline_depth_->append<ftl::operators::CrossSupport>("cross")->set("enabled", false);
- pipeline_depth_->append<ftl::operators::ColourMLS>("mls"); // Perform MLS (using smoothing channel)
+ pipeline_depth_->append<ftl::operators::AggreMLS>("mls"); // Perform MLS (using smoothing channel)
LOG(INFO) << "StereoVideo source ready...";
ready_ = true;
--
GitLab
From 275cd0bca283186591ba43c825c630a01ceeb0a0 Mon Sep 17 00:00:00 2001
From: Sebastian Hahta <joseha@utu.fi>
Date: Thu, 14 Nov 2019 11:08:57 +0200
Subject: [PATCH 4/7] error message if support1 channel missing
---
applications/reconstruct/src/main.cpp | 37 +++++++++++++++-----------
components/operators/src/smoothing.cpp | 5 ++++
2 files changed, 26 insertions(+), 16 deletions(-)
diff --git a/applications/reconstruct/src/main.cpp b/applications/reconstruct/src/main.cpp
index 0f8df2680..ef99a2cdb 100644
--- a/applications/reconstruct/src/main.cpp
+++ b/applications/reconstruct/src/main.cpp
@@ -99,20 +99,25 @@ class ConfigProxy {
auto config = json_t::parse(net_->call<string>(peers_[0], "get_cfg", uris_[0] + "/" + uri));
auto *proxy = ftl::create<ftl::Configurable>(root, name);
- for (auto &itm : config.get<json::object_t>()) {
- auto key = itm.first;
- auto value = itm.second;
- if (*key.begin() == '$') { continue; }
-
- proxy->set(key, value);
- proxy->on(key, [this, uri, key, value, proxy](const ftl::config::Event&) {
- for (size_t i = 0; i < uris_.size(); i++) {
- // TODO: check that config exists?
- auto peer = peers_[i];
- std::string name = uris_[i] + "/" + uri + "/" + key;
- net_->send(peer, "update_cfg", name, proxy->getConfig()[key].dump());
- }
- });
+ try {
+ for (auto &itm : config.get<json::object_t>()) {
+ auto key = itm.first;
+ auto value = itm.second;
+ if (*key.begin() == '$') { continue; }
+
+ proxy->set(key, value);
+ proxy->on(key, [this, uri, key, value, proxy](const ftl::config::Event&) {
+ for (size_t i = 0; i < uris_.size(); i++) {
+ // TODO: check that config exists?
+ auto peer = peers_[i];
+ std::string name = uris_[i] + "/" + uri + "/" + key;
+ net_->send(peer, "update_cfg", name, proxy->getConfig()[key].dump());
+ }
+ });
+ }
+ }
+ catch (nlohmann::detail::type_error) {
+ LOG(ERROR) << "Failed to add config proxy for: " << uri << "/" << name;
}
}
};
@@ -166,7 +171,7 @@ static void run(ftl::Configurable *root) {
LOG(ERROR) << "No sources configured!";
return;
}
-
+
ConfigProxy *configproxy = nullptr;
if (net->numberOfPeers() > 0) {
configproxy = new ConfigProxy(net); // TODO delete
@@ -291,7 +296,7 @@ static void run(ftl::Configurable *root) {
// TODO: Write pose+calibration+config packets
auto sources = group->sources();
- for (int i=0; i<sources.size(); ++i) {
+ for (size_t i=0; i<sources.size(); ++i) {
//writeSourceProperties(writer, i, sources[i]);
sources[i]->inject(Channel::Calibration, sources[i]->parameters(), Channel::Left, sources[i]->getCapabilities());
sources[i]->inject(sources[i]->getPose());
diff --git a/components/operators/src/smoothing.cpp b/components/operators/src/smoothing.cpp
index a4b23ab7c..2eafe7977 100644
--- a/components/operators/src/smoothing.cpp
+++ b/components/operators/src/smoothing.cpp
@@ -253,6 +253,11 @@ bool AggreMLS::apply(ftl::rgbd::Frame &in, ftl::rgbd::Frame &out, ftl::rgbd::Sou
return false;
}
+ if (!in.hasChannel(Channel::Support1)) {
+ LOG(ERROR) << "Required support channel missing for MLS";
+ return false;
+ }
+
auto size = in.get<GpuMat>(Channel::Depth).size();
centroid_horiz_.create(size.height, size.width);
normals_horiz_.create(size.height, size.width);
--
GitLab
From 3413003ba82acf08feea729634c537b93fd29423 Mon Sep 17 00:00:00 2001
From: Sebastian Hahta <joseha@utu.fi>
Date: Thu, 14 Nov 2019 11:22:31 +0200
Subject: [PATCH 5/7] stereovision.cpp: CullDiscontinuity
---
components/rgbd-sources/src/sources/stereovideo/stereovideo.cpp | 2 ++
1 file changed, 2 insertions(+)
diff --git a/components/rgbd-sources/src/sources/stereovideo/stereovideo.cpp b/components/rgbd-sources/src/sources/stereovideo/stereovideo.cpp
index fe3c40e3e..540c1b1a5 100644
--- a/components/rgbd-sources/src/sources/stereovideo/stereovideo.cpp
+++ b/components/rgbd-sources/src/sources/stereovideo/stereovideo.cpp
@@ -15,6 +15,7 @@
#include "ftl/operators/filling.hpp"
#include "ftl/operators/segmentation.hpp"
#include "ftl/operators/disparity.hpp"
+#include "ftl/operators/mask.hpp"
#include "ftl/threads.hpp"
#include "calibrate.hpp"
@@ -141,6 +142,7 @@ void StereoVideoSource::init(const string &file) {
pipeline_depth_->append<ftl::operators::ColourChannels>("colour"); // Convert BGR to BGRA
pipeline_depth_->append<ftl::operators::Normals>("normals"); // Estimate surface normals
pipeline_depth_->append<ftl::operators::CrossSupport>("cross")->set("enabled", false);
+ pipeline_depth_->append<ftl::operators::CullDiscontinuity>("remove_discontinuity");
pipeline_depth_->append<ftl::operators::AggreMLS>("mls"); // Perform MLS (using smoothing channel)
LOG(INFO) << "StereoVideo source ready...";
--
GitLab
From 8c5536fcc18df1919a24c546abd38ddf1e82ffa5 Mon Sep 17 00:00:00 2001
From: Sebastian Hahta <joseha@utu.fi>
Date: Thu, 14 Nov 2019 12:33:49 +0200
Subject: [PATCH 6/7] stereovideo DiscontinuityMask
---
.../rgbd-sources/src/sources/stereovideo/stereovideo.cpp | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/components/rgbd-sources/src/sources/stereovideo/stereovideo.cpp b/components/rgbd-sources/src/sources/stereovideo/stereovideo.cpp
index 540c1b1a5..84d1e574b 100644
--- a/components/rgbd-sources/src/sources/stereovideo/stereovideo.cpp
+++ b/components/rgbd-sources/src/sources/stereovideo/stereovideo.cpp
@@ -141,8 +141,8 @@ void StereoVideoSource::init(const string &file) {
pipeline_depth_->append<ftl::operators::DisparityToDepth>("calculate_depth");
pipeline_depth_->append<ftl::operators::ColourChannels>("colour"); // Convert BGR to BGRA
pipeline_depth_->append<ftl::operators::Normals>("normals"); // Estimate surface normals
- pipeline_depth_->append<ftl::operators::CrossSupport>("cross")->set("enabled", false);
- pipeline_depth_->append<ftl::operators::CullDiscontinuity>("remove_discontinuity");
+ pipeline_depth_->append<ftl::operators::CrossSupport>("cross");
+ pipeline_depth_->append<ftl::operators::DiscontinuityMask>("discontinuity_mask");
pipeline_depth_->append<ftl::operators::AggreMLS>("mls"); // Perform MLS (using smoothing channel)
LOG(INFO) << "StereoVideo source ready...";
--
GitLab
From 87aaa80e3273a1bf2bfc4443fe1876ac51e515aa Mon Sep 17 00:00:00 2001
From: Sebastian Hahta <joseha@utu.fi>
Date: Thu, 14 Nov 2019 12:39:16 +0200
Subject: [PATCH 7/7] comments
---
applications/reconstruct/src/main.cpp | 6 ++++--
components/operators/include/ftl/operators/mask.hpp | 12 ++++++------
2 files changed, 10 insertions(+), 8 deletions(-)
diff --git a/applications/reconstruct/src/main.cpp b/applications/reconstruct/src/main.cpp
index ef99a2cdb..c8cc100b7 100644
--- a/applications/reconstruct/src/main.cpp
+++ b/applications/reconstruct/src/main.cpp
@@ -75,8 +75,10 @@ static Eigen::Affine3d create_rotation_matrix(float ax, float ay, float az) {
return rz * rx * ry;
}
-// TODO: Remove this class (requires more general solution). Also does not
-// process disconnections/reconnections/types etc. correctly.
+// TODO: * Remove this class (requires more general solution). Also does not
+// process disconnections/reconnections/types etc. correctly.
+// * Update when new options become available.
+
class ConfigProxy {
private:
vector<ftl::UUID> peers_;
diff --git a/components/operators/include/ftl/operators/mask.hpp b/components/operators/include/ftl/operators/mask.hpp
index ef9758e39..579b1f6fe 100644
--- a/components/operators/include/ftl/operators/mask.hpp
+++ b/components/operators/include/ftl/operators/mask.hpp
@@ -14,12 +14,12 @@ namespace operators {
*/
class DiscontinuityMask : public ftl::operators::Operator {
public:
- explicit DiscontinuityMask(ftl::Configurable*);
- ~DiscontinuityMask();
+ explicit DiscontinuityMask(ftl::Configurable*);
+ ~DiscontinuityMask();
inline Operator::Type type() const override { return Operator::Type::OneToOne; }
- bool apply(ftl::rgbd::Frame &in, ftl::rgbd::Frame &out, ftl::rgbd::Source *src, cudaStream_t stream) override;
+ bool apply(ftl::rgbd::Frame &in, ftl::rgbd::Frame &out, ftl::rgbd::Source *src, cudaStream_t stream) override;
};
@@ -28,12 +28,12 @@ class DiscontinuityMask : public ftl::operators::Operator {
*/
class CullDiscontinuity : public ftl::operators::Operator {
public:
- explicit CullDiscontinuity(ftl::Configurable*);
- ~CullDiscontinuity();
+ explicit CullDiscontinuity(ftl::Configurable*);
+ ~CullDiscontinuity();
inline Operator::Type type() const override { return Operator::Type::OneToOne; }
- bool apply(ftl::rgbd::Frame &in, ftl::rgbd::Frame &out, ftl::rgbd::Source *src, cudaStream_t stream) override;
+ bool apply(ftl::rgbd::Frame &in, ftl::rgbd::Frame &out, ftl::rgbd::Source *src, cudaStream_t stream) override;
};
--
GitLab