diff --git a/applications/reconstruct/src/main.cpp b/applications/reconstruct/src/main.cpp
index 8202aff31a10a164edb330173634016e296ed508..4ca06248a336a51bf6ca2aacc25aced4ec970e34 100644
--- a/applications/reconstruct/src/main.cpp
+++ b/applications/reconstruct/src/main.cpp
@@ -246,11 +246,14 @@ static void run(ftl::Configurable *root) {
bool busy = false;
- auto *smooth = ftl::config::create<ftl::filters::MLSSmoother>(root, "filters");
+ // Create the source depth map filters
+ auto *filters = ftl::config::create<ftl::Filters>(root, "filters");
+ filters->create<ftl::filters::DepthSmoother>("hfnoise");
+ filters->create<ftl::filters::MLSSmoother>("mls");
group->setLatency(4);
group->setName("ReconGroup");
- group->sync([splat,virt,&busy,&slave,&scene_A,&scene_B,&align,controls,smooth](ftl::rgbd::FrameSet &fs) -> bool {
+ group->sync([splat,virt,&busy,&slave,&scene_A,&scene_B,&align,controls,filters](ftl::rgbd::FrameSet &fs) -> bool {
//cudaSetDevice(scene->getCUDADevice());
//if (slave.isPaused()) return true;
@@ -265,7 +268,7 @@ static void run(ftl::Configurable *root) {
// Swap the entire frameset to allow rapid return
fs.swapTo(scene_A);
- ftl::pool.push([&scene_B,&scene_A,&busy,&slave,&align, smooth](int id) {
+ ftl::pool.push([&scene_B,&scene_A,&busy,&slave,&align, filters](int id) {
//cudaSetDevice(scene->getCUDADevice());
// TODO: Release frameset here...
//cudaSafeCall(cudaStreamSynchronize(scene->getIntegrationStream()));
@@ -296,7 +299,7 @@ static void run(ftl::Configurable *root) {
cv::cuda::cvtColor(tmp,col, cv::COLOR_BGR2BGRA, 0);
}
- smooth->smooth(f, s);
+ filters->filter(f, s, 0);
/*ftl::cuda::smoothing_factor(
f.createTexture<float>(Channel::Depth),
diff --git a/components/codecs/src/nvpipe_encoder.cpp b/components/codecs/src/nvpipe_encoder.cpp
index 10901ef56993f25960cadd0e6f17778093f256a7..132a3209ad0849dd76f1a5f7438eba8f5655b854 100644
--- a/components/codecs/src/nvpipe_encoder.cpp
+++ b/components/codecs/src/nvpipe_encoder.cpp
@@ -59,6 +59,11 @@ bool NvPipeEncoder::encode(const cv::cuda::GpuMat &in, definition_t odefinition,
auto width = ftl::codecs::getWidth(definition);
auto height = ftl::codecs::getHeight(definition);
+ if (in.empty()) {
+ LOG(WARNING) << "No data";
+ return false;
+ }
+
cv::cuda::GpuMat tmp;
if (width != in.cols || height != in.rows) {
LOG(WARNING) << "Mismatch resolution with encoding resolution";
diff --git a/components/filters/CMakeLists.txt b/components/filters/CMakeLists.txt
index 92601b3714ac5e166d4e4f8eecf22c8f04edc58f..31fc04413f12bcfb4a0194b05516ccd63adb5cf9 100644
--- a/components/filters/CMakeLists.txt
+++ b/components/filters/CMakeLists.txt
@@ -1,6 +1,7 @@
add_library(ftlfilter
src/smoothing.cpp
- src/smoothing.cu
+ src/smoothing.cu
+ src/filter.cpp
)
# These cause errors in CI build and are being removed from PCL in newer versions
diff --git a/components/filters/include/ftl/filters/filter.hpp b/components/filters/include/ftl/filters/filter.hpp
new file mode 100644
index 0000000000000000000000000000000000000000..d11ba08e48dae5a6b15af34a4ebd6535fc4201ab
--- /dev/null
+++ b/components/filters/include/ftl/filters/filter.hpp
@@ -0,0 +1,65 @@
+#ifndef _FTL_FILTERS_FILTER_HPP_
+
+#include <list>
+#include <ftl/configurable.hpp>
+#include <ftl/configuration.hpp>
+#include <ftl/rgbd/frame.hpp>
+#include <ftl/rgbd/source.hpp>
+#include <ftl/cuda_common.hpp>
+
+namespace ftl {
+//namespace filters {
+
+/**
+ * An abstract frame filter interface. Any kind of filter that operates on a
+ * single frame should use this as a base class. An example of a filter would
+ * be MLS smoothing, or optical flow temporal smoothing. Some 'filters' might
+ * simply generate additional channels, such as a 'Normals' filter that
+ * generates a normals channel. Filters may also have internal data buffers,
+ * these may also persist between frames in some cases.
+ */
+class Filter : public ftl::Configurable {
+ public:
+ explicit Filter(nlohmann::json &config);
+ virtual ~Filter();
+
+ virtual void filter(ftl::rgbd::Frame &f, ftl::rgbd::Source *s, cudaStream_t stream)=0;
+
+ inline void enable() { enabled_ = true; }
+ inline void disable() { enabled_ = false; }
+ inline bool enabled() const { return enabled_; }
+ inline void enabled(bool e) { enabled_ = e; }
+
+ protected:
+ bool enabled_;
+};
+
+/**
+ * Represent a sequential collection of filters. Each filter created is
+ * added to an internal list and then applied to a frame in the order they were
+ * created.
+ */
+class Filters : public ftl::Configurable {
+ public:
+ explicit Filters(nlohmann::json &config);
+ ~Filters();
+
+ template <typename T>
+ Filter *create(const std::string &name);
+
+ void filter(ftl::rgbd::Frame &f, ftl::rgbd::Source *s, cudaStream_t stream=0);
+
+ private:
+ std::list<ftl::Filter*> filters_;
+ Filter *_create(Filter *f);
+};
+
+//}
+}
+
+template <typename T>
+ftl::Filter *ftl::Filters::create(const std::string &name) {
+ return _create(dynamic_cast<ftl::Filter*>(ftl::create<T>(this, name)));
+}
+
+#endif // _FTL_FILTERS_FILTER_HPP_
diff --git a/components/filters/include/ftl/filters/smoothing.hpp b/components/filters/include/ftl/filters/smoothing.hpp
index 06f8a594155c95b7d5f1f2ab40f30086c4aa8f25..f6722e9b38b8a9fa862ae28934b508f78f86e0b5 100644
--- a/components/filters/include/ftl/filters/smoothing.hpp
+++ b/components/filters/include/ftl/filters/smoothing.hpp
@@ -1,32 +1,29 @@
#ifndef _FTL_SMOOTHING_HPP_
#define _FTL_SMOOTHING_HPP_
-#include <ftl/configurable.hpp>
-#include <ftl/cuda_common.hpp>
-#include <ftl/rgbd/source.hpp>
-#include <ftl/rgbd/frame.hpp>
+#include <ftl/filters/filter.hpp>
namespace ftl {
namespace filters {
-class DepthSmoother : public ftl::Configurable {
+class DepthSmoother : public ftl::Filter {
public:
explicit DepthSmoother(nlohmann::json &config);
~DepthSmoother();
- void smooth(ftl::rgbd::Frame &frame, ftl::rgbd::Source *src);
+ void filter(ftl::rgbd::Frame &frame, ftl::rgbd::Source *src, cudaStream_t stream) override;
private:
cv::cuda::GpuMat temp_;
ftl::rgbd::Frame frames_[4];
};
-class MLSSmoother : public ftl::Configurable {
+class MLSSmoother : public ftl::Filter {
public:
explicit MLSSmoother(nlohmann::json &config);
~MLSSmoother();
- void smooth(ftl::rgbd::Frame &frame, ftl::rgbd::Source *src);
+ void filter(ftl::rgbd::Frame &frame, ftl::rgbd::Source *src, cudaStream_t stream) override;
private:
};
diff --git a/components/filters/src/filter.cpp b/components/filters/src/filter.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..4712ea7e7dcff9fc1e779ccba5493c1c00fd3d0a
--- /dev/null
+++ b/components/filters/src/filter.cpp
@@ -0,0 +1,35 @@
+#include <ftl/filters/filter.hpp>
+
+using ftl::Filter;
+using ftl::Filters;
+
+Filter::Filter(nlohmann::json &config) : ftl::Configurable(config) {
+ enabled_ = value("enabled", true);
+
+ on("enabled", [this](const ftl::config::Event &e) {
+ enabled_ = value("enabled", true);
+ });
+}
+
+Filter::~Filter() {}
+
+Filters::Filters(nlohmann::json &config) : ftl::Configurable(config) {
+
+}
+
+Filters::~Filters() {
+
+}
+
+void Filters::filter(ftl::rgbd::Frame &f, ftl::rgbd::Source *s, cudaStream_t stream) {
+ for (auto i : filters_) {
+ if (i->enabled()) {
+ i->filter(f, s, stream);
+ }
+ }
+}
+
+Filter *Filters::_create(Filter *f) {
+ filters_.push_back(f);
+ return f;
+}
diff --git a/components/filters/src/smoothing.cpp b/components/filters/src/smoothing.cpp
index e2805174544875aaec56cc8c092a31e5173dfaa3..f473018e69a9a7636fb9b94453b12dee1af713af 100644
--- a/components/filters/src/smoothing.cpp
+++ b/components/filters/src/smoothing.cpp
@@ -8,7 +8,7 @@ using ftl::filters::MLSSmoother;
using ftl::codecs::Channel;
using cv::cuda::GpuMat;
-DepthSmoother::DepthSmoother(nlohmann::json &config) : ftl::Configurable(config) {
+DepthSmoother::DepthSmoother(nlohmann::json &config) : ftl::Filter(config) {
}
@@ -16,13 +16,13 @@ DepthSmoother::~DepthSmoother() {
}
-void DepthSmoother::smooth(ftl::rgbd::Frame &f, ftl::rgbd::Source *s) {
+void DepthSmoother::filter(ftl::rgbd::Frame &f, ftl::rgbd::Source *s, cudaStream_t stream) {
float var_thresh = value("variance_threshold", 0.0002f);
- bool do_smooth = value("pre_smooth", false);
+ //bool do_smooth = value("pre_smooth", false);
int levels = max(0, min(value("levels",0), 4));
int iters = value("iterations",5);
- if (!do_smooth) return;
+ //if (!do_smooth) return;
for (int i=0; i<iters; ++i) {
ftl::cuda::smoothing_factor(
@@ -59,7 +59,7 @@ void DepthSmoother::smooth(ftl::rgbd::Frame &f, ftl::rgbd::Source *s) {
// ===== MLS ===================================================================
-MLSSmoother::MLSSmoother(nlohmann::json &config) : ftl::Configurable(config) {
+MLSSmoother::MLSSmoother(nlohmann::json &config) : ftl::Filter(config) {
}
@@ -67,13 +67,13 @@ MLSSmoother::~MLSSmoother() {
}
-void MLSSmoother::smooth(ftl::rgbd::Frame &f, ftl::rgbd::Source *s) {
- bool do_smooth = value("mls_smooth", false);
- if (!do_smooth) return;
-
+void MLSSmoother::filter(ftl::rgbd::Frame &f, ftl::rgbd::Source *s, cudaStream_t stream) {
+ //bool do_smooth = value("mls_smooth", false);
float thresh = value("mls_threshold", 0.04f);
int iters = value("mls_iterations", 1);
+ //if (!do_smooth) return;
+
for (int i=0; i<iters; ++i) {
if (i > 0 || !f.hasChannel(Channel::Normals)) {
ftl::cuda::normals(