diff --git a/components/operators/CMakeLists.txt b/components/operators/CMakeLists.txt
index d946fae15ff253467a2e4bc56b99bbb7f6045938..54da0eb4be9284b27ac9d5f9c338a7fa98eb6987 100644
--- a/components/operators/CMakeLists.txt
+++ b/components/operators/CMakeLists.txt
@@ -1,5 +1,5 @@
add_library(ftloperators
- src/smoothing.cpp
+ src/smoothing.cpp
src/smoothing.cu
src/mls.cu
src/smoothchan.cu
@@ -8,6 +8,7 @@ add_library(ftloperators
src/normals.cpp
src/filling.cpp
src/filling.cu
+ src/nvopticalflow.cpp
)
# These cause errors in CI build and are being removed from PCL in newer versions
diff --git a/components/operators/include/ftl/operators/operator.hpp b/components/operators/include/ftl/operators/operator.hpp
index fe9b52f2c6b7d6d3364c1d1722f70bd691602170..04f542b3b1b088b53adfdb79cb012d9fb3e89e9c 100644
--- a/components/operators/include/ftl/operators/operator.hpp
+++ b/components/operators/include/ftl/operators/operator.hpp
@@ -23,7 +23,7 @@ namespace operators {
class Operator {
public:
explicit Operator(ftl::Configurable *cfg);
- virtual ~Operator();
+ virtual ~Operator();
enum class Type {
OneToOne, // Frame to Frame (Filter or generator)
@@ -87,7 +87,7 @@ struct OperatorNode {
class Graph : public ftl::Configurable {
public:
explicit Graph(nlohmann::json &config);
- ~Graph();
+ ~Graph();
template <typename T>
ftl::Configurable *append(const std::string &name);
diff --git a/components/operators/include/ftl/operators/opticalflow.hpp b/components/operators/include/ftl/operators/opticalflow.hpp
new file mode 100644
index 0000000000000000000000000000000000000000..1cbd58b76a33e4956fc45fc94afc32804f742fb2
--- /dev/null
+++ b/components/operators/include/ftl/operators/opticalflow.hpp
@@ -0,0 +1,34 @@
+#pragma once
+
+#include <ftl/operators/operator.hpp>
+#include <opencv2/cudaoptflow.hpp>
+
+namespace ftl {
+namespace operators {
+
+class NVOpticalFlow : public ftl::operators::Operator {
+ public:
+ explicit NVOpticalFlow(ftl::Configurable*);
+ ~NVOpticalFlow();
+
+ inline Operator::Type type() const override { return Operator::Type::OneToOne; }
+
+ bool apply(ftl::rgbd::Frame &in, ftl::rgbd::Frame &out, ftl::rgbd::Source *src, cudaStream_t stream) override;
+
+ protected:
+ void init();
+
+ private:
+ cv::Size size_;
+
+ // TODO: Left to Flow always assumed, could also calculate something else?
+ const ftl::codecs::Channel channel_in_ = ftl::codecs::Channel::Left;
+ const ftl::codecs::Channel channel_out_ = ftl::codecs::Channel::Flow;
+
+ cv::Ptr<cv::cuda::NvidiaOpticalFlow_1_0> nvof_;
+ cv::cuda::GpuMat left_gray_;
+ cv::cuda::GpuMat left_gray_prev_;
+};
+
+}
+}
diff --git a/components/operators/include/ftl/operators/smoothing.hpp b/components/operators/include/ftl/operators/smoothing.hpp
index b5362ea6f1a6c16de81cdb7c55339058476da09d..f6683fcc3e665f67b708a8bf11140bfabf179060 100644
--- a/components/operators/include/ftl/operators/smoothing.hpp
+++ b/components/operators/include/ftl/operators/smoothing.hpp
@@ -13,17 +13,17 @@ namespace operators {
* first derivative. Only the depth channel is used and modified.
*/
class HFSmoother : public ftl::operators::Operator {
- public:
- explicit HFSmoother(ftl::Configurable*);
- ~HFSmoother();
+ public:
+ explicit HFSmoother(ftl::Configurable*);
+ ~HFSmoother();
inline Operator::Type type() const override { return Operator::Type::OneToOne; }
- bool apply(ftl::rgbd::Frame &in, ftl::rgbd::Frame &out, ftl::rgbd::Source *src, cudaStream_t stream) override;
+ bool apply(ftl::rgbd::Frame &in, ftl::rgbd::Frame &out, ftl::rgbd::Source *src, cudaStream_t stream) override;
- private:
- cv::cuda::GpuMat temp_;
- ftl::rgbd::Frame frames_[4];
+ private:
+ cv::cuda::GpuMat temp_;
+ ftl::rgbd::Frame frames_[4];
};
/**
@@ -34,13 +34,13 @@ class HFSmoother : public ftl::operators::Operator {
* no smoothing.
*/
class SmoothChannel : public ftl::operators::Operator {
- public:
- explicit SmoothChannel(ftl::Configurable*);
- ~SmoothChannel();
+ public:
+ explicit SmoothChannel(ftl::Configurable*);
+ ~SmoothChannel();
inline Operator::Type type() const override { return Operator::Type::OneToOne; }
- bool apply(ftl::rgbd::Frame &in, ftl::rgbd::Frame &out, ftl::rgbd::Source *src, cudaStream_t stream) override;
+ bool apply(ftl::rgbd::Frame &in, ftl::rgbd::Frame &out, ftl::rgbd::Source *src, cudaStream_t stream) override;
private:
ftl::rgbd::Frame temp_[6];
@@ -52,15 +52,15 @@ class SmoothChannel : public ftl::operators::Operator {
* Also outputs Depth + Normals.
*/
class SimpleMLS : public ftl::operators::Operator {
- public:
- explicit SimpleMLS(ftl::Configurable*);
- ~SimpleMLS();
+ public:
+ explicit SimpleMLS(ftl::Configurable*);
+ ~SimpleMLS();
inline Operator::Type type() const override { return Operator::Type::OneToOne; }
- bool apply(ftl::rgbd::Frame &in, ftl::rgbd::Frame &out, ftl::rgbd::Source *src, cudaStream_t stream) override;
+ bool apply(ftl::rgbd::Frame &in, ftl::rgbd::Frame &out, ftl::rgbd::Source *src, cudaStream_t stream) override;
- private:
+ private:
};
/**
@@ -68,15 +68,15 @@ class SimpleMLS : public ftl::operators::Operator {
* by a simple colour similarity weighting. In practice this is too naive.
*/
class ColourMLS : public ftl::operators::Operator {
- public:
- explicit ColourMLS(ftl::Configurable*);
- ~ColourMLS();
+ public:
+ explicit ColourMLS(ftl::Configurable*);
+ ~ColourMLS();
inline Operator::Type type() const override { return Operator::Type::OneToOne; }
- bool apply(ftl::rgbd::Frame &in, ftl::rgbd::Frame &out, ftl::rgbd::Source *src, cudaStream_t stream) override;
+ bool apply(ftl::rgbd::Frame &in, ftl::rgbd::Frame &out, ftl::rgbd::Source *src, cudaStream_t stream) override;
- private:
+ private:
};
/**
@@ -87,15 +87,15 @@ class ColourMLS : public ftl::operators::Operator {
* it can be only a few or even no pixels with a zero smoothing factor.
*/
class AdaptiveMLS : public ftl::operators::Operator {
- public:
- explicit AdaptiveMLS(ftl::Configurable*);
- ~AdaptiveMLS();
+ public:
+ explicit AdaptiveMLS(ftl::Configurable*);
+ ~AdaptiveMLS();
inline Operator::Type type() const override { return Operator::Type::OneToOne; }
- bool apply(ftl::rgbd::Frame &in, ftl::rgbd::Frame &out, ftl::rgbd::Source *src, cudaStream_t stream) override;
+ bool apply(ftl::rgbd::Frame &in, ftl::rgbd::Frame &out, ftl::rgbd::Source *src, cudaStream_t stream) override;
- private:
+ private:
};
}
diff --git a/components/operators/src/nvopticalflow.cpp b/components/operators/src/nvopticalflow.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..ba4fefa6bbe3c635c8bfadf880e31eb9f38d9eaf
--- /dev/null
+++ b/components/operators/src/nvopticalflow.cpp
@@ -0,0 +1,48 @@
+#include <ftl/operators/opticalflow.hpp>
+
+using ftl::rgbd::Frame;
+using ftl::rgbd::Source;
+using ftl::codecs::Channel;
+
+using ftl::operators::NVOpticalFlow;
+
+using cv::Size;
+using cv::cuda::GpuMat;
+
+NVOpticalFlow::NVOpticalFlow(ftl::Configurable* cfg) :
+ ftl::operators::Operator(cfg) {
+ size_ = Size(0, 0);
+}
+
+NVOpticalFlow::~NVOpticalFlow() {
+}
+
+void NVOpticalFlow::init() {
+ nvof_ = cv::cuda::NvidiaOpticalFlow_1_0::create(
+ size_.width, size_.height,
+ cv::cuda::NvidiaOpticalFlow_1_0::NV_OF_PERF_LEVEL_SLOW,
+ true, false, false, 0);
+
+ left_gray_.create(size_, CV_8UC1);
+ left_gray_prev_.create(size_, CV_8UC1);
+}
+
+bool NVOpticalFlow::apply(Frame &in, Frame &out, Source *src, cudaStream_t stream) {
+ if (!in.hasChannel(channel_in_)) {
+ return true; // false?
+ }
+
+ if (in.get<GpuMat>(channel_in_).size() != size_) {
+ size_ = in.get<GpuMat>(channel_in_).size();
+ init();
+ }
+
+ auto cvstream = cv::cuda::StreamAccessor::wrapStream(stream);
+ auto &flow = out.create<GpuMat>(channel_out_);
+
+ cv::cuda::cvtColor(in.get<GpuMat>(channel_in_), left_gray_, cv::COLOR_BGR2GRAY, 0, cvstream);
+
+ nvof_->calc(left_gray_, left_gray_prev_, flow, cvstream);
+
+ return true;
+}
\ No newline at end of file
diff --git a/components/rgbd-sources/CMakeLists.txt b/components/rgbd-sources/CMakeLists.txt
index 0dbd5ccf8cca5aecea9b270c6fc28d4b53b3b4ac..7b81e216d9adb11c5898493077bd02dd2121dfbb 100644
--- a/components/rgbd-sources/CMakeLists.txt
+++ b/components/rgbd-sources/CMakeLists.txt
@@ -69,7 +69,7 @@ set_property(TARGET ftlrgbd PROPERTY CUDA_SEPARABLE_COMPILATION OFF)
endif()
#target_include_directories(cv-node PUBLIC ${PROJECT_SOURCE_DIR}/include)
-target_link_libraries(ftlrgbd ftlcommon ${OpenCV_LIBS} ${LIBSGM_LIBRARIES} ${CUDA_LIBRARIES} Eigen3::Eigen ${REALSENSE_LIBRARY} ftlnet ${LibArchive_LIBRARIES} ftlcodecs)
+target_link_libraries(ftlrgbd ftlcommon ${OpenCV_LIBS} ${LIBSGM_LIBRARIES} ${CUDA_LIBRARIES} Eigen3::Eigen ${REALSENSE_LIBRARY} ftlnet ${LibArchive_LIBRARIES} ftlcodecs ftloperators)
add_subdirectory(test)
diff --git a/components/rgbd-sources/src/sources/stereovideo/stereovideo.cpp b/components/rgbd-sources/src/sources/stereovideo/stereovideo.cpp
index f8e08a9f6d1f7670adeb8be9310129e8353daf1b..8f0d6e345e1db51b0fbe9a1a867805d4e449d49a 100644
--- a/components/rgbd-sources/src/sources/stereovideo/stereovideo.cpp
+++ b/components/rgbd-sources/src/sources/stereovideo/stereovideo.cpp
@@ -1,6 +1,9 @@
#include <loguru.hpp>
#include "stereovideo.hpp"
+
#include <ftl/configuration.hpp>
+#include <ftl/operators/opticalflow.hpp>
+
#include <ftl/threads.hpp>
#include "calibrate.hpp"
#include "local.hpp"
@@ -58,16 +61,24 @@ void StereoVideoSource::init(const string &file)
lsrc_ = ftl::create<LocalSource>(host_, "feed");
}
+ // Create the source depth map pipeline
+ pipeline_ = ftl::config::create<ftl::operators::Graph>(host_, "disparity");
+ /*pipeline1->append<ftl::operators::ColourChannels>("colour"); // Convert BGR to BGRA
+ pipeline1->append<ftl::operators::HFSmoother>("hfnoise"); // Remove high-frequency noise
+ pipeline1->append<ftl::operators::Normals>("normals"); // Estimate surface normals
+ pipeline1->append<ftl::operators::SmoothChannel>("smoothing"); // Generate a smoothing channel
+ //pipeline1->append<ftl::operators::ScanFieldFill>("filling"); // Generate a smoothing channel
+ pipeline1->append<ftl::operators::ColourMLS>("mls"); // Perform MLS (using smoothing channel)
+ */
+
cv::Size size = cv::Size(lsrc_->width(), lsrc_->height());
frames_ = std::vector<Frame>(2);
#ifdef HAVE_OPTFLOW
+
use_optflow_ = host_->value("use_optflow", false);
LOG(INFO) << "Using optical flow: " << (use_optflow_ ? "true" : "false");
-
- nvof_ = cv::cuda::NvidiaOpticalFlow_1_0::create(size.width, size.height,
- cv::cuda::NvidiaOpticalFlow_1_0::NV_OF_PERF_LEVEL_SLOW,
- true, false, false, 0);
+ pipeline_->append<ftl::operators::NVOpticalFlow>("optflow");
#endif
@@ -179,10 +190,11 @@ bool StereoVideoSource::retrieve() {
auto &left = frame.create<cv::cuda::GpuMat>(Channel::Left);
auto &right = frame.create<cv::cuda::GpuMat>(Channel::Right);
lsrc_->get(left, right, calib_, stream2_);
+ pipeline_->apply(frame, frame, (ftl::rgbd::Source*) lsrc_, cv::cuda::StreamAccessor::getStream(stream2_));
#ifdef HAVE_OPTFLOW
// see comments in https://gitlab.utu.fi/nicolas.pope/ftl/issues/155
-
+ /*
if (use_optflow_)
{
auto &left_gray = frame.create<cv::cuda::GpuMat>(Channel::LeftGray);
@@ -201,7 +213,7 @@ bool StereoVideoSource::retrieve() {
// cv::cuda::resize() does not work wiht 2-channel input
// cv::cuda::resize(optflow_, optflow, left.size(), 0.0, 0.0, cv::INTER_NEAREST, stream2_);
}
- }
+ }*/
#endif
stream2_.waitForCompletion();
diff --git a/components/rgbd-sources/src/sources/stereovideo/stereovideo.hpp b/components/rgbd-sources/src/sources/stereovideo/stereovideo.hpp
index dfea7937f99777c23f753a7ab2b5bad8c68bb4af..eb29186770fd0fe2347be20653ff51ec9b5080b4 100644
--- a/components/rgbd-sources/src/sources/stereovideo/stereovideo.hpp
+++ b/components/rgbd-sources/src/sources/stereovideo/stereovideo.hpp
@@ -3,6 +3,7 @@
#define _FTL_RGBD_STEREOVIDEO_HPP_
#include <ftl/rgbd/source.hpp>
+#include <ftl/operators/operator.hpp>
#include <string>
namespace ftl {
@@ -39,7 +40,9 @@ class StereoVideoSource : public detail::Source {
LocalSource *lsrc_;
Calibrate *calib_;
Disparity *disp_;
-
+
+ ftl::operators::Graph *pipeline_;
+
bool ready_;
bool use_optflow_;