operator: optical flow

components/operators/CMakeLists.txt

 add_library(ftloperators
-    src/smoothing.cpp
+	src/smoothing.cpp
 	src/smoothing.cu
 	src/mls.cu
 	src/smoothchan.cu
@@ -8,6 +8,7 @@ add_library(ftloperators
 	src/normals.cpp
 	src/filling.cpp
 	src/filling.cu
+	src/nvopticalflow.cpp
 )
 
 # These cause errors in CI build and are being removed from PCL in newer versions

components/operators/include/ftl/operators/operator.hpp

@@ -23,7 +23,7 @@ namespace operators {
 class Operator {
 	public:
 	explicit Operator(ftl::Configurable *cfg);
-    virtual ~Operator();
+	virtual ~Operator();
 
 	enum class Type {
 		OneToOne,		// Frame to Frame (Filter or generator)
@@ -87,7 +87,7 @@ struct OperatorNode {
 class Graph : public ftl::Configurable {
 	public:
 	explicit Graph(nlohmann::json &config);
-    ~Graph();
+	~Graph();
 
 	template <typename T>
 	ftl::Configurable *append(const std::string &name);

components/operators/include/ftl/operators/opticalflow.hpp

+#pragma once
+
+#include <ftl/operators/operator.hpp>
+#include <opencv2/cudaoptflow.hpp>
+
+namespace ftl {
+namespace operators {
+
+class NVOpticalFlow : public ftl::operators::Operator {
+	public:
+	explicit NVOpticalFlow(ftl::Configurable*);
+	~NVOpticalFlow();
+
+	inline Operator::Type type() const override { return Operator::Type::OneToOne; }
+
+	bool apply(ftl::rgbd::Frame &in, ftl::rgbd::Frame &out, ftl::rgbd::Source *src, cudaStream_t stream) override;
+
+	protected:
+	void init();
+
+	private:
+	cv::Size size_;
+	
+	// TODO: Left to Flow always assumed, could also calculate something else?
+	const ftl::codecs::Channel channel_in_ = ftl::codecs::Channel::Left;
+	const ftl::codecs::Channel channel_out_ = ftl::codecs::Channel::Flow;
+
+	cv::Ptr<cv::cuda::NvidiaOpticalFlow_1_0> nvof_;
+	cv::cuda::GpuMat left_gray_;
+	cv::cuda::GpuMat left_gray_prev_;
+};
+
+}
+}

components/operators/include/ftl/operators/smoothing.hpp

@@ -13,17 +13,17 @@ namespace operators {
  * first derivative. Only the depth channel is used and modified.
  */
 class HFSmoother : public ftl::operators::Operator {
-    public:
-    explicit HFSmoother(ftl::Configurable*);
-    ~HFSmoother();
+	public:
+	explicit HFSmoother(ftl::Configurable*);
+	~HFSmoother();
 
 	inline Operator::Type type() const override { return Operator::Type::OneToOne; }
 
-    bool apply(ftl::rgbd::Frame &in, ftl::rgbd::Frame &out, ftl::rgbd::Source *src, cudaStream_t stream) override;
+	bool apply(ftl::rgbd::Frame &in, ftl::rgbd::Frame &out, ftl::rgbd::Source *src, cudaStream_t stream) override;
 
-    private:
-    cv::cuda::GpuMat temp_;
-    ftl::rgbd::Frame frames_[4];
+	private:
+	cv::cuda::GpuMat temp_;
+	ftl::rgbd::Frame frames_[4];
 };
 
 /**
@@ -34,13 +34,13 @@ class HFSmoother : public ftl::operators::Operator {
  * no smoothing.
  */
 class SmoothChannel : public ftl::operators::Operator {
-    public:
-    explicit SmoothChannel(ftl::Configurable*);
-    ~SmoothChannel();
+	public:
+	explicit SmoothChannel(ftl::Configurable*);
+	~SmoothChannel();
 
 	inline Operator::Type type() const override { return Operator::Type::OneToOne; }
 
-    bool apply(ftl::rgbd::Frame &in, ftl::rgbd::Frame &out, ftl::rgbd::Source *src, cudaStream_t stream) override;
+	bool apply(ftl::rgbd::Frame &in, ftl::rgbd::Frame &out, ftl::rgbd::Source *src, cudaStream_t stream) override;
 
 	private:
 	ftl::rgbd::Frame temp_[6];
@@ -52,15 +52,15 @@ class SmoothChannel : public ftl::operators::Operator {
  * Also outputs Depth + Normals.
  */
 class SimpleMLS : public ftl::operators::Operator {
-    public:
-    explicit SimpleMLS(ftl::Configurable*);
-    ~SimpleMLS();
+	public:
+	explicit SimpleMLS(ftl::Configurable*);
+	~SimpleMLS();
 
 	inline Operator::Type type() const override { return Operator::Type::OneToOne; }
 
-    bool apply(ftl::rgbd::Frame &in, ftl::rgbd::Frame &out, ftl::rgbd::Source *src, cudaStream_t stream) override;
+	bool apply(ftl::rgbd::Frame &in, ftl::rgbd::Frame &out, ftl::rgbd::Source *src, cudaStream_t stream) override;
 
-    private:
+	private:
 };
 
 /**
@@ -68,15 +68,15 @@ class SimpleMLS : public ftl::operators::Operator {
  * by a simple colour similarity weighting. In practice this is too naive.
  */
 class ColourMLS : public ftl::operators::Operator {
-    public:
-    explicit ColourMLS(ftl::Configurable*);
-    ~ColourMLS();
+	public:
+	explicit ColourMLS(ftl::Configurable*);
+	~ColourMLS();
 
 	inline Operator::Type type() const override { return Operator::Type::OneToOne; }
 
-    bool apply(ftl::rgbd::Frame &in, ftl::rgbd::Frame &out, ftl::rgbd::Source *src, cudaStream_t stream) override;
+	bool apply(ftl::rgbd::Frame &in, ftl::rgbd::Frame &out, ftl::rgbd::Source *src, cudaStream_t stream) override;
 
-    private:
+	private:
 };
 
 /**
@@ -87,15 +87,15 @@ class ColourMLS : public ftl::operators::Operator {
  * it can be only a few or even no pixels with a zero smoothing factor.
  */
 class AdaptiveMLS : public ftl::operators::Operator {
-    public:
-    explicit AdaptiveMLS(ftl::Configurable*);
-    ~AdaptiveMLS();
+	public:
+	explicit AdaptiveMLS(ftl::Configurable*);
+	~AdaptiveMLS();
 
 	inline Operator::Type type() const override { return Operator::Type::OneToOne; }
 
-    bool apply(ftl::rgbd::Frame &in, ftl::rgbd::Frame &out, ftl::rgbd::Source *src, cudaStream_t stream) override;
+	bool apply(ftl::rgbd::Frame &in, ftl::rgbd::Frame &out, ftl::rgbd::Source *src, cudaStream_t stream) override;
 
-    private:
+	private:
 };
 
 }

components/operators/src/nvopticalflow.cpp

+#include <ftl/operators/opticalflow.hpp>
+
+using ftl::rgbd::Frame;
+using ftl::rgbd::Source;
+using ftl::codecs::Channel;
+
+using ftl::operators::NVOpticalFlow;
+
+using cv::Size;
+using cv::cuda::GpuMat;
+
+NVOpticalFlow::NVOpticalFlow(ftl::Configurable* cfg) :
+		ftl::operators::Operator(cfg) {
+	size_ = Size(0, 0);
+}
+
+NVOpticalFlow::~NVOpticalFlow() {
+}
+
+void NVOpticalFlow::init() {
+	nvof_ = cv::cuda::NvidiaOpticalFlow_1_0::create(
+				size_.width, size_.height, 
+				cv::cuda::NvidiaOpticalFlow_1_0::NV_OF_PERF_LEVEL_SLOW,
+				true, false, false, 0);
+	
+	left_gray_.create(size_, CV_8UC1);
+	left_gray_prev_.create(size_, CV_8UC1);
+}
+
+bool NVOpticalFlow::apply(Frame &in, Frame &out, Source *src, cudaStream_t stream) {
+	if (!in.hasChannel(channel_in_)) {
+		return true; // false?
+	}
+
+	if (in.get<GpuMat>(channel_in_).size() != size_) {
+		size_ = in.get<GpuMat>(channel_in_).size();
+		init();
+	}
+	
+	auto cvstream = cv::cuda::StreamAccessor::wrapStream(stream);
+	auto &flow = out.create<GpuMat>(channel_out_);
+
+	cv::cuda::cvtColor(in.get<GpuMat>(channel_in_), left_gray_, cv::COLOR_BGR2GRAY, 0, cvstream);
+
+	nvof_->calc(left_gray_, left_gray_prev_, flow, cvstream);
+
+	return true;
+}
\ No newline at end of file

components/rgbd-sources/CMakeLists.txt

@@ -69,7 +69,7 @@ set_property(TARGET ftlrgbd PROPERTY CUDA_SEPARABLE_COMPILATION OFF)
 endif()
 
 #target_include_directories(cv-node PUBLIC ${PROJECT_SOURCE_DIR}/include)
-target_link_libraries(ftlrgbd ftlcommon ${OpenCV_LIBS} ${LIBSGM_LIBRARIES} ${CUDA_LIBRARIES} Eigen3::Eigen ${REALSENSE_LIBRARY} ftlnet ${LibArchive_LIBRARIES} ftlcodecs)
+target_link_libraries(ftlrgbd ftlcommon ${OpenCV_LIBS} ${LIBSGM_LIBRARIES} ${CUDA_LIBRARIES} Eigen3::Eigen ${REALSENSE_LIBRARY} ftlnet ${LibArchive_LIBRARIES} ftlcodecs ftloperators)
 
 add_subdirectory(test)
 

components/rgbd-sources/src/sources/stereovideo/stereovideo.cpp

 #include <loguru.hpp>
 #include "stereovideo.hpp"
+
 #include <ftl/configuration.hpp>
+#include <ftl/operators/opticalflow.hpp>
+
 #include <ftl/threads.hpp>
 #include "calibrate.hpp"
 #include "local.hpp"
@@ -58,16 +61,24 @@ void StereoVideoSource::init(const string &file)
 		lsrc_ = ftl::create<LocalSource>(host_, "feed");
 	}
 
+	// Create the source depth map pipeline
+	pipeline_ = ftl::config::create<ftl::operators::Graph>(host_, "disparity");
+	/*pipeline1->append<ftl::operators::ColourChannels>("colour");  // Convert BGR to BGRA
+	pipeline1->append<ftl::operators::HFSmoother>("hfnoise");  // Remove high-frequency noise
+	pipeline1->append<ftl::operators::Normals>("normals");  // Estimate surface normals
+	pipeline1->append<ftl::operators::SmoothChannel>("smoothing");  // Generate a smoothing channel
+	//pipeline1->append<ftl::operators::ScanFieldFill>("filling");  // Generate a smoothing channel
+	pipeline1->append<ftl::operators::ColourMLS>("mls");  // Perform MLS (using smoothing channel)
+	*/
+
 	cv::Size size = cv::Size(lsrc_->width(), lsrc_->height());
 	frames_ = std::vector<Frame>(2);
 
 #ifdef HAVE_OPTFLOW
+
 	use_optflow_ =  host_->value("use_optflow", false);
 	LOG(INFO) << "Using optical flow: " << (use_optflow_ ? "true" : "false");
-
-	nvof_ = cv::cuda::NvidiaOpticalFlow_1_0::create(size.width, size.height,
-													cv::cuda::NvidiaOpticalFlow_1_0::NV_OF_PERF_LEVEL_SLOW,
-													true, false, false, 0);
+	pipeline_->append<ftl::operators::NVOpticalFlow>("optflow");
 
 #endif
 
@@ -179,10 +190,11 @@ bool StereoVideoSource::retrieve() {
 	auto &left = frame.create<cv::cuda::GpuMat>(Channel::Left);
 	auto &right = frame.create<cv::cuda::GpuMat>(Channel::Right);
 	lsrc_->get(left, right, calib_, stream2_);
+	pipeline_->apply(frame, frame, (ftl::rgbd::Source*) lsrc_, cv::cuda::StreamAccessor::getStream(stream2_));
 
 #ifdef HAVE_OPTFLOW
 	// see comments in https://gitlab.utu.fi/nicolas.pope/ftl/issues/155
-	
+	/*
 	if (use_optflow_)
 	{
 		auto &left_gray = frame.create<cv::cuda::GpuMat>(Channel::LeftGray);
@@ -201,7 +213,7 @@ bool StereoVideoSource::retrieve() {
 			// cv::cuda::resize() does not work wiht 2-channel input
 			// cv::cuda::resize(optflow_, optflow, left.size(), 0.0, 0.0, cv::INTER_NEAREST, stream2_);
 		}
-	}
+	}*/
 #endif
 
 	stream2_.waitForCompletion();

components/rgbd-sources/src/sources/stereovideo/stereovideo.hpp

@@ -3,6 +3,7 @@
 #define _FTL_RGBD_STEREOVIDEO_HPP_
 
 #include <ftl/rgbd/source.hpp>
+#include <ftl/operators/operator.hpp>
 #include <string>
 
 namespace ftl {
@@ -39,7 +40,9 @@ class StereoVideoSource : public detail::Source {
 	LocalSource *lsrc_;
 	Calibrate *calib_;
 	Disparity *disp_;
-	
+
+	ftl::operators::Graph *pipeline_;
+
 	bool ready_;
 	bool use_optflow_;