done: opticalflow operator

components/operators/include/ftl/operators/opticalflow.hpp

@@ -6,6 +6,10 @@
 namespace ftl {
 namespace operators {
 
+/*
+ * Compute Optical flow from Channel::Colour (Left) and save the result in
+ * Channel::Flow using NVidia Optical Flow 1.0 (via OpenCV wrapper).
+ */
 class NVOpticalFlow : public ftl::operators::Operator {
 	public:
 	explicit NVOpticalFlow(ftl::Configurable*);
@@ -21,8 +25,8 @@ class NVOpticalFlow : public ftl::operators::Operator {
 	private:
 	cv::Size size_;
 	
-	// TODO: Left to Flow always assumed, could also calculate something else?
-	const ftl::codecs::Channel channel_in_ = ftl::codecs::Channel::Left;
+	// TODO: Colour to Flow always assumed, could also calculate something else?
+	const ftl::codecs::Channel channel_in_ = ftl::codecs::Channel::Colour;
 	const ftl::codecs::Channel channel_out_ = ftl::codecs::Channel::Flow;
 
 	cv::Ptr<cv::cuda::NvidiaOpticalFlow_1_0> nvof_;

components/rgbd-sources/src/sources/stereovideo/stereovideo.cpp

@@ -2,7 +2,10 @@
 #include "stereovideo.hpp"
 
 #include <ftl/configuration.hpp>
+
+#ifdef HAVE_OPTFLOW
 #include <ftl/operators/opticalflow.hpp>
+#endif
 
 #include <ftl/threads.hpp>
 #include "calibrate.hpp"
@@ -61,29 +64,10 @@ void StereoVideoSource::init(const string &file)
 		lsrc_ = ftl::create<LocalSource>(host_, "feed");
 	}
 
-	// Create the source depth map pipeline
-	pipeline_ = ftl::config::create<ftl::operators::Graph>(host_, "disparity_pipeline");
-	/*pipeline1->append<ftl::operators::ColourChannels>("colour");  // Convert BGR to BGRA
-	pipeline1->append<ftl::operators::HFSmoother>("hfnoise");  // Remove high-frequency noise
-	pipeline1->append<ftl::operators::Normals>("normals");  // Estimate surface normals
-	pipeline1->append<ftl::operators::SmoothChannel>("smoothing");  // Generate a smoothing channel
-	//pipeline1->append<ftl::operators::ScanFieldFill>("filling");  // Generate a smoothing channel
-	pipeline1->append<ftl::operators::ColourMLS>("mls");  // Perform MLS (using smoothing channel)
-	*/
-
 	cv::Size size = cv::Size(lsrc_->width(), lsrc_->height());
 	frames_ = std::vector<Frame>(2);
 
-#ifdef HAVE_OPTFLOW
-
-	use_optflow_ =  host_->value("use_optflow", false);
-	LOG(INFO) << "Using optical flow: " << (use_optflow_ ? "true" : "false");
-	pipeline_->append<ftl::operators::NVOpticalFlow>("optflow");
-
-#endif
-
 	calib_ = ftl::create<Calibrate>(host_, "calibration", size, stream_);
-
 	if (!calib_->isCalibrated()) LOG(WARNING) << "Cameras are not calibrated!";
 
 	// Generate camera parameters from camera matrix
@@ -128,17 +112,22 @@ void StereoVideoSource::init(const string &file)
 	});
 	
 	// left and right masks (areas outside rectified images)
-	// only left mask used
+	// only left mask used (not used)
 	cv::cuda::GpuMat mask_r_gpu(lsrc_->height(), lsrc_->width(), CV_8U, 255);
 	cv::cuda::GpuMat mask_l_gpu(lsrc_->height(), lsrc_->width(), CV_8U, 255);
-	
 	calib_->rectifyStereo(mask_l_gpu, mask_r_gpu, stream_);
 	stream_.waitForCompletion();
-
 	cv::Mat mask_l;
 	mask_l_gpu.download(mask_l);
 	mask_l_ = (mask_l == 0);
 	
+	pipeline_input_ = ftl::config::create<ftl::operators::Graph>(host_, "pipeline_input");
+	#ifdef HAVE_OPTFLOW
+	pipeline_input_->append<ftl::operators::NVOpticalFlow>("optflow");
+	#endif
+
+	pipeline_depth_ = ftl::config::create<ftl::operators::Graph>(host_, "pipeline_disparity");
+
 	disp_ = Disparity::create(host_, "disparity");
 	if (!disp_) LOG(FATAL) << "Unknown disparity algorithm : " << *host_->get<ftl::config::json_t>("disparity");
 	disp_->setMask(mask_l_);
@@ -190,33 +179,10 @@ bool StereoVideoSource::retrieve() {
 	auto &left = frame.create<cv::cuda::GpuMat>(Channel::Left);
 	auto &right = frame.create<cv::cuda::GpuMat>(Channel::Right);
 	lsrc_->get(left, right, calib_, stream2_);
-	pipeline_->apply(frame, frame, (ftl::rgbd::Source*) lsrc_, cv::cuda::StreamAccessor::getStream(stream2_));
-
-#ifdef HAVE_OPTFLOW
-	// see comments in https://gitlab.utu.fi/nicolas.pope/ftl/issues/155
-	/*
-	if (use_optflow_)
-	{
-		auto &left_gray = frame.create<cv::cuda::GpuMat>(Channel::LeftGray);
-		auto &right_gray = frame.create<cv::cuda::GpuMat>(Channel::RightGray);
-
-		cv::cuda::cvtColor(left, left_gray, cv::COLOR_BGR2GRAY, 0, stream2_);
-		cv::cuda::cvtColor(right, right_gray, cv::COLOR_BGR2GRAY, 0, stream2_);
-
-		if (frames_[1].hasChannel(Channel::LeftGray))
-		{
-			//frames_[1].download(Channel::LeftGray);
-			auto &left_gray_prev = frames_[1].get<cv::cuda::GpuMat>(Channel::LeftGray);
-			auto &optflow = frame.create<cv::cuda::GpuMat>(Channel::Flow);
-			nvof_->calc(left_gray, left_gray_prev, optflow, stream2_);
-			// nvof_->upSampler() isn't implemented with CUDA
-			// cv::cuda::resize() does not work wiht 2-channel input
-			// cv::cuda::resize(optflow_, optflow, left.size(), 0.0, 0.0, cv::INTER_NEAREST, stream2_);
-		}
-	}*/
-#endif
 
+	pipeline_input_->apply(frame, frame, (ftl::rgbd::Source*) lsrc_, cv::cuda::StreamAccessor::getStream(stream2_));
 	stream2_.waitForCompletion();
+	
 	return true;
 }
 

components/rgbd-sources/src/sources/stereovideo/stereovideo.hpp

@@ -34,17 +34,15 @@ class StereoVideoSource : public detail::Source {
 	bool isReady();
 	Camera parameters(ftl::codecs::Channel chan);
 
-	//const cv::Mat &getRight() const { return right_; }
-
 	private:
 	LocalSource *lsrc_;
 	Calibrate *calib_;
 	Disparity *disp_;
 
-	ftl::operators::Graph *pipeline_;
+	ftl::operators::Graph *pipeline_input_;
+	ftl::operators::Graph *pipeline_depth_;
 
 	bool ready_;
-	bool use_optflow_;
 	
 	cv::cuda::Stream stream_;
 	cv::cuda::Stream stream2_;
@@ -53,12 +51,6 @@ class StereoVideoSource : public detail::Source {
 
 	cv::Mat mask_l_;
 
-#ifdef HAVE_OPTFLOW
-	// see comments in https://gitlab.utu.fi/nicolas.pope/ftl/issues/155
-	cv::Ptr<cv::cuda::NvidiaOpticalFlow_1_0> nvof_;
-	cv::cuda::GpuMat optflow_;
-#endif
-
 	void init(const std::string &);
 };