diff --git a/components/operators/include/ftl/operators/opticalflow.hpp b/components/operators/include/ftl/operators/opticalflow.hpp
index 1cbd58b76a33e4956fc45fc94afc32804f742fb2..70df5e49e0dd5ef52b3cccaba5f6017703e2fbbd 100644
--- a/components/operators/include/ftl/operators/opticalflow.hpp
+++ b/components/operators/include/ftl/operators/opticalflow.hpp
@@ -6,6 +6,10 @@
namespace ftl {
namespace operators {
+/*
+ * Compute Optical flow from Channel::Colour (Left) and save the result in
+ * Channel::Flow using NVidia Optical Flow 1.0 (via OpenCV wrapper).
+ */
class NVOpticalFlow : public ftl::operators::Operator {
public:
explicit NVOpticalFlow(ftl::Configurable*);
@@ -21,8 +25,8 @@ class NVOpticalFlow : public ftl::operators::Operator {
private:
cv::Size size_;
- // TODO: Left to Flow always assumed, could also calculate something else?
- const ftl::codecs::Channel channel_in_ = ftl::codecs::Channel::Left;
+ // TODO: Colour to Flow always assumed, could also calculate something else?
+ const ftl::codecs::Channel channel_in_ = ftl::codecs::Channel::Colour;
const ftl::codecs::Channel channel_out_ = ftl::codecs::Channel::Flow;
cv::Ptr<cv::cuda::NvidiaOpticalFlow_1_0> nvof_;
diff --git a/components/rgbd-sources/src/sources/stereovideo/stereovideo.cpp b/components/rgbd-sources/src/sources/stereovideo/stereovideo.cpp
index b7c0c73c8af2bf5d66d9b2572f849e4d1a794b20..3bc0b2fa5c4642b3bce9580e52cd4c918d73711a 100644
--- a/components/rgbd-sources/src/sources/stereovideo/stereovideo.cpp
+++ b/components/rgbd-sources/src/sources/stereovideo/stereovideo.cpp
@@ -2,7 +2,10 @@
#include "stereovideo.hpp"
#include <ftl/configuration.hpp>
+
+#ifdef HAVE_OPTFLOW
#include <ftl/operators/opticalflow.hpp>
+#endif
#include <ftl/threads.hpp>
#include "calibrate.hpp"
@@ -61,29 +64,10 @@ void StereoVideoSource::init(const string &file)
lsrc_ = ftl::create<LocalSource>(host_, "feed");
}
- // Create the source depth map pipeline
- pipeline_ = ftl::config::create<ftl::operators::Graph>(host_, "disparity_pipeline");
- /*pipeline1->append<ftl::operators::ColourChannels>("colour"); // Convert BGR to BGRA
- pipeline1->append<ftl::operators::HFSmoother>("hfnoise"); // Remove high-frequency noise
- pipeline1->append<ftl::operators::Normals>("normals"); // Estimate surface normals
- pipeline1->append<ftl::operators::SmoothChannel>("smoothing"); // Generate a smoothing channel
- //pipeline1->append<ftl::operators::ScanFieldFill>("filling"); // Generate a smoothing channel
- pipeline1->append<ftl::operators::ColourMLS>("mls"); // Perform MLS (using smoothing channel)
- */
-
cv::Size size = cv::Size(lsrc_->width(), lsrc_->height());
frames_ = std::vector<Frame>(2);
-#ifdef HAVE_OPTFLOW
-
- use_optflow_ = host_->value("use_optflow", false);
- LOG(INFO) << "Using optical flow: " << (use_optflow_ ? "true" : "false");
- pipeline_->append<ftl::operators::NVOpticalFlow>("optflow");
-
-#endif
-
calib_ = ftl::create<Calibrate>(host_, "calibration", size, stream_);
-
if (!calib_->isCalibrated()) LOG(WARNING) << "Cameras are not calibrated!";
// Generate camera parameters from camera matrix
@@ -128,17 +112,22 @@ void StereoVideoSource::init(const string &file)
});
// left and right masks (areas outside rectified images)
- // only left mask used
+ // only left mask used (not used)
cv::cuda::GpuMat mask_r_gpu(lsrc_->height(), lsrc_->width(), CV_8U, 255);
cv::cuda::GpuMat mask_l_gpu(lsrc_->height(), lsrc_->width(), CV_8U, 255);
-
calib_->rectifyStereo(mask_l_gpu, mask_r_gpu, stream_);
stream_.waitForCompletion();
-
cv::Mat mask_l;
mask_l_gpu.download(mask_l);
mask_l_ = (mask_l == 0);
+ pipeline_input_ = ftl::config::create<ftl::operators::Graph>(host_, "pipeline_input");
+ #ifdef HAVE_OPTFLOW
+ pipeline_input_->append<ftl::operators::NVOpticalFlow>("optflow");
+ #endif
+
+ pipeline_depth_ = ftl::config::create<ftl::operators::Graph>(host_, "pipeline_disparity");
+
disp_ = Disparity::create(host_, "disparity");
if (!disp_) LOG(FATAL) << "Unknown disparity algorithm : " << *host_->get<ftl::config::json_t>("disparity");
disp_->setMask(mask_l_);
@@ -190,33 +179,10 @@ bool StereoVideoSource::retrieve() {
auto &left = frame.create<cv::cuda::GpuMat>(Channel::Left);
auto &right = frame.create<cv::cuda::GpuMat>(Channel::Right);
lsrc_->get(left, right, calib_, stream2_);
- pipeline_->apply(frame, frame, (ftl::rgbd::Source*) lsrc_, cv::cuda::StreamAccessor::getStream(stream2_));
-
-#ifdef HAVE_OPTFLOW
- // see comments in https://gitlab.utu.fi/nicolas.pope/ftl/issues/155
- /*
- if (use_optflow_)
- {
- auto &left_gray = frame.create<cv::cuda::GpuMat>(Channel::LeftGray);
- auto &right_gray = frame.create<cv::cuda::GpuMat>(Channel::RightGray);
-
- cv::cuda::cvtColor(left, left_gray, cv::COLOR_BGR2GRAY, 0, stream2_);
- cv::cuda::cvtColor(right, right_gray, cv::COLOR_BGR2GRAY, 0, stream2_);
-
- if (frames_[1].hasChannel(Channel::LeftGray))
- {
- //frames_[1].download(Channel::LeftGray);
- auto &left_gray_prev = frames_[1].get<cv::cuda::GpuMat>(Channel::LeftGray);
- auto &optflow = frame.create<cv::cuda::GpuMat>(Channel::Flow);
- nvof_->calc(left_gray, left_gray_prev, optflow, stream2_);
- // nvof_->upSampler() isn't implemented with CUDA
- // cv::cuda::resize() does not work wiht 2-channel input
- // cv::cuda::resize(optflow_, optflow, left.size(), 0.0, 0.0, cv::INTER_NEAREST, stream2_);
- }
- }*/
-#endif
+ pipeline_input_->apply(frame, frame, (ftl::rgbd::Source*) lsrc_, cv::cuda::StreamAccessor::getStream(stream2_));
stream2_.waitForCompletion();
+
return true;
}
diff --git a/components/rgbd-sources/src/sources/stereovideo/stereovideo.hpp b/components/rgbd-sources/src/sources/stereovideo/stereovideo.hpp
index eb29186770fd0fe2347be20653ff51ec9b5080b4..44014a88d5b33c47a94adb363cb039211faa87b9 100644
--- a/components/rgbd-sources/src/sources/stereovideo/stereovideo.hpp
+++ b/components/rgbd-sources/src/sources/stereovideo/stereovideo.hpp
@@ -34,17 +34,15 @@ class StereoVideoSource : public detail::Source {
bool isReady();
Camera parameters(ftl::codecs::Channel chan);
- //const cv::Mat &getRight() const { return right_; }
-
private:
LocalSource *lsrc_;
Calibrate *calib_;
Disparity *disp_;
- ftl::operators::Graph *pipeline_;
+ ftl::operators::Graph *pipeline_input_;
+ ftl::operators::Graph *pipeline_depth_;
bool ready_;
- bool use_optflow_;
cv::cuda::Stream stream_;
cv::cuda::Stream stream2_;
@@ -53,12 +51,6 @@ class StereoVideoSource : public detail::Source {
cv::Mat mask_l_;
-#ifdef HAVE_OPTFLOW
- // see comments in https://gitlab.utu.fi/nicolas.pope/ftl/issues/155
- cv::Ptr<cv::cuda::NvidiaOpticalFlow_1_0> nvof_;
- cv::cuda::GpuMat optflow_;
-#endif
-
void init(const std::string &);
};