calculate optical flow

components/rgbd-sources/src/stereovideo.cpp

@@ -33,8 +33,6 @@ StereoVideoSource::~StereoVideoSource() {
 
 void StereoVideoSource::init(const string &file)
 {
-	frames_ = std::vector<Frame>(2);
-
 	capabilities_ = kCapVideo | kCapStereo;
 
 	if (ftl::is_video(file)) {
@@ -60,6 +58,14 @@ void StereoVideoSource::init(const string &file)
 	}
 
 	cv::Size size = cv::Size(lsrc_->width(), lsrc_->height());
+	frames_ = std::vector<StereoVideoSource::Frame>(2); // triple-buffering for optical flow
+
+#ifdef HAVE_OPTFLOW
+	nvof_ = cv::cuda::NvidiaOpticalFlow_1_0::create(size.width, size.height,
+													cv::cuda::NvidiaOpticalFlow_1_0::NV_OF_PERF_LEVEL_SLOW,
+													true, false, false, 0);
+#endif
+
 	calib_ = ftl::create<Calibrate>(host_, "calibration", size, stream_);
 
 	if (!calib_->isCalibrated()) LOG(WARNING) << "Cameras are not calibrated!";
@@ -163,9 +169,27 @@ bool StereoVideoSource::capture(int64_t ts) {
 }
 
 bool StereoVideoSource::retrieve() {
-	auto frame = frames_[0];
+	auto &frame = frames_[0];
 	frame.reset();
 	lsrc_->get(frame.left, frame.right, calib_, stream2_);
+
+#ifdef HAVE_OPTFLOW
+	if (nvof_)
+	{
+		cv::cuda::cvtColor(frame.left, frame.left_gray, cv::COLOR_BGR2GRAY, 0, stream2_);
+		cv::cuda::cvtColor(frame.right, frame.right_gray, cv::COLOR_BGR2GRAY, 0, stream2_);
+		frame.left_gray_ready = frame.right_gray_ready = true;
+
+		if (frames_[1].left_gray_ready)
+		{
+			nvof_->calc(frame.left_gray, frames_[1].left_gray, frame.optflow_, stream2_);
+			// nvof_->upSampler() isn't implemented with CUDA
+			cv::cuda::resize(frame.optflow_, frame.optflow, frame.left.size(), 0.0, 0.0, cv::INTER_NEAREST, stream2_);
+			frame.optflow_ready = true;
+		}
+	}
+#endif
+
 	stream2_.waitForCompletion();
 	return true;
 }

components/rgbd-sources/src/stereovideo.hpp

@@ -76,6 +76,10 @@ class StereoVideoSource : public detail::Source {
 
 	cv::Mat mask_l_;
 
+#ifdef HAVE_OPTFLOW
+	cv::Ptr<cv::cuda::NvidiaOpticalFlow_1_0> nvof_;
+#endif
+
 	void init(const std::string &);
 };