From a2292897c44a208d158f9f3a2309b5e89bf3b353 Mon Sep 17 00:00:00 2001
From: Sebastian Hahta <joseha@utu.fi>
Date: Wed, 4 Sep 2019 13:08:18 +0300
Subject: [PATCH] optical flow in stereovideo
---
.../rgbd-sources/include/ftl/rgbd/frame.hpp | 4 ++-
components/rgbd-sources/src/stereovideo.cpp | 25 +++++++++++--------
components/rgbd-sources/src/stereovideo.hpp | 1 +
3 files changed, 18 insertions(+), 12 deletions(-)
diff --git a/components/rgbd-sources/include/ftl/rgbd/frame.hpp b/components/rgbd-sources/include/ftl/rgbd/frame.hpp
index aec182503..e17b05702 100644
--- a/components/rgbd-sources/include/ftl/rgbd/frame.hpp
+++ b/components/rgbd-sources/include/ftl/rgbd/frame.hpp
@@ -21,11 +21,13 @@ static const channel_t kChanNormals = 0x0020;
static const channel_t kChanConfidence = 0x0040;
static const channel_t kChanFlow = 0x0080;
static const channel_t kChanEnergy = 0x0100;
+static const channel_t kChanLeftGray = 0x0200;
+static const channel_t kChanRightGray = 0x0400;
static const channel_t kChanOverlay1 = 0x1000;
// maximum number of available channels
-static const unsigned int n_channels = 11;
+static const unsigned int n_channels = 13;
inline bool isFloatChannel(ftl::rgbd::channel_t chan) {
return (chan == ftl::rgbd::kChanDepth || chan == ftl::rgbd::kChanEnergy);
diff --git a/components/rgbd-sources/src/stereovideo.cpp b/components/rgbd-sources/src/stereovideo.cpp
index 58ad0d923..0935e238a 100644
--- a/components/rgbd-sources/src/stereovideo.cpp
+++ b/components/rgbd-sources/src/stereovideo.cpp
@@ -62,7 +62,9 @@ void StereoVideoSource::init(const string &file)
#ifdef HAVE_OPTFLOW
// TODO could be calculated at later step too if have access to old frames
-
+ use_optflow_ = host_->value("use_optflow", false);
+ LOG(INFO) << "Using optical flow: " << (use_optflow_ ? "true" : "false");
+
nvof_ = cv::cuda::NvidiaOpticalFlow_1_0::create(size.width, size.height,
cv::cuda::NvidiaOpticalFlow_1_0::NV_OF_PERF_LEVEL_SLOW,
true, false, false, 0);
@@ -178,22 +180,23 @@ bool StereoVideoSource::retrieve() {
lsrc_->get(left, right, calib_, stream2_);
#ifdef HAVE_OPTFLOW
-/*
- if (nvof_)
+ if (use_optflow_)
{
- cv::cuda::cvtColor(frame.left, frame.left_gray, cv::COLOR_BGR2GRAY, 0, stream2_);
- cv::cuda::cvtColor(frame.right, frame.right_gray, cv::COLOR_BGR2GRAY, 0, stream2_);
- frame.left_gray_ready = frame.right_gray_ready = true;
+ auto &left_gray = frame.setChannel<cv::cuda::GpuMat>(kChanLeftGray);
+ auto &right_gray = frame.setChannel<cv::cuda::GpuMat>(kChanRightGray);
+
+ cv::cuda::cvtColor(left, left_gray, cv::COLOR_BGR2GRAY, 0, stream2_);
+ cv::cuda::cvtColor(right, right_gray, cv::COLOR_BGR2GRAY, 0, stream2_);
- if (frames_[1].left_gray_ready)
+ if (frames_[1].hasChannel(kChanLeftGray))
{
- nvof_->calc(frame.left_gray, frames_[1].left_gray, frame.optflow_, stream2_);
+ auto &left_gray_prev = frame.getChannel<cv::cuda::GpuMat>(kChanLeftGray, stream2_);
+ auto &optflow = frame.setChannel<cv::cuda::GpuMat>(kChanFlow);
+ nvof_->calc(left_gray, left_gray_prev, optflow_, stream2_);
// nvof_->upSampler() isn't implemented with CUDA
- cv::cuda::resize(frame.optflow_, frame.optflow, frame.left.size(), 0.0, 0.0, cv::INTER_NEAREST, stream2_);
- frame.optflow_ready = true;
+ cv::cuda::resize(optflow_, optflow, left.size(), 0.0, 0.0, cv::INTER_NEAREST, stream2_);
}
}
-*/
#endif
stream2_.waitForCompletion();
diff --git a/components/rgbd-sources/src/stereovideo.hpp b/components/rgbd-sources/src/stereovideo.hpp
index 2f9fa1831..934aeaf56 100644
--- a/components/rgbd-sources/src/stereovideo.hpp
+++ b/components/rgbd-sources/src/stereovideo.hpp
@@ -52,6 +52,7 @@ class StereoVideoSource : public detail::Source {
#ifdef HAVE_OPTFLOW
cv::Ptr<cv::cuda::NvidiaOpticalFlow_1_0> nvof_;
+ cv::cuda::GpuMat optflow_;
#endif
void init(const std::string &);
--
GitLab