Skip to content
Snippets Groups Projects
Commit 8d4575d4 authored by Sebastian Hahta's avatar Sebastian Hahta
Browse files

operator: optical flow

parent c2f26c57
No related branches found
No related tags found
No related merge requests found
Pipeline #16240 passed
add_library(ftloperators add_library(ftloperators
src/smoothing.cpp src/smoothing.cpp
src/smoothing.cu src/smoothing.cu
src/mls.cu src/mls.cu
src/smoothchan.cu src/smoothchan.cu
...@@ -8,6 +8,7 @@ add_library(ftloperators ...@@ -8,6 +8,7 @@ add_library(ftloperators
src/normals.cpp src/normals.cpp
src/filling.cpp src/filling.cpp
src/filling.cu src/filling.cu
src/nvopticalflow.cpp
) )
# These cause errors in CI build and are being removed from PCL in newer versions # These cause errors in CI build and are being removed from PCL in newer versions
......
...@@ -23,7 +23,7 @@ namespace operators { ...@@ -23,7 +23,7 @@ namespace operators {
class Operator { class Operator {
public: public:
explicit Operator(ftl::Configurable *cfg); explicit Operator(ftl::Configurable *cfg);
virtual ~Operator(); virtual ~Operator();
enum class Type { enum class Type {
OneToOne, // Frame to Frame (Filter or generator) OneToOne, // Frame to Frame (Filter or generator)
...@@ -87,7 +87,7 @@ struct OperatorNode { ...@@ -87,7 +87,7 @@ struct OperatorNode {
class Graph : public ftl::Configurable { class Graph : public ftl::Configurable {
public: public:
explicit Graph(nlohmann::json &config); explicit Graph(nlohmann::json &config);
~Graph(); ~Graph();
template <typename T> template <typename T>
ftl::Configurable *append(const std::string &name); ftl::Configurable *append(const std::string &name);
......
#pragma once
#include <ftl/operators/operator.hpp>
#include <opencv2/cudaoptflow.hpp>
namespace ftl {
namespace operators {
class NVOpticalFlow : public ftl::operators::Operator {
public:
explicit NVOpticalFlow(ftl::Configurable*);
~NVOpticalFlow();
inline Operator::Type type() const override { return Operator::Type::OneToOne; }
bool apply(ftl::rgbd::Frame &in, ftl::rgbd::Frame &out, ftl::rgbd::Source *src, cudaStream_t stream) override;
protected:
void init();
private:
cv::Size size_;
// TODO: Left to Flow always assumed, could also calculate something else?
const ftl::codecs::Channel channel_in_ = ftl::codecs::Channel::Left;
const ftl::codecs::Channel channel_out_ = ftl::codecs::Channel::Flow;
cv::Ptr<cv::cuda::NvidiaOpticalFlow_1_0> nvof_;
cv::cuda::GpuMat left_gray_;
cv::cuda::GpuMat left_gray_prev_;
};
}
}
...@@ -13,17 +13,17 @@ namespace operators { ...@@ -13,17 +13,17 @@ namespace operators {
* first derivative. Only the depth channel is used and modified. * first derivative. Only the depth channel is used and modified.
*/ */
class HFSmoother : public ftl::operators::Operator { class HFSmoother : public ftl::operators::Operator {
public: public:
explicit HFSmoother(ftl::Configurable*); explicit HFSmoother(ftl::Configurable*);
~HFSmoother(); ~HFSmoother();
inline Operator::Type type() const override { return Operator::Type::OneToOne; } inline Operator::Type type() const override { return Operator::Type::OneToOne; }
bool apply(ftl::rgbd::Frame &in, ftl::rgbd::Frame &out, ftl::rgbd::Source *src, cudaStream_t stream) override; bool apply(ftl::rgbd::Frame &in, ftl::rgbd::Frame &out, ftl::rgbd::Source *src, cudaStream_t stream) override;
private: private:
cv::cuda::GpuMat temp_; cv::cuda::GpuMat temp_;
ftl::rgbd::Frame frames_[4]; ftl::rgbd::Frame frames_[4];
}; };
/** /**
...@@ -34,13 +34,13 @@ class HFSmoother : public ftl::operators::Operator { ...@@ -34,13 +34,13 @@ class HFSmoother : public ftl::operators::Operator {
* no smoothing. * no smoothing.
*/ */
class SmoothChannel : public ftl::operators::Operator { class SmoothChannel : public ftl::operators::Operator {
public: public:
explicit SmoothChannel(ftl::Configurable*); explicit SmoothChannel(ftl::Configurable*);
~SmoothChannel(); ~SmoothChannel();
inline Operator::Type type() const override { return Operator::Type::OneToOne; } inline Operator::Type type() const override { return Operator::Type::OneToOne; }
bool apply(ftl::rgbd::Frame &in, ftl::rgbd::Frame &out, ftl::rgbd::Source *src, cudaStream_t stream) override; bool apply(ftl::rgbd::Frame &in, ftl::rgbd::Frame &out, ftl::rgbd::Source *src, cudaStream_t stream) override;
private: private:
ftl::rgbd::Frame temp_[6]; ftl::rgbd::Frame temp_[6];
...@@ -52,15 +52,15 @@ class SmoothChannel : public ftl::operators::Operator { ...@@ -52,15 +52,15 @@ class SmoothChannel : public ftl::operators::Operator {
* Also outputs Depth + Normals. * Also outputs Depth + Normals.
*/ */
class SimpleMLS : public ftl::operators::Operator { class SimpleMLS : public ftl::operators::Operator {
public: public:
explicit SimpleMLS(ftl::Configurable*); explicit SimpleMLS(ftl::Configurable*);
~SimpleMLS(); ~SimpleMLS();
inline Operator::Type type() const override { return Operator::Type::OneToOne; } inline Operator::Type type() const override { return Operator::Type::OneToOne; }
bool apply(ftl::rgbd::Frame &in, ftl::rgbd::Frame &out, ftl::rgbd::Source *src, cudaStream_t stream) override; bool apply(ftl::rgbd::Frame &in, ftl::rgbd::Frame &out, ftl::rgbd::Source *src, cudaStream_t stream) override;
private: private:
}; };
/** /**
...@@ -68,15 +68,15 @@ class SimpleMLS : public ftl::operators::Operator { ...@@ -68,15 +68,15 @@ class SimpleMLS : public ftl::operators::Operator {
* by a simple colour similarity weighting. In practice this is too naive. * by a simple colour similarity weighting. In practice this is too naive.
*/ */
class ColourMLS : public ftl::operators::Operator { class ColourMLS : public ftl::operators::Operator {
public: public:
explicit ColourMLS(ftl::Configurable*); explicit ColourMLS(ftl::Configurable*);
~ColourMLS(); ~ColourMLS();
inline Operator::Type type() const override { return Operator::Type::OneToOne; } inline Operator::Type type() const override { return Operator::Type::OneToOne; }
bool apply(ftl::rgbd::Frame &in, ftl::rgbd::Frame &out, ftl::rgbd::Source *src, cudaStream_t stream) override; bool apply(ftl::rgbd::Frame &in, ftl::rgbd::Frame &out, ftl::rgbd::Source *src, cudaStream_t stream) override;
private: private:
}; };
/** /**
...@@ -87,15 +87,15 @@ class ColourMLS : public ftl::operators::Operator { ...@@ -87,15 +87,15 @@ class ColourMLS : public ftl::operators::Operator {
* it can be only a few or even no pixels with a zero smoothing factor. * it can be only a few or even no pixels with a zero smoothing factor.
*/ */
class AdaptiveMLS : public ftl::operators::Operator { class AdaptiveMLS : public ftl::operators::Operator {
public: public:
explicit AdaptiveMLS(ftl::Configurable*); explicit AdaptiveMLS(ftl::Configurable*);
~AdaptiveMLS(); ~AdaptiveMLS();
inline Operator::Type type() const override { return Operator::Type::OneToOne; } inline Operator::Type type() const override { return Operator::Type::OneToOne; }
bool apply(ftl::rgbd::Frame &in, ftl::rgbd::Frame &out, ftl::rgbd::Source *src, cudaStream_t stream) override; bool apply(ftl::rgbd::Frame &in, ftl::rgbd::Frame &out, ftl::rgbd::Source *src, cudaStream_t stream) override;
private: private:
}; };
} }
......
#include <ftl/operators/opticalflow.hpp>
using ftl::rgbd::Frame;
using ftl::rgbd::Source;
using ftl::codecs::Channel;
using ftl::operators::NVOpticalFlow;
using cv::Size;
using cv::cuda::GpuMat;
NVOpticalFlow::NVOpticalFlow(ftl::Configurable* cfg) :
ftl::operators::Operator(cfg) {
size_ = Size(0, 0);
}
NVOpticalFlow::~NVOpticalFlow() {
}
void NVOpticalFlow::init() {
nvof_ = cv::cuda::NvidiaOpticalFlow_1_0::create(
size_.width, size_.height,
cv::cuda::NvidiaOpticalFlow_1_0::NV_OF_PERF_LEVEL_SLOW,
true, false, false, 0);
left_gray_.create(size_, CV_8UC1);
left_gray_prev_.create(size_, CV_8UC1);
}
bool NVOpticalFlow::apply(Frame &in, Frame &out, Source *src, cudaStream_t stream) {
if (!in.hasChannel(channel_in_)) {
return true; // false?
}
if (in.get<GpuMat>(channel_in_).size() != size_) {
size_ = in.get<GpuMat>(channel_in_).size();
init();
}
auto cvstream = cv::cuda::StreamAccessor::wrapStream(stream);
auto &flow = out.create<GpuMat>(channel_out_);
cv::cuda::cvtColor(in.get<GpuMat>(channel_in_), left_gray_, cv::COLOR_BGR2GRAY, 0, cvstream);
nvof_->calc(left_gray_, left_gray_prev_, flow, cvstream);
return true;
}
\ No newline at end of file
...@@ -69,7 +69,7 @@ set_property(TARGET ftlrgbd PROPERTY CUDA_SEPARABLE_COMPILATION OFF) ...@@ -69,7 +69,7 @@ set_property(TARGET ftlrgbd PROPERTY CUDA_SEPARABLE_COMPILATION OFF)
endif() endif()
#target_include_directories(cv-node PUBLIC ${PROJECT_SOURCE_DIR}/include) #target_include_directories(cv-node PUBLIC ${PROJECT_SOURCE_DIR}/include)
target_link_libraries(ftlrgbd ftlcommon ${OpenCV_LIBS} ${LIBSGM_LIBRARIES} ${CUDA_LIBRARIES} Eigen3::Eigen ${REALSENSE_LIBRARY} ftlnet ${LibArchive_LIBRARIES} ftlcodecs) target_link_libraries(ftlrgbd ftlcommon ${OpenCV_LIBS} ${LIBSGM_LIBRARIES} ${CUDA_LIBRARIES} Eigen3::Eigen ${REALSENSE_LIBRARY} ftlnet ${LibArchive_LIBRARIES} ftlcodecs ftloperators)
add_subdirectory(test) add_subdirectory(test)
#include <loguru.hpp> #include <loguru.hpp>
#include "stereovideo.hpp" #include "stereovideo.hpp"
#include <ftl/configuration.hpp> #include <ftl/configuration.hpp>
#include <ftl/operators/opticalflow.hpp>
#include <ftl/threads.hpp> #include <ftl/threads.hpp>
#include "calibrate.hpp" #include "calibrate.hpp"
#include "local.hpp" #include "local.hpp"
...@@ -58,16 +61,24 @@ void StereoVideoSource::init(const string &file) ...@@ -58,16 +61,24 @@ void StereoVideoSource::init(const string &file)
lsrc_ = ftl::create<LocalSource>(host_, "feed"); lsrc_ = ftl::create<LocalSource>(host_, "feed");
} }
// Create the source depth map pipeline
pipeline_ = ftl::config::create<ftl::operators::Graph>(host_, "disparity");
/*pipeline1->append<ftl::operators::ColourChannels>("colour"); // Convert BGR to BGRA
pipeline1->append<ftl::operators::HFSmoother>("hfnoise"); // Remove high-frequency noise
pipeline1->append<ftl::operators::Normals>("normals"); // Estimate surface normals
pipeline1->append<ftl::operators::SmoothChannel>("smoothing"); // Generate a smoothing channel
//pipeline1->append<ftl::operators::ScanFieldFill>("filling"); // Generate a smoothing channel
pipeline1->append<ftl::operators::ColourMLS>("mls"); // Perform MLS (using smoothing channel)
*/
cv::Size size = cv::Size(lsrc_->width(), lsrc_->height()); cv::Size size = cv::Size(lsrc_->width(), lsrc_->height());
frames_ = std::vector<Frame>(2); frames_ = std::vector<Frame>(2);
#ifdef HAVE_OPTFLOW #ifdef HAVE_OPTFLOW
use_optflow_ = host_->value("use_optflow", false); use_optflow_ = host_->value("use_optflow", false);
LOG(INFO) << "Using optical flow: " << (use_optflow_ ? "true" : "false"); LOG(INFO) << "Using optical flow: " << (use_optflow_ ? "true" : "false");
pipeline_->append<ftl::operators::NVOpticalFlow>("optflow");
nvof_ = cv::cuda::NvidiaOpticalFlow_1_0::create(size.width, size.height,
cv::cuda::NvidiaOpticalFlow_1_0::NV_OF_PERF_LEVEL_SLOW,
true, false, false, 0);
#endif #endif
...@@ -179,10 +190,11 @@ bool StereoVideoSource::retrieve() { ...@@ -179,10 +190,11 @@ bool StereoVideoSource::retrieve() {
auto &left = frame.create<cv::cuda::GpuMat>(Channel::Left); auto &left = frame.create<cv::cuda::GpuMat>(Channel::Left);
auto &right = frame.create<cv::cuda::GpuMat>(Channel::Right); auto &right = frame.create<cv::cuda::GpuMat>(Channel::Right);
lsrc_->get(left, right, calib_, stream2_); lsrc_->get(left, right, calib_, stream2_);
pipeline_->apply(frame, frame, (ftl::rgbd::Source*) lsrc_, cv::cuda::StreamAccessor::getStream(stream2_));
#ifdef HAVE_OPTFLOW #ifdef HAVE_OPTFLOW
// see comments in https://gitlab.utu.fi/nicolas.pope/ftl/issues/155 // see comments in https://gitlab.utu.fi/nicolas.pope/ftl/issues/155
/*
if (use_optflow_) if (use_optflow_)
{ {
auto &left_gray = frame.create<cv::cuda::GpuMat>(Channel::LeftGray); auto &left_gray = frame.create<cv::cuda::GpuMat>(Channel::LeftGray);
...@@ -201,7 +213,7 @@ bool StereoVideoSource::retrieve() { ...@@ -201,7 +213,7 @@ bool StereoVideoSource::retrieve() {
// cv::cuda::resize() does not work wiht 2-channel input // cv::cuda::resize() does not work wiht 2-channel input
// cv::cuda::resize(optflow_, optflow, left.size(), 0.0, 0.0, cv::INTER_NEAREST, stream2_); // cv::cuda::resize(optflow_, optflow, left.size(), 0.0, 0.0, cv::INTER_NEAREST, stream2_);
} }
} }*/
#endif #endif
stream2_.waitForCompletion(); stream2_.waitForCompletion();
......
...@@ -3,6 +3,7 @@ ...@@ -3,6 +3,7 @@
#define _FTL_RGBD_STEREOVIDEO_HPP_ #define _FTL_RGBD_STEREOVIDEO_HPP_
#include <ftl/rgbd/source.hpp> #include <ftl/rgbd/source.hpp>
#include <ftl/operators/operator.hpp>
#include <string> #include <string>
namespace ftl { namespace ftl {
...@@ -39,7 +40,9 @@ class StereoVideoSource : public detail::Source { ...@@ -39,7 +40,9 @@ class StereoVideoSource : public detail::Source {
LocalSource *lsrc_; LocalSource *lsrc_;
Calibrate *calib_; Calibrate *calib_;
Disparity *disp_; Disparity *disp_;
ftl::operators::Graph *pipeline_;
bool ready_; bool ready_;
bool use_optflow_; bool use_optflow_;
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment