Skip to content
Snippets Groups Projects

Compare revisions

Changes are shown as if the source revision was being merged into the target revision. Learn more about comparing revisions.

Source

Select target project
No results found

Target

Select target project
  • nicolaspope/ftl
1 result
Show changes
Commits on Source (3)
Hide whitespace changes
Inline Side-by-side
Showing
with 136 additions and 37 deletions
components/operators/CMakeLists.txt
View file @ 39d6bef7
add_library(ftloperators
src/smoothing.cpp
src/smoothing.cpp
src/smoothing.cu
src/mls.cu
src/smoothchan.cu
......@@ -8,6 +8,7 @@ add_library(ftloperators
src/normals.cpp
src/filling.cpp
src/filling.cu
src/nvopticalflow.cpp
)
# These cause errors in CI build and are being removed from PCL in newer versions
......
components/operators/include/ftl/operators/operator.hpp
View file @ 39d6bef7
......@@ -23,7 +23,7 @@ namespace operators {
class Operator {
public:
explicit Operator(ftl::Configurable *cfg);
virtual ~Operator();
virtual ~Operator();
enum class Type {
OneToOne, // Frame to Frame (Filter or generator)
......@@ -87,7 +87,7 @@ struct OperatorNode {
class Graph : public ftl::Configurable {
public:
explicit Graph(nlohmann::json &config);
~Graph();
~Graph();
template <typename T>
ftl::Configurable *append(const std::string &name);
......
components/operators/include/ftl/operators/opticalflow.hpp 0 → 100644
View file @ 39d6bef7
#pragma once
#include <ftl/operators/operator.hpp>
#include <opencv2/cudaoptflow.hpp>
namespace ftl {
namespace operators {
class NVOpticalFlow : public ftl::operators::Operator {
public:
explicit NVOpticalFlow(ftl::Configurable*);
~NVOpticalFlow();
inline Operator::Type type() const override { return Operator::Type::OneToOne; }
bool apply(ftl::rgbd::Frame &in, ftl::rgbd::Frame &out, ftl::rgbd::Source *src, cudaStream_t stream) override;
protected:
void init();
private:
cv::Size size_;
// TODO: Left to Flow always assumed, could also calculate something else?
const ftl::codecs::Channel channel_in_ = ftl::codecs::Channel::Left;
const ftl::codecs::Channel channel_out_ = ftl::codecs::Channel::Flow;
cv::Ptr<cv::cuda::NvidiaOpticalFlow_1_0> nvof_;
cv::cuda::GpuMat left_gray_;
cv::cuda::GpuMat left_gray_prev_;
};
}
}
components/operators/include/ftl/operators/smoothing.hpp
View file @ 39d6bef7
......@@ -13,17 +13,17 @@ namespace operators {
* first derivative. Only the depth channel is used and modified.
*/
class HFSmoother : public ftl::operators::Operator {
public:
explicit HFSmoother(ftl::Configurable*);
~HFSmoother();
public:
explicit HFSmoother(ftl::Configurable*);
~HFSmoother();
inline Operator::Type type() const override { return Operator::Type::OneToOne; }
bool apply(ftl::rgbd::Frame &in, ftl::rgbd::Frame &out, ftl::rgbd::Source *src, cudaStream_t stream) override;
bool apply(ftl::rgbd::Frame &in, ftl::rgbd::Frame &out, ftl::rgbd::Source *src, cudaStream_t stream) override;
private:
cv::cuda::GpuMat temp_;
ftl::rgbd::Frame frames_[4];
private:
cv::cuda::GpuMat temp_;
ftl::rgbd::Frame frames_[4];
};
/**
......@@ -34,13 +34,13 @@ class HFSmoother : public ftl::operators::Operator {
* no smoothing.
*/
class SmoothChannel : public ftl::operators::Operator {
public:
explicit SmoothChannel(ftl::Configurable*);
~SmoothChannel();
public:
explicit SmoothChannel(ftl::Configurable*);
~SmoothChannel();
inline Operator::Type type() const override { return Operator::Type::OneToOne; }
bool apply(ftl::rgbd::Frame &in, ftl::rgbd::Frame &out, ftl::rgbd::Source *src, cudaStream_t stream) override;
bool apply(ftl::rgbd::Frame &in, ftl::rgbd::Frame &out, ftl::rgbd::Source *src, cudaStream_t stream) override;
private:
ftl::rgbd::Frame temp_[6];
......@@ -52,15 +52,15 @@ class SmoothChannel : public ftl::operators::Operator {
* Also outputs Depth + Normals.
*/
class SimpleMLS : public ftl::operators::Operator {
public:
explicit SimpleMLS(ftl::Configurable*);
~SimpleMLS();
public:
explicit SimpleMLS(ftl::Configurable*);
~SimpleMLS();
inline Operator::Type type() const override { return Operator::Type::OneToOne; }
bool apply(ftl::rgbd::Frame &in, ftl::rgbd::Frame &out, ftl::rgbd::Source *src, cudaStream_t stream) override;
bool apply(ftl::rgbd::Frame &in, ftl::rgbd::Frame &out, ftl::rgbd::Source *src, cudaStream_t stream) override;
private:
private:
};
/**
......@@ -68,15 +68,15 @@ class SimpleMLS : public ftl::operators::Operator {
* by a simple colour similarity weighting. In practice this is too naive.
*/
class ColourMLS : public ftl::operators::Operator {
public:
explicit ColourMLS(ftl::Configurable*);
~ColourMLS();
public:
explicit ColourMLS(ftl::Configurable*);
~ColourMLS();
inline Operator::Type type() const override { return Operator::Type::OneToOne; }
bool apply(ftl::rgbd::Frame &in, ftl::rgbd::Frame &out, ftl::rgbd::Source *src, cudaStream_t stream) override;
bool apply(ftl::rgbd::Frame &in, ftl::rgbd::Frame &out, ftl::rgbd::Source *src, cudaStream_t stream) override;
private:
private:
};
/**
......@@ -87,15 +87,15 @@ class ColourMLS : public ftl::operators::Operator {
* it can be only a few or even no pixels with a zero smoothing factor.
*/
class AdaptiveMLS : public ftl::operators::Operator {
public:
explicit AdaptiveMLS(ftl::Configurable*);
~AdaptiveMLS();
public:
explicit AdaptiveMLS(ftl::Configurable*);
~AdaptiveMLS();
inline Operator::Type type() const override { return Operator::Type::OneToOne; }
bool apply(ftl::rgbd::Frame &in, ftl::rgbd::Frame &out, ftl::rgbd::Source *src, cudaStream_t stream) override;
bool apply(ftl::rgbd::Frame &in, ftl::rgbd::Frame &out, ftl::rgbd::Source *src, cudaStream_t stream) override;
private:
private:
};
}
......
components/operators/src/nvopticalflow.cpp 0 → 100644
View file @ 39d6bef7
#include <ftl/operators/opticalflow.hpp>
using ftl::rgbd::Frame;
using ftl::rgbd::Source;
using ftl::codecs::Channel;
using ftl::operators::NVOpticalFlow;
using cv::Size;
using cv::cuda::GpuMat;
NVOpticalFlow::NVOpticalFlow(ftl::Configurable* cfg) :
ftl::operators::Operator(cfg) {
size_ = Size(0, 0);
}
NVOpticalFlow::~NVOpticalFlow() {
}
void NVOpticalFlow::init() {
nvof_ = cv::cuda::NvidiaOpticalFlow_1_0::create(
size_.width, size_.height,
cv::cuda::NvidiaOpticalFlow_1_0::NV_OF_PERF_LEVEL_SLOW,
true, false, false, 0);
left_gray_.create(size_, CV_8UC1);
left_gray_prev_.create(size_, CV_8UC1);
}
bool NVOpticalFlow::apply(Frame &in, Frame &out, Source *src, cudaStream_t stream) {
if (!in.hasChannel(channel_in_)) {
return true; // false?
}
if (in.get<GpuMat>(channel_in_).size() != size_) {
size_ = in.get<GpuMat>(channel_in_).size();
init();
}
auto cvstream = cv::cuda::StreamAccessor::wrapStream(stream);
auto &flow = out.create<GpuMat>(channel_out_);
cv::cuda::cvtColor(in.get<GpuMat>(channel_in_), left_gray_, cv::COLOR_BGR2GRAY, 0, cvstream);
nvof_->calc(left_gray_, left_gray_prev_, flow, cvstream);
std::swap(left_gray_, left_gray_prev_);
return true;
}
\ No newline at end of file
components/rgbd-sources/CMakeLists.txt
View file @ 39d6bef7
......@@ -69,7 +69,7 @@ set_property(TARGET ftlrgbd PROPERTY CUDA_SEPARABLE_COMPILATION OFF)
endif()
#target_include_directories(cv-node PUBLIC ${PROJECT_SOURCE_DIR}/include)
target_link_libraries(ftlrgbd ftlcommon ${OpenCV_LIBS} ${LIBSGM_LIBRARIES} ${CUDA_LIBRARIES} Eigen3::Eigen ${REALSENSE_LIBRARY} ftlnet ${LibArchive_LIBRARIES} ftlcodecs)
target_link_libraries(ftlrgbd ftlcommon ${OpenCV_LIBS} ${LIBSGM_LIBRARIES} ${CUDA_LIBRARIES} Eigen3::Eigen ${REALSENSE_LIBRARY} ftlnet ${LibArchive_LIBRARIES} ftlcodecs ftloperators)
add_subdirectory(test)
components/rgbd-sources/src/sources/stereovideo/stereovideo.cpp
View file @ 39d6bef7
#include <loguru.hpp>
#include "stereovideo.hpp"
#include <ftl/configuration.hpp>
#include <ftl/operators/opticalflow.hpp>
#include <ftl/threads.hpp>
#include "calibrate.hpp"
#include "local.hpp"
......@@ -58,16 +61,24 @@ void StereoVideoSource::init(const string &file)
lsrc_ = ftl::create<LocalSource>(host_, "feed");
}
// Create the source depth map pipeline
pipeline_ = ftl::config::create<ftl::operators::Graph>(host_, "disparity_pipeline");
/*pipeline1->append<ftl::operators::ColourChannels>("colour"); // Convert BGR to BGRA
pipeline1->append<ftl::operators::HFSmoother>("hfnoise"); // Remove high-frequency noise
pipeline1->append<ftl::operators::Normals>("normals"); // Estimate surface normals
pipeline1->append<ftl::operators::SmoothChannel>("smoothing"); // Generate a smoothing channel
//pipeline1->append<ftl::operators::ScanFieldFill>("filling"); // Generate a smoothing channel
pipeline1->append<ftl::operators::ColourMLS>("mls"); // Perform MLS (using smoothing channel)
*/
cv::Size size = cv::Size(lsrc_->width(), lsrc_->height());
frames_ = std::vector<Frame>(2);
#ifdef HAVE_OPTFLOW
use_optflow_ = host_->value("use_optflow", false);
LOG(INFO) << "Using optical flow: " << (use_optflow_ ? "true" : "false");
nvof_ = cv::cuda::NvidiaOpticalFlow_1_0::create(size.width, size.height,
cv::cuda::NvidiaOpticalFlow_1_0::NV_OF_PERF_LEVEL_SLOW,
true, false, false, 0);
pipeline_->append<ftl::operators::NVOpticalFlow>("optflow");
#endif
......@@ -179,10 +190,11 @@ bool StereoVideoSource::retrieve() {
auto &left = frame.create<cv::cuda::GpuMat>(Channel::Left);
auto &right = frame.create<cv::cuda::GpuMat>(Channel::Right);
lsrc_->get(left, right, calib_, stream2_);
pipeline_->apply(frame, frame, (ftl::rgbd::Source*) lsrc_, cv::cuda::StreamAccessor::getStream(stream2_));
#ifdef HAVE_OPTFLOW
// see comments in https://gitlab.utu.fi/nicolas.pope/ftl/issues/155
/*
if (use_optflow_)
{
auto &left_gray = frame.create<cv::cuda::GpuMat>(Channel::LeftGray);
......@@ -201,7 +213,7 @@ bool StereoVideoSource::retrieve() {
// cv::cuda::resize() does not work wiht 2-channel input
// cv::cuda::resize(optflow_, optflow, left.size(), 0.0, 0.0, cv::INTER_NEAREST, stream2_);
}
}
}*/
#endif
stream2_.waitForCompletion();
......
components/rgbd-sources/src/sources/stereovideo/stereovideo.hpp
View file @ 39d6bef7
......@@ -3,6 +3,7 @@
#define _FTL_RGBD_STEREOVIDEO_HPP_
#include <ftl/rgbd/source.hpp>
#include <ftl/operators/operator.hpp>
#include <string>
namespace ftl {
......@@ -39,7 +40,9 @@ class StereoVideoSource : public detail::Source {
LocalSource *lsrc_;
Calibrate *calib_;
Disparity *disp_;
ftl::operators::Graph *pipeline_;
bool ready_;
bool use_optflow_;
......