From 95d51f0b92f90e2c123bcd865ca2baf364670425 Mon Sep 17 00:00:00 2001
From: Sebastian Hahta <joseha@utu.fi>
Date: Wed, 4 Sep 2019 11:03:41 +0300
Subject: [PATCH] more refactoring: disparity class use new frame class
---
.../include/ftl/rgbd/detail/source.hpp | 25 +------
.../rgbd-sources/include/ftl/rgbd/frame.hpp | 65 ++++++++++++++++---
.../src/algorithms/fixstars_sgm.cpp | 12 +++-
.../src/algorithms/fixstars_sgm.hpp | 2 +-
components/rgbd-sources/src/disparity.hpp | 11 +++-
components/rgbd-sources/src/frame.cpp | 8 +--
components/rgbd-sources/src/stereovideo.cpp | 10 +--
7 files changed, 86 insertions(+), 47 deletions(-)
diff --git a/components/rgbd-sources/include/ftl/rgbd/detail/source.hpp b/components/rgbd-sources/include/ftl/rgbd/detail/source.hpp
index a75fb1879..29edf722f 100644
--- a/components/rgbd-sources/include/ftl/rgbd/detail/source.hpp
+++ b/components/rgbd-sources/include/ftl/rgbd/detail/source.hpp
@@ -4,36 +4,13 @@
#include <Eigen/Eigen>
#include <opencv2/opencv.hpp>
#include <ftl/rgbd/camera.hpp>
+#include <ftl/rgbd/frame.hpp>
namespace ftl{
namespace rgbd {
class Source;
-// TODO: frame.hpp has to know the exact amount of channels
-// (remove hardcoded value by providing exact value elsewhere).
-// Possibly move channels to frame.hpp?
-
-typedef unsigned int channel_t;
-
-static const channel_t kChanNone = 0;
-static const channel_t kChanLeft = 0x0001;
-static const channel_t kChanDepth = 0x0002;
-static const channel_t kChanRight = 0x0004;
-static const channel_t kChanDisparity = 0x0008;
-static const channel_t kChanDeviation = 0x0010;
-static const channel_t kChanNormals = 0x0020;
-static const channel_t kChanConfidence = 0x0040;
-static const channel_t kChanFlow = 0x0080;
-static const channel_t kChanEnergy = 0x0100;
-
-static const channel_t kChanOverlay1 = 0x1000;
-
-inline bool isFloatChannel(ftl::rgbd::channel_t chan) {
- return (chan == ftl::rgbd::kChanDepth || chan == ftl::rgbd::kChanEnergy);
-}
-
-
typedef unsigned int capability_t;
static const capability_t kCapMovable = 0x0001; // A movable virtual cam
diff --git a/components/rgbd-sources/include/ftl/rgbd/frame.hpp b/components/rgbd-sources/include/ftl/rgbd/frame.hpp
index ddc624d9b..993b49aa5 100644
--- a/components/rgbd-sources/include/ftl/rgbd/frame.hpp
+++ b/components/rgbd-sources/include/ftl/rgbd/frame.hpp
@@ -1,17 +1,44 @@
#pragma once
+#ifndef _FTL_RGBD_FRAME_HPP_
+#define _FTL_RGBD_FRAME_HPP_
-#include <ftl/rgbd/detail/source.hpp>
#include <ftl/configuration.hpp>
#include <opencv2/core.hpp>
+#include <opencv2/core/cuda.hpp>
namespace ftl {
namespace rgbd {
+typedef unsigned int channel_t;
+
+static const channel_t kChanNone = 0;
+static const channel_t kChanLeft = 0x0001;
+static const channel_t kChanDepth = 0x0002;
+static const channel_t kChanRight = 0x0004;
+static const channel_t kChanDisparity = 0x0008;
+static const channel_t kChanDeviation = 0x0010;
+static const channel_t kChanNormals = 0x0020;
+static const channel_t kChanConfidence = 0x0040;
+static const channel_t kChanFlow = 0x0080;
+static const channel_t kChanEnergy = 0x0100;
+
+static const channel_t kChanOverlay1 = 0x1000;
+
+// maximum number of available channels
+static const unsigned int n_channels = 11;
+
+inline bool isFloatChannel(ftl::rgbd::channel_t chan) {
+ return (chan == ftl::rgbd::kChanDepth || chan == ftl::rgbd::kChanEnergy);
+}
+
+class Frame;
+
class Frame {
public:
- /* @todo TODO: REMOVE HARDCODED CHANNEL COUNT!
- */
- Frame() : channels_host_(11), channels_gpu_(11), available_(11, 0) {}
+ Frame() : channels_host_(n_channels),
+ channels_gpu_(n_channels),
+ available_(n_channels, 0)
+ {}
/* @brief Reset all channels without releasing memory.
*/
@@ -27,16 +54,27 @@ public:
return available_[_channelIdx(channel)];
}
- /* @brief get reference to the channel contents
- * @param channel type
- * @returns const reference to channel data
+ /* @brief Method to get reference to the channel content
+ * @param Channel type
+ * @returns Const reference to channel data
*
- * Methods automatically copy between host/gpu if the data is only available
- * in the other. Results are cached. Result is valid only if hasChannel() is
- * true.
+ * Result is valid only if hasChannel() is true. Host/Gpu transfer is
+ * performed, if necessary, but only once unless channel contents is
+ * changed by calling setChannel(). Return value valid only if
+ * hasChannel(channel) is true.
*/
template <typename T> const T& getChannel(const ftl::rgbd::channel_t& channel);
+
+
+ /* @brief Method to set/modify channel content
+ * @param Channel type
+ * @returns Reference to channel data
+ *
+ * Returns non-const reference to channel data. Invalidates other copies of
+ * the data (host/gpu) for the specified channel, next time getChannel() is
+ * called a memory transfer may occur.
+ */
template <typename T> T& setChannel(const ftl::rgbd::channel_t& channel);
private:
@@ -70,5 +108,12 @@ private:
std::vector<uint> available_;
};
+template<> const cv::Mat& Frame::getChannel(const ftl::rgbd::channel_t& channel);
+template<> cv::Mat& Frame::setChannel(const ftl::rgbd::channel_t& channel);
+template<> const cv::cuda::GpuMat& Frame::getChannel(const ftl::rgbd::channel_t& channel);
+template<> cv::cuda::GpuMat& Frame::setChannel(const ftl::rgbd::channel_t& channel);
+
}
}
+
+#endif // _FTL_RGBD_FRAME_HPP_
\ No newline at end of file
diff --git a/components/rgbd-sources/src/algorithms/fixstars_sgm.cpp b/components/rgbd-sources/src/algorithms/fixstars_sgm.cpp
index 71f714c3a..b37c7b078 100644
--- a/components/rgbd-sources/src/algorithms/fixstars_sgm.cpp
+++ b/components/rgbd-sources/src/algorithms/fixstars_sgm.cpp
@@ -64,9 +64,17 @@ void FixstarsSGM::init(const cv::Size size) {
);
}
-void FixstarsSGM::compute(const cv::cuda::GpuMat &l, const cv::cuda::GpuMat &r,
- cv::cuda::GpuMat &disp, cv::cuda::Stream &stream)
+void FixstarsSGM::compute(ftl::rgbd::Frame &frame, cv::cuda::Stream &stream)
{
+ const auto &l = frame.getChannel<GpuMat>(ftl::rgbd::kChanLeft);
+ const auto &r = frame.getChannel<GpuMat>(ftl::rgbd::kChanRight);
+ auto &disp = frame.setChannel<GpuMat>(ftl::rgbd::kChanDisparity);
+
+ if (disp.size() != l.size())
+ {
+ disp = GpuMat(l.size(), CV_32FC1);
+ }
+
if (l.size() != size_) {
// re-use same buffer for l/r
cv::cuda::resize(r, l_downscaled_, size_, 0.0, 0.0, cv::INTER_CUBIC, stream);
diff --git a/components/rgbd-sources/src/algorithms/fixstars_sgm.hpp b/components/rgbd-sources/src/algorithms/fixstars_sgm.hpp
index d773446d7..14f80bf89 100644
--- a/components/rgbd-sources/src/algorithms/fixstars_sgm.hpp
+++ b/components/rgbd-sources/src/algorithms/fixstars_sgm.hpp
@@ -30,7 +30,7 @@ namespace ftl {
public:
explicit FixstarsSGM(nlohmann::json &config);
- void compute(const cv::cuda::GpuMat &l, const cv::cuda::GpuMat &r, cv::cuda::GpuMat &disp, cv::cuda::Stream &stream) override;
+ void compute(ftl::rgbd::Frame &frame, cv::cuda::Stream &stream) override;
void setMask(cv::Mat &mask) override;
/* Factory creator */
diff --git a/components/rgbd-sources/src/disparity.hpp b/components/rgbd-sources/src/disparity.hpp
index e7e78b277..a5e7efea7 100644
--- a/components/rgbd-sources/src/disparity.hpp
+++ b/components/rgbd-sources/src/disparity.hpp
@@ -8,6 +8,7 @@
#include <opencv2/opencv.hpp>
#include <nlohmann/json.hpp>
#include <ftl/configurable.hpp>
+#include <ftl/rgbd/frame.hpp>
namespace ftl {
namespace rgbd {
@@ -32,7 +33,15 @@ class Disparity : public ftl::Configurable {
* Pure virtual function representing the actual computation of
* disparity from left and right images to be implemented.
*/
- virtual void compute(const cv::cuda::GpuMat &l, const cv::cuda::GpuMat &r, cv::cuda::GpuMat &disp, cv::cuda::Stream &stream)=0;
+ virtual void compute(Frame &frame, cv::cuda::Stream &stream)=0;
+ virtual void compute(cv::cuda::GpuMat &l, cv::cuda::GpuMat &r, cv::cuda::GpuMat &disp, cv::cuda::Stream &stream)
+ {
+ ftl::rgbd::Frame frame;
+ frame.setChannel<cv::cuda::GpuMat>(kChanLeft) = l;
+ frame.setChannel<cv::cuda::GpuMat>(kChanRight) = r;
+ frame.setChannel<cv::cuda::GpuMat>(kChanDisparity) = disp;
+ }
+
/**
* Factory registration class.
diff --git a/components/rgbd-sources/src/frame.cpp b/components/rgbd-sources/src/frame.cpp
index 62b3795ff..ea6a94f4f 100644
--- a/components/rgbd-sources/src/frame.cpp
+++ b/components/rgbd-sources/src/frame.cpp
@@ -6,7 +6,7 @@ namespace rgbd {
template<> const cv::Mat& Frame::getChannel(const ftl::rgbd::channel_t& channel)
{
- auto idx = _channelIdx(channel);
+ size_t idx = _channelIdx(channel);
if (!(available_[idx] & mask_host))
{
if (available_[idx] & mask_gpu)
@@ -21,14 +21,14 @@ template<> const cv::Mat& Frame::getChannel(const ftl::rgbd::channel_t& channel)
template<> cv::Mat& Frame::setChannel(const ftl::rgbd::channel_t& channel)
{
- auto idx = _channelIdx(channel);
+ size_t idx = _channelIdx(channel);
available_[idx] = mask_host;
return channels_host_[idx];
}
template<> const cv::cuda::GpuMat& Frame::getChannel(const ftl::rgbd::channel_t& channel)
{
- auto idx = _channelIdx(channel);
+ size_t idx = _channelIdx(channel);
if (!(available_[idx] & mask_gpu))
{
if (available_[idx] & mask_host)
@@ -43,7 +43,7 @@ template<> const cv::cuda::GpuMat& Frame::getChannel(const ftl::rgbd::channel_t&
template<> cv::cuda::GpuMat& Frame::setChannel(const ftl::rgbd::channel_t& channel)
{
- auto idx = _channelIdx(channel);
+ size_t idx = _channelIdx(channel);
available_[idx] = mask_gpu;
return channels_gpu_[idx];
}
diff --git a/components/rgbd-sources/src/stereovideo.cpp b/components/rgbd-sources/src/stereovideo.cpp
index 9e2de4615..5a1ecfdb6 100644
--- a/components/rgbd-sources/src/stereovideo.cpp
+++ b/components/rgbd-sources/src/stereovideo.cpp
@@ -217,16 +217,16 @@ bool StereoVideoSource::compute(int n, int b) {
if (left.empty() || right.empty()) return false;
if (chan == ftl::rgbd::kChanDepth) {
+ disp_->compute(frame, stream_);
+
+ auto &disp = frame.getChannel<cv::cuda::GpuMat>(ftl::rgbd::kChanDisparity);
auto &depth = frame.setChannel<cv::cuda::GpuMat>(ftl::rgbd::kChanDepth);
- auto &disp = frame.setChannel<cv::cuda::GpuMat>(ftl::rgbd::kChanDisparity);
-
if (depth.empty()) depth = cv::cuda::GpuMat(left.size(), CV_32FC1);
- if (disp.empty()) disp = cv::cuda::GpuMat(left.size(), CV_32FC1);
- disp_->compute(left, right, disp, stream_);
+
ftl::cuda::disparity_to_depth(disp, depth, params_, stream_);
+
left.download(rgb_, stream_);
depth.download(depth_, stream_);
-
stream_.waitForCompletion(); // TODO:(Nick) Move to getFrames
} else if (chan == ftl::rgbd::kChanRight) {
left.download(rgb_, stream_);
--
GitLab