From 0c1e5ad18823c8ae429902dd48fdbad1d10f3718 Mon Sep 17 00:00:00 2001
From: Nicolas Pope <nwpope@utu.fi>
Date: Sun, 14 Jun 2020 13:49:02 +0300
Subject: [PATCH] WIP refactor to use new frame class
---
.../codecs/include/ftl/codecs/channels.hpp | 1 +
.../include/ftl/operators/smoothing.hpp | 12 +-
.../src/disparity/bilateral_filter.cpp | 2 +-
.../operators/src/disparity/libstereo.cpp | 2 +-
components/operators/src/fusion/mvmls.cpp | 22 +-
components/operators/src/mask.cpp | 2 +-
components/operators/src/operator.cpp | 2 +-
components/operators/src/smoothing.cpp | 60 ++--
.../cpp/include/ftl/render/CUDARender.hpp | 3 +-
.../cpp/include/ftl/render/overlay.hpp | 2 +-
components/renderers/cpp/src/CUDARender.cpp | 49 +--
components/renderers/cpp/src/colouriser.cpp | 2 +-
components/renderers/cpp/src/overlay.cpp | 31 +-
components/rgbd-sources/CMakeLists.txt | 6 +-
.../rgbd-sources/include/ftl/rgbd/frame.hpp | 287 +++++-------------
.../include/ftl/rgbd/frameset.hpp | 173 +----------
.../rgbd-sources/include/ftl/rgbd/source.hpp | 18 +-
components/rgbd-sources/src/frame.cpp | 204 ++-----------
components/rgbd-sources/src/source.cpp | 20 +-
.../structures/include/ftl/data/new_frame.hpp | 23 +-
components/structures/src/new_frame.cpp | 21 ++
21 files changed, 272 insertions(+), 670 deletions(-)
diff --git a/components/codecs/include/ftl/codecs/channels.hpp b/components/codecs/include/ftl/codecs/channels.hpp
index 43ef166a2..d2c65328f 100644
--- a/components/codecs/include/ftl/codecs/channels.hpp
+++ b/components/codecs/include/ftl/codecs/channels.hpp
@@ -53,6 +53,7 @@ enum struct Channel : int {
Index = 68,
Control = 69, // For stream and encoder control
Settings3 = 70,
+ Name = 71,
Data = 2048, // Custom data, any codec.
Faces = 2049, // Data about detected faces
diff --git a/components/operators/include/ftl/operators/smoothing.hpp b/components/operators/include/ftl/operators/smoothing.hpp
index 0dc463d2e..c42f801ed 100644
--- a/components/operators/include/ftl/operators/smoothing.hpp
+++ b/components/operators/include/ftl/operators/smoothing.hpp
@@ -23,7 +23,7 @@ class HFSmoother : public ftl::operators::Operator {
private:
cv::cuda::GpuMat temp_;
- ftl::rgbd::Frame frames_[4];
+ //ftl::rgbd::Frame frames_[4];
};
/**
@@ -43,7 +43,7 @@ class SmoothChannel : public ftl::operators::Operator {
bool apply(ftl::rgbd::Frame &in, ftl::rgbd::Frame &out, cudaStream_t stream) override;
private:
- ftl::rgbd::Frame temp_[6];
+ ftl::cuda::TextureObject<uchar4> temp_[6];
};
/**
@@ -61,7 +61,7 @@ class SimpleMLS : public ftl::operators::Operator {
bool apply(ftl::rgbd::Frame &in, ftl::rgbd::Frame &out, cudaStream_t stream) override;
private:
- ftl::rgbd::Frame temp_;
+ ftl::data::Frame temp_;
};
/**
@@ -78,7 +78,7 @@ class ColourMLS : public ftl::operators::Operator {
bool apply(ftl::rgbd::Frame &in, ftl::rgbd::Frame &out, cudaStream_t stream) override;
private:
- ftl::rgbd::Frame temp_;
+ ftl::data::Frame temp_;
};
/**
@@ -125,7 +125,7 @@ class AggreMLS : public ftl::operators::Operator {
ftl::cuda::TextureObject<float4> centroid_vert_;
ftl::cuda::TextureObject<half4> normals_horiz_;
- ftl::rgbd::Frame temp_;
+ ftl::data::Frame temp_;
};
/**
@@ -145,7 +145,7 @@ class AdaptiveMLS : public ftl::operators::Operator {
bool apply(ftl::rgbd::Frame &in, ftl::rgbd::Frame &out, cudaStream_t stream) override;
private:
- ftl::rgbd::Frame temp_;
+ ftl::data::Frame temp_;
};
}
diff --git a/components/operators/src/disparity/bilateral_filter.cpp b/components/operators/src/disparity/bilateral_filter.cpp
index 4b14c3bb5..7ffce560c 100644
--- a/components/operators/src/disparity/bilateral_filter.cpp
+++ b/components/operators/src/disparity/bilateral_filter.cpp
@@ -56,7 +56,7 @@ bool DisparityBilateralFilter::apply(ftl::rgbd::Frame &in, ftl::rgbd::Frame &out
auto cvstream = cv::cuda::StreamAccessor::wrapStream(stream);
const GpuMat &rgb = in.get<GpuMat>(Channel::Colour);
- GpuMat &disp_in = in.get<GpuMat>(Channel::Disparity);
+ const GpuMat &disp_in = in.get<GpuMat>(Channel::Disparity);
GpuMat &disp_out = out.create<GpuMat>(Channel::Disparity);
disp_int_.create(disp_in.size(), disp_in.type());
diff --git a/components/operators/src/disparity/libstereo.cpp b/components/operators/src/disparity/libstereo.cpp
index 22aaac630..8531dd98f 100644
--- a/components/operators/src/disparity/libstereo.cpp
+++ b/components/operators/src/disparity/libstereo.cpp
@@ -56,7 +56,7 @@ bool StereoDisparity::apply(Frame &in, Frame &out, cudaStream_t stream) {
disp32f_.create(l.size(), CV_32FC1);
bool has_estimate = in.hasChannel(Channel::Disparity);
- auto &disp = (!has_estimate) ? out.create<GpuMat>(Channel::Disparity, Format<short>(l.size())) : in.get<GpuMat>(Channel::Disparity);
+ auto &disp = (!has_estimate) ? out.create<ftl::rgbd::VideoFrame>(Channel::Disparity).createGPU(Format<short>(l.size())) : in.get<GpuMat>(Channel::Disparity);
auto cvstream = cv::cuda::StreamAccessor::wrapStream(stream);
diff --git a/components/operators/src/fusion/mvmls.cpp b/components/operators/src/fusion/mvmls.cpp
index 0dfe26b14..a444d8d1a 100644
--- a/components/operators/src/fusion/mvmls.cpp
+++ b/components/operators/src/fusion/mvmls.cpp
@@ -62,7 +62,7 @@ bool MultiViewMLS::apply(ftl::rgbd::FrameSet &in, ftl::rgbd::FrameSet &out, cuda
for (size_t i=0; i<in.frames.size(); ++i) {
if (!in.hasFrame(i)) continue;
- auto &f = in.frames[i];
+ auto &f = in.frames[i].cast<ftl::rgbd::Frame>();
auto size = f.get<GpuMat>(Channel::Depth).size();
centroid_horiz_[i]->create(size.height, size.width);
normals_horiz_[i]->create(size.height, size.width);
@@ -77,12 +77,12 @@ bool MultiViewMLS::apply(ftl::rgbd::FrameSet &in, ftl::rgbd::FrameSet &out, cuda
}
// Create required channels
- f.create<GpuMat>(Channel::Confidence, Format<float>(size));
+ f.create<ftl::rgbd::VideoFrame>(Channel::Confidence).createGPU(Format<float>(size));
f.createTexture<float>(Channel::Confidence);
- f.create<GpuMat>(Channel::Screen, Format<short2>(size));
+ f.create<ftl::rgbd::VideoFrame>(Channel::Screen).createGPU(Format<short2>(size));
f.createTexture<short2>(Channel::Screen);
- f.get<GpuMat>(Channel::Confidence).setTo(cv::Scalar(0.0f), cvstream);
+ f.set<GpuMat>(Channel::Confidence).setTo(cv::Scalar(0.0f), cvstream);
}
//for (int iter=0; iter<iters; ++iter) {
@@ -95,7 +95,7 @@ bool MultiViewMLS::apply(ftl::rgbd::FrameSet &in, ftl::rgbd::FrameSet &out, cuda
for (size_t i=0; i<in.frames.size(); ++i) {
if (!in.hasFrame(i)) continue;
- auto &f1 = in.frames[i];
+ auto &f1 = in.frames[i].cast<ftl::rgbd::Frame>();
//f1.get<GpuMat>(Channel::Depth2).setTo(cv::Scalar(0.0f), cvstream);
//f1.get<GpuMat>(Channel::Confidence).setTo(cv::Scalar(0.0f), cvstream);
@@ -108,7 +108,7 @@ bool MultiViewMLS::apply(ftl::rgbd::FrameSet &in, ftl::rgbd::FrameSet &out, cuda
//LOG(INFO) << "Running phase1";
- auto &f2 = in.frames[j];
+ auto &f2 = in.frames[j].cast<ftl::rgbd::Frame>();
//auto s1 = in.sources[i];
//auto s2 = in.sources[j];
@@ -345,7 +345,7 @@ bool MultiViewMLS::apply(ftl::rgbd::FrameSet &in, ftl::rgbd::FrameSet &out, cuda
// Redo normals
for (size_t i=0; i<in.frames.size(); ++i) {
if (!in.hasFrame(i)) continue;
- auto &f = in.frames[i];
+ auto &f = in.frames[i].cast<ftl::rgbd::Frame>();
ftl::cuda::normals(
f.getTexture<half4>(Channel::Normals),
f.getTexture<float>(Channel::Depth),
@@ -411,7 +411,7 @@ bool MultiViewMLS::apply(ftl::rgbd::FrameSet &in, ftl::rgbd::FrameSet &out, cuda
for (size_t i=0; i<in.frames.size(); ++i) {
if (!in.hasFrame(i)) continue;
- auto &f = in.frames[i];
+ auto &f = in.frames[i].cast<ftl::rgbd::Frame>();
//auto *s = in.sources[i];
// Clear data
@@ -465,7 +465,7 @@ bool MultiViewMLS::apply(ftl::rgbd::FrameSet &in, ftl::rgbd::FrameSet &out, cuda
if (do_aggr) {
for (size_t i=0; i<in.frames.size(); ++i) {
if (!in.hasFrame(i)) continue;
- auto &f1 = in.frames[i];
+ auto &f1 = in.frames[i].cast<ftl::rgbd::Frame>();
//f1.get<GpuMat>(Channel::Depth2).setTo(cv::Scalar(0.0f), cvstream);
//f1.get<GpuMat>(Channel::Confidence).setTo(cv::Scalar(0.0f), cvstream);
@@ -478,7 +478,7 @@ bool MultiViewMLS::apply(ftl::rgbd::FrameSet &in, ftl::rgbd::FrameSet &out, cuda
//LOG(INFO) << "Running phase1";
- auto &f2 = in.frames[j];
+ auto &f2 = in.frames[j].cast<ftl::rgbd::Frame>();
//auto s1 = in.sources[i];
//auto s2 = in.sources[j];
@@ -520,7 +520,7 @@ bool MultiViewMLS::apply(ftl::rgbd::FrameSet &in, ftl::rgbd::FrameSet &out, cuda
// Normalise aggregations and move the points
for (size_t i=0; i<in.frames.size(); ++i) {
if (!in.hasFrame(i)) continue;
- auto &f = in.frames[i];
+ auto &f = in.frames[i].cast<ftl::rgbd::Frame>();
//auto *s = in.sources[i];
auto size = f.get<GpuMat>(Channel::Depth).size();
diff --git a/components/operators/src/mask.cpp b/components/operators/src/mask.cpp
index e39fcabea..c9f1dad53 100644
--- a/components/operators/src/mask.cpp
+++ b/components/operators/src/mask.cpp
@@ -99,7 +99,7 @@ bool CullDiscontinuity::apply(ftl::rgbd::Frame &in, ftl::rgbd::Frame &out, cudaS
unsigned int radius = config()->value("radius", 0);
bool inverted = config()->value("invert", false);
- out.clearPackets(Channel::Depth); // Force reset
+ out.set<ftl::rgbd::VideoFrame>(Channel::Depth); // Force reset
ftl::cuda::cull_mask(
in.createTexture<uint8_t>(Channel::Mask),
out.createTexture<float>(Channel::Depth),
diff --git a/components/operators/src/operator.cpp b/components/operators/src/operator.cpp
index 617514afd..2baf2f41b 100644
--- a/components/operators/src/operator.cpp
+++ b/components/operators/src/operator.cpp
@@ -71,7 +71,7 @@ bool Graph::apply(FrameSet &in, FrameSet &out, cudaStream_t stream) {
if (instance->enabled()) {
try {
- instance->apply(in.frames[j], out.frames[j], stream_actual);
+ instance->apply(in.frames[j].cast<ftl::rgbd::Frame>(), out.frames[j].cast<ftl::rgbd::Frame>(), stream_actual);
} catch (const std::exception &e) {
LOG(ERROR) << "Operator exception for '" << instance->config()->getID() << "': " << e.what();
success = false;
diff --git a/components/operators/src/smoothing.cpp b/components/operators/src/smoothing.cpp
index 739162622..0ca343057 100644
--- a/components/operators/src/smoothing.cpp
+++ b/components/operators/src/smoothing.cpp
@@ -89,7 +89,7 @@ bool SmoothChannel::apply(ftl::rgbd::Frame &in, ftl::rgbd::Frame &out, cudaStrea
float scale = 1.0f;
// Clear to max smoothing
- out.create<GpuMat>(Channel::Smoothing, Format<float>(width, height)).setTo(cv::Scalar(1.0f));
+ out.create<ftl::rgbd::VideoFrame>(Channel::Smoothing).createGPU(Format<float>(width, height)).setTo(cv::Scalar(1.0f));
// Reduce to nearest
ftl::cuda::smooth_channel(
@@ -108,14 +108,16 @@ bool SmoothChannel::apply(ftl::rgbd::Frame &in, ftl::rgbd::Frame &out, cudaStrea
height /= 2;
scale *= 2.0f;
+ temp_[i].create(width,height);
+
ftl::rgbd::Camera scaledCam = in.getLeftCamera().scaled(width, height);
// Downscale images for next pass
- cv::cuda::resize(in.get<GpuMat>(Channel::Colour), temp_[i].create<GpuMat>(Channel::Colour), cv::Size(width, height), 0.0, 0.0, cv::INTER_LINEAR);
+ cv::cuda::resize(in.get<GpuMat>(Channel::Colour), temp_[i].to_gpumat(), cv::Size(width, height), 0.0, 0.0, cv::INTER_LINEAR);
//cv::cuda::resize(in.get<GpuMat>(Channel::Depth), temp_[i].create<GpuMat>(Channel::Depth), cv::Size(width, height), 0.0, 0.0, cv::INTER_NEAREST);
ftl::cuda::smooth_channel(
- temp_[i].createTexture<uchar4>(Channel::Colour),
+ temp_[i],
//temp_[i].createTexture<float>(Channel::Depth),
out.getTexture<float>(Channel::Smoothing),
scaledCam,
@@ -133,7 +135,7 @@ bool SmoothChannel::apply(ftl::rgbd::Frame &in, ftl::rgbd::Frame &out, cudaStrea
// ===== MLS ===================================================================
-SimpleMLS::SimpleMLS(ftl::Configurable *cfg) : ftl::operators::Operator(cfg) {
+SimpleMLS::SimpleMLS(ftl::Configurable *cfg) : ftl::operators::Operator(cfg), temp_(ftl::data::Frame::make_standalone()) {
}
@@ -146,6 +148,8 @@ bool SimpleMLS::apply(ftl::rgbd::Frame &in, ftl::rgbd::Frame &out, cudaStream_t
int iters = config()->value("mls_iterations", 1);
int radius = config()->value("mls_radius",2);
+ auto &temp = temp_.cast<ftl::rgbd::Frame>();
+
if (!in.hasChannel(Channel::Normals)) {
/*ftl::cuda::normals(
in.createTexture<float4>(Channel::Normals, ftl::rgbd::Format<float4>(in.get<cv::cuda::GpuMat>(Channel::Depth).size())),
@@ -160,9 +164,9 @@ bool SimpleMLS::apply(ftl::rgbd::Frame &in, ftl::rgbd::Frame &out, cudaStream_t
for (int i=0; i<iters; ++i) {
ftl::cuda::mls_smooth(
in.createTexture<half4>(Channel::Normals),
- temp_.createTexture<half4>(Channel::Normals, ftl::rgbd::Format<half4>(in.get<cv::cuda::GpuMat>(Channel::Depth).size())),
+ temp.createTexture<half4>(Channel::Normals, ftl::rgbd::Format<half4>(in.get<cv::cuda::GpuMat>(Channel::Depth).size())),
in.createTexture<float>(Channel::Depth),
- temp_.createTexture<float>(Channel::Depth, ftl::rgbd::Format<float>(in.get<cv::cuda::GpuMat>(Channel::Depth).size())),
+ temp.createTexture<float>(Channel::Depth, ftl::rgbd::Format<float>(in.get<cv::cuda::GpuMat>(Channel::Depth).size())),
thresh,
radius,
in.getLeftCamera(),
@@ -171,7 +175,8 @@ bool SimpleMLS::apply(ftl::rgbd::Frame &in, ftl::rgbd::Frame &out, cudaStream_t
//in.swapChannels(Channel::Depth, Channel::Depth2);
//in.swapChannels(Channel::Normals, Channel::Points);
- temp_.swapChannels(Channel::Normals + Channel::Depth, in);
+ temp.swapChannel(Channel::Normals, in);
+ temp.swapChannel(Channel::Depth, in);
}
return true;
@@ -179,7 +184,7 @@ bool SimpleMLS::apply(ftl::rgbd::Frame &in, ftl::rgbd::Frame &out, cudaStream_t
-ColourMLS::ColourMLS(ftl::Configurable *cfg) : ftl::operators::Operator(cfg) {
+ColourMLS::ColourMLS(ftl::Configurable *cfg) : ftl::operators::Operator(cfg), temp_(ftl::data::Frame::make_standalone()) {
}
@@ -200,14 +205,16 @@ bool ColourMLS::apply(ftl::rgbd::Frame &in, ftl::rgbd::Frame &out, cudaStream_t
return false;
}
+ auto &temp = temp_.cast<ftl::rgbd::Frame>();
+
// FIXME: Assume in and out are the same frame.
for (int i=0; i<iters; ++i) {
if (!crosssup) {
ftl::cuda::colour_mls_smooth(
in.createTexture<half4>(Channel::Normals),
- temp_.createTexture<half4>(Channel::Normals, ftl::rgbd::Format<half4>(in.get<cv::cuda::GpuMat>(Channel::Depth).size())),
+ temp.createTexture<half4>(Channel::Normals, ftl::rgbd::Format<half4>(in.get<cv::cuda::GpuMat>(Channel::Depth).size())),
in.createTexture<float>(Channel::Depth),
- temp_.createTexture<float>(Channel::Depth, ftl::rgbd::Format<float>(in.get<cv::cuda::GpuMat>(Channel::Depth).size())),
+ temp.createTexture<float>(Channel::Depth, ftl::rgbd::Format<float>(in.get<cv::cuda::GpuMat>(Channel::Depth).size())),
in.createTexture<uchar4>(Channel::Colour),
thresh,
col_smooth,
@@ -219,9 +226,9 @@ bool ColourMLS::apply(ftl::rgbd::Frame &in, ftl::rgbd::Frame &out, cudaStream_t
ftl::cuda::colour_mls_smooth_csr(
in.createTexture<uchar4>(Channel::Support1),
in.createTexture<half4>(Channel::Normals),
- temp_.createTexture<half4>(Channel::Normals, ftl::rgbd::Format<half4>(in.get<cv::cuda::GpuMat>(Channel::Depth).size())),
+ temp.createTexture<half4>(Channel::Normals, ftl::rgbd::Format<half4>(in.get<cv::cuda::GpuMat>(Channel::Depth).size())),
in.createTexture<float>(Channel::Depth),
- temp_.createTexture<float>(Channel::Depth, ftl::rgbd::Format<float>(in.get<cv::cuda::GpuMat>(Channel::Depth).size())),
+ temp.createTexture<float>(Channel::Depth, ftl::rgbd::Format<float>(in.get<cv::cuda::GpuMat>(Channel::Depth).size())),
in.createTexture<uchar4>(Channel::Colour),
thresh,
col_smooth,
@@ -233,7 +240,8 @@ bool ColourMLS::apply(ftl::rgbd::Frame &in, ftl::rgbd::Frame &out, cudaStream_t
//in.swapChannels(Channel::Depth, Channel::Depth2);
//in.swapChannels(Channel::Normals, Channel::Points);
- temp_.swapChannels(Channel::Normals + Channel::Depth, in);
+ temp_.swapChannel(Channel::Depth, in);
+ temp_.swapChannel(Channel::Normals, in);
}
return true;
@@ -242,7 +250,7 @@ bool ColourMLS::apply(ftl::rgbd::Frame &in, ftl::rgbd::Frame &out, cudaStream_t
// ====== Aggregating MLS ======================================================
-AggreMLS::AggreMLS(ftl::Configurable *cfg) : ftl::operators::Operator(cfg) {
+AggreMLS::AggreMLS(ftl::Configurable *cfg) : ftl::operators::Operator(cfg), temp_(ftl::data::Frame::make_standalone()) {
}
@@ -267,6 +275,8 @@ bool AggreMLS::apply(ftl::rgbd::Frame &in, ftl::rgbd::Frame &out, cudaStream_t s
return false;
}
+ auto &temp = temp_.cast<ftl::rgbd::Frame>();
+
auto size = in.get<GpuMat>(Channel::Depth).size();
centroid_horiz_.create(size.height, size.width);
normals_horiz_.create(size.height, size.width);
@@ -306,7 +316,7 @@ bool AggreMLS::apply(ftl::rgbd::Frame &in, ftl::rgbd::Frame &out, cudaStream_t s
ftl::cuda::mls_adjust_depth(
in.createTexture<half4>(Channel::Normals),
centroid_vert_,
- temp_.createTexture<float>(Channel::Depth, ftl::rgbd::Format<float>(size)),
+ temp.createTexture<float>(Channel::Depth, ftl::rgbd::Format<float>(size)),
in.getTexture<float>(Channel::Depth),
in.getLeftCamera(),
stream
@@ -314,15 +324,15 @@ bool AggreMLS::apply(ftl::rgbd::Frame &in, ftl::rgbd::Frame &out, cudaStream_t s
//in.swapChannels(Channel::Depth, Channel::Depth2);
//in.swapChannels(Channel::Normals, Channel::Points);
- temp_.swapChannels(ftl::codecs::Channels<0>(Channel::Depth), in);
+ temp_.swapChannel(Channel::Depth, in);
} else {
ftl::cuda::colour_mls_smooth_csr(
in.createTexture<uchar4>(Channel::Support1),
in.createTexture<half4>(Channel::Normals),
- temp_.createTexture<half4>(Channel::Normals, ftl::rgbd::Format<half4>(in.get<cv::cuda::GpuMat>(Channel::Depth).size())),
+ temp.createTexture<half4>(Channel::Normals, ftl::rgbd::Format<half4>(in.get<cv::cuda::GpuMat>(Channel::Depth).size())),
in.createTexture<float>(Channel::Depth),
- temp_.createTexture<float>(Channel::Depth, ftl::rgbd::Format<float>(in.get<cv::cuda::GpuMat>(Channel::Depth).size())),
+ temp.createTexture<float>(Channel::Depth, ftl::rgbd::Format<float>(in.get<cv::cuda::GpuMat>(Channel::Depth).size())),
in.createTexture<uchar4>(Channel::Colour),
thresh,
col_smooth,
@@ -331,7 +341,8 @@ bool AggreMLS::apply(ftl::rgbd::Frame &in, ftl::rgbd::Frame &out, cudaStream_t s
stream
);
- temp_.swapChannels(Channel::Normals + Channel::Depth, in);
+ temp_.swapChannel(Channel::Depth, in);
+ temp_.swapChannel(Channel::Normals, in);
}
}
@@ -341,7 +352,7 @@ bool AggreMLS::apply(ftl::rgbd::Frame &in, ftl::rgbd::Frame &out, cudaStream_t s
// ====== Adaptive MLS =========================================================
-AdaptiveMLS::AdaptiveMLS(ftl::Configurable *cfg) : ftl::operators::Operator(cfg) {
+AdaptiveMLS::AdaptiveMLS(ftl::Configurable *cfg) : ftl::operators::Operator(cfg), temp_(ftl::data::Frame::make_standalone()) {
}
@@ -358,20 +369,23 @@ bool AdaptiveMLS::apply(ftl::rgbd::Frame &in, ftl::rgbd::Frame &out, cudaStream_
return false;
}
+ auto &temp = temp_.cast<ftl::rgbd::Frame>();
+
// FIXME: Assume in and out are the same frame.
for (int i=0; i<iters; ++i) {
ftl::cuda::adaptive_mls_smooth(
in.createTexture<half4>(Channel::Normals),
- temp_.createTexture<half4>(Channel::Normals, ftl::rgbd::Format<half4>(in.get<cv::cuda::GpuMat>(Channel::Depth).size())),
+ temp.createTexture<half4>(Channel::Normals, ftl::rgbd::Format<half4>(in.get<cv::cuda::GpuMat>(Channel::Depth).size())),
in.createTexture<float>(Channel::Depth),
- temp_.createTexture<float>(Channel::Depth, ftl::rgbd::Format<float>(in.get<cv::cuda::GpuMat>(Channel::Depth).size())),
+ temp.createTexture<float>(Channel::Depth, ftl::rgbd::Format<float>(in.get<cv::cuda::GpuMat>(Channel::Depth).size())),
in.createTexture<float>(Channel::Smoothing),
radius,
in.getLeftCamera(),
stream
);
- temp_.swapChannels(Channel::Normals + Channel::Depth, in);
+ temp_.swapChannel(Channel::Depth, in);
+ temp_.swapChannel(Channel::Normals, in);
}
diff --git a/components/renderers/cpp/include/ftl/render/CUDARender.hpp b/components/renderers/cpp/include/ftl/render/CUDARender.hpp
index cfdb4cf40..8743137c2 100644
--- a/components/renderers/cpp/include/ftl/render/CUDARender.hpp
+++ b/components/renderers/cpp/include/ftl/render/CUDARender.hpp
@@ -44,7 +44,8 @@ class CUDARender : public ftl::render::FSRenderer {
private:
int device_;
- ftl::rgbd::Frame temp_;
+ ftl::data::Frame temp_d_;
+ ftl::rgbd::Frame &temp_;
//ftl::rgbd::Frame accum_;
ftl::cuda::TextureObject<float4> accum_; // 2 is number of channels can render together
ftl::cuda::TextureObject<int> contrib_;
diff --git a/components/renderers/cpp/include/ftl/render/overlay.hpp b/components/renderers/cpp/include/ftl/render/overlay.hpp
index be4f36d31..1bb8d7703 100644
--- a/components/renderers/cpp/include/ftl/render/overlay.hpp
+++ b/components/renderers/cpp/include/ftl/render/overlay.hpp
@@ -24,7 +24,7 @@ class Overlay : public ftl::Configurable {
//void apply(ftl::rgbd::FrameSet &fs, cv::Mat &out, ftl::rgbd::FrameState &state);
- void draw(ftl::rgbd::FrameSet &fs, ftl::rgbd::FrameState &state, const Eigen::Vector2f &);
+ void draw(ftl::data::FrameSet &fs, ftl::rgbd::Frame &frame, const Eigen::Vector2f &);
private:
nanogui::GLShader oShader;
diff --git a/components/renderers/cpp/src/CUDARender.cpp b/components/renderers/cpp/src/CUDARender.cpp
index 03ea37f63..07cc666fd 100644
--- a/components/renderers/cpp/src/CUDARender.cpp
+++ b/components/renderers/cpp/src/CUDARender.cpp
@@ -25,13 +25,14 @@ using ftl::render::CUDARender;
using ftl::codecs::Channel;
using ftl::codecs::Channels;
using ftl::rgbd::Format;
+using ftl::rgbd::VideoFrame;
using cv::cuda::GpuMat;
using std::stoul;
using ftl::cuda::Mask;
using ftl::render::parseCUDAColour;
using ftl::render::parseCVColour;
-CUDARender::CUDARender(nlohmann::json &config) : ftl::render::FSRenderer(config), scene_(nullptr) {
+CUDARender::CUDARender(nlohmann::json &config) : ftl::render::FSRenderer(config), temp_d_(ftl::data::Frame::make_standalone()), temp_(temp_d_.cast<ftl::rgbd::Frame>()), scene_(nullptr) {
/*if (config["clipping"].is_object()) {
auto &c = config["clipping"];
float rx = c.value("pitch", 0.0f);
@@ -111,7 +112,7 @@ void CUDARender::_renderChannel(ftl::rgbd::Frame &output, ftl::codecs::Channel i
for (size_t i=0; i < scene_->frames.size(); ++i) {
if (!scene_->hasFrame(i)) continue;
- auto &f = scene_->frames[i];
+ auto &f = scene_->frames[i].cast<ftl::rgbd::Frame>();
if (!f.hasChannel(in)) {
LOG(ERROR) << "Reprojecting unavailable channel";
@@ -169,14 +170,14 @@ void CUDARender::_renderChannel(ftl::rgbd::Frame &output, ftl::codecs::Channel i
void CUDARender::_dibr(ftl::rgbd::Frame &out, const Eigen::Matrix4d &t, cudaStream_t stream) {
cv::cuda::Stream cvstream = cv::cuda::StreamAccessor::wrapStream(stream);
- temp_.get<GpuMat>(Channel::Depth2).setTo(cv::Scalar(0x7FFFFFFF), cvstream);
+ temp_.set<GpuMat>(Channel::Depth2).setTo(cv::Scalar(0x7FFFFFFF), cvstream);
for (size_t i=0; i < scene_->frames.size(); ++i) {
if (!scene_->hasFrame(i)) continue;
- auto &f = scene_->frames[i];
+ auto &f = scene_->frames[i].cast<ftl::rgbd::Frame>();
//auto *s = scene_->sources[i];
- if (f.empty(Channel::Colour)) {
+ if (f.has(Channel::Colour)) {
LOG(ERROR) << "Missing required channel";
continue;
}
@@ -233,19 +234,19 @@ void CUDARender::_mesh(ftl::rgbd::Frame &out, const Eigen::Matrix4d &t, cudaStre
bool do_blend = value("mesh_blend", false);
float blend_alpha = value("blend_alpha", 0.02f);
if (do_blend) {
- temp_.get<GpuMat>(Channel::Depth).setTo(cv::Scalar(0x7FFFFFFF), cvstream);
- temp_.get<GpuMat>(Channel::Weights).setTo(cv::Scalar(0.0f), cvstream);
+ temp_.set<GpuMat>(Channel::Depth).setTo(cv::Scalar(0x7FFFFFFF), cvstream);
+ temp_.set<GpuMat>(Channel::Weights).setTo(cv::Scalar(0.0f), cvstream);
} else {
- temp_.get<GpuMat>(Channel::Depth2).setTo(cv::Scalar(0x7FFFFFFF), cvstream);
+ temp_.set<GpuMat>(Channel::Depth2).setTo(cv::Scalar(0x7FFFFFFF), cvstream);
}
// For each source depth map
for (size_t i=0; i < scene_->frames.size(); ++i) {
if (!scene_->hasFrame(i)) continue;
- auto &f = scene_->frames[i];
+ auto &f = scene_->frames[i].cast<ftl::rgbd::Frame>();
//auto *s = scene_->sources[i];
- if (f.empty(Channel::Colour)) {
+ if (f.has(Channel::Colour)) {
LOG(ERROR) << "Missing required channel";
continue;
}
@@ -285,7 +286,7 @@ void CUDARender::_mesh(ftl::rgbd::Frame &out, const Eigen::Matrix4d &t, cudaStre
// Must reset depth channel if blending
if (do_blend) {
- temp_.get<GpuMat>(Channel::Depth).setTo(cv::Scalar(0x7FFFFFFF), cvstream);
+ temp_.set<GpuMat>(Channel::Depth).setTo(cv::Scalar(0x7FFFFFFF), cvstream);
}
// Decide on and render triangles around each point
@@ -347,27 +348,27 @@ void CUDARender::_allocateChannels(ftl::rgbd::Frame &out, ftl::codecs::Channel c
// Allocate left channel buffers and clear them
if (chan == Channel::Colour) {
//if (!out.hasChannel(Channel::Depth)) {
- out.create<GpuMat>(Channel::Depth, Format<float>(camera.width, camera.height));
- out.create<GpuMat>(Channel::Colour, Format<uchar4>(camera.width, camera.height));
- out.create<GpuMat>(Channel::Normals, Format<half4>(camera.width, camera.height));
+ out.create<VideoFrame>(Channel::Depth).createGPU(Format<float>(camera.width, camera.height));
+ out.create<VideoFrame>(Channel::Colour).createGPU(Format<uchar4>(camera.width, camera.height));
+ out.create<VideoFrame>(Channel::Normals).createGPU(Format<half4>(camera.width, camera.height));
out.createTexture<uchar4>(Channel::Colour, true); // Force interpolated colour
- out.get<GpuMat>(Channel::Depth).setTo(cv::Scalar(1000.0f), cvstream);
+ out.set<GpuMat>(Channel::Depth).setTo(cv::Scalar(1000.0f), cvstream);
//}
// Allocate right channel buffers and clear them
} else {
if (!out.hasChannel(Channel::Depth2)) {
- out.create<GpuMat>(Channel::Depth2, Format<float>(camera.width, camera.height));
- out.create<GpuMat>(Channel::Colour2, Format<uchar4>(camera.width, camera.height));
- out.create<GpuMat>(Channel::Normals2, Format<half4>(camera.width, camera.height));
+ out.create<VideoFrame>(Channel::Depth2).createGPU(Format<float>(camera.width, camera.height));
+ out.create<VideoFrame>(Channel::Colour2).createGPU(Format<uchar4>(camera.width, camera.height));
+ out.create<VideoFrame>(Channel::Normals2).createGPU(Format<half4>(camera.width, camera.height));
out.createTexture<uchar4>(Channel::Colour2, true); // Force interpolated colour
- out.get<GpuMat>(Channel::Depth2).setTo(cv::Scalar(1000.0f), cvstream);
+ out.set<GpuMat>(Channel::Depth2).setTo(cv::Scalar(1000.0f), cvstream);
}
}
- temp_.create<GpuMat>(Channel::Depth, Format<int>(camera.width, camera.height));
- temp_.create<GpuMat>(Channel::Depth2, Format<int>(camera.width, camera.height));
- temp_.create<GpuMat>(Channel::Normals, Format<half4>(camera.width, camera.height));
- temp_.create<GpuMat>(Channel::Weights, Format<float>(camera.width, camera.height));
+ temp_.create<VideoFrame>(Channel::Depth).createGPU(Format<int>(camera.width, camera.height));
+ temp_.create<VideoFrame>(Channel::Depth2).createGPU(Format<int>(camera.width, camera.height));
+ temp_.create<VideoFrame>(Channel::Normals).createGPU(Format<half4>(camera.width, camera.height));
+ temp_.create<VideoFrame>(Channel::Weights).createGPU(Format<float>(camera.width, camera.height));
temp_.createTexture<int>(Channel::Depth);
accum_.create(camera.width, camera.height);
@@ -492,7 +493,7 @@ void CUDARender::begin(ftl::rgbd::Frame &out, ftl::codecs::Channel chan) {
// Apply a colour background
if (env_image_.empty() || !value("environment_enabled", false)) {
- out.get<GpuMat>(chan).setTo(background_, cvstream);
+ out.set<GpuMat>(chan).setTo(background_, cvstream);
} else {
auto pose = poseInverse_.getFloat3x3();
ftl::cuda::equirectangular_reproject(
diff --git a/components/renderers/cpp/src/colouriser.cpp b/components/renderers/cpp/src/colouriser.cpp
index 6341f1a86..c4dfbc19c 100644
--- a/components/renderers/cpp/src/colouriser.cpp
+++ b/components/renderers/cpp/src/colouriser.cpp
@@ -192,7 +192,7 @@ TextureObject<uchar4> &Colouriser::_processColour(ftl::rgbd::Frame &f, Channel c
auto &buf = _getBuffer(size.width, size.height);
if (colour_sources) {
- auto colour = HSVtoRGB(360 / 8 * f.id, 0.6, 0.85);
+ auto colour = HSVtoRGB(360 / 8 * f.source(), 0.6, 0.85);
buf.to_gpumat().setTo(colour, cvstream);
}
diff --git a/components/renderers/cpp/src/overlay.cpp b/components/renderers/cpp/src/overlay.cpp
index 7054c275a..1513bc36f 100644
--- a/components/renderers/cpp/src/overlay.cpp
+++ b/components/renderers/cpp/src/overlay.cpp
@@ -241,11 +241,11 @@ void Overlay::_drawAxis(const Eigen::Matrix4d &pose, const Eigen::Vector3f &scal
(const void *)(loffset * sizeof(uint32_t)));
}
-void Overlay::draw(ftl::rgbd::FrameSet &fs, ftl::rgbd::FrameState &state, const Eigen::Vector2f &screenSize) {
+void Overlay::draw(ftl::data::FrameSet &fs, ftl::rgbd::Frame &frame, const Eigen::Vector2f &screenSize) {
if (!value("enabled", false)) return;
double zfar = 8.0f;
- auto intrin = state.getLeft();
+ auto intrin = frame.getLeft();
intrin = intrin.scaled(screenSize[0], screenSize[1]);
if (!init_) {
@@ -263,7 +263,7 @@ void Overlay::draw(ftl::rgbd::FrameSet &fs, ftl::rgbd::FrameState &state, const
{0.8f, -0.4f, 2.0f}
};
- auto pose = MatrixConversion::toCUDA(state.getPose().cast<float>().inverse());
+ auto pose = MatrixConversion::toCUDA(frame.getPose().cast<float>().inverse());
tris[0] = pose * tris[0];
tris[1] = pose * tris[1];
@@ -295,11 +295,13 @@ void Overlay::draw(ftl::rgbd::FrameSet &fs, ftl::rgbd::FrameState &state, const
if (value("show_poses", false)) {
for (size_t i=0; i<fs.frames.size(); ++i) {
- auto pose = fs.frames[i].getPose(); //.inverse() * state.getPose();
+ auto &f = fs.frames[i].cast<ftl::rgbd::Frame>();
- auto name = fs.frames[i].get<std::string>("name");
- _drawOutlinedShape(Shape::CAMERA, state.getPose().inverse() * pose, Eigen::Vector3f(0.2f,0.2f,0.2f), make_uchar4(255,0,0,80), make_uchar4(255,0,0,255));
- _drawAxis(state.getPose().inverse() * pose, Eigen::Vector3f(0.2f, 0.2f, 0.2f));
+ auto pose = f.getPose(); //.inverse() * state.getPose();
+
+ auto name = fs.frames[i].get<std::string>(Channel::Name);
+ _drawOutlinedShape(Shape::CAMERA, frame.getPose().inverse() * pose, Eigen::Vector3f(0.2f,0.2f,0.2f), make_uchar4(255,0,0,80), make_uchar4(255,0,0,255));
+ _drawAxis(frame.getPose().inverse() * pose, Eigen::Vector3f(0.2f, 0.2f, 0.2f));
//ftl::overlay::drawCamera(state.getLeft(), out, over_depth_, fs.frames[i].getLeftCamera(), pose, cv::Scalar(0,0,255,255), 0.2,value("show_frustrum", false));
//if (name) ftl::overlay::drawText(state.getLeft(), out, over_depth_, *name, pos, 0.5, cv::Scalar(0,0,255,255));
@@ -308,15 +310,15 @@ void Overlay::draw(ftl::rgbd::FrameSet &fs, ftl::rgbd::FrameState &state, const
if (value("show_xz_plane", false)) {
float gscale = value("grid_scale",0.5f);
- _drawOutlinedShape(Shape::XZPLANE, state.getPose().inverse(), Eigen::Vector3f(gscale,gscale,gscale), make_uchar4(200,200,200,50), make_uchar4(255,255,255,100));
+ _drawOutlinedShape(Shape::XZPLANE, frame.getPose().inverse(), Eigen::Vector3f(gscale,gscale,gscale), make_uchar4(200,200,200,50), make_uchar4(255,255,255,100));
}
if (value("show_axis", true)) {
- _drawAxis(state.getPose().inverse(), Eigen::Vector3f(0.5f, 0.5f, 0.5f));
+ _drawAxis(frame.getPose().inverse(), Eigen::Vector3f(0.5f, 0.5f, 0.5f));
}
if (value("show_shapes", false)) {
- if (fs.hasChannel(Channel::Shapes3D)) {
+ /*if (fs.hasChannel(Channel::Shapes3D)) {
std::vector<ftl::codecs::Shape3D> shapes;
fs.get(Channel::Shapes3D, shapes);
@@ -340,12 +342,11 @@ void Overlay::draw(ftl::rgbd::FrameSet &fs, ftl::rgbd::FrameState &state, const
//ftl::overlay::drawBox(state.getLeft(), out, over_depth_, pose, cv::Scalar(0,0,255,255), s.size.cast<double>());
//ftl::overlay::drawText(state.getLeft(), out, over_depth_, s.label, pos, 0.5, cv::Scalar(0,0,255,100));
}
- }
+ }*/
for (size_t i=0; i<fs.frames.size(); ++i) {
if (fs.frames[i].hasChannel(Channel::Shapes3D)) {
- std::vector<ftl::codecs::Shape3D> shapes;
- fs.frames[i].get(Channel::Shapes3D, shapes);
+ const auto &shapes = fs.frames[i].get<std::vector<ftl::codecs::Shape3D>>(Channel::Shapes3D);
for (auto &s : shapes) {
auto pose = s.pose.cast<double>(); //.inverse() * state.getPose();
@@ -355,8 +356,8 @@ void Overlay::draw(ftl::rgbd::FrameSet &fs, ftl::rgbd::FrameState &state, const
Eigen::Vector3f scale(s.size[0]/2.0f, s.size[1]/2.0f, s.size[2]/2.0f);
switch (s.type) {
- case ftl::codecs::Shape3DType::CLIPPING: _drawOutlinedShape(Shape::BOX, state.getPose().inverse() * pose, scale, make_uchar4(255,0,255,80), make_uchar4(255,0,255,255)); break;
- case ftl::codecs::Shape3DType::ARUCO: _drawAxis(state.getPose().inverse() * pose, Eigen::Vector3f(0.2f, 0.2f, 0.2f)); break;
+ case ftl::codecs::Shape3DType::CLIPPING: _drawOutlinedShape(Shape::BOX, frame.getPose().inverse() * pose, scale, make_uchar4(255,0,255,80), make_uchar4(255,0,255,255)); break;
+ case ftl::codecs::Shape3DType::ARUCO: _drawAxis(frame.getPose().inverse() * pose, Eigen::Vector3f(0.2f, 0.2f, 0.2f)); break;
default: break;
}
diff --git a/components/rgbd-sources/CMakeLists.txt b/components/rgbd-sources/CMakeLists.txt
index 7e47d77e4..30f9cb3cb 100644
--- a/components/rgbd-sources/CMakeLists.txt
+++ b/components/rgbd-sources/CMakeLists.txt
@@ -3,11 +3,11 @@ set(RGBDSRC
src/sources/stereovideo/opencv.cpp
src/source.cpp
src/frame.cpp
- src/frameset.cpp
+ #src/frameset.cpp
src/sources/stereovideo/stereovideo.cpp
#src/colour.cpp
- src/group.cpp
- src/cb_segmentation.cpp
+ #src/group.cpp
+ #src/cb_segmentation.cpp
#src/abr.cpp
src/sources/screencapture/screencapture.cpp
src/camera.cpp
diff --git a/components/rgbd-sources/include/ftl/rgbd/frame.hpp b/components/rgbd-sources/include/ftl/rgbd/frame.hpp
index 8fc747cd7..2cd518e3d 100644
--- a/components/rgbd-sources/include/ftl/rgbd/frame.hpp
+++ b/components/rgbd-sources/include/ftl/rgbd/frame.hpp
@@ -8,7 +8,7 @@
#include <opencv2/core/cuda.hpp>
#include <opencv2/core/cuda_stream_accessor.hpp>
-#include <ftl/data/frame.hpp>
+#include <ftl/data/new_frame.hpp>
#include <ftl/codecs/channels.hpp>
#include <ftl/rgbd/format.hpp>
@@ -16,7 +16,6 @@
#include <ftl/codecs/codecs.hpp>
#include <ftl/codecs/packet.hpp>
#include <ftl/utility/vectorbuffer.hpp>
-#include <ftl/data/framestate.hpp>
#include <ftl/cuda_common.hpp>
#include <type_traits>
@@ -28,264 +27,146 @@
namespace ftl {
namespace rgbd {
-typedef ftl::data::FrameState<ftl::rgbd::Camera,2> FrameState;
+//typedef ftl::data::Frame Frame;
-struct VideoData {
- ftl::cuda::TextureObjectBase tex;
- cv::cuda::GpuMat gpu;
- cv::Mat host;
- bool isgpu;
- bool validhost;
- std::list<ftl::codecs::Packet> encoded;
+/*inline const ftl::rgbd::Camera &getLeftCamera(const Frame &f) { return f.get<ftl::rgbd::Camera>(ftl::codecs::Channel::Calibration); }
+inline const ftl::rgbd::Camera &getRightCamera(const Frame &f) { return f.get<ftl::rgbd::Camera>(ftl::codecs::Channel::Calibration2); }
+inline const ftl::rgbd::Camera &getLeft(const Frame &f) { return f.get<ftl::rgbd::Camera>(ftl::codecs::Channel::Calibration); }
+inline const ftl::rgbd::Camera &getRight(const Frame &f) { return f.get<ftl::rgbd::Camera>(ftl::codecs::Channel::Calibration2); }
+inline const Eigen::Matrix4d &getPose(const Frame &f) { return f.get<Eigen::Matrix4d>(ftl::codecs::Channel::Pose); }*/
- template <typename T>
- T &as() {
- throw FTL_Error("Unsupported type for Video data channel");
- };
+class VideoFrame {
+ public:
+ VideoFrame() {}
+
+ // Manually add copy constructor since default is removed
+ VideoFrame(const VideoFrame &);
+ VideoFrame &operator=(const VideoFrame &);
template <typename T>
- const T &as() const {
- throw FTL_Error("Unsupported type for Video data channel");
- };
+ ftl::cuda::TextureObject<T> &createTexture(const ftl::rgbd::Format<T> &f, bool interpolated);
template <typename T>
- T &make() {
- throw FTL_Error("Unsupported type for Video data channel");
- };
+ ftl::cuda::TextureObject<T> &createTexture(bool interpolated=false) const;
- inline void reset() {
- validhost = false;
- encoded.clear();
- }
+ cv::cuda::GpuMat &createGPU();
+ cv::cuda::GpuMat &createGPU(const ftl::rgbd::FormatBase &f);
+
+ template <typename T> ftl::cuda::TextureObject<T> &getTexture() const;
+
+ cv::Mat &createCPU();
+ cv::Mat &createCPU(const ftl::rgbd::FormatBase &f);
+
+ inline bool isGPU() const { return isgpu; };
+
+
+ private:
+ mutable ftl::cuda::TextureObjectBase tex;
+ cv::cuda::GpuMat gpu;
+ cv::Mat host;
+ bool isgpu=false;
+ bool validhost=false;
};
-// Specialisations for cv mat types
-template <> cv::Mat &VideoData::as<cv::Mat>();
-template <> const cv::Mat &VideoData::as<cv::Mat>() const;
-template <> cv::cuda::GpuMat &VideoData::as<cv::cuda::GpuMat>();
-template <> const cv::cuda::GpuMat &VideoData::as<cv::cuda::GpuMat>() const;
-
-template <> cv::Mat &VideoData::make<cv::Mat>();
-template <> cv::cuda::GpuMat &VideoData::make<cv::cuda::GpuMat>();
-
-/**
- * Manage a set of image channels corresponding to a single camera frame.
- */
-class Frame : public ftl::data::Frame<0,32,ftl::rgbd::FrameState,VideoData> {
-//class Frame {
-public:
- using ftl::data::Frame<0,32,ftl::rgbd::FrameState,VideoData>::create;
-
- Frame();
- Frame(Frame &&f);
- ~Frame();
-
- Frame &operator=(Frame &&f);
-
- // Prevent frame copy, instead use a move.
- //Frame(const Frame &)=delete;
- //Frame &operator=(const Frame &)=delete;
-
- void download(ftl::codecs::Channel c, cv::cuda::Stream stream);
- void upload(ftl::codecs::Channel c, cv::cuda::Stream stream);
- void download(ftl::codecs::Channels<0> c, cv::cuda::Stream stream);
- void upload(ftl::codecs::Channels<0> c, cv::cuda::Stream stream);
-
- inline void download(ftl::codecs::Channel c, cudaStream_t stream=0) { download(c, cv::cuda::StreamAccessor::wrapStream(stream)); };
- inline void upload(ftl::codecs::Channel c, cudaStream_t stream=0) { upload(c, cv::cuda::StreamAccessor::wrapStream(stream)); };
- inline void download(const ftl::codecs::Channels<0> &c, cudaStream_t stream=0) { download(c, cv::cuda::StreamAccessor::wrapStream(stream)); };
- inline void upload(const ftl::codecs::Channels<0> &c, cudaStream_t stream=0) { upload(c, cv::cuda::StreamAccessor::wrapStream(stream)); };
-
- /**
- * Special case optional download. If a host memory version still exists,
- * use that. Only download if no host version exists. This assumes that
- * the GPU version has not been modified since the host version was created,
- * in otherwords that both version are still the same. It also does not
- * actually mark the channel as downloaded.
- */
- cv::Mat &fastDownload(ftl::codecs::Channel c, cv::cuda::Stream stream);
-
- /**
- * Get an existing CUDA texture object.
- */
- template <typename T> const ftl::cuda::TextureObject<T> &getTexture(ftl::codecs::Channel) const;
-
- /**
- * Get an existing CUDA texture object.
- */
- template <typename T> ftl::cuda::TextureObject<T> &getTexture(ftl::codecs::Channel);
-
- /**
- * Create a channel with a given format. This will discard any existing
- * data associated with the channel and ensure all data structures and
- * memory allocations match the new format.
- */
- template <typename T> T &create(ftl::codecs::Channel c, const ftl::rgbd::FormatBase &f);
-
- /**
- * Create a CUDA texture object for a channel. This version takes a format
- * argument to also create (or recreate) the associated GpuMat.
- */
- template <typename T>
- ftl::cuda::TextureObject<T> &createTexture(ftl::codecs::Channel c, const ftl::rgbd::Format<T> &f, bool interpolated=false);
+class Frame : public ftl::data::Frame {
+ public:
+ inline const ftl::rgbd::Camera &getLeftCamera() { return this->get<ftl::rgbd::Camera>(ftl::codecs::Channel::Calibration); }
+ inline const ftl::rgbd::Camera &getRightCamera() { return this->get<ftl::rgbd::Camera>(ftl::codecs::Channel::Calibration2); }
+ inline const ftl::rgbd::Camera &getLeft() { return this->get<ftl::rgbd::Camera>(ftl::codecs::Channel::Calibration); }
+ inline const ftl::rgbd::Camera &getRight() { return this->get<ftl::rgbd::Camera>(ftl::codecs::Channel::Calibration2); }
+ inline const Eigen::Matrix4d &getPose() { return this->get<Eigen::Matrix4d>(ftl::codecs::Channel::Pose); }
- /**
- * Create a CUDA texture object for a channel. With this version the GpuMat
- * must already exist and be of the correct type.
- */
template <typename T>
- ftl::cuda::TextureObject<T> &createTexture(ftl::codecs::Channel c, bool interpolated=false);
-
- /**
- * Append encoded data for a channel. This will move the data, invalidating
- * the original packet structure. It is to be used to allow data that is
- * already encoded to be transmitted or saved again without re-encoding.
- * A called to `create` will clear all encoded data for that channel.
- */
- void pushPacket(ftl::codecs::Channel c, ftl::codecs::Packet &pkt);
-
- /**
- * Obtain a list of any existing encodings for this channel.
- */
- const std::list<ftl::codecs::Packet> &getPackets(ftl::codecs::Channel c) const;
-
- /**
- * Clear any existing encoded packets. Used when the channel data is
- * modified and the encodings are therefore out-of-date.
- */
- void clearPackets(ftl::codecs::Channel c);
-
- /**
- * Packets from multiple frames are merged together in sequence. An example
- * case is if a frame gets dropped but the original encoding is inter-frame
- * and hence still requires the dropped frames encoding data.
- */
- void mergeEncoding(ftl::rgbd::Frame &f);
-
- void resetTexture(ftl::codecs::Channel c);
-
- /**
- * Check if any specified channels are empty or missing.
- */
- bool empty(ftl::codecs::Channels<0> c);
-
- /**
- * Check if a specific channel is missing or has no memory allocated.
- */
- inline bool empty(ftl::codecs::Channel c) {
- auto &m = getData(c);
- return !hasChannel(c) || (m.host.empty() && m.gpu.empty());
- }
-
- /**
- * Obtain a mask of all available channels in the frame.
- */
- inline ftl::codecs::Channels<0> getVideoChannels() const { return getChannels(); }
+ ftl::cuda::TextureObject<T> &getTexture(ftl::codecs::Channel c) { return this->get<VideoFrame>(c).getTexture<T>(); }
- inline const ftl::rgbd::Camera &getLeftCamera() const { return getLeft(); }
- inline const ftl::rgbd::Camera &getRightCamera() const { return getRight(); }
+ template <typename T>
+ ftl::cuda::TextureObject<T> &createTexture(ftl::codecs::Channel c, bool interpolated=false) { return this->get<VideoFrame>(c).createTexture<T>(interpolated); }
- /**
- * Is the channel data currently located on GPU. This also returns false if
- * the channel does not exist.
- */
- inline bool isGPU(ftl::codecs::Channel channel) const {
- return hasChannel(channel) && getData(channel).isgpu;
- }
+ template <typename T>
+ ftl::cuda::TextureObject<T> &createTexture(ftl::codecs::Channel c, const ftl::rgbd::Format<T> &fmt, bool interpolated=false) { return this->create<VideoFrame>(c).createTexture<T>(fmt, interpolated); }
- /**
- * Is the channel data currently located on CPU memory. This also returns
- * false if the channel does not exist.
- */
- inline bool isCPU(ftl::codecs::Channel channel) const {
- return hasChannel(channel) && !getData(channel).isgpu;
- }
};
-// Specialisations
-
-template <> cv::Mat &Frame::create(ftl::codecs::Channel c, const ftl::rgbd::FormatBase &);
-template <> cv::cuda::GpuMat &Frame::create(ftl::codecs::Channel c, const ftl::rgbd::FormatBase &);
template <typename T>
-ftl::cuda::TextureObject<T> &Frame::getTexture(ftl::codecs::Channel c) {
- if (!hasChannel(c)) throw FTL_Error("Texture channel does not exist: " << (int)c);
-
- auto &m = getData(c);
- if (!m.isgpu) throw FTL_Error("Texture channel is not on GPU");
+ftl::cuda::TextureObject<T> &VideoFrame::getTexture() const {
+ if (!isgpu) throw FTL_Error("Texture channel is not on GPU");
- if (m.tex.cvType() != ftl::traits::OpenCVType<T>::value || m.tex.width() != static_cast<size_t>(m.gpu.cols) || m.tex.height() != static_cast<size_t>(m.gpu.rows) || m.gpu.type() != m.tex.cvType()) {
- throw FTL_Error("Texture has not been created properly for this channel: " << (int)c);
+ if (tex.cvType() != ftl::traits::OpenCVType<T>::value || tex.width() != static_cast<size_t>(gpu.cols) || tex.height() != static_cast<size_t>(gpu.rows) || gpu.type() != tex.cvType()) {
+ throw FTL_Error("Texture has not been created properly");
}
- return ftl::cuda::TextureObject<T>::cast(m.tex);
+ return ftl::cuda::TextureObject<T>::cast(tex);
}
template <typename T>
-ftl::cuda::TextureObject<T> &Frame::createTexture(ftl::codecs::Channel c, const ftl::rgbd::Format<T> &f, bool interpolated) {
- //if (!hasChannel(c)) channels_ += c;
- //using ftl::data::Frame<0,32,ftl::rgbd::FrameState,VideoData>::create;
-
- create<cv::cuda::GpuMat>(c);
- auto &m = getData(c);
+ftl::cuda::TextureObject<T> &VideoFrame::createTexture(const ftl::rgbd::Format<T> &f, bool interpolated) {
+ createGPU();
if (f.empty()) {
throw FTL_Error("createTexture needs a non-empty format");
} else {
- m.gpu.create(f.size(), f.cvType);
+ gpu.create(f.size(), f.cvType);
}
- if (m.gpu.type() != ftl::traits::OpenCVType<T>::value) {
- throw FTL_Error("Texture type mismatch: " << (int)c << " " << m.gpu.type() << " != " << ftl::traits::OpenCVType<T>::value);
+ if (gpu.type() != ftl::traits::OpenCVType<T>::value) {
+ throw FTL_Error("Texture type mismatch: " << gpu.type() << " != " << ftl::traits::OpenCVType<T>::value);
}
// TODO: Check tex cvType
- if (m.tex.devicePtr() == nullptr) {
+ if (tex.devicePtr() == nullptr) {
//LOG(INFO) << "Creating texture object";
- m.tex = ftl::cuda::TextureObject<T>(m.gpu, interpolated);
- } else if (m.tex.cvType() != ftl::traits::OpenCVType<T>::value || m.tex.width() != static_cast<size_t>(m.gpu.cols) || m.tex.height() != static_cast<size_t>(m.gpu.rows)) {
+ tex = ftl::cuda::TextureObject<T>(gpu, interpolated);
+ } else if (tex.cvType() != ftl::traits::OpenCVType<T>::value || tex.width() != static_cast<size_t>(gpu.cols) || tex.height() != static_cast<size_t>(gpu.rows)) {
//LOG(INFO) << "Recreating texture object for '" << ftl::codecs::name(c) << "'";
- m.tex.free();
- m.tex = ftl::cuda::TextureObject<T>(m.gpu, interpolated);
+ tex.free();
+ tex = ftl::cuda::TextureObject<T>(gpu, interpolated);
}
- return ftl::cuda::TextureObject<T>::cast(m.tex);
+ return ftl::cuda::TextureObject<T>::cast(tex);
}
template <typename T>
-ftl::cuda::TextureObject<T> &Frame::createTexture(ftl::codecs::Channel c, bool interpolated) {
- if (!hasChannel(c)) throw FTL_Error("createTexture needs a format if the channel does not exist: " << (int)c);
-
- auto &m = getData(c);
-
- if (!m.isgpu && !m.host.empty()) {
- m.gpu.create(m.host.size(), m.host.type());
+ftl::cuda::TextureObject<T> &VideoFrame::createTexture(bool interpolated) const {
+ if (!isgpu && !host.empty()) {
+ //gpu.create(host.size(), host.type());
// TODO: Should this upload to GPU or not?
//gpu_ += c;
- } else if (!m.isgpu || (m.isgpu && m.gpu.empty())) {
+ throw FTL_Error("Cannot create a texture on a host frame");
+ } else if (!isgpu || (isgpu && gpu.empty())) {
throw FTL_Error("createTexture needs a format if no memory is allocated");
}
- if (m.gpu.type() != ftl::traits::OpenCVType<T>::value) {
- throw FTL_Error("Texture type mismatch: " << (int)c << " " << m.gpu.type() << " != " << ftl::traits::OpenCVType<T>::value);
+ if (gpu.type() != ftl::traits::OpenCVType<T>::value) {
+ throw FTL_Error("Texture type mismatch: " << gpu.type() << " != " << ftl::traits::OpenCVType<T>::value);
}
// TODO: Check tex cvType
- if (m.tex.devicePtr() == nullptr) {
+ if (tex.devicePtr() == nullptr) {
//LOG(INFO) << "Creating texture object";
- m.tex = ftl::cuda::TextureObject<T>(m.gpu, interpolated);
- } else if (m.tex.cvType() != ftl::traits::OpenCVType<T>::value || m.tex.width() != static_cast<size_t>(m.gpu.cols) || m.tex.height() != static_cast<size_t>(m.gpu.rows) || m.tex.devicePtr() != m.gpu.data) {
+ tex = ftl::cuda::TextureObject<T>(gpu, interpolated);
+ } else if (tex.cvType() != ftl::traits::OpenCVType<T>::value || tex.width() != static_cast<size_t>(gpu.cols) || tex.height() != static_cast<size_t>(gpu.rows) || tex.devicePtr() != gpu.data) {
//LOG(INFO) << "Recreating texture object for '" << ftl::codecs::name(c) << "'.";
- m.tex.free();
- m.tex = ftl::cuda::TextureObject<T>(m.gpu, interpolated);
+ tex.free();
+ tex = ftl::cuda::TextureObject<T>(gpu, interpolated);
}
- return ftl::cuda::TextureObject<T>::cast(m.tex);
+ return ftl::cuda::TextureObject<T>::cast(tex);
+}
+
+}
}
+template <>
+cv::Mat &ftl::data::Frame::create<cv::Mat>(ftl::codecs::Channel c) {
+ return create<ftl::rgbd::VideoFrame>(c).createCPU();
}
+
+template <>
+cv::cuda::GpuMat &ftl::data::Frame::create<cv::cuda::GpuMat>(ftl::codecs::Channel c) {
+ return create<ftl::rgbd::VideoFrame>(c).createGPU();
}
#endif // _FTL_RGBD_FRAME_HPP_
\ No newline at end of file
diff --git a/components/rgbd-sources/include/ftl/rgbd/frameset.hpp b/components/rgbd-sources/include/ftl/rgbd/frameset.hpp
index 30ced7b65..02f638e7b 100644
--- a/components/rgbd-sources/include/ftl/rgbd/frameset.hpp
+++ b/components/rgbd-sources/include/ftl/rgbd/frameset.hpp
@@ -4,7 +4,7 @@
#include <ftl/threads.hpp>
#include <ftl/timer.hpp>
#include <ftl/rgbd/frame.hpp>
-#include <ftl/data/frameset.hpp>
+#include <ftl/data/new_frameset.hpp>
//#include <opencv2/core.hpp>
#include <vector>
@@ -16,176 +16,7 @@ namespace rgbd {
static const size_t kMaxFramesets = 15;
static const size_t kMaxFramesInSet = 32;
-class Source;
-
-typedef ftl::data::FrameSet<ftl::rgbd::Frame> FrameSet;
-
-/**
- * Represents a set of synchronised frames, each with two channels. This is
- * used to collect all frames from multiple computers that have the same
- * timestamp.
- */
-//struct FrameSet {
-// int id=0;
-// int64_t timestamp; // Millisecond timestamp of all frames
-// std::vector<ftl::rgbd::Frame> frames;
-// std::atomic<int> count; // Number of valid frames
-// std::atomic<unsigned int> mask; // Mask of all sources that contributed
-// bool stale; // True if buffers have been invalidated
-// SHARED_MUTEX mtx;
-
- /**
- * Upload all specified host memory channels to GPU memory.
- */
-// void upload(ftl::codecs::Channels<0>, cudaStream_t stream=0);
-
- /**
- * Download all specified GPU memory channels to host memory.
- */
-// void download(ftl::codecs::Channels<0>, cudaStream_t stream=0);
-
- /**
- * Move the entire frameset to another frameset object. This will
- * invalidate the current frameset object as all memory buffers will be
- * moved.
- */
-// void swapTo(ftl::rgbd::FrameSet &);
-
- /**
- * Clear all channels and all memory allocations within those channels.
- * This will perform a resetFull on all frames in the frameset.
- */
-// void resetFull();
-//};
-
-/**
- * Callback type for receiving video frames.
- */
-typedef std::function<bool(ftl::rgbd::FrameSet &)> VideoCallback;
-
-/**
- * Abstract class for any generator of FrameSet structures. A generator
- * produces (decoded) frame sets at regular frame intervals depending on the
- * global timer settings. The `onFrameSet` callback may be triggered from any
- * thread and also may drop frames and not be called for a given timestamp.
- */
-class Generator {
- public:
- Generator() {}
- virtual ~Generator() {}
-
- /** Number of frames in last frameset. This can change over time. */
- virtual size_t size()=0;
-
- /**
- * Get the persistent state object for a frame. An exception is thrown
- * for a bad index.
- */
- virtual ftl::rgbd::FrameState &state(size_t ix)=0;
-
- inline ftl::rgbd::FrameState &operator[](int ix) { return state(ix); }
-
- /** Register a callback to receive new frame sets. */
- virtual void onFrameSet(const ftl::rgbd::VideoCallback &)=0;
-};
-
-/**
- * Accept frames and generate framesets as they become completed. This can
- * directly act as a generator of framesets, each frameset being generated
- * by the global timer. Once the expected number of frames have been received,
- * a frameset is marked as complete and can then be passed to the callback at
- * the appropriate time. If frames are generated faster than consumed then they
- * are buffered and merged into a single frameset. The buffer has a limited size
- * so a longer delay in a callback will cause buffer failures. If frames are
- * generated below framerate then the on frameset callback is just not called.
- */
-class Builder : public Generator {
- public:
- Builder();
- ~Builder();
-
- size_t size() override;
-
- ftl::rgbd::FrameState &state(size_t ix) override;
-
- inline void setID(int id) { id_ = id; }
-
- void onFrameSet(const ftl::rgbd::VideoCallback &) override;
-
- /**
- * Add a new frame at a given timestamp.
- */
- //void push(int64_t timestamp, size_t ix, ftl::rgbd::Frame &f);
-
- /**
- * Instead of pushing a frame, find or create a direct reference to one.
- */
- ftl::rgbd::Frame &get(int64_t timestamp, size_t ix);
-
- /**
- * Get the entire frameset for a given timestamp.
- */
- ftl::rgbd::FrameSet *get(int64_t timestamp);
-
- /**
- * Mark a frame as completed.
- */
- void completed(int64_t ts, size_t ix);
-
- void markPartial(int64_t ts);
-
- void setName(const std::string &name);
-
- void setBufferSize(size_t n) { bufferSize_ = n; }
-
- /**
- * Retrieve an fps + latency pair, averaged since last call to this
- * function.
- */
- static std::pair<float,float> getStatistics();
-
- private:
- std::list<FrameSet*> framesets_; // Active framesets
- std::list<FrameSet*> allocated_; // Keep memory allocations
-
- size_t head_;
- ftl::rgbd::VideoCallback cb_;
- MUTEX mutex_;
- int mspf_;
- int64_t last_ts_;
- int64_t last_frame_;
- int id_;
- std::atomic<int> jobs_;
- volatile bool skip_;
- ftl::Handle main_id_;
- size_t size_;
- size_t bufferSize_;
- std::vector<ftl::rgbd::FrameState*> states_;
-
- std::string name_;
-
- static MUTEX msg_mutex__;
- static float latency__;
- static float fps__;
- static int stats_count__;
-
- /* Insert a new frameset into the buffer, along with all intermediate
- * framesets between the last in buffer and the new one.
- */
- ftl::rgbd::FrameSet *_addFrameset(int64_t timestamp);
-
- /* Find a frameset with given latency in frames. */
- ftl::rgbd::FrameSet *_getFrameset();
- ftl::rgbd::FrameSet *_get(int64_t timestamp);
-
- /* Search for a matching frameset. */
- ftl::rgbd::FrameSet *_findFrameset(int64_t ts);
- void _freeFrameset(ftl::rgbd::FrameSet *);
-
- void _schedule();
-
- void _recordStats(float fps, float latency);
-};
+typedef ftl::data::FrameSet FrameSet;
}
}
diff --git a/components/rgbd-sources/include/ftl/rgbd/source.hpp b/components/rgbd-sources/include/ftl/rgbd/source.hpp
index 868e1a9a0..b7f4441f6 100644
--- a/components/rgbd-sources/include/ftl/rgbd/source.hpp
+++ b/components/rgbd-sources/include/ftl/rgbd/source.hpp
@@ -13,7 +13,7 @@
#include <map>
#include <ftl/cuda_common.hpp>
-#include <ftl/rgbd/frame.hpp>
+#include <ftl/data/new_frame.hpp>
namespace ftl {
@@ -24,7 +24,6 @@ class Universe;
namespace rgbd {
class BaseSourceImpl;
-typedef std::function<void(int64_t,ftl::rgbd::Frame&)> FrameCallback;
static inline bool isValidDepth(float d) { return (d > 0.01f) && (d < 39.99f); }
@@ -80,14 +79,14 @@ class Source : public ftl::Configurable {
/**
* Download captured frame. This could be a blocking IO operation.
*/
- bool retrieve();
+ bool retrieve(ftl::data::Frame &);
/**
* Generate a thread job using the provided callback for the most recently
* retrieved frame (matching the provided timestamp). If already busy
* dispatching, returns false.
*/
- bool dispatch(int64_t ts);
+ bool dispatch(ftl::data::Frame &);
/**
* Between frames, do any required buffer swaps.
@@ -139,19 +138,14 @@ class Source : public ftl::Configurable {
std::string getURI() { return value("uri", std::string("")); }
- ftl::rgbd::FrameState &state();
-
SHARED_MUTEX &mutex() { return mutex_; }
- const FrameCallback &callback() { return callback_; }
-
/**
* Set the callback that receives decoded frames as they are generated.
* There can be only a single such callback as the buffers can be swapped
* by the callback.
*/
- void setCallback(const FrameCallback &cb);
- void removeCallback() { callback_ = nullptr; }
+ ftl::Handle setCallback(const std::function<bool(ftl::data::Frame&)> &cb);
/**
* Notify of a decoded or available pair of frames. This calls the source
@@ -168,8 +162,8 @@ class Source : public ftl::Configurable {
SHARED_MUTEX mutex_;
ftl::codecs::Channel channel_;
cudaStream_t stream_;
- FrameCallback callback_;
- ftl::rgbd::Frame frames_[2];
+ ftl::SingletonHandler<ftl::data::Frame> callback_;
+ //ftl::rgbd::Frame frames_[2];
bool is_dispatching;
bool is_retrieving;
diff --git a/components/rgbd-sources/src/frame.cpp b/components/rgbd-sources/src/frame.cpp
index 8c809c026..275175dd2 100644
--- a/components/rgbd-sources/src/frame.cpp
+++ b/components/rgbd-sources/src/frame.cpp
@@ -4,126 +4,27 @@
#define LOGURU_REPLACE_GLOG 1
#include <loguru.hpp>
-using ftl::rgbd::Frame;
-using ftl::rgbd::FrameState;
using ftl::codecs::Channels;
using ftl::codecs::Channel;
-using ftl::rgbd::VideoData;
+using ftl::rgbd::VideoFrame;
-static cv::Mat none;
-static cv::cuda::GpuMat noneGPU;
-static std::atomic<int> frame_count = 0;
-
-template <>
-cv::Mat &VideoData::as<cv::Mat>() {
- if (isgpu) throw FTL_Error("Host request for GPU data without download");
- return host;
-}
-
-template <>
-const cv::Mat &VideoData::as<cv::Mat>() const {
- if (isgpu) throw FTL_Error("Host request for GPU data without download");
- return host;
-}
-
-template <>
-cv::cuda::GpuMat &VideoData::as<cv::cuda::GpuMat>() {
- if (!isgpu) throw FTL_Error("GPU request for Host data without upload");
- return gpu;
-}
-
-template <>
-const cv::cuda::GpuMat &VideoData::as<cv::cuda::GpuMat>() const {
- if (!isgpu) throw FTL_Error("GPU request for Host data without upload");
- return gpu;
-}
-
-template <>
-cv::Mat &VideoData::make<cv::Mat>() {
- validhost = true;
- isgpu = false;
- encoded.clear();
- return host;
-}
-
-template <>
-cv::cuda::GpuMat &VideoData::make<cv::cuda::GpuMat>() {
- isgpu = true;
- encoded.clear();
- return gpu;
-}
-
-// =============================================================================
-
-/*void Frame::reset() {
- origin_ = nullptr;
- channels_.clear();
- gpu_.clear();
- data_channels_.clear();
- for (size_t i=0u; i<Channels<0>::kMax; ++i) {
- data_[i].encoded.clear();
- }
-}*/
-
-/*void Frame::resetFull() {
- origin_ = nullptr;
- channels_.clear();
- gpu_.clear();
- for (size_t i=0u; i<Channels<0>::kMax; ++i) {
- data_[i].gpu = cv::cuda::GpuMat();
- data_[i].host = cv::Mat();
- data_[i].encoded.clear();
- }
-}*/
-
-Frame::Frame() {
- ++frame_count;
- //LOG(INFO) << "Frames: " << frame_count;
+VideoFrame::VideoFrame(const VideoFrame &f) {
+ gpu = f.gpu;
+ host = f.host;
+ isgpu = f.isgpu;
+ validhost = f.validhost;
}
-Frame::Frame(Frame &&f) : ftl::data::Frame<0,32,ftl::rgbd::FrameState,VideoData>(std::move(f)) {
-
-}
-
-Frame &Frame::operator=(Frame &&f) {
- ftl::data::Frame<0,32,ftl::rgbd::FrameState,VideoData>::operator=(std::move(f));
+VideoFrame &VideoFrame::operator=(const VideoFrame &f) {
+ gpu = f.gpu;
+ host = f.host;
+ isgpu = f.isgpu;
+ validhost = f.validhost;
return *this;
}
-Frame::~Frame() {
- --frame_count;
-}
-
-void Frame::download(Channel c, cv::cuda::Stream stream) {
- download(Channels(c), stream);
-}
-
-void Frame::upload(Channel c, cv::cuda::Stream stream) {
- upload(Channels(c), stream);
-}
-
-void Frame::download(Channels<0> c, cv::cuda::Stream stream) {
- for (size_t i=0u; i<Channels<0>::kMax; ++i) {
- if (c.has(i) && hasChannel(static_cast<Channel>(i)) && isGPU(static_cast<Channel>(i))) {
- auto &data = getData(static_cast<Channel>(i));
- data.validhost = true;
- data.gpu.download(data.host, stream);
- data.isgpu = false;
- }
- }
-}
-void Frame::upload(Channels<0> c, cv::cuda::Stream stream) {
- for (size_t i=0u; i<Channels<0>::kMax; ++i) {
- if (c.has(i) && hasChannel(static_cast<Channel>(i)) && !isGPU(static_cast<Channel>(i))) {
- auto &data = getData(static_cast<Channel>(i));
- data.gpu.upload(data.host, stream);
- data.isgpu = true;
- }
- }
-}
-
-cv::Mat &Frame::fastDownload(ftl::codecs::Channel c, cv::cuda::Stream stream) {
+/*cv::Mat &Frame::fastDownload(ftl::codecs::Channel c, cv::cuda::Stream stream) {
if (hasChannel(c)) {
auto &data = getData(static_cast<Channel>(c));
if (!data.isgpu) return data.host;
@@ -136,86 +37,23 @@ cv::Mat &Frame::fastDownload(ftl::codecs::Channel c, cv::cuda::Stream stream) {
return data.host;
}
throw FTL_Error("Fast download channel does not exist: " << (int)c);
-}
-
-void Frame::pushPacket(ftl::codecs::Channel c, ftl::codecs::Packet &pkt) {
- if (hasChannel(c)) {
- auto &m1 = getData(c);
- m1.encoded.emplace_back() = std::move(pkt);
- } else {
- throw FTL_Error("Channel " << (int)c << " doesn't exist for packet push");
- }
-}
-
-const std::list<ftl::codecs::Packet> &Frame::getPackets(ftl::codecs::Channel c) const {
- if (!hasChannel(c)) {
- throw FTL_Error("Frame channel does not exist: " << (int)c);
- }
-
- auto &m1 = getData(c);
- return m1.encoded;
-}
-
-void Frame::mergeEncoding(ftl::rgbd::Frame &f) {
- //LOG(INFO) << "MERGE " << (unsigned int)f.channels_;
- for (auto c : getChannels()) {
- //if (!f.hasChannel(c)) f.create<cv::cuda::GpuMat>(c);
- if (f.hasChannel(c)) {
- auto &m1 = getData(c);
- auto &m2 = f.getData(c);
- m1.encoded.splice(m1.encoded.begin(), m2.encoded);
- //LOG(INFO) << "SPLICED: " << m1.encoded.size();
- }
- }
-}
-
-bool Frame::empty(ftl::codecs::Channels<0> channels) {
- for (auto c : channels) {
- if (empty(c)) return true;
- }
- return false;
-}
-
-template <> cv::Mat &Frame::create(ftl::codecs::Channel c, const ftl::rgbd::FormatBase &f) {
- if (c == Channel::None) {
- throw FTL_Error("Cannot create a None channel");
- }
-
- create<cv::Mat>(c);
- auto &m = getData(c);
-
- m.encoded.clear(); // Remove all old encoded data
+}*/
+cv::Mat &VideoFrame::createCPU(const ftl::rgbd::FormatBase &f) {
if (!f.empty()) {
- m.host.create(f.size(), f.cvType);
+ host.create(f.size(), f.cvType);
}
+ isgpu = false;
- return m.host;
+ return host;
}
-template <> cv::cuda::GpuMat &Frame::create(ftl::codecs::Channel c, const ftl::rgbd::FormatBase &f) {
- if (c == Channel::None) {
- throw FTL_Error("Cannot create a None channel");
- }
-
- create<cv::cuda::GpuMat>(c);
- auto &m = getData(c);
-
- m.encoded.clear(); // Remove all old encoded data
-
+cv::cuda::GpuMat &VideoFrame::create(const ftl::rgbd::FormatBase &f) {
if (!f.empty()) {
- m.gpu.create(f.size(), f.cvType);
+ gpu.create(f.size(), f.cvType);
}
+ isgpu = true;
- return m.gpu;
-}
-
-void Frame::clearPackets(ftl::codecs::Channel c) {
- auto &m = getData(c);
- m.encoded.clear();
+ return gpu;
}
-void Frame::resetTexture(ftl::codecs::Channel c) {
- auto &m = getData(c);
- m.tex.free();
-}
diff --git a/components/rgbd-sources/src/source.cpp b/components/rgbd-sources/src/source.cpp
index 2a9af5889..0ee4a2f1a 100644
--- a/components/rgbd-sources/src/source.cpp
+++ b/components/rgbd-sources/src/source.cpp
@@ -30,7 +30,6 @@ using ftl::rgbd::capability_t;
using ftl::codecs::Channel;
//using ftl::rgbd::detail::FileSource;
using ftl::rgbd::Camera;
-using ftl::rgbd::FrameCallback;
Source::Source(ftl::config::json_t &cfg) : Configurable(cfg), pose_(Eigen::Matrix4d::Identity()), net_(nullptr) {
@@ -185,34 +184,34 @@ bool Source::capture(int64_t ts) {
else return true;
}
-bool Source::retrieve() {
+bool Source::retrieve(ftl::data::Frame &f) {
is_retrieving = true;
bool status = false;
- if (impl_) status = impl_->retrieve(frames_[0]);
+ if (impl_) status = impl_->retrieve(f);
is_retrieving = false;
return status;
}
-bool Source::dispatch(int64_t ts) {
+bool Source::dispatch(ftl::data::Frame &f) {
if (!callback_) return false;
if (is_dispatching || is_retrieving) {
LOG(WARNING) << "Previous distance not completed";
return false;
}
is_dispatching = true;
- _swap();
+ //_swap();
ftl::pool.push([this,ts](int id) {
- callback_(ts, frames_[1]);
+ callback_.trigger(f);
is_dispatching = false;
});
return true;
}
-void Source::_swap() {
+/*void Source::_swap() {
auto tmp = std::move(frames_[0]);
frames_[0] = std::move(frames_[1]);
frames_[1] = std::move(tmp);
-}
+}*/
bool Source::setChannel(ftl::codecs::Channel c) {
channel_ = c;
@@ -224,9 +223,8 @@ const ftl::rgbd::Camera Source::parameters(ftl::codecs::Channel chan) const {
return (impl_) ? impl_->parameters(chan) : parameters();
}
-void Source::setCallback(const FrameCallback &cb) {
- if (bool(callback_)) LOG(ERROR) << "Source already has a callback: " << getURI();
- callback_ = cb;
+ftl::Handle Source::setCallback(const std::function<bool(ftl::data::Frame&)> &cb) {
+ return callback_.on(cb);
}
diff --git a/components/structures/include/ftl/data/new_frame.hpp b/components/structures/include/ftl/data/new_frame.hpp
index 84a598b66..25997f5a9 100644
--- a/components/structures/include/ftl/data/new_frame.hpp
+++ b/components/structures/include/ftl/data/new_frame.hpp
@@ -26,7 +26,8 @@ class Pool;
enum FrameMode {
PRIMARY,
- RESPONSE
+ RESPONSE,
+ STANDALONE
};
enum FrameStatus {
@@ -102,6 +103,8 @@ class Frame {
bool has(ftl::codecs::Channel c) const;
+ inline bool hasChannel(ftl::codecs::Channel c) const { return has(c); }
+
inline bool hasOwn(ftl::codecs::Channel c) const;
inline bool changed(ftl::codecs::Channel c) const;
@@ -204,6 +207,8 @@ class Frame {
void swapChannel(ftl::codecs::Channel, Frame &);
+ void swapChannels(ftl::codecs::Channel, ftl::codecs::Channel);
+
/**
* Discard all change status without removing the data.
*/
@@ -248,6 +253,15 @@ class Frame {
*/
Frame response();
+ template <typename T>
+ T &cast();
+
+ /**
+ * Used to create isolated frame objects for buffer purposes. This is
+ * deliberately separate from default constructor to force its explicit use.
+ */
+ static Frame make_standalone();
+
protected:
std::any &createAnyChange(ftl::codecs::Channel c, ftl::data::ChangeType t);
@@ -320,6 +334,13 @@ class Session : public Frame {
MUTEX &ftl::data::Frame::mutex() { return parent_->mutex(); }
+template <typename T>
+T &ftl::data::Frame::cast() {
+ static_assert(std::is_base_of<Frame, T>::value, "Can only cast to type inheriting Frame");
+ static_assert(sizeof(T) == sizeof(Frame), "Casting type must not have additional data members");
+ return *reinterpret_cast<T*>(this);
+}
+
bool ftl::data::Frame::hasOwn(ftl::codecs::Channel c) const {
const auto &i = data_.find(c);
return (i != data_.end() && i->second.status != ftl::data::ChannelStatus::INVALID);
diff --git a/components/structures/src/new_frame.cpp b/components/structures/src/new_frame.cpp
index b33a41424..dc2823c79 100644
--- a/components/structures/src/new_frame.cpp
+++ b/components/structures/src/new_frame.cpp
@@ -229,6 +229,21 @@ void Frame::swapChannel(ftl::codecs::Channel c, Frame &f) {
}
}
+void Frame::swapChannels(ftl::codecs::Channel c1, ftl::codecs::Channel c2) {
+ if (hasOwn(c1) && hasOwn(c2)) {
+ auto &d1 = data_[c1];
+ auto &d2 = data_[c2];
+ d2.data.swap(d1.data);
+
+ auto status = d1.status;
+ d1.status = d2.status;
+ d2.status = status;
+
+ changed_[c1] = (mode_ == FrameMode::PRIMARY) ? ChangeType::PRIMARY : ChangeType::RESPONSE;
+ changed_[c2] = (mode_ == FrameMode::PRIMARY) ? ChangeType::PRIMARY : ChangeType::RESPONSE;
+ }
+}
+
void Frame::reset() {
for (auto &d : data_) {
d.second.status = ChannelStatus::INVALID;
@@ -251,6 +266,12 @@ Frame Frame::response() {
return f;
}
+Frame Frame::make_standalone() {
+ Frame f(nullptr, nullptr, 0, 0);
+ f.mode_ = FrameMode::STANDALONE;
+ return f;
+}
+
// ==== Session ================================================================
ftl::Handle Session::onChange(uint32_t pid, ftl::codecs::Channel c, const std::function<bool(Frame&,ftl::codecs::Channel)> &cb) {
--
GitLab