From c2fdc8fe274bfa4cbfdf63cb42d5fcc1d4e29164 Mon Sep 17 00:00:00 2001
From: Sebastian Hahta <joseha@utu.fi>
Date: Thu, 5 Dec 2019 15:07:00 +0200
Subject: [PATCH] WIP: resolution
---
.../reconstruct/src/reconstruction.cpp | 26 +++++++++++++++++++
.../reconstruct/src/reconstruction.hpp | 3 +++
components/codecs/src/opencv_decoder.cpp | 3 +--
components/codecs/src/opencv_encoder.cpp | 5 ++--
.../rgbd-sources/src/sources/net/net.cpp | 6 +++--
5 files changed, 37 insertions(+), 6 deletions(-)
diff --git a/applications/reconstruct/src/reconstruction.cpp b/applications/reconstruct/src/reconstruction.cpp
index bbaf7f8b5..c38eaa62a 100644
--- a/applications/reconstruct/src/reconstruction.cpp
+++ b/applications/reconstruct/src/reconstruction.cpp
@@ -50,10 +50,36 @@ Reconstruction::Reconstruction(nlohmann::json &config, const std::string name) :
ftl::pool.push([this](int id) {
UNIQUE_LOCK(fs_align_.mtx, lk);
+
+ rgb_.resize(fs_align_.frames.size());
+ for (size_t i = 0; i < rgb_.size(); i++) {
+ auto &depth = fs_align_.frames[i].get<cv::cuda::GpuMat>(ftl::codecs::Channel::Depth);
+ auto &color = fs_align_.frames[i].get<cv::cuda::GpuMat>(ftl::codecs::Channel::Colour);
+
+ if (depth.size() != color.size()) {
+ std::swap(rgb_[i], color);
+ cv::cuda::resize(rgb_[i], color, depth.size(), 0.0, 0.0, cv::INTER_LINEAR);
+ }
+ }
+
pipeline_->apply(fs_align_, fs_align_, 0);
// TODO: To use second GPU, could do a download, swap, device change,
// then upload to other device. Or some direct device-2-device copy.
+ /*
+ for (size_t i = 0; i < rgb_.size(); i++) {
+ auto &depth = fs_align_.frames[i].get<cv::cuda::GpuMat>(ftl::codecs::Channel::Depth);
+ auto &color = fs_align_.frames[i].get<cv::cuda::GpuMat>(ftl::codecs::Channel::Colour);
+ auto &tmp = rgb_[i];
+
+ // TODO doesn't always work correctly if resolution changes
+ if (!tmp.empty() && (depth.size() != tmp.size())) {
+ std::swap(tmp, color);
+ fs_align_.frames[i].resetTexture(ftl::codecs::Channel::Colour);
+ fs_align_.frames[i].createTexture<uchar4>(ftl::codecs::Channel::Colour, true);
+ }
+ }*/
+
fs_align_.swapTo(fs_render_);
LOG(INFO) << "Align complete... " << fs_align_.timestamp;
diff --git a/applications/reconstruct/src/reconstruction.hpp b/applications/reconstruct/src/reconstruction.hpp
index 340e65bf2..8571b8c72 100644
--- a/applications/reconstruct/src/reconstruction.hpp
+++ b/applications/reconstruct/src/reconstruction.hpp
@@ -25,11 +25,14 @@ class Reconstruction : public ftl::Configurable {
private:
bool busy_;
+
ftl::rgbd::FrameSet fs_render_;
ftl::rgbd::FrameSet fs_align_;
ftl::rgbd::Group *group_;
ftl::operators::Graph *pipeline_;
ftl::render::Triangular *renderer_;
+
+ std::vector<cv::cuda::GpuMat> rgb_;
};
}
diff --git a/components/codecs/src/opencv_decoder.cpp b/components/codecs/src/opencv_decoder.cpp
index 0b9feea46..c3c5e9567 100644
--- a/components/codecs/src/opencv_decoder.cpp
+++ b/components/codecs/src/opencv_decoder.cpp
@@ -18,7 +18,7 @@ bool OpenCVDecoder::accepts(const ftl::codecs::Packet &pkt) {
}
bool OpenCVDecoder::decode(const ftl::codecs::Packet &pkt, cv::cuda::GpuMat &out) {
-
+ //CHECK(cv::Size(ftl::codecs::getWidth(pkt.definition), ftl::codecs::getHeight(pkt.definition)) == out.size());
int chunk_dim = std::sqrt(pkt.block_total);
int chunk_width = out.cols / chunk_dim;
int chunk_height = out.rows / chunk_dim;
@@ -37,7 +37,6 @@ bool OpenCVDecoder::decode(const ftl::codecs::Packet &pkt, cv::cuda::GpuMat &out
// Apply colour correction to chunk
//ftl::rgbd::colourCorrection(tmp_rgb, gamma_, temperature_);
-
// TODO:(Nick) Decode directly into double buffer if no scaling
// Can either check JPG/PNG headers or just use pkt definition.
diff --git a/components/codecs/src/opencv_encoder.cpp b/components/codecs/src/opencv_encoder.cpp
index 5dc1995a8..6398c72b0 100644
--- a/components/codecs/src/opencv_encoder.cpp
+++ b/components/codecs/src/opencv_encoder.cpp
@@ -17,7 +17,7 @@ OpenCVEncoder::OpenCVEncoder(ftl::codecs::definition_t maxdef,
}
OpenCVEncoder::~OpenCVEncoder() {
-
+
}
bool OpenCVEncoder::supports(ftl::codecs::codec_t codec) {
@@ -33,6 +33,7 @@ bool OpenCVEncoder::encode(const cv::cuda::GpuMat &in, definition_t definition,
current_definition_ = definition;
in.download(tmp_);
+ //CHECK(cv::Size(ftl::codecs::getWidth(definition), ftl::codecs::getHeight(definition)) == in.size());
// Scale down image to match requested definition...
if (ftl::codecs::getHeight(current_definition_) < in.rows) {
@@ -42,7 +43,7 @@ bool OpenCVEncoder::encode(const cv::cuda::GpuMat &in, definition_t definition,
}
// Represent float at 16bit int
- if (!is_colour) {
+ if (!is_colour) {
tmp_.convertTo(tmp_, CV_16UC1, 1000);
}
diff --git a/components/rgbd-sources/src/sources/net/net.cpp b/components/rgbd-sources/src/sources/net/net.cpp
index e4073536a..04b093490 100644
--- a/components/rgbd-sources/src/sources/net/net.cpp
+++ b/components/rgbd-sources/src/sources/net/net.cpp
@@ -276,8 +276,9 @@ void NetSource::_recvPacket(short ttimeoff, const ftl::codecs::StreamPacket &spk
LOG(WARNING) << "Missing calibration, skipping frame";
return;
}
-
- NetFrame &frame = queue_.getFrame(spkt.timestamp, cv::Size(params_.width, params_.height), CV_8UC3, (isFloatChannel(chan) ? CV_32FC1 : CV_8UC3));
+
+ const cv::Size size = cv::Size(ftl::codecs::getWidth(pkt.definition), ftl::codecs::getHeight(pkt.definition));
+ NetFrame &frame = queue_.getFrame(spkt.timestamp, size, CV_8UC3, (isFloatChannel(chan) ? CV_32FC1 : CV_8UC3));
// Update frame statistics
frame.tx_size += pkt.data.size();
@@ -291,6 +292,7 @@ void NetSource::_recvPacket(short ttimeoff, const ftl::codecs::StreamPacket &spk
return;
}
+ frame.channel[channum].create(size, frame.channel[channum].type()); // TODO!!! queue_.getFrame() assumes same size
decoder->decode(pkt, frame.channel[channum]);
} else if (chan != Channel::None && rchan != Channel::Colour) {
// Didn't receive correct second channel so just clear the images
--
GitLab