diff --git a/applications/reconstruct/src/reconstruction.cpp b/applications/reconstruct/src/reconstruction.cpp index bbaf7f8b5e6414140233aa9480ccc1239c7de262..c38eaa62a1584a88e687522327ae41de0caa20af 100644 --- a/applications/reconstruct/src/reconstruction.cpp +++ b/applications/reconstruct/src/reconstruction.cpp @@ -50,10 +50,36 @@ Reconstruction::Reconstruction(nlohmann::json &config, const std::string name) : ftl::pool.push([this](int id) { UNIQUE_LOCK(fs_align_.mtx, lk); + + rgb_.resize(fs_align_.frames.size()); + for (size_t i = 0; i < rgb_.size(); i++) { + auto &depth = fs_align_.frames[i].get<cv::cuda::GpuMat>(ftl::codecs::Channel::Depth); + auto &color = fs_align_.frames[i].get<cv::cuda::GpuMat>(ftl::codecs::Channel::Colour); + + if (depth.size() != color.size()) { + std::swap(rgb_[i], color); + cv::cuda::resize(rgb_[i], color, depth.size(), 0.0, 0.0, cv::INTER_LINEAR); + } + } + pipeline_->apply(fs_align_, fs_align_, 0); // TODO: To use second GPU, could do a download, swap, device change, // then upload to other device. Or some direct device-2-device copy. + /* + for (size_t i = 0; i < rgb_.size(); i++) { + auto &depth = fs_align_.frames[i].get<cv::cuda::GpuMat>(ftl::codecs::Channel::Depth); + auto &color = fs_align_.frames[i].get<cv::cuda::GpuMat>(ftl::codecs::Channel::Colour); + auto &tmp = rgb_[i]; + + // TODO doesn't always work correctly if resolution changes + if (!tmp.empty() && (depth.size() != tmp.size())) { + std::swap(tmp, color); + fs_align_.frames[i].resetTexture(ftl::codecs::Channel::Colour); + fs_align_.frames[i].createTexture<uchar4>(ftl::codecs::Channel::Colour, true); + } + }*/ + fs_align_.swapTo(fs_render_); LOG(INFO) << "Align complete... " << fs_align_.timestamp; diff --git a/applications/reconstruct/src/reconstruction.hpp b/applications/reconstruct/src/reconstruction.hpp index 340e65bf2f47650400d4c6850c1ae1310a0458aa..8571b8c7293defc1b537ff66537cb77e9340956f 100644 --- a/applications/reconstruct/src/reconstruction.hpp +++ b/applications/reconstruct/src/reconstruction.hpp @@ -25,11 +25,14 @@ class Reconstruction : public ftl::Configurable { private: bool busy_; + ftl::rgbd::FrameSet fs_render_; ftl::rgbd::FrameSet fs_align_; ftl::rgbd::Group *group_; ftl::operators::Graph *pipeline_; ftl::render::Triangular *renderer_; + + std::vector<cv::cuda::GpuMat> rgb_; }; } diff --git a/components/codecs/src/opencv_decoder.cpp b/components/codecs/src/opencv_decoder.cpp index 0b9feea46e5925f16ce5ab323747d94d8bdb1d2a..c3c5e9567deb6752bf2395d0c36e345fcead50ee 100644 --- a/components/codecs/src/opencv_decoder.cpp +++ b/components/codecs/src/opencv_decoder.cpp @@ -18,7 +18,7 @@ bool OpenCVDecoder::accepts(const ftl::codecs::Packet &pkt) { } bool OpenCVDecoder::decode(const ftl::codecs::Packet &pkt, cv::cuda::GpuMat &out) { - + //CHECK(cv::Size(ftl::codecs::getWidth(pkt.definition), ftl::codecs::getHeight(pkt.definition)) == out.size()); int chunk_dim = std::sqrt(pkt.block_total); int chunk_width = out.cols / chunk_dim; int chunk_height = out.rows / chunk_dim; @@ -37,7 +37,6 @@ bool OpenCVDecoder::decode(const ftl::codecs::Packet &pkt, cv::cuda::GpuMat &out // Apply colour correction to chunk //ftl::rgbd::colourCorrection(tmp_rgb, gamma_, temperature_); - // TODO:(Nick) Decode directly into double buffer if no scaling // Can either check JPG/PNG headers or just use pkt definition. diff --git a/components/codecs/src/opencv_encoder.cpp b/components/codecs/src/opencv_encoder.cpp index 5dc1995a82e6147184572d6e45d20a8a49561ddc..6398c72b036c137d9a5b994e971995ed7a170ef7 100644 --- a/components/codecs/src/opencv_encoder.cpp +++ b/components/codecs/src/opencv_encoder.cpp @@ -17,7 +17,7 @@ OpenCVEncoder::OpenCVEncoder(ftl::codecs::definition_t maxdef, } OpenCVEncoder::~OpenCVEncoder() { - + } bool OpenCVEncoder::supports(ftl::codecs::codec_t codec) { @@ -33,6 +33,7 @@ bool OpenCVEncoder::encode(const cv::cuda::GpuMat &in, definition_t definition, current_definition_ = definition; in.download(tmp_); + //CHECK(cv::Size(ftl::codecs::getWidth(definition), ftl::codecs::getHeight(definition)) == in.size()); // Scale down image to match requested definition... if (ftl::codecs::getHeight(current_definition_) < in.rows) { @@ -42,7 +43,7 @@ bool OpenCVEncoder::encode(const cv::cuda::GpuMat &in, definition_t definition, } // Represent float at 16bit int - if (!is_colour) { + if (!is_colour) { tmp_.convertTo(tmp_, CV_16UC1, 1000); } diff --git a/components/rgbd-sources/src/sources/net/net.cpp b/components/rgbd-sources/src/sources/net/net.cpp index e4073536a574255965de81ab5f2294e008695032..04b093490ca86125a38a221f0ddbcfd524809e10 100644 --- a/components/rgbd-sources/src/sources/net/net.cpp +++ b/components/rgbd-sources/src/sources/net/net.cpp @@ -276,8 +276,9 @@ void NetSource::_recvPacket(short ttimeoff, const ftl::codecs::StreamPacket &spk LOG(WARNING) << "Missing calibration, skipping frame"; return; } - - NetFrame &frame = queue_.getFrame(spkt.timestamp, cv::Size(params_.width, params_.height), CV_8UC3, (isFloatChannel(chan) ? CV_32FC1 : CV_8UC3)); + + const cv::Size size = cv::Size(ftl::codecs::getWidth(pkt.definition), ftl::codecs::getHeight(pkt.definition)); + NetFrame &frame = queue_.getFrame(spkt.timestamp, size, CV_8UC3, (isFloatChannel(chan) ? CV_32FC1 : CV_8UC3)); // Update frame statistics frame.tx_size += pkt.data.size(); @@ -291,6 +292,7 @@ void NetSource::_recvPacket(short ttimeoff, const ftl::codecs::StreamPacket &spk return; } + frame.channel[channum].create(size, frame.channel[channum].type()); // TODO!!! queue_.getFrame() assumes same size decoder->decode(pkt, frame.channel[channum]); } else if (chan != Channel::None && rchan != Channel::Colour) { // Didn't receive correct second channel so just clear the images