diff --git a/applications/gui/src/camera.cpp b/applications/gui/src/camera.cpp
index f35b4e539e8216b32106762b69ea7e2b028706eb..cca8a40f35c6e679597185668661fe4a814e504f 100644
--- a/applications/gui/src/camera.cpp
+++ b/applications/gui/src/camera.cpp
@@ -148,17 +148,20 @@ ftl::gui::Camera::Camera(ftl::gui::Screen *screen, ftl::rgbd::Source *src) : scr
posewin_->setTheme(screen->windowtheme);
posewin_->setVisible(false);
- src->setCallback([this](int64_t ts, cv::Mat &channel1, cv::Mat &channel2) {
+ src->setCallback([this](int64_t ts, cv::cuda::GpuMat &channel1, cv::cuda::GpuMat &channel2) {
UNIQUE_LOCK(mutex_, lk);
im1_.create(channel1.size(), channel1.type());
im2_.create(channel2.size(), channel2.type());
//cv::swap(channel1, im1_);
//cv::swap(channel2, im2_);
+
+ channel1.download(im1_);
+ channel2.download(im2_);
// OpenGL (0,0) bottom left
- cv::flip(channel1, im1_, 0);
- cv::flip(channel2, im2_, 0);
+ cv::flip(im1_, im1_, 0);
+ cv::flip(im2_, im2_, 0);
});
}
diff --git a/applications/player/src/main.cpp b/applications/player/src/main.cpp
index 60d2793c1c4b0484dbb226bff12deebfb6e1fc2e..7eeb6e6bf5f96fd8f32360949c566c93860518db 100644
--- a/applications/player/src/main.cpp
+++ b/applications/player/src/main.cpp
@@ -77,11 +77,13 @@ int main(int argc, char **argv) {
//LOG(INFO) << "Reading packet: (" << (int)spkt.streamID << "," << (int)spkt.channel << ") " << (int)pkt.codec << ", " << (int)pkt.definition;
- cv::Mat frame(cv::Size(ftl::codecs::getWidth(pkt.definition),ftl::codecs::getHeight(pkt.definition)), (spkt.channel == Channel::Depth) ? CV_32F : CV_8UC3);
+ cv::cuda::GpuMat gframe(cv::Size(ftl::codecs::getWidth(pkt.definition),ftl::codecs::getHeight(pkt.definition)), (spkt.channel == Channel::Depth) ? CV_32F : CV_8UC3);
+ cv::Mat frame;
createDecoder(pkt);
try {
- decoder->decode(pkt, frame);
+ decoder->decode(pkt, gframe);
+ gframe.download(frame);
} catch (std::exception &e) {
LOG(INFO) << "Decoder exception: " << e.what();
}
diff --git a/components/codecs/include/ftl/codecs/encoder.hpp b/components/codecs/include/ftl/codecs/encoder.hpp
index fed8d95755859d2c9c71ee1475f89d024bf38811..9c3aa8fefc64810bf7660e323b44a3c4440d5098 100644
--- a/components/codecs/include/ftl/codecs/encoder.hpp
+++ b/components/codecs/include/ftl/codecs/encoder.hpp
@@ -60,7 +60,7 @@ class Encoder {
/**
* Wrapper encode to allow use of presets.
*/
- virtual bool encode(const cv::Mat &in, ftl::codecs::preset_t preset,
+ virtual bool encode(const cv::cuda::GpuMat &in, ftl::codecs::preset_t preset,
const std::function<void(const ftl::codecs::Packet&)> &cb);
/**
@@ -77,7 +77,7 @@ class Encoder {
* @return True if succeeded with encoding.
*/
virtual bool encode(
- const cv::Mat &in,
+ const cv::cuda::GpuMat &in,
ftl::codecs::definition_t definition,
ftl::codecs::bitrate_t bitrate,
const std::function<void(const ftl::codecs::Packet&)> &cb)=0;
diff --git a/components/codecs/include/ftl/codecs/nvpipe_decoder.hpp b/components/codecs/include/ftl/codecs/nvpipe_decoder.hpp
index a6ae2e36e41f40415d3f24daa4cc4029bf07a645..7734998e2d80ecb0aa01930adadf01e8e87c31e3 100644
--- a/components/codecs/include/ftl/codecs/nvpipe_decoder.hpp
+++ b/components/codecs/include/ftl/codecs/nvpipe_decoder.hpp
@@ -24,6 +24,7 @@ class NvPipeDecoder : public ftl::codecs::Decoder {
ftl::codecs::definition_t last_definition_;
MUTEX mutex_;
bool seen_iframe_;
+ cv::cuda::GpuMat tmp_;
};
}
diff --git a/components/codecs/include/ftl/codecs/nvpipe_encoder.hpp b/components/codecs/include/ftl/codecs/nvpipe_encoder.hpp
index c2fe379dc985488f014e911dd1e096431bdee96e..3b5515f296c06978428b8cfe8f0854129791d83a 100644
--- a/components/codecs/include/ftl/codecs/nvpipe_encoder.hpp
+++ b/components/codecs/include/ftl/codecs/nvpipe_encoder.hpp
@@ -13,12 +13,12 @@ class NvPipeEncoder : public ftl::codecs::Encoder {
ftl::codecs::definition_t mindef);
~NvPipeEncoder();
- bool encode(const cv::Mat &in, ftl::codecs::preset_t preset,
+ bool encode(const cv::cuda::GpuMat &in, ftl::codecs::preset_t preset,
const std::function<void(const ftl::codecs::Packet&)> &cb) {
return Encoder::encode(in, preset, cb);
}
- bool encode(const cv::Mat &in, ftl::codecs::definition_t definition, ftl::codecs::bitrate_t bitrate,
+ bool encode(const cv::cuda::GpuMat &in, ftl::codecs::definition_t definition, ftl::codecs::bitrate_t bitrate,
const std::function<void(const ftl::codecs::Packet&)>&) override;
//bool encode(const cv::cuda::GpuMat &in, std::vector<uint8_t> &out, bitrate_t bix, bool);
@@ -35,10 +35,12 @@ class NvPipeEncoder : public ftl::codecs::Encoder {
bool is_float_channel_;
bool was_reset_;
ftl::codecs::codec_t preference_;
+ cv::cuda::GpuMat tmp_;
+ cv::cuda::GpuMat tmp2_;
- bool _encoderMatch(const cv::Mat &in, definition_t def);
- bool _createEncoder(const cv::Mat &in, definition_t def, bitrate_t rate);
- ftl::codecs::definition_t _verifiedDefinition(ftl::codecs::definition_t def, const cv::Mat &in);
+ bool _encoderMatch(const cv::cuda::GpuMat &in, definition_t def);
+ bool _createEncoder(const cv::cuda::GpuMat &in, definition_t def, bitrate_t rate);
+ ftl::codecs::definition_t _verifiedDefinition(ftl::codecs::definition_t def, const cv::cuda::GpuMat &in);
};
}
diff --git a/components/codecs/include/ftl/codecs/opencv_decoder.hpp b/components/codecs/include/ftl/codecs/opencv_decoder.hpp
index 0f085b90b9fdca3bc899aee2899c40e423bb1086..53b61f683bc1d0a93a9c21049e0d312c133c83f7 100644
--- a/components/codecs/include/ftl/codecs/opencv_decoder.hpp
+++ b/components/codecs/include/ftl/codecs/opencv_decoder.hpp
@@ -14,6 +14,9 @@ class OpenCVDecoder : public ftl::codecs::Decoder {
bool decode(const ftl::codecs::Packet &pkt, cv::cuda::GpuMat &out) override;
bool accepts(const ftl::codecs::Packet &pkt);
+
+ private:
+ cv::Mat tmp_;
};
}
diff --git a/components/codecs/include/ftl/codecs/opencv_encoder.hpp b/components/codecs/include/ftl/codecs/opencv_encoder.hpp
index aabe9247c81d2dbcb365ad259d3f44bfce6b7f95..82b0a5cccf32398ed6e94d9a7a84d0c0ee1b9f25 100644
--- a/components/codecs/include/ftl/codecs/opencv_encoder.hpp
+++ b/components/codecs/include/ftl/codecs/opencv_encoder.hpp
@@ -20,12 +20,12 @@ class OpenCVEncoder : public ftl::codecs::Encoder {
ftl::codecs::definition_t mindef);
~OpenCVEncoder();
- bool encode(const cv::Mat &in, ftl::codecs::preset_t preset,
+ bool encode(const cv::cuda::GpuMat &in, ftl::codecs::preset_t preset,
const std::function<void(const ftl::codecs::Packet&)> &cb) {
return Encoder::encode(in, preset, cb);
}
- bool encode(const cv::Mat &in, ftl::codecs::definition_t definition, ftl::codecs::bitrate_t bitrate,
+ bool encode(const cv::cuda::GpuMat &in, ftl::codecs::definition_t definition, ftl::codecs::bitrate_t bitrate,
const std::function<void(const ftl::codecs::Packet&)>&) override;
bool supports(ftl::codecs::codec_t codec) override;
@@ -40,6 +40,7 @@ class OpenCVEncoder : public ftl::codecs::Encoder {
std::atomic<int> jobs_;
std::mutex job_mtx_;
std::condition_variable job_cv_;
+ cv::Mat tmp_;
bool _encodeBlock(const cv::Mat &in, ftl::codecs::Packet &pkt, ftl::codecs::bitrate_t);
};
diff --git a/components/codecs/src/encoder.cpp b/components/codecs/src/encoder.cpp
index 1863222d6944e5bc5dfbea9fd8aa90d4fdc821f7..9a7eac72def3a20b966e4332d45d8073f57c47f6 100644
--- a/components/codecs/src/encoder.cpp
+++ b/components/codecs/src/encoder.cpp
@@ -70,7 +70,7 @@ Encoder::~Encoder() {
}
-bool Encoder::encode(const cv::Mat &in, preset_t preset,
+bool Encoder::encode(const cv::cuda::GpuMat &in, preset_t preset,
const std::function<void(const ftl::codecs::Packet&)> &cb) {
const auto &settings = ftl::codecs::getPreset(preset);
const definition_t definition = (in.type() == CV_32F) ? settings.depth_res : settings.colour_res;
diff --git a/components/codecs/src/nvpipe_decoder.cpp b/components/codecs/src/nvpipe_decoder.cpp
index 54540be1cd72e7a53c88b8faae8f9754b45783e9..eeb89f93c9e5194260360fefcbb57e45719ab2cb 100644
--- a/components/codecs/src/nvpipe_decoder.cpp
+++ b/components/codecs/src/nvpipe_decoder.cpp
@@ -69,38 +69,38 @@ bool NvPipeDecoder::decode(const ftl::codecs::Packet &pkt, cv::cuda::GpuMat &out
// No I-Frame yet so don't attempt to decode P-Frames.
if (!seen_iframe_) return false;
- int rc = NvPipe_Decode(nv_decoder_, pkt.data.data(), pkt.data.size(), tmp.data, tmp.cols, tmp.rows);
+ int rc = NvPipe_Decode(nv_decoder_, pkt.data.data(), pkt.data.size(), tmp_.data, tmp_.cols, tmp_.rows);
if (rc == 0) LOG(ERROR) << "NvPipe decode error: " << NvPipe_GetError(nv_decoder_);
if (is_float_frame) {
// Is the received frame the same size as requested output?
if (out.rows == ftl::codecs::getHeight(pkt.definition)) {
- tmp.convertTo(out, CV_32FC1, 1.0f/1000.0f);
+ tmp_.convertTo(out, CV_32FC1, 1.0f/1000.0f);
} else {
- LOG(WARNING) << "Resizing decoded frame from " << tmp.size() << " to " << out.size();
+ LOG(WARNING) << "Resizing decoded frame from " << tmp_.size() << " to " << out.size();
// FIXME: This won't work on GPU
- tmp.convertTo(tmp, CV_32FC1, 1.0f/1000.0f);
- cv::cuda::resize(tmp, out, out.size(), 0, 0, cv::INTER_NEAREST);
+ tmp_.convertTo(tmp_, CV_32FC1, 1.0f/1000.0f);
+ cv::cuda::resize(tmp_, out, out.size(), 0, 0, cv::INTER_NEAREST);
}
} else {
// Is the received frame the same size as requested output?
if (out.rows == ftl::codecs::getHeight(pkt.definition)) {
// Flag 0x1 means frame is in RGB so needs conversion to BGR
if (pkt.flags & 0x1) {
- cv::cvtColor(tmp, out, cv::COLOR_RGBA2BGR);
+ cv::cuda::cvtColor(tmp_, out, cv::COLOR_RGBA2BGR);
} else {
- cv::cvtColor(tmp, out, cv::COLOR_BGRA2BGR);
+ cv::cuda::cvtColor(tmp_, out, cv::COLOR_BGRA2BGR);
}
} else {
- LOG(WARNING) << "Resizing decoded frame from " << tmp.size() << " to " << out.size();
+ LOG(WARNING) << "Resizing decoded frame from " << tmp_.size() << " to " << out.size();
// FIXME: This won't work on GPU, plus it allocates extra memory...
// Flag 0x1 means frame is in RGB so needs conversion to BGR
if (pkt.flags & 0x1) {
- cv::cuda::cvtColor(tmp, tmp, cv::COLOR_RGBA2BGR);
+ cv::cuda::cvtColor(tmp_, tmp_, cv::COLOR_RGBA2BGR);
} else {
- cv::cuda::cvtColor(tmp, tmp, cv::COLOR_BGRA2BGR);
+ cv::cuda::cvtColor(tmp_, tmp_, cv::COLOR_BGRA2BGR);
}
- cv::cuda::resize(tmp, out, out.size());
+ cv::cuda::resize(tmp_, out, out.size());
}
}
diff --git a/components/codecs/src/nvpipe_encoder.cpp b/components/codecs/src/nvpipe_encoder.cpp
index 28407a6583d39f3f9091dd6e5fd52c991d562e3e..3ed25448b9138e103563ece1221da860fb9a0a3f 100644
--- a/components/codecs/src/nvpipe_encoder.cpp
+++ b/components/codecs/src/nvpipe_encoder.cpp
@@ -40,7 +40,7 @@ bool NvPipeEncoder::supports(ftl::codecs::codec_t codec) {
}
/* Check preset resolution is not better than actual resolution. */
-definition_t NvPipeEncoder::_verifiedDefinition(definition_t def, const cv::Mat &in) {
+definition_t NvPipeEncoder::_verifiedDefinition(definition_t def, const cv::cuda::GpuMat &in) {
int height = ftl::codecs::getHeight(def);
// FIXME: Make sure this can't go forever
@@ -52,48 +52,22 @@ definition_t NvPipeEncoder::_verifiedDefinition(definition_t def, const cv::Mat
return def;
}
-void scaleDownAndPad(cv::Mat &in, cv::Mat &out) {
- const auto isize = in.size();
- const auto osize = out.size();
- cv::Mat tmp;
-
- if (isize != osize) {
- double x_scale = ((double) isize.width) / osize.width;
- double y_scale = ((double) isize.height) / osize.height;
- double x_scalei = 1.0 / x_scale;
- double y_scalei = 1.0 / y_scale;
-
- if (x_scale > 1.0 || y_scale > 1.0) {
- if (x_scale > y_scale) {
- cv::resize(in, tmp, cv::Size(osize.width, osize.height * x_scalei));
- } else {
- cv::resize(in, tmp, cv::Size(osize.width * y_scalei, osize.height));
- }
- }
- else { tmp = in; }
-
- if (tmp.size().width < osize.width || tmp.size().height < osize.height) {
- tmp.copyTo(out(cv::Rect(cv::Point2i(0, 0), tmp.size())));
- }
- else { out = tmp; }
- }
-}
-
-bool NvPipeEncoder::encode(const cv::Mat &in, definition_t odefinition, bitrate_t bitrate, const std::function<void(const ftl::codecs::Packet&)> &cb) {
+bool NvPipeEncoder::encode(const cv::cuda::GpuMat &in, definition_t odefinition, bitrate_t bitrate, const std::function<void(const ftl::codecs::Packet&)> &cb) {
cudaSetDevice(0);
auto definition = odefinition; //_verifiedDefinition(odefinition, in);
auto width = ftl::codecs::getWidth(definition);
auto height = ftl::codecs::getHeight(definition);
- cv::Mat tmp;
+ cv::cuda::GpuMat tmp;
if (width != in.cols || height != in.rows) {
LOG(WARNING) << "Mismatch resolution with encoding resolution";
if (in.type() == CV_32F) {
- cv::resize(in, tmp, cv::Size(width,height), 0.0, 0.0, cv::INTER_NEAREST);
+ cv::cuda::resize(in, tmp_, cv::Size(width,height), 0.0, 0.0, cv::INTER_NEAREST);
} else {
- cv::resize(in, tmp, cv::Size(width,height));
+ cv::cuda::resize(in, tmp_, cv::Size(width,height));
}
+ tmp = tmp_;
} else {
tmp = in;
}
@@ -110,21 +84,16 @@ bool NvPipeEncoder::encode(const cv::Mat &in, definition_t odefinition, bitrate_
//cv::Mat tmp;
if (tmp.type() == CV_32F) {
- tmp.convertTo(tmp, CV_16UC1, 1000);
+ tmp.convertTo(tmp2_, CV_16UC1, 1000);
} else if (tmp.type() == CV_8UC3) {
- cv::cvtColor(tmp, tmp, cv::COLOR_BGR2RGBA);
+ cv::cuda::cvtColor(tmp, tmp2_, cv::COLOR_BGR2RGBA);
} else if (tmp.type() == CV_8UC4) {
- cv::cvtColor(tmp, tmp, cv::COLOR_BGRA2RGBA);
+ cv::cuda::cvtColor(tmp, tmp2_, cv::COLOR_BGRA2RGBA);
} else {
LOG(ERROR) << "Unsupported cv::Mat type in Nvidia encoder";
return false;
}
- // scale/pad to fit output format
- //cv::Mat tmp2 = cv::Mat::zeros(getHeight(odefinition), getWidth(odefinition), tmp.type());
- //scaleDownAndPad(tmp, tmp2);
- //std::swap(tmp, tmp2);
-
Packet pkt;
pkt.codec = (preference_ == codec_t::Any) ? codec_t::HEVC : preference_;
pkt.definition = definition;
@@ -135,12 +104,12 @@ bool NvPipeEncoder::encode(const cv::Mat &in, definition_t odefinition, bitrate_
pkt.data.resize(ftl::codecs::kVideoBufferSize);
uint64_t cs = NvPipe_Encode(
nvenc_,
- tmp.data,
- tmp.step,
+ tmp2_.data,
+ tmp2_.step,
pkt.data.data(),
ftl::codecs::kVideoBufferSize,
- tmp.cols,
- tmp.rows,
+ tmp2_.cols,
+ tmp2_.rows,
was_reset_ // Force IFrame!
);
pkt.data.resize(cs);
@@ -155,7 +124,7 @@ bool NvPipeEncoder::encode(const cv::Mat &in, definition_t odefinition, bitrate_
}
}
-bool NvPipeEncoder::_encoderMatch(const cv::Mat &in, definition_t def) {
+bool NvPipeEncoder::_encoderMatch(const cv::cuda::GpuMat &in, definition_t def) {
return ((in.type() == CV_32F && is_float_channel_) ||
((in.type() == CV_8UC3 || in.type() == CV_8UC4) && !is_float_channel_)) && current_definition_ == def;
}
@@ -183,7 +152,7 @@ static uint64_t calculateBitrate(definition_t def, bitrate_t rate) {
return uint64_t(bitrate * 1000.0f * 1000.0f * scale);
}
-bool NvPipeEncoder::_createEncoder(const cv::Mat &in, definition_t def, bitrate_t rate) {
+bool NvPipeEncoder::_createEncoder(const cv::cuda::GpuMat &in, definition_t def, bitrate_t rate) {
if (_encoderMatch(in, def) && nvenc_) return true;
uint64_t bitrate = calculateBitrate(def, rate);
diff --git a/components/codecs/src/opencv_decoder.cpp b/components/codecs/src/opencv_decoder.cpp
index c7e54531a0ee19975033868b3a0e80e1f43ecdff..0b9feea46e5925f16ce5ab323747d94d8bdb1d2a 100644
--- a/components/codecs/src/opencv_decoder.cpp
+++ b/components/codecs/src/opencv_decoder.cpp
@@ -17,8 +17,7 @@ bool OpenCVDecoder::accepts(const ftl::codecs::Packet &pkt) {
return (pkt.codec == codec_t::JPG || pkt.codec == codec_t::PNG);
}
-bool OpenCVDecoder::decode(const ftl::codecs::Packet &pkt, cv::Mat &out) {
- cv::Mat tmp;
+bool OpenCVDecoder::decode(const ftl::codecs::Packet &pkt, cv::cuda::GpuMat &out) {
int chunk_dim = std::sqrt(pkt.block_total);
int chunk_width = out.cols / chunk_dim;
@@ -28,12 +27,12 @@ bool OpenCVDecoder::decode(const ftl::codecs::Packet &pkt, cv::Mat &out) {
int cx = (pkt.block_number % chunk_dim) * chunk_width;
int cy = (pkt.block_number / chunk_dim) * chunk_height;
cv::Rect roi(cx,cy,chunk_width,chunk_height);
- cv::Mat chunkHead = out(roi);
+ cv::cuda::GpuMat chunkHead = out(roi);
//LOG(INFO) << "DECODE JPEG " << (int)pkt.block_number << "/" << chunk_dim;
// Decode in temporary buffers to prevent long locks
- cv::imdecode(pkt.data, cv::IMREAD_UNCHANGED, &tmp);
+ cv::imdecode(pkt.data, cv::IMREAD_UNCHANGED, &tmp_);
// Apply colour correction to chunk
//ftl::rgbd::colourCorrection(tmp_rgb, gamma_, temperature_);
@@ -43,21 +42,25 @@ bool OpenCVDecoder::decode(const ftl::codecs::Packet &pkt, cv::Mat &out) {
// Can either check JPG/PNG headers or just use pkt definition.
// Original size so just copy
- if (tmp.cols == chunkHead.cols) {
- if (!tmp.empty() && tmp.type() == CV_16U && chunkHead.type() == CV_32F) {
- tmp.convertTo(chunkHead, CV_32FC1, 1.0f/1000.0f); //(16.0f*10.0f));
- } else if (!tmp.empty() && tmp.type() == CV_8UC3 && chunkHead.type() == CV_8UC3) {
- tmp.copyTo(chunkHead);
+ if (tmp_.cols == chunkHead.cols) {
+ if (!tmp_.empty() && tmp_.type() == CV_16U && chunkHead.type() == CV_32F) {
+ tmp_.convertTo(tmp_, CV_32FC1, 1.0f/1000.0f);
+ chunkHead.upload(tmp_);
+ } else if (!tmp_.empty() && tmp_.type() == CV_8UC3 && chunkHead.type() == CV_8UC3) {
+ //tmp_.copyTo(chunkHead);
+ chunkHead.upload(tmp_);
} else {
// Silent ignore?
}
// Downsized so needs a scale up
} else {
- if (!tmp.empty() && tmp.type() == CV_16U && chunkHead.type() == CV_32F) {
- tmp.convertTo(tmp, CV_32FC1, 1.0f/1000.0f); //(16.0f*10.0f));
- cv::resize(tmp, chunkHead, chunkHead.size(), 0, 0, cv::INTER_NEAREST);
- } else if (!tmp.empty() && tmp.type() == CV_8UC3 && chunkHead.type() == CV_8UC3) {
- cv::resize(tmp, chunkHead, chunkHead.size());
+ if (!tmp_.empty() && tmp_.type() == CV_16U && chunkHead.type() == CV_32F) {
+ tmp_.convertTo(tmp_, CV_32FC1, 1.0f/1000.0f); //(16.0f*10.0f));
+ cv::resize(tmp_, tmp_, chunkHead.size(), 0, 0, cv::INTER_NEAREST);
+ chunkHead.upload(tmp_);
+ } else if (!tmp_.empty() && tmp_.type() == CV_8UC3 && chunkHead.type() == CV_8UC3) {
+ cv::resize(tmp_, tmp_, chunkHead.size());
+ chunkHead.upload(tmp_);
} else {
// Silent ignore?
}
diff --git a/components/codecs/src/opencv_encoder.cpp b/components/codecs/src/opencv_encoder.cpp
index 1ab7a7a3150d99cdec475deb2497f7ff7b013b1e..5dc1995a82e6147184572d6e45d20a8a49561ddc 100644
--- a/components/codecs/src/opencv_encoder.cpp
+++ b/components/codecs/src/opencv_encoder.cpp
@@ -28,21 +28,22 @@ bool OpenCVEncoder::supports(ftl::codecs::codec_t codec) {
}
}
-bool OpenCVEncoder::encode(const cv::Mat &in, definition_t definition, bitrate_t bitrate, const std::function<void(const ftl::codecs::Packet&)> &cb) {
- cv::Mat tmp;
+bool OpenCVEncoder::encode(const cv::cuda::GpuMat &in, definition_t definition, bitrate_t bitrate, const std::function<void(const ftl::codecs::Packet&)> &cb) {
bool is_colour = in.type() != CV_32F;
current_definition_ = definition;
+ in.download(tmp_);
+
// Scale down image to match requested definition...
if (ftl::codecs::getHeight(current_definition_) < in.rows) {
- cv::resize(in, tmp, cv::Size(ftl::codecs::getWidth(current_definition_), ftl::codecs::getHeight(current_definition_)), 0, 0, (is_colour) ? 1 : cv::INTER_NEAREST);
+ cv::resize(tmp_, tmp_, cv::Size(ftl::codecs::getWidth(current_definition_), ftl::codecs::getHeight(current_definition_)), 0, 0, (is_colour) ? 1 : cv::INTER_NEAREST);
} else {
- tmp = in;
+
}
// Represent float at 16bit int
if (!is_colour) {
- tmp.convertTo(tmp, CV_16UC1, 1000);
+ tmp_.convertTo(tmp_, CV_16UC1, 1000);
}
chunk_dim_ = (definition == definition_t::LD360) ? 1 : 4;
@@ -51,7 +52,7 @@ bool OpenCVEncoder::encode(const cv::Mat &in, definition_t definition, bitrate_t
for (int i=0; i<chunk_count_; ++i) {
// Add chunk job to thread pool
- ftl::pool.push([this,i,&tmp,cb,is_colour,bitrate](int id) {
+ ftl::pool.push([this,i,cb,is_colour,bitrate](int id) {
ftl::codecs::Packet pkt;
pkt.block_number = i;
pkt.block_total = chunk_count_;
@@ -59,7 +60,7 @@ bool OpenCVEncoder::encode(const cv::Mat &in, definition_t definition, bitrate_t
pkt.codec = (is_colour) ? codec_t::JPG : codec_t::PNG;
try {
- _encodeBlock(tmp, pkt, bitrate);
+ _encodeBlock(tmp_, pkt, bitrate);
} catch(...) {
LOG(ERROR) << "OpenCV encode block exception: " << i;
}
diff --git a/components/codecs/test/nvpipe_codec_unit.cpp b/components/codecs/test/nvpipe_codec_unit.cpp
index 86f773df64f8fcdd55d493305ffa2505d356a22c..609ce56a50059978af931b718de57098201a0c1a 100644
--- a/components/codecs/test/nvpipe_codec_unit.cpp
+++ b/components/codecs/test/nvpipe_codec_unit.cpp
@@ -24,7 +24,7 @@ namespace ftl {
TEST_CASE( "NvPipeEncoder::encode() - A colour test image at preset 0" ) {
ftl::codecs::NvPipeEncoder encoder(definition_t::HD1080, definition_t::SD480);
- cv::Mat m(cv::Size(1920,1080), CV_8UC3, cv::Scalar(0,0,0));
+ cv::cuda::GpuMat m(cv::Size(1920,1080), CV_8UC3, cv::Scalar(0,0,0));
int block_total = 0;
std::atomic<int> block_count = 0;
@@ -46,7 +46,7 @@ TEST_CASE( "NvPipeEncoder::encode() - A colour test image at preset 0" ) {
TEST_CASE( "NvPipeEncoder::encode() - A depth test image at preset 0" ) {
ftl::codecs::NvPipeEncoder encoder(definition_t::HD1080, definition_t::SD480);
- cv::Mat m(cv::Size(1920,1080), CV_32F, cv::Scalar(0.0f));
+ cv::cuda::GpuMat m(cv::Size(1920,1080), CV_32F, cv::Scalar(0.0f));
int block_total = 0;
std::atomic<int> block_count = 0;
@@ -70,13 +70,13 @@ TEST_CASE( "NvPipeDecoder::decode() - A colour test image" ) {
ftl::codecs::NvPipeEncoder encoder(definition_t::HD1080, definition_t::SD480);
ftl::codecs::NvPipeDecoder decoder;
- cv::Mat in;
- cv::Mat out;
+ cv::cuda::GpuMat in;
+ cv::cuda::GpuMat out;
bool r = false;
SECTION("FHD in and out, FHD encoding") {
- in = cv::Mat(cv::Size(1920,1080), CV_8UC3, cv::Scalar(255,0,0));
- out = cv::Mat(cv::Size(1920,1080), CV_8UC3, cv::Scalar(0,0,0));
+ in = cv::cuda::GpuMat(cv::Size(1920,1080), CV_8UC3, cv::Scalar(255,0,0));
+ out = cv::cuda::GpuMat(cv::Size(1920,1080), CV_8UC3, cv::Scalar(0,0,0));
r = encoder.encode(in, ftl::codecs::kPreset0, [&out,&decoder](const ftl::codecs::Packet &pkt) {
REQUIRE( decoder.decode(pkt, out) );
@@ -84,8 +84,8 @@ TEST_CASE( "NvPipeDecoder::decode() - A colour test image" ) {
}
SECTION("Full HD in, 720 out, FHD encoding") {
- in = cv::Mat(cv::Size(1920,1080), CV_8UC3, cv::Scalar(255,0,0));
- out = cv::Mat(cv::Size(1280,720), CV_8UC3, cv::Scalar(0,0,0));
+ in = cv::cuda::GpuMat(cv::Size(1920,1080), CV_8UC3, cv::Scalar(255,0,0));
+ out = cv::cuda::GpuMat(cv::Size(1280,720), CV_8UC3, cv::Scalar(0,0,0));
r = encoder.encode(in, ftl::codecs::kPreset0, [&out,&decoder](const ftl::codecs::Packet &pkt) {
REQUIRE( decoder.decode(pkt, out) );
@@ -95,8 +95,8 @@ TEST_CASE( "NvPipeDecoder::decode() - A colour test image" ) {
}
SECTION("HHD in, FHD out, FHD encoding") {
- in = cv::Mat(cv::Size(1280,720), CV_8UC3, cv::Scalar(255,0,0));
- out = cv::Mat(cv::Size(1920,1080), CV_8UC3, cv::Scalar(0,0,0));
+ in = cv::cuda::GpuMat(cv::Size(1280,720), CV_8UC3, cv::Scalar(255,0,0));
+ out = cv::cuda::GpuMat(cv::Size(1920,1080), CV_8UC3, cv::Scalar(0,0,0));
r = encoder.encode(in, ftl::codecs::kPreset0, [&out,&decoder](const ftl::codecs::Packet &pkt) {
REQUIRE( decoder.decode(pkt, out) );
@@ -106,8 +106,8 @@ TEST_CASE( "NvPipeDecoder::decode() - A colour test image" ) {
}
SECTION("FHD in, HHD out, SD encoding") {
- in = cv::Mat(cv::Size(1920,1080), CV_8UC3, cv::Scalar(255,0,0));
- out = cv::Mat(cv::Size(1280,720), CV_8UC3, cv::Scalar(0,0,0));
+ in = cv::cuda::GpuMat(cv::Size(1920,1080), CV_8UC3, cv::Scalar(255,0,0));
+ out = cv::cuda::GpuMat(cv::Size(1280,720), CV_8UC3, cv::Scalar(0,0,0));
r = encoder.encode(in, ftl::codecs::kPreset4, [&out,&decoder](const ftl::codecs::Packet &pkt) {
REQUIRE( decoder.decode(pkt, out) );
@@ -117,5 +117,5 @@ TEST_CASE( "NvPipeDecoder::decode() - A colour test image" ) {
}
REQUIRE( r );
- REQUIRE( (cv::sum(out) != cv::Scalar(0,0,0)) );
+ REQUIRE( (cv::cuda::sum(out) != cv::Scalar(0,0,0)) );
}
diff --git a/components/codecs/test/opencv_codec_unit.cpp b/components/codecs/test/opencv_codec_unit.cpp
index dd85140dbfcddbd9c48f0c83795a84b0d4914882..2505eeb8994e1397bc19175f7f0903bdb3000c9d 100644
--- a/components/codecs/test/opencv_codec_unit.cpp
+++ b/components/codecs/test/opencv_codec_unit.cpp
@@ -24,7 +24,7 @@ namespace ftl {
TEST_CASE( "OpenCVEncoder::encode() - A colour test image at preset 0" ) {
ftl::codecs::OpenCVEncoder encoder(definition_t::HD1080, definition_t::SD480);
- cv::Mat m(cv::Size(1024,576), CV_8UC3, cv::Scalar(0,0,0));
+ cv::cuda::GpuMat m(cv::Size(1024,576), CV_8UC3, cv::Scalar(0,0,0));
int block_total = 0;
std::atomic<int> block_count = 0;
@@ -53,7 +53,7 @@ TEST_CASE( "OpenCVEncoder::encode() - A colour test image at preset 0" ) {
TEST_CASE( "OpenCVEncoder::encode() - A depth test image at preset 0" ) {
ftl::codecs::OpenCVEncoder encoder(definition_t::HD1080, definition_t::SD480);
- cv::Mat m(cv::Size(1024,576), CV_32F, cv::Scalar(0.0f));
+ cv::cuda::GpuMat m(cv::Size(1024,576), CV_32F, cv::Scalar(0.0f));
int block_total = 0;
std::atomic<int> block_count = 0;
@@ -82,8 +82,8 @@ TEST_CASE( "OpenCVEncoder::encode() - A depth test image at preset 0" ) {
TEST_CASE( "OpenCVDecoder::decode() - A colour test image no resolution change" ) {
ftl::codecs::OpenCVEncoder encoder(definition_t::HD1080, definition_t::SD480);
ftl::codecs::OpenCVDecoder decoder;
- cv::Mat in(cv::Size(1024,576), CV_8UC3, cv::Scalar(255,0,0));
- cv::Mat out(cv::Size(1024,576), CV_8UC3, cv::Scalar(0,0,0));
+ cv::cuda::GpuMat in(cv::Size(1024,576), CV_8UC3, cv::Scalar(255,0,0));
+ cv::cuda::GpuMat out(cv::Size(1024,576), CV_8UC3, cv::Scalar(0,0,0));
std::mutex mtx;
@@ -92,5 +92,5 @@ TEST_CASE( "OpenCVDecoder::decode() - A colour test image no resolution change"
REQUIRE( decoder.decode(pkt, out) );
});
- REQUIRE( (cv::sum(out) != cv::Scalar(0,0,0)) );
+ REQUIRE( (cv::cuda::sum(out) != cv::Scalar(0,0,0)) );
}
diff --git a/components/rgbd-sources/include/ftl/rgbd/detail/source.hpp b/components/rgbd-sources/include/ftl/rgbd/detail/source.hpp
index 0d188c0b163777e842f3bd8c31f91ddd51435269..712d29170739be2ee966208439df6ba342df752e 100644
--- a/components/rgbd-sources/include/ftl/rgbd/detail/source.hpp
+++ b/components/rgbd-sources/include/ftl/rgbd/detail/source.hpp
@@ -62,8 +62,8 @@ class Source {
capability_t capabilities_;
ftl::rgbd::Source *host_;
ftl::rgbd::Camera params_;
- cv::Mat rgb_;
- cv::Mat depth_;
+ cv::cuda::GpuMat rgb_;
+ cv::cuda::GpuMat depth_;
int64_t timestamp_;
//Eigen::Matrix4f pose_;
};
diff --git a/components/rgbd-sources/src/group.cpp b/components/rgbd-sources/src/group.cpp
index 1eae2837a31746df611c324882699ae7a04b20f7..b2236ef3c83821d38ff16e8132748a635dd2712f 100644
--- a/components/rgbd-sources/src/group.cpp
+++ b/components/rgbd-sources/src/group.cpp
@@ -79,7 +79,7 @@ void Group::addSource(ftl::rgbd::Source *src) {
//fs.channel1[ix].create(rgb.size(), rgb.type());
//fs.channel2[ix].create(depth.size(), depth.type());
fs.frames[ix].create<cv::cuda::GpuMat>(Channel::Colour, Format<uchar3>(rgb.size())); //.create(rgb.size(), rgb.type());
- if (chan != Channel::None) fs.frames[ix].create<cv::cuda::Mat>(chan, ftl::rgbd::FormatBase(depth.cols, depth.rows, depth.type())); //.create(depth.size(), depth.type());
+ if (chan != Channel::None) fs.frames[ix].create<cv::cuda::GpuMat>(chan, ftl::rgbd::FormatBase(depth.cols, depth.rows, depth.type())); //.create(depth.size(), depth.type());
//cv::swap(rgb, fs.channel1[ix]);
//cv::swap(depth, fs.channel2[ix]);
diff --git a/components/rgbd-sources/src/sources/middlebury/middlebury_source.cpp b/components/rgbd-sources/src/sources/middlebury/middlebury_source.cpp
index e82167fdcf6b4e2bfd1a38a00b50e3cdceeb2aef..d152d25e5a7d8830e8b555851f94d97b70463e29 100644
--- a/components/rgbd-sources/src/sources/middlebury/middlebury_source.cpp
+++ b/components/rgbd-sources/src/sources/middlebury/middlebury_source.cpp
@@ -129,30 +129,30 @@ MiddleburySource::MiddleburySource(ftl::rgbd::Source *host, const string &dir)
disp_->setMask(mask_l_);
// Load image files...
- cv::Mat right_tmp;
- rgb_ = cv::imread(dir+"/im0.png", cv::IMREAD_COLOR);
+ cv::Mat left_tmp, right_tmp;
+ left_tmp = cv::imread(dir+"/im0.png", cv::IMREAD_COLOR);
right_tmp = cv::imread(dir+"/im1.png", cv::IMREAD_COLOR);
- cv::resize(rgb_, rgb_, cv::Size(params_.width, params_.height));
+ cv::resize(left_tmp, left_tmp, cv::Size(params_.width, params_.height));
cv::resize(right_tmp, right_tmp, cv::Size(params_.width, params_.height));
- left_.upload(rgb_);
- right_.upload(right_tmp);
+ rgb_.upload(left_tmp, stream_);
+ right_.upload(right_tmp, stream_);
_performDisparity();
ready_ = true;
}
void MiddleburySource::_performDisparity() {
- if (depth_tmp_.empty()) depth_tmp_ = cv::cuda::GpuMat(left_.size(), CV_32FC1);
- if (disp_tmp_.empty()) disp_tmp_ = cv::cuda::GpuMat(left_.size(), CV_32FC1);
+ depth_.create(left_.size(), CV_32FC1);
+ disp_tmp_.create(left_.size(), CV_32FC1);
//calib_->rectifyStereo(left_, right_, stream_);
- disp_->compute(left_, right_, disp_tmp_, stream_);
+ disp_->compute(rgb_, right_, disp_tmp_, stream_);
//disparityToDepth(disp_tmp_, depth_tmp_, params_, stream_);
- ftl::cuda::disparity_to_depth(disp_tmp_, depth_tmp_, params_, stream_);
+ ftl::cuda::disparity_to_depth(disp_tmp_, depth_, params_, stream_);
//left_.download(rgb_, stream_);
//rgb_ = lsrc_->cachedLeft();
- depth_tmp_.download(depth_, stream_);
+ //depth_tmp_.download(depth_, stream_);
stream_.waitForCompletion();
diff --git a/components/rgbd-sources/src/sources/net/net.cpp b/components/rgbd-sources/src/sources/net/net.cpp
index 56b6355eb9696c6684c46a80a6fb3ff1e0584d5a..e4073536a574255965de81ab5f2294e008695032 100644
--- a/components/rgbd-sources/src/sources/net/net.cpp
+++ b/components/rgbd-sources/src/sources/net/net.cpp
@@ -422,9 +422,9 @@ bool NetSource::compute(int n, int b) {
// Verify depth destination is of required type
if (isFloatChannel(chan) && depth_.type() != CV_32F) {
- depth_ = cv::Mat(cv::Size(params_.width, params_.height), CV_32FC1, 0.0f);
+ depth_.create(cv::Size(params_.width, params_.height), CV_32FC1); // 0.0f
} else if (!isFloatChannel(chan) && depth_.type() != CV_8UC3) {
- depth_ = cv::Mat(cv::Size(params_.width, params_.height), CV_8UC3, cv::Scalar(0,0,0));
+ depth_.create(cv::Size(params_.width, params_.height), CV_8UC3); // cv::Scalar(0,0,0)
}
if (prev_chan_ != chan) {
diff --git a/components/rgbd-sources/src/sources/realsense/realsense_source.cpp b/components/rgbd-sources/src/sources/realsense/realsense_source.cpp
index b458aa3e77c8557ee828a8f71431bb5e4e665066..2b55356c853cdafc2d6aa3be523e07376e7ac0f8 100644
--- a/components/rgbd-sources/src/sources/realsense/realsense_source.cpp
+++ b/components/rgbd-sources/src/sources/realsense/realsense_source.cpp
@@ -54,9 +54,11 @@ bool RealsenseSource::compute(int n, int b) {
float h = depth.get_height();
rscolour_ = frames.first(RS2_STREAM_COLOR); //.get_color_frame();
- cv::Mat tmp(cv::Size((int)w, (int)h), CV_16UC1, (void*)depth.get_data(), depth.get_stride_in_bytes());
- tmp.convertTo(depth_, CV_32FC1, scale_);
- rgb_ = cv::Mat(cv::Size(w, h), CV_8UC4, (void*)rscolour_.get_data(), cv::Mat::AUTO_STEP);
+ cv::Mat tmp_depth(cv::Size((int)w, (int)h), CV_16UC1, (void*)depth.get_data(), depth.get_stride_in_bytes());
+ tmp_depth.convertTo(tmp_depth, CV_32FC1, scale_);
+ depth_.upload(tmp_depth);
+ cv::Mat tmp_rgb(cv::Size(w, h), CV_8UC4, (void*)rscolour_.get_data(), cv::Mat::AUTO_STEP);
+ rgb_.upload(tmp_rgb);
auto cb = host_->callback();
if (cb) cb(timestamp_, rgb_, depth_);
diff --git a/components/rgbd-sources/src/sources/virtual/virtual.cpp b/components/rgbd-sources/src/sources/virtual/virtual.cpp
index e9fe781b58eb47593c3b5d3c5ae0fbec800bc535..fbead541591bbd5398ce81d8121ae2946fd1928f 100644
--- a/components/rgbd-sources/src/sources/virtual/virtual.cpp
+++ b/components/rgbd-sources/src/sources/virtual/virtual.cpp
@@ -91,16 +91,16 @@ class VirtualImpl : public ftl::rgbd::detail::Source {
}
if (frame.hasChannel(Channel::Colour)) {
- frame.download(Channel::Colour);
- cv::swap(frame.get<cv::Mat>(Channel::Colour), rgb_);
+ //frame.download(Channel::Colour);
+ cv::cuda::swap(frame.get<cv::cuda::GpuMat>(Channel::Colour), rgb_);
} else {
LOG(ERROR) << "Channel 1 frame in rendering";
}
if ((host_->getChannel() != Channel::None) &&
frame.hasChannel(host_->getChannel())) {
- frame.download(host_->getChannel());
- cv::swap(frame.get<cv::Mat>(host_->getChannel()), depth_);
+ //frame.download(host_->getChannel());
+ cv::cuda::swap(frame.get<cv::cuda::GpuMat>(host_->getChannel()), depth_);
}
auto cb = host_->callback();
diff --git a/components/rgbd-sources/src/streamer.cpp b/components/rgbd-sources/src/streamer.cpp
index 8ecda51b7a53c17552a9a51fd91ceb658e2982be..e05e7faf9305ac0a3ede6dade21596bfe8251db7 100644
--- a/components/rgbd-sources/src/streamer.cpp
+++ b/components/rgbd-sources/src/streamer.cpp
@@ -468,7 +468,7 @@ void Streamer::_process(ftl::rgbd::FrameSet &fs) {
auto chan = fs.sources[j]->getChannel();
- enc2->encode(fs.frames[j].get<cv::Mat>(chan), src->hq_bitrate, [this,src,hasChan2,chan](const ftl::codecs::Packet &blk){
+ enc2->encode(fs.frames[j].get<cv::cuda::GpuMat>(chan), src->hq_bitrate, [this,src,hasChan2,chan](const ftl::codecs::Packet &blk){
_transmitPacket(src, blk, chan, hasChan2, Quality::High);
});
} else {
@@ -477,7 +477,7 @@ void Streamer::_process(ftl::rgbd::FrameSet &fs) {
// TODO: Stagger the reset between nodes... random phasing
if (fs.timestamp % (10*ftl::timer::getInterval()) == 0) enc1->reset();
- enc1->encode(fs.frames[j].get<cv::Mat>(Channel::Colour), src->hq_bitrate, [this,src,hasChan2](const ftl::codecs::Packet &blk){
+ enc1->encode(fs.frames[j].get<cv::cuda::GpuMat>(Channel::Colour), src->hq_bitrate, [this,src,hasChan2](const ftl::codecs::Packet &blk){
_transmitPacket(src, blk, Channel::Colour, hasChan2, Quality::High);
});
}
@@ -500,14 +500,14 @@ void Streamer::_process(ftl::rgbd::FrameSet &fs) {
if (hasChan2) {
auto chan = fs.sources[j]->getChannel();
- enc2->encode(fs.frames[j].get<cv::Mat>(chan), src->lq_bitrate, [this,src,hasChan2,chan](const ftl::codecs::Packet &blk){
+ enc2->encode(fs.frames[j].get<cv::cuda::GpuMat>(chan), src->lq_bitrate, [this,src,hasChan2,chan](const ftl::codecs::Packet &blk){
_transmitPacket(src, blk, chan, hasChan2, Quality::Low);
});
} else {
if (enc2) enc2->reset();
}
- enc1->encode(fs.frames[j].get<cv::Mat>(Channel::Colour), src->lq_bitrate, [this,src,hasChan2](const ftl::codecs::Packet &blk){
+ enc1->encode(fs.frames[j].get<cv::cuda::GpuMat>(Channel::Colour), src->lq_bitrate, [this,src,hasChan2](const ftl::codecs::Packet &blk){
_transmitPacket(src, blk, Channel::Colour, hasChan2, Quality::Low);
});
}