diff --git a/applications/gui/src/camera.cpp b/applications/gui/src/camera.cpp
index f35b4e539e8216b32106762b69ea7e2b028706eb..cca8a40f35c6e679597185668661fe4a814e504f 100644
--- a/applications/gui/src/camera.cpp
+++ b/applications/gui/src/camera.cpp
@@ -148,17 +148,20 @@ ftl::gui::Camera::Camera(ftl::gui::Screen *screen, ftl::rgbd::Source *src) : scr
posewin_->setTheme(screen->windowtheme);
posewin_->setVisible(false);
- src->setCallback([this](int64_t ts, cv::Mat &channel1, cv::Mat &channel2) {
+ src->setCallback([this](int64_t ts, cv::cuda::GpuMat &channel1, cv::cuda::GpuMat &channel2) {
UNIQUE_LOCK(mutex_, lk);
im1_.create(channel1.size(), channel1.type());
im2_.create(channel2.size(), channel2.type());
//cv::swap(channel1, im1_);
//cv::swap(channel2, im2_);
+
+ channel1.download(im1_);
+ channel2.download(im2_);
// OpenGL (0,0) bottom left
- cv::flip(channel1, im1_, 0);
- cv::flip(channel2, im2_, 0);
+ cv::flip(im1_, im1_, 0);
+ cv::flip(im2_, im2_, 0);
});
}
diff --git a/applications/player/src/main.cpp b/applications/player/src/main.cpp
index 60d2793c1c4b0484dbb226bff12deebfb6e1fc2e..7eeb6e6bf5f96fd8f32360949c566c93860518db 100644
--- a/applications/player/src/main.cpp
+++ b/applications/player/src/main.cpp
@@ -77,11 +77,13 @@ int main(int argc, char **argv) {
//LOG(INFO) << "Reading packet: (" << (int)spkt.streamID << "," << (int)spkt.channel << ") " << (int)pkt.codec << ", " << (int)pkt.definition;
- cv::Mat frame(cv::Size(ftl::codecs::getWidth(pkt.definition),ftl::codecs::getHeight(pkt.definition)), (spkt.channel == Channel::Depth) ? CV_32F : CV_8UC3);
+ cv::cuda::GpuMat gframe(cv::Size(ftl::codecs::getWidth(pkt.definition),ftl::codecs::getHeight(pkt.definition)), (spkt.channel == Channel::Depth) ? CV_32F : CV_8UC3);
+ cv::Mat frame;
createDecoder(pkt);
try {
- decoder->decode(pkt, frame);
+ decoder->decode(pkt, gframe);
+ gframe.download(frame);
} catch (std::exception &e) {
LOG(INFO) << "Decoder exception: " << e.what();
}
diff --git a/applications/reconstruct/src/ilw/ilw.cpp b/applications/reconstruct/src/ilw/ilw.cpp
index 93fd75a188b5e48e48e387c5ddf96ea43bcb1eec..dfa43267ed5d3ac09309a466b150c0498fb72967 100644
--- a/applications/reconstruct/src/ilw/ilw.cpp
+++ b/applications/reconstruct/src/ilw/ilw.cpp
@@ -144,7 +144,7 @@ ILW::~ILW() {
bool ILW::process(ftl::rgbd::FrameSet &fs) {
if (!enabled_) return false;
- fs.upload(Channel::Colour + Channel::Depth, stream_);
+ //fs.upload(Channel::Colour + Channel::Depth, stream_);
_phase0(fs, stream_);
params_.range = value("search_range", 0.05f);
diff --git a/components/codecs/include/ftl/codecs/decoder.hpp b/components/codecs/include/ftl/codecs/decoder.hpp
index e6883680f34085f8b39fbcd02959b19ea4a2ca9e..7684990e204a1231af0cd8c114eabbf4ef9290d5 100644
--- a/components/codecs/include/ftl/codecs/decoder.hpp
+++ b/components/codecs/include/ftl/codecs/decoder.hpp
@@ -35,7 +35,7 @@ class Decoder {
Decoder() {};
virtual ~Decoder() {};
- virtual bool decode(const ftl::codecs::Packet &pkt, cv::Mat &out)=0;
+ virtual bool decode(const ftl::codecs::Packet &pkt, cv::cuda::GpuMat &out)=0;
virtual bool accepts(const ftl::codecs::Packet &)=0;
};
diff --git a/components/codecs/include/ftl/codecs/encoder.hpp b/components/codecs/include/ftl/codecs/encoder.hpp
index fed8d95755859d2c9c71ee1475f89d024bf38811..9c3aa8fefc64810bf7660e323b44a3c4440d5098 100644
--- a/components/codecs/include/ftl/codecs/encoder.hpp
+++ b/components/codecs/include/ftl/codecs/encoder.hpp
@@ -60,7 +60,7 @@ class Encoder {
/**
* Wrapper encode to allow use of presets.
*/
- virtual bool encode(const cv::Mat &in, ftl::codecs::preset_t preset,
+ virtual bool encode(const cv::cuda::GpuMat &in, ftl::codecs::preset_t preset,
const std::function<void(const ftl::codecs::Packet&)> &cb);
/**
@@ -77,7 +77,7 @@ class Encoder {
* @return True if succeeded with encoding.
*/
virtual bool encode(
- const cv::Mat &in,
+ const cv::cuda::GpuMat &in,
ftl::codecs::definition_t definition,
ftl::codecs::bitrate_t bitrate,
const std::function<void(const ftl::codecs::Packet&)> &cb)=0;
diff --git a/components/codecs/include/ftl/codecs/nvpipe_decoder.hpp b/components/codecs/include/ftl/codecs/nvpipe_decoder.hpp
index 75807f05ee45c66d7a8b231c5d755190754f63df..987915ea2595c6b1e88cf8959b239cb7eb7a3f09 100644
--- a/components/codecs/include/ftl/codecs/nvpipe_decoder.hpp
+++ b/components/codecs/include/ftl/codecs/nvpipe_decoder.hpp
@@ -14,7 +14,7 @@ class NvPipeDecoder : public ftl::codecs::Decoder {
NvPipeDecoder();
~NvPipeDecoder();
- bool decode(const ftl::codecs::Packet &pkt, cv::Mat &out);
+ bool decode(const ftl::codecs::Packet &pkt, cv::cuda::GpuMat &out) override;
bool accepts(const ftl::codecs::Packet &pkt);
@@ -24,6 +24,8 @@ class NvPipeDecoder : public ftl::codecs::Decoder {
ftl::codecs::definition_t last_definition_;
MUTEX mutex_;
bool seen_iframe_;
+ cv::cuda::GpuMat tmp_;
+ cv::cuda::Stream stream_;
};
}
diff --git a/components/codecs/include/ftl/codecs/nvpipe_encoder.hpp b/components/codecs/include/ftl/codecs/nvpipe_encoder.hpp
index c2fe379dc985488f014e911dd1e096431bdee96e..5d04068c53cf3b46dee73c63cf8e2fcf674f148d 100644
--- a/components/codecs/include/ftl/codecs/nvpipe_encoder.hpp
+++ b/components/codecs/include/ftl/codecs/nvpipe_encoder.hpp
@@ -13,12 +13,12 @@ class NvPipeEncoder : public ftl::codecs::Encoder {
ftl::codecs::definition_t mindef);
~NvPipeEncoder();
- bool encode(const cv::Mat &in, ftl::codecs::preset_t preset,
+ bool encode(const cv::cuda::GpuMat &in, ftl::codecs::preset_t preset,
const std::function<void(const ftl::codecs::Packet&)> &cb) {
return Encoder::encode(in, preset, cb);
}
- bool encode(const cv::Mat &in, ftl::codecs::definition_t definition, ftl::codecs::bitrate_t bitrate,
+ bool encode(const cv::cuda::GpuMat &in, ftl::codecs::definition_t definition, ftl::codecs::bitrate_t bitrate,
const std::function<void(const ftl::codecs::Packet&)>&) override;
//bool encode(const cv::cuda::GpuMat &in, std::vector<uint8_t> &out, bitrate_t bix, bool);
@@ -35,10 +35,13 @@ class NvPipeEncoder : public ftl::codecs::Encoder {
bool is_float_channel_;
bool was_reset_;
ftl::codecs::codec_t preference_;
+ cv::cuda::GpuMat tmp_;
+ cv::cuda::GpuMat tmp2_;
+ cv::cuda::Stream stream_;
- bool _encoderMatch(const cv::Mat &in, definition_t def);
- bool _createEncoder(const cv::Mat &in, definition_t def, bitrate_t rate);
- ftl::codecs::definition_t _verifiedDefinition(ftl::codecs::definition_t def, const cv::Mat &in);
+ bool _encoderMatch(const cv::cuda::GpuMat &in, definition_t def);
+ bool _createEncoder(const cv::cuda::GpuMat &in, definition_t def, bitrate_t rate);
+ ftl::codecs::definition_t _verifiedDefinition(ftl::codecs::definition_t def, const cv::cuda::GpuMat &in);
};
}
diff --git a/components/codecs/include/ftl/codecs/opencv_decoder.hpp b/components/codecs/include/ftl/codecs/opencv_decoder.hpp
index c4ef5ab057b1e60da12f36fd603642ea16026888..53b61f683bc1d0a93a9c21049e0d312c133c83f7 100644
--- a/components/codecs/include/ftl/codecs/opencv_decoder.hpp
+++ b/components/codecs/include/ftl/codecs/opencv_decoder.hpp
@@ -11,9 +11,12 @@ class OpenCVDecoder : public ftl::codecs::Decoder {
OpenCVDecoder();
~OpenCVDecoder();
- bool decode(const ftl::codecs::Packet &pkt, cv::Mat &out);
+ bool decode(const ftl::codecs::Packet &pkt, cv::cuda::GpuMat &out) override;
bool accepts(const ftl::codecs::Packet &pkt);
+
+ private:
+ cv::Mat tmp_;
};
}
diff --git a/components/codecs/include/ftl/codecs/opencv_encoder.hpp b/components/codecs/include/ftl/codecs/opencv_encoder.hpp
index aabe9247c81d2dbcb365ad259d3f44bfce6b7f95..82b0a5cccf32398ed6e94d9a7a84d0c0ee1b9f25 100644
--- a/components/codecs/include/ftl/codecs/opencv_encoder.hpp
+++ b/components/codecs/include/ftl/codecs/opencv_encoder.hpp
@@ -20,12 +20,12 @@ class OpenCVEncoder : public ftl::codecs::Encoder {
ftl::codecs::definition_t mindef);
~OpenCVEncoder();
- bool encode(const cv::Mat &in, ftl::codecs::preset_t preset,
+ bool encode(const cv::cuda::GpuMat &in, ftl::codecs::preset_t preset,
const std::function<void(const ftl::codecs::Packet&)> &cb) {
return Encoder::encode(in, preset, cb);
}
- bool encode(const cv::Mat &in, ftl::codecs::definition_t definition, ftl::codecs::bitrate_t bitrate,
+ bool encode(const cv::cuda::GpuMat &in, ftl::codecs::definition_t definition, ftl::codecs::bitrate_t bitrate,
const std::function<void(const ftl::codecs::Packet&)>&) override;
bool supports(ftl::codecs::codec_t codec) override;
@@ -40,6 +40,7 @@ class OpenCVEncoder : public ftl::codecs::Encoder {
std::atomic<int> jobs_;
std::mutex job_mtx_;
std::condition_variable job_cv_;
+ cv::Mat tmp_;
bool _encodeBlock(const cv::Mat &in, ftl::codecs::Packet &pkt, ftl::codecs::bitrate_t);
};
diff --git a/components/codecs/src/encoder.cpp b/components/codecs/src/encoder.cpp
index 1863222d6944e5bc5dfbea9fd8aa90d4fdc821f7..9a7eac72def3a20b966e4332d45d8073f57c47f6 100644
--- a/components/codecs/src/encoder.cpp
+++ b/components/codecs/src/encoder.cpp
@@ -70,7 +70,7 @@ Encoder::~Encoder() {
}
-bool Encoder::encode(const cv::Mat &in, preset_t preset,
+bool Encoder::encode(const cv::cuda::GpuMat &in, preset_t preset,
const std::function<void(const ftl::codecs::Packet&)> &cb) {
const auto &settings = ftl::codecs::getPreset(preset);
const definition_t definition = (in.type() == CV_32F) ? settings.depth_res : settings.colour_res;
diff --git a/components/codecs/src/nvpipe_decoder.cpp b/components/codecs/src/nvpipe_decoder.cpp
index 4c9f515066870f4519b7148c6acc31e32cb84b11..77a3105f88b84f2b9c00f5dba152bbc9814c70db 100644
--- a/components/codecs/src/nvpipe_decoder.cpp
+++ b/components/codecs/src/nvpipe_decoder.cpp
@@ -22,7 +22,7 @@ NvPipeDecoder::~NvPipeDecoder() {
}
}
-bool NvPipeDecoder::decode(const ftl::codecs::Packet &pkt, cv::Mat &out) {
+bool NvPipeDecoder::decode(const ftl::codecs::Packet &pkt, cv::cuda::GpuMat &out) {
cudaSetDevice(0);
UNIQUE_LOCK(mutex_,lk);
if (pkt.codec != codec_t::HEVC && pkt.codec != codec_t::H264) return false;
@@ -57,7 +57,7 @@ bool NvPipeDecoder::decode(const ftl::codecs::Packet &pkt, cv::Mat &out) {
}
// TODO: (Nick) Move to member variable to prevent re-creation
- cv::Mat tmp(cv::Size(ftl::codecs::getWidth(pkt.definition),ftl::codecs::getHeight(pkt.definition)), (is_float_frame) ? CV_16U : CV_8UC4);
+ tmp_.create(cv::Size(ftl::codecs::getWidth(pkt.definition),ftl::codecs::getHeight(pkt.definition)), (is_float_frame) ? CV_16U : CV_8UC4);
// Check for an I-Frame
if (pkt.codec == ftl::codecs::codec_t::HEVC) {
@@ -69,39 +69,43 @@ bool NvPipeDecoder::decode(const ftl::codecs::Packet &pkt, cv::Mat &out) {
// No I-Frame yet so don't attempt to decode P-Frames.
if (!seen_iframe_) return false;
- int rc = NvPipe_Decode(nv_decoder_, pkt.data.data(), pkt.data.size(), tmp.data, tmp.cols, tmp.rows);
+ int rc = NvPipe_Decode(nv_decoder_, pkt.data.data(), pkt.data.size(), tmp_.data, tmp_.cols, tmp_.rows, tmp_.step);
if (rc == 0) LOG(ERROR) << "NvPipe decode error: " << NvPipe_GetError(nv_decoder_);
if (is_float_frame) {
// Is the received frame the same size as requested output?
if (out.rows == ftl::codecs::getHeight(pkt.definition)) {
- tmp.convertTo(out, CV_32FC1, 1.0f/1000.0f);
+ tmp_.convertTo(out, CV_32FC1, 1.0f/1000.0f, stream_);
} else {
- LOG(WARNING) << "Resizing decoded frame from " << tmp.size() << " to " << out.size();
- tmp.convertTo(tmp, CV_32FC1, 1.0f/1000.0f);
- cv::resize(tmp, out, out.size(), 0, 0, cv::INTER_NEAREST);
+ LOG(WARNING) << "Resizing decoded frame from " << tmp_.size() << " to " << out.size();
+ // FIXME: This won't work on GPU
+ tmp_.convertTo(tmp_, CV_32FC1, 1.0f/1000.0f, stream_);
+ cv::cuda::resize(tmp_, out, out.size(), 0, 0, cv::INTER_NEAREST, stream_);
}
} else {
// Is the received frame the same size as requested output?
if (out.rows == ftl::codecs::getHeight(pkt.definition)) {
// Flag 0x1 means frame is in RGB so needs conversion to BGR
if (pkt.flags & 0x1) {
- cv::cvtColor(tmp, out, cv::COLOR_RGBA2BGR);
+ cv::cuda::cvtColor(tmp_, out, cv::COLOR_RGBA2BGR, 0, stream_);
} else {
- cv::cvtColor(tmp, out, cv::COLOR_BGRA2BGR);
+ cv::cuda::cvtColor(tmp_, out, cv::COLOR_BGRA2BGR, 0, stream_);
}
} else {
- LOG(WARNING) << "Resizing decoded frame from " << tmp.size() << " to " << out.size();
+ LOG(WARNING) << "Resizing decoded frame from " << tmp_.size() << " to " << out.size();
+ // FIXME: This won't work on GPU, plus it allocates extra memory...
// Flag 0x1 means frame is in RGB so needs conversion to BGR
if (pkt.flags & 0x1) {
- cv::cvtColor(tmp, tmp, cv::COLOR_RGBA2BGR);
+ cv::cuda::cvtColor(tmp_, tmp_, cv::COLOR_RGBA2BGR, 0, stream_);
} else {
- cv::cvtColor(tmp, tmp, cv::COLOR_BGRA2BGR);
+ cv::cuda::cvtColor(tmp_, tmp_, cv::COLOR_BGRA2BGR, 0, stream_);
}
- cv::resize(tmp, out, out.size());
+ cv::cuda::resize(tmp_, out, out.size(), 0.0, 0.0, cv::INTER_LINEAR, stream_);
}
}
+ stream_.waitForCompletion();
+
return rc > 0;
}
diff --git a/components/codecs/src/nvpipe_encoder.cpp b/components/codecs/src/nvpipe_encoder.cpp
index 28407a6583d39f3f9091dd6e5fd52c991d562e3e..10901ef56993f25960cadd0e6f17778093f256a7 100644
--- a/components/codecs/src/nvpipe_encoder.cpp
+++ b/components/codecs/src/nvpipe_encoder.cpp
@@ -40,7 +40,7 @@ bool NvPipeEncoder::supports(ftl::codecs::codec_t codec) {
}
/* Check preset resolution is not better than actual resolution. */
-definition_t NvPipeEncoder::_verifiedDefinition(definition_t def, const cv::Mat &in) {
+definition_t NvPipeEncoder::_verifiedDefinition(definition_t def, const cv::cuda::GpuMat &in) {
int height = ftl::codecs::getHeight(def);
// FIXME: Make sure this can't go forever
@@ -52,48 +52,22 @@ definition_t NvPipeEncoder::_verifiedDefinition(definition_t def, const cv::Mat
return def;
}
-void scaleDownAndPad(cv::Mat &in, cv::Mat &out) {
- const auto isize = in.size();
- const auto osize = out.size();
- cv::Mat tmp;
-
- if (isize != osize) {
- double x_scale = ((double) isize.width) / osize.width;
- double y_scale = ((double) isize.height) / osize.height;
- double x_scalei = 1.0 / x_scale;
- double y_scalei = 1.0 / y_scale;
-
- if (x_scale > 1.0 || y_scale > 1.0) {
- if (x_scale > y_scale) {
- cv::resize(in, tmp, cv::Size(osize.width, osize.height * x_scalei));
- } else {
- cv::resize(in, tmp, cv::Size(osize.width * y_scalei, osize.height));
- }
- }
- else { tmp = in; }
-
- if (tmp.size().width < osize.width || tmp.size().height < osize.height) {
- tmp.copyTo(out(cv::Rect(cv::Point2i(0, 0), tmp.size())));
- }
- else { out = tmp; }
- }
-}
-
-bool NvPipeEncoder::encode(const cv::Mat &in, definition_t odefinition, bitrate_t bitrate, const std::function<void(const ftl::codecs::Packet&)> &cb) {
+bool NvPipeEncoder::encode(const cv::cuda::GpuMat &in, definition_t odefinition, bitrate_t bitrate, const std::function<void(const ftl::codecs::Packet&)> &cb) {
cudaSetDevice(0);
auto definition = odefinition; //_verifiedDefinition(odefinition, in);
auto width = ftl::codecs::getWidth(definition);
auto height = ftl::codecs::getHeight(definition);
- cv::Mat tmp;
+ cv::cuda::GpuMat tmp;
if (width != in.cols || height != in.rows) {
LOG(WARNING) << "Mismatch resolution with encoding resolution";
if (in.type() == CV_32F) {
- cv::resize(in, tmp, cv::Size(width,height), 0.0, 0.0, cv::INTER_NEAREST);
+ cv::cuda::resize(in, tmp_, cv::Size(width,height), 0.0, 0.0, cv::INTER_NEAREST, stream_);
} else {
- cv::resize(in, tmp, cv::Size(width,height));
+ cv::cuda::resize(in, tmp_, cv::Size(width,height), 0.0, 0.0, cv::INTER_LINEAR, stream_);
}
+ tmp = tmp_;
} else {
tmp = in;
}
@@ -110,20 +84,18 @@ bool NvPipeEncoder::encode(const cv::Mat &in, definition_t odefinition, bitrate_
//cv::Mat tmp;
if (tmp.type() == CV_32F) {
- tmp.convertTo(tmp, CV_16UC1, 1000);
+ tmp.convertTo(tmp2_, CV_16UC1, 1000, stream_);
} else if (tmp.type() == CV_8UC3) {
- cv::cvtColor(tmp, tmp, cv::COLOR_BGR2RGBA);
+ cv::cuda::cvtColor(tmp, tmp2_, cv::COLOR_BGR2RGBA, 0, stream_);
} else if (tmp.type() == CV_8UC4) {
- cv::cvtColor(tmp, tmp, cv::COLOR_BGRA2RGBA);
+ cv::cuda::cvtColor(tmp, tmp2_, cv::COLOR_BGRA2RGBA, 0, stream_);
} else {
LOG(ERROR) << "Unsupported cv::Mat type in Nvidia encoder";
return false;
}
- // scale/pad to fit output format
- //cv::Mat tmp2 = cv::Mat::zeros(getHeight(odefinition), getWidth(odefinition), tmp.type());
- //scaleDownAndPad(tmp, tmp2);
- //std::swap(tmp, tmp2);
+ // Make sure conversions complete...
+ stream_.waitForCompletion();
Packet pkt;
pkt.codec = (preference_ == codec_t::Any) ? codec_t::HEVC : preference_;
@@ -135,12 +107,12 @@ bool NvPipeEncoder::encode(const cv::Mat &in, definition_t odefinition, bitrate_
pkt.data.resize(ftl::codecs::kVideoBufferSize);
uint64_t cs = NvPipe_Encode(
nvenc_,
- tmp.data,
- tmp.step,
+ tmp2_.data,
+ tmp2_.step,
pkt.data.data(),
ftl::codecs::kVideoBufferSize,
- tmp.cols,
- tmp.rows,
+ tmp2_.cols,
+ tmp2_.rows,
was_reset_ // Force IFrame!
);
pkt.data.resize(cs);
@@ -155,7 +127,7 @@ bool NvPipeEncoder::encode(const cv::Mat &in, definition_t odefinition, bitrate_
}
}
-bool NvPipeEncoder::_encoderMatch(const cv::Mat &in, definition_t def) {
+bool NvPipeEncoder::_encoderMatch(const cv::cuda::GpuMat &in, definition_t def) {
return ((in.type() == CV_32F && is_float_channel_) ||
((in.type() == CV_8UC3 || in.type() == CV_8UC4) && !is_float_channel_)) && current_definition_ == def;
}
@@ -183,7 +155,7 @@ static uint64_t calculateBitrate(definition_t def, bitrate_t rate) {
return uint64_t(bitrate * 1000.0f * 1000.0f * scale);
}
-bool NvPipeEncoder::_createEncoder(const cv::Mat &in, definition_t def, bitrate_t rate) {
+bool NvPipeEncoder::_createEncoder(const cv::cuda::GpuMat &in, definition_t def, bitrate_t rate) {
if (_encoderMatch(in, def) && nvenc_) return true;
uint64_t bitrate = calculateBitrate(def, rate);
diff --git a/components/codecs/src/opencv_decoder.cpp b/components/codecs/src/opencv_decoder.cpp
index c7e54531a0ee19975033868b3a0e80e1f43ecdff..0b9feea46e5925f16ce5ab323747d94d8bdb1d2a 100644
--- a/components/codecs/src/opencv_decoder.cpp
+++ b/components/codecs/src/opencv_decoder.cpp
@@ -17,8 +17,7 @@ bool OpenCVDecoder::accepts(const ftl::codecs::Packet &pkt) {
return (pkt.codec == codec_t::JPG || pkt.codec == codec_t::PNG);
}
-bool OpenCVDecoder::decode(const ftl::codecs::Packet &pkt, cv::Mat &out) {
- cv::Mat tmp;
+bool OpenCVDecoder::decode(const ftl::codecs::Packet &pkt, cv::cuda::GpuMat &out) {
int chunk_dim = std::sqrt(pkt.block_total);
int chunk_width = out.cols / chunk_dim;
@@ -28,12 +27,12 @@ bool OpenCVDecoder::decode(const ftl::codecs::Packet &pkt, cv::Mat &out) {
int cx = (pkt.block_number % chunk_dim) * chunk_width;
int cy = (pkt.block_number / chunk_dim) * chunk_height;
cv::Rect roi(cx,cy,chunk_width,chunk_height);
- cv::Mat chunkHead = out(roi);
+ cv::cuda::GpuMat chunkHead = out(roi);
//LOG(INFO) << "DECODE JPEG " << (int)pkt.block_number << "/" << chunk_dim;
// Decode in temporary buffers to prevent long locks
- cv::imdecode(pkt.data, cv::IMREAD_UNCHANGED, &tmp);
+ cv::imdecode(pkt.data, cv::IMREAD_UNCHANGED, &tmp_);
// Apply colour correction to chunk
//ftl::rgbd::colourCorrection(tmp_rgb, gamma_, temperature_);
@@ -43,21 +42,25 @@ bool OpenCVDecoder::decode(const ftl::codecs::Packet &pkt, cv::Mat &out) {
// Can either check JPG/PNG headers or just use pkt definition.
// Original size so just copy
- if (tmp.cols == chunkHead.cols) {
- if (!tmp.empty() && tmp.type() == CV_16U && chunkHead.type() == CV_32F) {
- tmp.convertTo(chunkHead, CV_32FC1, 1.0f/1000.0f); //(16.0f*10.0f));
- } else if (!tmp.empty() && tmp.type() == CV_8UC3 && chunkHead.type() == CV_8UC3) {
- tmp.copyTo(chunkHead);
+ if (tmp_.cols == chunkHead.cols) {
+ if (!tmp_.empty() && tmp_.type() == CV_16U && chunkHead.type() == CV_32F) {
+ tmp_.convertTo(tmp_, CV_32FC1, 1.0f/1000.0f);
+ chunkHead.upload(tmp_);
+ } else if (!tmp_.empty() && tmp_.type() == CV_8UC3 && chunkHead.type() == CV_8UC3) {
+ //tmp_.copyTo(chunkHead);
+ chunkHead.upload(tmp_);
} else {
// Silent ignore?
}
// Downsized so needs a scale up
} else {
- if (!tmp.empty() && tmp.type() == CV_16U && chunkHead.type() == CV_32F) {
- tmp.convertTo(tmp, CV_32FC1, 1.0f/1000.0f); //(16.0f*10.0f));
- cv::resize(tmp, chunkHead, chunkHead.size(), 0, 0, cv::INTER_NEAREST);
- } else if (!tmp.empty() && tmp.type() == CV_8UC3 && chunkHead.type() == CV_8UC3) {
- cv::resize(tmp, chunkHead, chunkHead.size());
+ if (!tmp_.empty() && tmp_.type() == CV_16U && chunkHead.type() == CV_32F) {
+ tmp_.convertTo(tmp_, CV_32FC1, 1.0f/1000.0f); //(16.0f*10.0f));
+ cv::resize(tmp_, tmp_, chunkHead.size(), 0, 0, cv::INTER_NEAREST);
+ chunkHead.upload(tmp_);
+ } else if (!tmp_.empty() && tmp_.type() == CV_8UC3 && chunkHead.type() == CV_8UC3) {
+ cv::resize(tmp_, tmp_, chunkHead.size());
+ chunkHead.upload(tmp_);
} else {
// Silent ignore?
}
diff --git a/components/codecs/src/opencv_encoder.cpp b/components/codecs/src/opencv_encoder.cpp
index 1ab7a7a3150d99cdec475deb2497f7ff7b013b1e..5dc1995a82e6147184572d6e45d20a8a49561ddc 100644
--- a/components/codecs/src/opencv_encoder.cpp
+++ b/components/codecs/src/opencv_encoder.cpp
@@ -28,21 +28,22 @@ bool OpenCVEncoder::supports(ftl::codecs::codec_t codec) {
}
}
-bool OpenCVEncoder::encode(const cv::Mat &in, definition_t definition, bitrate_t bitrate, const std::function<void(const ftl::codecs::Packet&)> &cb) {
- cv::Mat tmp;
+bool OpenCVEncoder::encode(const cv::cuda::GpuMat &in, definition_t definition, bitrate_t bitrate, const std::function<void(const ftl::codecs::Packet&)> &cb) {
bool is_colour = in.type() != CV_32F;
current_definition_ = definition;
+ in.download(tmp_);
+
// Scale down image to match requested definition...
if (ftl::codecs::getHeight(current_definition_) < in.rows) {
- cv::resize(in, tmp, cv::Size(ftl::codecs::getWidth(current_definition_), ftl::codecs::getHeight(current_definition_)), 0, 0, (is_colour) ? 1 : cv::INTER_NEAREST);
+ cv::resize(tmp_, tmp_, cv::Size(ftl::codecs::getWidth(current_definition_), ftl::codecs::getHeight(current_definition_)), 0, 0, (is_colour) ? 1 : cv::INTER_NEAREST);
} else {
- tmp = in;
+
}
// Represent float at 16bit int
if (!is_colour) {
- tmp.convertTo(tmp, CV_16UC1, 1000);
+ tmp_.convertTo(tmp_, CV_16UC1, 1000);
}
chunk_dim_ = (definition == definition_t::LD360) ? 1 : 4;
@@ -51,7 +52,7 @@ bool OpenCVEncoder::encode(const cv::Mat &in, definition_t definition, bitrate_t
for (int i=0; i<chunk_count_; ++i) {
// Add chunk job to thread pool
- ftl::pool.push([this,i,&tmp,cb,is_colour,bitrate](int id) {
+ ftl::pool.push([this,i,cb,is_colour,bitrate](int id) {
ftl::codecs::Packet pkt;
pkt.block_number = i;
pkt.block_total = chunk_count_;
@@ -59,7 +60,7 @@ bool OpenCVEncoder::encode(const cv::Mat &in, definition_t definition, bitrate_t
pkt.codec = (is_colour) ? codec_t::JPG : codec_t::PNG;
try {
- _encodeBlock(tmp, pkt, bitrate);
+ _encodeBlock(tmp_, pkt, bitrate);
} catch(...) {
LOG(ERROR) << "OpenCV encode block exception: " << i;
}
diff --git a/components/codecs/test/nvpipe_codec_unit.cpp b/components/codecs/test/nvpipe_codec_unit.cpp
index 86f773df64f8fcdd55d493305ffa2505d356a22c..609ce56a50059978af931b718de57098201a0c1a 100644
--- a/components/codecs/test/nvpipe_codec_unit.cpp
+++ b/components/codecs/test/nvpipe_codec_unit.cpp
@@ -24,7 +24,7 @@ namespace ftl {
TEST_CASE( "NvPipeEncoder::encode() - A colour test image at preset 0" ) {
ftl::codecs::NvPipeEncoder encoder(definition_t::HD1080, definition_t::SD480);
- cv::Mat m(cv::Size(1920,1080), CV_8UC3, cv::Scalar(0,0,0));
+ cv::cuda::GpuMat m(cv::Size(1920,1080), CV_8UC3, cv::Scalar(0,0,0));
int block_total = 0;
std::atomic<int> block_count = 0;
@@ -46,7 +46,7 @@ TEST_CASE( "NvPipeEncoder::encode() - A colour test image at preset 0" ) {
TEST_CASE( "NvPipeEncoder::encode() - A depth test image at preset 0" ) {
ftl::codecs::NvPipeEncoder encoder(definition_t::HD1080, definition_t::SD480);
- cv::Mat m(cv::Size(1920,1080), CV_32F, cv::Scalar(0.0f));
+ cv::cuda::GpuMat m(cv::Size(1920,1080), CV_32F, cv::Scalar(0.0f));
int block_total = 0;
std::atomic<int> block_count = 0;
@@ -70,13 +70,13 @@ TEST_CASE( "NvPipeDecoder::decode() - A colour test image" ) {
ftl::codecs::NvPipeEncoder encoder(definition_t::HD1080, definition_t::SD480);
ftl::codecs::NvPipeDecoder decoder;
- cv::Mat in;
- cv::Mat out;
+ cv::cuda::GpuMat in;
+ cv::cuda::GpuMat out;
bool r = false;
SECTION("FHD in and out, FHD encoding") {
- in = cv::Mat(cv::Size(1920,1080), CV_8UC3, cv::Scalar(255,0,0));
- out = cv::Mat(cv::Size(1920,1080), CV_8UC3, cv::Scalar(0,0,0));
+ in = cv::cuda::GpuMat(cv::Size(1920,1080), CV_8UC3, cv::Scalar(255,0,0));
+ out = cv::cuda::GpuMat(cv::Size(1920,1080), CV_8UC3, cv::Scalar(0,0,0));
r = encoder.encode(in, ftl::codecs::kPreset0, [&out,&decoder](const ftl::codecs::Packet &pkt) {
REQUIRE( decoder.decode(pkt, out) );
@@ -84,8 +84,8 @@ TEST_CASE( "NvPipeDecoder::decode() - A colour test image" ) {
}
SECTION("Full HD in, 720 out, FHD encoding") {
- in = cv::Mat(cv::Size(1920,1080), CV_8UC3, cv::Scalar(255,0,0));
- out = cv::Mat(cv::Size(1280,720), CV_8UC3, cv::Scalar(0,0,0));
+ in = cv::cuda::GpuMat(cv::Size(1920,1080), CV_8UC3, cv::Scalar(255,0,0));
+ out = cv::cuda::GpuMat(cv::Size(1280,720), CV_8UC3, cv::Scalar(0,0,0));
r = encoder.encode(in, ftl::codecs::kPreset0, [&out,&decoder](const ftl::codecs::Packet &pkt) {
REQUIRE( decoder.decode(pkt, out) );
@@ -95,8 +95,8 @@ TEST_CASE( "NvPipeDecoder::decode() - A colour test image" ) {
}
SECTION("HHD in, FHD out, FHD encoding") {
- in = cv::Mat(cv::Size(1280,720), CV_8UC3, cv::Scalar(255,0,0));
- out = cv::Mat(cv::Size(1920,1080), CV_8UC3, cv::Scalar(0,0,0));
+ in = cv::cuda::GpuMat(cv::Size(1280,720), CV_8UC3, cv::Scalar(255,0,0));
+ out = cv::cuda::GpuMat(cv::Size(1920,1080), CV_8UC3, cv::Scalar(0,0,0));
r = encoder.encode(in, ftl::codecs::kPreset0, [&out,&decoder](const ftl::codecs::Packet &pkt) {
REQUIRE( decoder.decode(pkt, out) );
@@ -106,8 +106,8 @@ TEST_CASE( "NvPipeDecoder::decode() - A colour test image" ) {
}
SECTION("FHD in, HHD out, SD encoding") {
- in = cv::Mat(cv::Size(1920,1080), CV_8UC3, cv::Scalar(255,0,0));
- out = cv::Mat(cv::Size(1280,720), CV_8UC3, cv::Scalar(0,0,0));
+ in = cv::cuda::GpuMat(cv::Size(1920,1080), CV_8UC3, cv::Scalar(255,0,0));
+ out = cv::cuda::GpuMat(cv::Size(1280,720), CV_8UC3, cv::Scalar(0,0,0));
r = encoder.encode(in, ftl::codecs::kPreset4, [&out,&decoder](const ftl::codecs::Packet &pkt) {
REQUIRE( decoder.decode(pkt, out) );
@@ -117,5 +117,5 @@ TEST_CASE( "NvPipeDecoder::decode() - A colour test image" ) {
}
REQUIRE( r );
- REQUIRE( (cv::sum(out) != cv::Scalar(0,0,0)) );
+ REQUIRE( (cv::cuda::sum(out) != cv::Scalar(0,0,0)) );
}
diff --git a/components/codecs/test/opencv_codec_unit.cpp b/components/codecs/test/opencv_codec_unit.cpp
index dd85140dbfcddbd9c48f0c83795a84b0d4914882..2505eeb8994e1397bc19175f7f0903bdb3000c9d 100644
--- a/components/codecs/test/opencv_codec_unit.cpp
+++ b/components/codecs/test/opencv_codec_unit.cpp
@@ -24,7 +24,7 @@ namespace ftl {
TEST_CASE( "OpenCVEncoder::encode() - A colour test image at preset 0" ) {
ftl::codecs::OpenCVEncoder encoder(definition_t::HD1080, definition_t::SD480);
- cv::Mat m(cv::Size(1024,576), CV_8UC3, cv::Scalar(0,0,0));
+ cv::cuda::GpuMat m(cv::Size(1024,576), CV_8UC3, cv::Scalar(0,0,0));
int block_total = 0;
std::atomic<int> block_count = 0;
@@ -53,7 +53,7 @@ TEST_CASE( "OpenCVEncoder::encode() - A colour test image at preset 0" ) {
TEST_CASE( "OpenCVEncoder::encode() - A depth test image at preset 0" ) {
ftl::codecs::OpenCVEncoder encoder(definition_t::HD1080, definition_t::SD480);
- cv::Mat m(cv::Size(1024,576), CV_32F, cv::Scalar(0.0f));
+ cv::cuda::GpuMat m(cv::Size(1024,576), CV_32F, cv::Scalar(0.0f));
int block_total = 0;
std::atomic<int> block_count = 0;
@@ -82,8 +82,8 @@ TEST_CASE( "OpenCVEncoder::encode() - A depth test image at preset 0" ) {
TEST_CASE( "OpenCVDecoder::decode() - A colour test image no resolution change" ) {
ftl::codecs::OpenCVEncoder encoder(definition_t::HD1080, definition_t::SD480);
ftl::codecs::OpenCVDecoder decoder;
- cv::Mat in(cv::Size(1024,576), CV_8UC3, cv::Scalar(255,0,0));
- cv::Mat out(cv::Size(1024,576), CV_8UC3, cv::Scalar(0,0,0));
+ cv::cuda::GpuMat in(cv::Size(1024,576), CV_8UC3, cv::Scalar(255,0,0));
+ cv::cuda::GpuMat out(cv::Size(1024,576), CV_8UC3, cv::Scalar(0,0,0));
std::mutex mtx;
@@ -92,5 +92,5 @@ TEST_CASE( "OpenCVDecoder::decode() - A colour test image no resolution change"
REQUIRE( decoder.decode(pkt, out) );
});
- REQUIRE( (cv::sum(out) != cv::Scalar(0,0,0)) );
+ REQUIRE( (cv::cuda::sum(out) != cv::Scalar(0,0,0)) );
}
diff --git a/components/renderers/cpp/src/tri_render.cpp b/components/renderers/cpp/src/tri_render.cpp
index a14aa6b167748eca3532d61f50c8555d61580d12..cfd81347c6bb1c2e2d4547e0445f0a24d554185b 100644
--- a/components/renderers/cpp/src/tri_render.cpp
+++ b/components/renderers/cpp/src/tri_render.cpp
@@ -357,7 +357,7 @@ bool Triangular::render(ftl::rgbd::VirtualSource *src, ftl::rgbd::Frame &out) {
SHARED_LOCK(scene_->mtx, lk);
if (!src->isReady()) return false;
- scene_->upload(Channel::Colour + Channel::Depth, stream_);
+ //scene_->upload(Channel::Colour + Channel::Depth, stream_);
const auto &camera = src->parameters();
//cudaSafeCall(cudaSetDevice(scene_->getCUDADevice()));
diff --git a/components/rgbd-sources/include/ftl/rgbd/detail/netframe.hpp b/components/rgbd-sources/include/ftl/rgbd/detail/netframe.hpp
index eb9c64b99a03c66f331381a595d308f1345fa100..995848ff01fdcd87b1e0d1e01e0b9cde61e267f3 100644
--- a/components/rgbd-sources/include/ftl/rgbd/detail/netframe.hpp
+++ b/components/rgbd-sources/include/ftl/rgbd/detail/netframe.hpp
@@ -14,7 +14,7 @@ namespace detail {
* Also maintains statistics about the frame transmission for later analysis.
*/
struct NetFrame {
- cv::Mat channel[2];
+ cv::cuda::GpuMat channel[2];
volatile int64_t timestamp;
std::atomic<int> chunk_count[2];
std::atomic<int> channel_count;
diff --git a/components/rgbd-sources/include/ftl/rgbd/detail/source.hpp b/components/rgbd-sources/include/ftl/rgbd/detail/source.hpp
index 0d188c0b163777e842f3bd8c31f91ddd51435269..712d29170739be2ee966208439df6ba342df752e 100644
--- a/components/rgbd-sources/include/ftl/rgbd/detail/source.hpp
+++ b/components/rgbd-sources/include/ftl/rgbd/detail/source.hpp
@@ -62,8 +62,8 @@ class Source {
capability_t capabilities_;
ftl::rgbd::Source *host_;
ftl::rgbd::Camera params_;
- cv::Mat rgb_;
- cv::Mat depth_;
+ cv::cuda::GpuMat rgb_;
+ cv::cuda::GpuMat depth_;
int64_t timestamp_;
//Eigen::Matrix4f pose_;
};
diff --git a/components/rgbd-sources/include/ftl/rgbd/frame.hpp b/components/rgbd-sources/include/ftl/rgbd/frame.hpp
index b08673c973f42253670b51a3da8e28735406c952..52bbe9022987a85c3bf4398e9e3287011943ecac 100644
--- a/components/rgbd-sources/include/ftl/rgbd/frame.hpp
+++ b/components/rgbd-sources/include/ftl/rgbd/frame.hpp
@@ -256,7 +256,7 @@ ftl::cuda::TextureObject<T> &Frame::createTexture(ftl::codecs::Channel c) {
//LOG(INFO) << "Creating texture object";
m.tex = ftl::cuda::TextureObject<T>(m.gpu);
} else if (m.tex.cvType() != ftl::traits::OpenCVType<T>::value || m.tex.width() != m.gpu.cols || m.tex.height() != m.gpu.rows || m.tex.devicePtr() != m.gpu.data) {
- LOG(INFO) << "Recreating texture object for '" << ftl::codecs::name(c) << "'";
+ LOG(INFO) << "Recreating texture object for '" << ftl::codecs::name(c) << "'.";
m.tex.free();
m.tex = ftl::cuda::TextureObject<T>(m.gpu);
}
diff --git a/components/rgbd-sources/include/ftl/rgbd/source.hpp b/components/rgbd-sources/include/ftl/rgbd/source.hpp
index 6892e9cc7e77a52ede955d6fc7a398af13da790f..7484788796dba398c525a208ad44708deadf3b70 100644
--- a/components/rgbd-sources/include/ftl/rgbd/source.hpp
+++ b/components/rgbd-sources/include/ftl/rgbd/source.hpp
@@ -174,14 +174,14 @@ class Source : public ftl::Configurable {
SHARED_MUTEX &mutex() { return mutex_; }
- std::function<void(int64_t, cv::Mat &, cv::Mat &)> &callback() { return callback_; }
+ std::function<void(int64_t, cv::cuda::GpuMat &, cv::cuda::GpuMat &)> &callback() { return callback_; }
/**
* Set the callback that receives decoded frames as they are generated.
* There can be only a single such callback as the buffers can be swapped
* by the callback.
*/
- void setCallback(std::function<void(int64_t, cv::Mat &, cv::Mat &)> cb);
+ void setCallback(std::function<void(int64_t, cv::cuda::GpuMat &, cv::cuda::GpuMat &)> cb);
void removeCallback() { callback_ = nullptr; }
/**
@@ -205,7 +205,7 @@ class Source : public ftl::Configurable {
* Notify of a decoded or available pair of frames. This calls the source
* callback after having verified the correct resolution of the frames.
*/
- void notify(int64_t ts, cv::Mat &c1, cv::Mat &c2);
+ void notify(int64_t ts, cv::cuda::GpuMat &c1, cv::cuda::GpuMat &c2);
// ==== Inject Data into stream ============================================
@@ -225,7 +225,7 @@ class Source : public ftl::Configurable {
SHARED_MUTEX mutex_;
ftl::codecs::Channel channel_;
cudaStream_t stream_;
- std::function<void(int64_t, cv::Mat &, cv::Mat &)> callback_;
+ std::function<void(int64_t, cv::cuda::GpuMat &, cv::cuda::GpuMat &)> callback_;
std::list<std::function<void(ftl::rgbd::Source*, const ftl::codecs::StreamPacket &spkt, const ftl::codecs::Packet &pkt)>> rawcallbacks_;
detail::Source *_createImplementation();
diff --git a/components/rgbd-sources/src/group.cpp b/components/rgbd-sources/src/group.cpp
index 4e178c0245e815da6b836fbd2de44df4dcc84e1b..b2236ef3c83821d38ff16e8132748a635dd2712f 100644
--- a/components/rgbd-sources/src/group.cpp
+++ b/components/rgbd-sources/src/group.cpp
@@ -50,7 +50,7 @@ void Group::addSource(ftl::rgbd::Source *src) {
size_t ix = sources_.size();
sources_.push_back(src);
- src->setCallback([this,ix,src](int64_t timestamp, cv::Mat &rgb, cv::Mat &depth) {
+ src->setCallback([this,ix,src](int64_t timestamp, cv::cuda::GpuMat &rgb, cv::cuda::GpuMat &depth) {
if (timestamp == 0) return;
auto chan = src->getChannel();
@@ -78,13 +78,13 @@ void Group::addSource(ftl::rgbd::Source *src) {
// Ensure channels match source mat format
//fs.channel1[ix].create(rgb.size(), rgb.type());
//fs.channel2[ix].create(depth.size(), depth.type());
- fs.frames[ix].create<cv::Mat>(Channel::Colour, Format<uchar3>(rgb.size())); //.create(rgb.size(), rgb.type());
- if (chan != Channel::None) fs.frames[ix].create<cv::Mat>(chan, ftl::rgbd::FormatBase(depth.cols, depth.rows, depth.type())); //.create(depth.size(), depth.type());
+ fs.frames[ix].create<cv::cuda::GpuMat>(Channel::Colour, Format<uchar3>(rgb.size())); //.create(rgb.size(), rgb.type());
+ if (chan != Channel::None) fs.frames[ix].create<cv::cuda::GpuMat>(chan, ftl::rgbd::FormatBase(depth.cols, depth.rows, depth.type())); //.create(depth.size(), depth.type());
//cv::swap(rgb, fs.channel1[ix]);
//cv::swap(depth, fs.channel2[ix]);
- cv::swap(rgb, fs.frames[ix].get<cv::Mat>(Channel::Colour));
- if (chan != Channel::None) cv::swap(depth, fs.frames[ix].get<cv::Mat>(chan));
+ cv::cuda::swap(rgb, fs.frames[ix].get<cv::cuda::GpuMat>(Channel::Colour));
+ if (chan != Channel::None) cv::cuda::swap(depth, fs.frames[ix].get<cv::cuda::GpuMat>(chan));
++fs.count;
fs.mask |= (1 << ix);
diff --git a/components/rgbd-sources/src/source.cpp b/components/rgbd-sources/src/source.cpp
index 52dae351ed27d134119cfbd3bc572c4ae819f355..a7b6b2cb0e3081efa8371479a216d6dc21dce801 100644
--- a/components/rgbd-sources/src/source.cpp
+++ b/components/rgbd-sources/src/source.cpp
@@ -247,7 +247,7 @@ const ftl::rgbd::Camera Source::parameters(ftl::codecs::Channel chan) const {
return (impl_) ? impl_->parameters(chan) : parameters();
}
-void Source::setCallback(std::function<void(int64_t, cv::Mat &, cv::Mat &)> cb) {
+void Source::setCallback(std::function<void(int64_t, cv::cuda::GpuMat &, cv::cuda::GpuMat &)> cb) {
if (bool(callback_)) LOG(ERROR) << "Source already has a callback: " << getURI();
callback_ = cb;
}
@@ -297,7 +297,7 @@ static Camera scaled(Camera &cam, int width, int height) {
return newcam;
}
-void Source::notify(int64_t ts, cv::Mat &c1, cv::Mat &c2) {
+void Source::notify(int64_t ts, cv::cuda::GpuMat &c1, cv::cuda::GpuMat &c2) {
// Ensure correct scaling of images and parameters.
int max_width = max(impl_->params_.width, max(c1.cols, c2.cols));
int max_height = max(impl_->params_.height, max(c1.rows, c2.rows));
@@ -309,15 +309,17 @@ void Source::notify(int64_t ts, cv::Mat &c1, cv::Mat &c2) {
// Should channel 1 be scaled?
if (c1.cols < max_width || c1.rows < max_height) {
- cv::resize(c1, c1, cv::Size(max_width, max_height));
+ LOG(WARNING) << "Resizing on GPU";
+ cv::cuda::resize(c1, c1, cv::Size(max_width, max_height));
}
// Should channel 2 be scaled?
- if (c2.cols < max_width || c2.rows < max_height) {
+ if (!c2.empty() && (c2.cols < max_width || c2.rows < max_height)) {
+ LOG(WARNING) << "Resizing on GPU";
if (c2.type() == CV_32F) {
- cv::resize(c2, c2, cv::Size(max_width, max_height), 0.0, 0.0, cv::INTER_NEAREST);
+ cv::cuda::resize(c2, c2, cv::Size(max_width, max_height), 0.0, 0.0, cv::INTER_NEAREST);
} else {
- cv::resize(c2, c2, cv::Size(max_width, max_height));
+ cv::cuda::resize(c2, c2, cv::Size(max_width, max_height));
}
}
diff --git a/components/rgbd-sources/src/sources/middlebury/middlebury_source.cpp b/components/rgbd-sources/src/sources/middlebury/middlebury_source.cpp
index e82167fdcf6b4e2bfd1a38a00b50e3cdceeb2aef..d152d25e5a7d8830e8b555851f94d97b70463e29 100644
--- a/components/rgbd-sources/src/sources/middlebury/middlebury_source.cpp
+++ b/components/rgbd-sources/src/sources/middlebury/middlebury_source.cpp
@@ -129,30 +129,30 @@ MiddleburySource::MiddleburySource(ftl::rgbd::Source *host, const string &dir)
disp_->setMask(mask_l_);
// Load image files...
- cv::Mat right_tmp;
- rgb_ = cv::imread(dir+"/im0.png", cv::IMREAD_COLOR);
+ cv::Mat left_tmp, right_tmp;
+ left_tmp = cv::imread(dir+"/im0.png", cv::IMREAD_COLOR);
right_tmp = cv::imread(dir+"/im1.png", cv::IMREAD_COLOR);
- cv::resize(rgb_, rgb_, cv::Size(params_.width, params_.height));
+ cv::resize(left_tmp, left_tmp, cv::Size(params_.width, params_.height));
cv::resize(right_tmp, right_tmp, cv::Size(params_.width, params_.height));
- left_.upload(rgb_);
- right_.upload(right_tmp);
+ rgb_.upload(left_tmp, stream_);
+ right_.upload(right_tmp, stream_);
_performDisparity();
ready_ = true;
}
void MiddleburySource::_performDisparity() {
- if (depth_tmp_.empty()) depth_tmp_ = cv::cuda::GpuMat(left_.size(), CV_32FC1);
- if (disp_tmp_.empty()) disp_tmp_ = cv::cuda::GpuMat(left_.size(), CV_32FC1);
+ depth_.create(left_.size(), CV_32FC1);
+ disp_tmp_.create(left_.size(), CV_32FC1);
//calib_->rectifyStereo(left_, right_, stream_);
- disp_->compute(left_, right_, disp_tmp_, stream_);
+ disp_->compute(rgb_, right_, disp_tmp_, stream_);
//disparityToDepth(disp_tmp_, depth_tmp_, params_, stream_);
- ftl::cuda::disparity_to_depth(disp_tmp_, depth_tmp_, params_, stream_);
+ ftl::cuda::disparity_to_depth(disp_tmp_, depth_, params_, stream_);
//left_.download(rgb_, stream_);
//rgb_ = lsrc_->cachedLeft();
- depth_tmp_.download(depth_, stream_);
+ //depth_tmp_.download(depth_, stream_);
stream_.waitForCompletion();
diff --git a/components/rgbd-sources/src/sources/net/net.cpp b/components/rgbd-sources/src/sources/net/net.cpp
index 56b6355eb9696c6684c46a80a6fb3ff1e0584d5a..e4073536a574255965de81ab5f2294e008695032 100644
--- a/components/rgbd-sources/src/sources/net/net.cpp
+++ b/components/rgbd-sources/src/sources/net/net.cpp
@@ -422,9 +422,9 @@ bool NetSource::compute(int n, int b) {
// Verify depth destination is of required type
if (isFloatChannel(chan) && depth_.type() != CV_32F) {
- depth_ = cv::Mat(cv::Size(params_.width, params_.height), CV_32FC1, 0.0f);
+ depth_.create(cv::Size(params_.width, params_.height), CV_32FC1); // 0.0f
} else if (!isFloatChannel(chan) && depth_.type() != CV_8UC3) {
- depth_ = cv::Mat(cv::Size(params_.width, params_.height), CV_8UC3, cv::Scalar(0,0,0));
+ depth_.create(cv::Size(params_.width, params_.height), CV_8UC3); // cv::Scalar(0,0,0)
}
if (prev_chan_ != chan) {
diff --git a/components/rgbd-sources/src/sources/realsense/realsense_source.cpp b/components/rgbd-sources/src/sources/realsense/realsense_source.cpp
index b458aa3e77c8557ee828a8f71431bb5e4e665066..2b55356c853cdafc2d6aa3be523e07376e7ac0f8 100644
--- a/components/rgbd-sources/src/sources/realsense/realsense_source.cpp
+++ b/components/rgbd-sources/src/sources/realsense/realsense_source.cpp
@@ -54,9 +54,11 @@ bool RealsenseSource::compute(int n, int b) {
float h = depth.get_height();
rscolour_ = frames.first(RS2_STREAM_COLOR); //.get_color_frame();
- cv::Mat tmp(cv::Size((int)w, (int)h), CV_16UC1, (void*)depth.get_data(), depth.get_stride_in_bytes());
- tmp.convertTo(depth_, CV_32FC1, scale_);
- rgb_ = cv::Mat(cv::Size(w, h), CV_8UC4, (void*)rscolour_.get_data(), cv::Mat::AUTO_STEP);
+ cv::Mat tmp_depth(cv::Size((int)w, (int)h), CV_16UC1, (void*)depth.get_data(), depth.get_stride_in_bytes());
+ tmp_depth.convertTo(tmp_depth, CV_32FC1, scale_);
+ depth_.upload(tmp_depth);
+ cv::Mat tmp_rgb(cv::Size(w, h), CV_8UC4, (void*)rscolour_.get_data(), cv::Mat::AUTO_STEP);
+ rgb_.upload(tmp_rgb);
auto cb = host_->callback();
if (cb) cb(timestamp_, rgb_, depth_);
diff --git a/components/rgbd-sources/src/sources/snapshot/snapshot_source.cpp b/components/rgbd-sources/src/sources/snapshot/snapshot_source.cpp
index 73db6b86c3861cd0aa1105f4d9ff7dcd9cbe9fe3..136a2e7dfcc7b279228cdd5c6efc0bfb8d303baa 100644
--- a/components/rgbd-sources/src/sources/snapshot/snapshot_source.cpp
+++ b/components/rgbd-sources/src/sources/snapshot/snapshot_source.cpp
@@ -59,8 +59,10 @@ bool SnapshotSource::compute(int n, int b) {
snapshot_.getLeftRGB(camera_idx_, frame_idx_, snap_rgb_);
snapshot_.getLeftDepth(camera_idx_, frame_idx_, snap_depth_);
- snap_rgb_.copyTo(rgb_);
- snap_depth_.copyTo(depth_);
+ //snap_rgb_.copyTo(rgb_);
+ //snap_depth_.copyTo(depth_);
+ rgb_.upload(snap_rgb_);
+ depth_.upload(snap_depth_);
auto cb = host_->callback();
if (cb) cb(timestamp_, rgb_, depth_);
diff --git a/components/rgbd-sources/src/sources/stereovideo/stereovideo.cpp b/components/rgbd-sources/src/sources/stereovideo/stereovideo.cpp
index b84385fcecb3a8c00a94398d998d872c68929951..f8e08a9f6d1f7670adeb8be9310129e8353daf1b 100644
--- a/components/rgbd-sources/src/sources/stereovideo/stereovideo.cpp
+++ b/components/rgbd-sources/src/sources/stereovideo/stereovideo.cpp
@@ -231,21 +231,24 @@ bool StereoVideoSource::compute(int n, int b) {
ftl::cuda::disparity_to_depth(disp, depth, params_, stream_);
- left.download(rgb_, stream_);
- depth.download(depth_, stream_);
+ //left.download(rgb_, stream_);
+ //depth.download(depth_, stream_);
//frame.download(Channel::Left + Channel::Depth);
- stream_.waitForCompletion(); // TODO:(Nick) Move to getFrames
+ stream_.waitForCompletion();
+ host_->notify(timestamp_, left, depth);
} else if (chan == Channel::Right) {
- left.download(rgb_, stream_);
- right.download(depth_, stream_);
+ //left.download(rgb_, stream_);
+ //right.download(depth_, stream_);
stream_.waitForCompletion(); // TODO:(Nick) Move to getFrames
+ host_->notify(timestamp_, left, right);
} else {
- left.download(rgb_, stream_);
+ //left.download(rgb_, stream_);
stream_.waitForCompletion(); // TODO:(Nick) Move to getFrames
+ //LOG(INFO) << "NO SECOND CHANNEL: " << (bool)depth_.empty();
+ depth_.create(left.size(), left.type());
+ host_->notify(timestamp_, left, depth_);
}
- auto cb = host_->callback();
- if (cb) cb(timestamp_, rgb_, depth_);
return true;
}
diff --git a/components/rgbd-sources/src/sources/virtual/virtual.cpp b/components/rgbd-sources/src/sources/virtual/virtual.cpp
index e9fe781b58eb47593c3b5d3c5ae0fbec800bc535..a1d1040c65caac8833397d39ef3949c7778cdebf 100644
--- a/components/rgbd-sources/src/sources/virtual/virtual.cpp
+++ b/components/rgbd-sources/src/sources/virtual/virtual.cpp
@@ -91,20 +91,19 @@ class VirtualImpl : public ftl::rgbd::detail::Source {
}
if (frame.hasChannel(Channel::Colour)) {
- frame.download(Channel::Colour);
- cv::swap(frame.get<cv::Mat>(Channel::Colour), rgb_);
+ //frame.download(Channel::Colour);
+ cv::cuda::swap(frame.get<cv::cuda::GpuMat>(Channel::Colour), rgb_);
} else {
LOG(ERROR) << "Channel 1 frame in rendering";
}
if ((host_->getChannel() != Channel::None) &&
frame.hasChannel(host_->getChannel())) {
- frame.download(host_->getChannel());
- cv::swap(frame.get<cv::Mat>(host_->getChannel()), depth_);
+ //frame.download(host_->getChannel());
+ cv::cuda::swap(frame.get<cv::cuda::GpuMat>(host_->getChannel()), depth_);
}
- auto cb = host_->callback();
- if (cb) cb(timestamp_, rgb_, depth_);
+ host_->notify(timestamp_, rgb_, depth_);
}
return true;
}
diff --git a/components/rgbd-sources/src/streamer.cpp b/components/rgbd-sources/src/streamer.cpp
index 8ecda51b7a53c17552a9a51fd91ceb658e2982be..e05e7faf9305ac0a3ede6dade21596bfe8251db7 100644
--- a/components/rgbd-sources/src/streamer.cpp
+++ b/components/rgbd-sources/src/streamer.cpp
@@ -468,7 +468,7 @@ void Streamer::_process(ftl::rgbd::FrameSet &fs) {
auto chan = fs.sources[j]->getChannel();
- enc2->encode(fs.frames[j].get<cv::Mat>(chan), src->hq_bitrate, [this,src,hasChan2,chan](const ftl::codecs::Packet &blk){
+ enc2->encode(fs.frames[j].get<cv::cuda::GpuMat>(chan), src->hq_bitrate, [this,src,hasChan2,chan](const ftl::codecs::Packet &blk){
_transmitPacket(src, blk, chan, hasChan2, Quality::High);
});
} else {
@@ -477,7 +477,7 @@ void Streamer::_process(ftl::rgbd::FrameSet &fs) {
// TODO: Stagger the reset between nodes... random phasing
if (fs.timestamp % (10*ftl::timer::getInterval()) == 0) enc1->reset();
- enc1->encode(fs.frames[j].get<cv::Mat>(Channel::Colour), src->hq_bitrate, [this,src,hasChan2](const ftl::codecs::Packet &blk){
+ enc1->encode(fs.frames[j].get<cv::cuda::GpuMat>(Channel::Colour), src->hq_bitrate, [this,src,hasChan2](const ftl::codecs::Packet &blk){
_transmitPacket(src, blk, Channel::Colour, hasChan2, Quality::High);
});
}
@@ -500,14 +500,14 @@ void Streamer::_process(ftl::rgbd::FrameSet &fs) {
if (hasChan2) {
auto chan = fs.sources[j]->getChannel();
- enc2->encode(fs.frames[j].get<cv::Mat>(chan), src->lq_bitrate, [this,src,hasChan2,chan](const ftl::codecs::Packet &blk){
+ enc2->encode(fs.frames[j].get<cv::cuda::GpuMat>(chan), src->lq_bitrate, [this,src,hasChan2,chan](const ftl::codecs::Packet &blk){
_transmitPacket(src, blk, chan, hasChan2, Quality::Low);
});
} else {
if (enc2) enc2->reset();
}
- enc1->encode(fs.frames[j].get<cv::Mat>(Channel::Colour), src->lq_bitrate, [this,src,hasChan2](const ftl::codecs::Packet &blk){
+ enc1->encode(fs.frames[j].get<cv::cuda::GpuMat>(Channel::Colour), src->lq_bitrate, [this,src,hasChan2](const ftl::codecs::Packet &blk){
_transmitPacket(src, blk, Channel::Colour, hasChan2, Quality::Low);
});
}