From 4f85c55a2f1736e301010f02b15eef1b13289442 Mon Sep 17 00:00:00 2001
From: Nicolas Pope <nicolas.pope@utu.fi>
Date: Sun, 2 Feb 2020 11:38:12 +0200
Subject: [PATCH] Reduce screen capture latency
---
.../sources/screencapture/screencapture.cpp | 23 ++++++++++++++-----
.../sources/screencapture/screencapture.hpp | 4 ++++
.../streams/include/ftl/streams/sender.hpp | 4 ++--
components/streams/src/sender.cpp | 11 +++++++--
4 files changed, 32 insertions(+), 10 deletions(-)
diff --git a/components/rgbd-sources/src/sources/screencapture/screencapture.cpp b/components/rgbd-sources/src/sources/screencapture/screencapture.cpp
index baf7091bb..306b12878 100644
--- a/components/rgbd-sources/src/sources/screencapture/screencapture.cpp
+++ b/components/rgbd-sources/src/sources/screencapture/screencapture.cpp
@@ -87,6 +87,9 @@ ScreenCapture::ScreenCapture(ftl::rgbd::Source *host)
return;
}
+ // Page lock the shared memory...
+ cudaSafeCall(cudaHostRegister(s.shminfo.shmaddr, s.ximg->bytes_per_line * s.ximg->height, cudaHostRegisterDefault));
+
ready_ = true;
#endif
@@ -101,6 +104,11 @@ ScreenCapture::ScreenCapture(ftl::rgbd::Source *host)
state_.getLeft() = params_;
+ host_->on("depth", [this](const ftl::config::Event &e) {
+ params_.maxDepth = host_->value("depth", 1.0f);
+ state_.getLeft() = params_;
+ });
+
}
ScreenCapture::~ScreenCapture() {
@@ -109,7 +117,15 @@ ScreenCapture::~ScreenCapture() {
#endif
}
+void ScreenCapture::swap() {
+
+}
+
bool ScreenCapture::retrieve() {
+ return true;
+}
+
+bool ScreenCapture::compute(int n, int b) {
if (!ready_) return false;
cv::Mat img;
@@ -122,14 +138,9 @@ bool ScreenCapture::retrieve() {
frame_.setOrigin(&state_);
if (!img.empty()) {
- frame_.create<cv::cuda::GpuMat>(Channel::Colour).upload(img);
+ frame_.create<cv::Mat>(Channel::Colour) = img;
}
- return true;
-}
-
-bool ScreenCapture::compute(int n, int b) {
- if (!ready_) return false;
host_->notify(timestamp_, frame_);
return true;
}
diff --git a/components/rgbd-sources/src/sources/screencapture/screencapture.hpp b/components/rgbd-sources/src/sources/screencapture/screencapture.hpp
index e1511cb9a..4c8ee78c4 100644
--- a/components/rgbd-sources/src/sources/screencapture/screencapture.hpp
+++ b/components/rgbd-sources/src/sources/screencapture/screencapture.hpp
@@ -23,12 +23,16 @@ class ScreenCapture : public ftl::rgbd::detail::Source {
~ScreenCapture();
bool capture(int64_t ts) { timestamp_ = ts; return true; };
+ void swap() override;
bool retrieve();
bool compute(int n=-1, int b=-1);
bool isReady();
private:
bool ready_;
+ int64_t cap_ts_;
+ int64_t cur_ts_;
+ ftl::rgbd::Frame sframe_;
ImplState *impl_state_;
};
diff --git a/components/streams/include/ftl/streams/sender.hpp b/components/streams/include/ftl/streams/sender.hpp
index c9c4091f5..2a8afc8ce 100644
--- a/components/streams/include/ftl/streams/sender.hpp
+++ b/components/streams/include/ftl/streams/sender.hpp
@@ -26,7 +26,7 @@ class Sender : public ftl::Configurable {
* Encode and transmit an entire frame set. Frames may already contain
* an encoded form, in which case that is used instead.
*/
- void post(const ftl::rgbd::FrameSet &fs);
+ void post(ftl::rgbd::FrameSet &fs);
/**
* Encode and transmit a set of audio channels.
@@ -55,7 +55,7 @@ class Sender : public ftl::Configurable {
std::unordered_map<int, EncodingState> state_;
//ftl::codecs::Encoder *_getEncoder(int fsid, int fid, ftl::codecs::Channel c);
- void _encodeChannel(const ftl::rgbd::FrameSet &fs, ftl::codecs::Channel c, bool reset);
+ void _encodeChannel(ftl::rgbd::FrameSet &fs, ftl::codecs::Channel c, bool reset);
int _generateTiles(const ftl::rgbd::FrameSet &fs, int offset, ftl::codecs::Channel c, cv::cuda::Stream &stream, bool);
EncodingState &_getTile(int fsid, ftl::codecs::Channel c);
cv::Rect _generateROI(const ftl::rgbd::FrameSet &fs, ftl::codecs::Channel c, int offset);
diff --git a/components/streams/src/sender.cpp b/components/streams/src/sender.cpp
index 80b6799b1..d5af31e8e 100644
--- a/components/streams/src/sender.cpp
+++ b/components/streams/src/sender.cpp
@@ -114,7 +114,7 @@ static void mergeNALUnits(const std::list<ftl::codecs::Packet> &pkts, ftl::codec
}
}
-void Sender::post(const ftl::rgbd::FrameSet &fs) {
+void Sender::post(ftl::rgbd::FrameSet &fs) {
if (!stream_) return;
Channels selected;
@@ -222,7 +222,7 @@ void Sender::post(const ftl::rgbd::FrameSet &fs) {
//do_inject_ = false;
}
-void Sender::_encodeChannel(const ftl::rgbd::FrameSet &fs, Channel c, bool reset) {
+void Sender::_encodeChannel(ftl::rgbd::FrameSet &fs, Channel c, bool reset) {
bool lossless = value("lossless", false);
int max_bitrate = std::max(0, std::min(255, value("max_bitrate", 255)));
//int min_bitrate = std::max(0, std::min(255, value("min_bitrate", 0))); // TODO: Use this
@@ -251,6 +251,13 @@ void Sender::_encodeChannel(const ftl::rgbd::FrameSet &fs, Channel c, bool reset
return;
}
+ // Upload if in host memory
+ for (auto &f : fs.frames) {
+ if (f.isCPU(c)) {
+ f.upload(Channels<0>(c), cv::cuda::StreamAccessor::getStream(enc->stream()));
+ }
+ }
+
int count = _generateTiles(fs, offset, c, enc->stream(), lossless);
if (count <= 0) {
LOG(ERROR) << "Could not generate tiles.";
--
GitLab