Merge branch 'exp/lowlatencycapture' into 'master'

Reduce screen capture latency

See merge request nicolas.pope/ftl!240

components/rgbd-sources/src/sources/screencapture/screencapture.cpp

@@ -87,6 +87,9 @@ ScreenCapture::ScreenCapture(ftl::rgbd::Source *host)
 		return;
 	}
 
+	// Page lock the shared memory...
+	cudaSafeCall(cudaHostRegister(s.shminfo.shmaddr, s.ximg->bytes_per_line * s.ximg->height, cudaHostRegisterDefault));
+
 	ready_ = true;
 
 	#endif
@@ -101,6 +104,11 @@ ScreenCapture::ScreenCapture(ftl::rgbd::Source *host)
 
 	state_.getLeft() = params_;
 
+	host_->on("depth", [this](const ftl::config::Event &e) {
+		params_.maxDepth = host_->value("depth", 1.0f);
+		state_.getLeft() = params_;
+	});
+
 }
 
 ScreenCapture::~ScreenCapture() {
@@ -109,7 +117,15 @@ ScreenCapture::~ScreenCapture() {
 	#endif
 }
 
+void ScreenCapture::swap() {
+
+}
+
 bool ScreenCapture::retrieve() {
+	return true;
+}
+
+bool ScreenCapture::compute(int n, int b) {
 	if (!ready_) return false;
 	cv::Mat img;
 
@@ -122,14 +138,9 @@ bool ScreenCapture::retrieve() {
 	frame_.setOrigin(&state_);
 
 	if (!img.empty()) {
-		frame_.create<cv::cuda::GpuMat>(Channel::Colour).upload(img);
+		frame_.create<cv::Mat>(Channel::Colour) = img;
 	}
 
-	return true;
-}
-
-bool ScreenCapture::compute(int n, int b) {
-	if (!ready_) return false;
 	host_->notify(timestamp_, frame_);
     return true;
 }

components/rgbd-sources/src/sources/screencapture/screencapture.hpp

@@ -23,12 +23,16 @@ class ScreenCapture : public ftl::rgbd::detail::Source {
 	~ScreenCapture();
 
 	bool capture(int64_t ts) { timestamp_ = ts; return true; };
+	void swap() override;
 	bool retrieve();
 	bool compute(int n=-1, int b=-1);
 	bool isReady();
 
 	private:
 	bool ready_;
+	int64_t cap_ts_;
+	int64_t cur_ts_;
+	ftl::rgbd::Frame sframe_;
 
 	ImplState *impl_state_;
 };

components/streams/include/ftl/streams/sender.hpp

@@ -26,7 +26,7 @@ class Sender : public ftl::Configurable {
 	 * Encode and transmit an entire frame set. Frames may already contain
 	 * an encoded form, in which case that is used instead.
 	 */
-	void post(const ftl::rgbd::FrameSet &fs);
+	void post(ftl::rgbd::FrameSet &fs);
 
 	/**
 	 * Encode and transmit a set of audio channels.
@@ -55,7 +55,7 @@ class Sender : public ftl::Configurable {
 	std::unordered_map<int, EncodingState> state_;
 
 	//ftl::codecs::Encoder *_getEncoder(int fsid, int fid, ftl::codecs::Channel c);
-	void _encodeChannel(const ftl::rgbd::FrameSet &fs, ftl::codecs::Channel c, bool reset);
+	void _encodeChannel(ftl::rgbd::FrameSet &fs, ftl::codecs::Channel c, bool reset);
 	int _generateTiles(const ftl::rgbd::FrameSet &fs, int offset, ftl::codecs::Channel c, cv::cuda::Stream &stream, bool);
 	EncodingState &_getTile(int fsid, ftl::codecs::Channel c);
 	cv::Rect _generateROI(const ftl::rgbd::FrameSet &fs, ftl::codecs::Channel c, int offset);

components/streams/src/sender.cpp

@@ -114,7 +114,7 @@ static void mergeNALUnits(const std::list<ftl::codecs::Packet> &pkts, ftl::codec
 	}
 }
 
-void Sender::post(const ftl::rgbd::FrameSet &fs) {
+void Sender::post(ftl::rgbd::FrameSet &fs) {
     if (!stream_) return;
 
     Channels selected;
@@ -222,7 +222,7 @@ void Sender::post(const ftl::rgbd::FrameSet &fs) {
 	//do_inject_ = false;
 }
 
-void Sender::_encodeChannel(const ftl::rgbd::FrameSet &fs, Channel c, bool reset) {
+void Sender::_encodeChannel(ftl::rgbd::FrameSet &fs, Channel c, bool reset) {
 	bool lossless = value("lossless", false);
 	int max_bitrate = std::max(0, std::min(255, value("max_bitrate", 255)));
 	//int min_bitrate = std::max(0, std::min(255, value("min_bitrate", 0)));  // TODO: Use this
@@ -251,6 +251,13 @@ void Sender::_encodeChannel(const ftl::rgbd::FrameSet &fs, Channel c, bool reset
 			return;
 		}
 
+		// Upload if in host memory
+		for (auto &f : fs.frames) {
+			if (f.isCPU(c)) {
+				f.upload(Channels<0>(c), cv::cuda::StreamAccessor::getStream(enc->stream()));
+			}
+		}
+
 		int count = _generateTiles(fs, offset, c, enc->stream(), lossless);
 		if (count <= 0) {
 			LOG(ERROR) << "Could not generate tiles.";