From dbf972f649dafbfb5948ad489d5257cd19b0cb18 Mon Sep 17 00:00:00 2001
From: Nicolas Pope <nwpope@utu.fi>
Date: Sat, 6 Jun 2020 21:01:28 +0300
Subject: [PATCH] Rework pipeline to lower latency by 50ms
---
.../include/ftl/rgbd/detail/source.hpp | 2 +-
.../rgbd-sources/include/ftl/rgbd/group.hpp | 3 +-
.../rgbd-sources/include/ftl/rgbd/source.hpp | 6 ++--
components/rgbd-sources/src/group.cpp | 33 ++++++++++++++-----
components/rgbd-sources/src/source.cpp | 4 +--
.../rgbd-sources/src/sources/image/image.hpp | 2 +-
.../rgbd-sources/src/sources/pylon/pylon.cpp | 4 +--
.../rgbd-sources/src/sources/pylon/pylon.hpp | 2 +-
.../sources/realsense/realsense_source.cpp | 26 +++++++++++----
.../sources/realsense/realsense_source.hpp | 6 ++--
.../sources/screencapture/screencapture.cpp | 4 +--
.../sources/screencapture/screencapture.hpp | 2 +-
.../src/sources/stereovideo/stereovideo.cpp | 4 +--
.../src/sources/stereovideo/stereovideo.hpp | 2 +-
components/rgbd-sources/test/source_unit.cpp | 18 +++++-----
15 files changed, 75 insertions(+), 43 deletions(-)
diff --git a/components/rgbd-sources/include/ftl/rgbd/detail/source.hpp b/components/rgbd-sources/include/ftl/rgbd/detail/source.hpp
index a5949ff51..3839fd6c0 100644
--- a/components/rgbd-sources/include/ftl/rgbd/detail/source.hpp
+++ b/components/rgbd-sources/include/ftl/rgbd/detail/source.hpp
@@ -46,7 +46,7 @@ class Source {
* @param n Number of frames to request in batch. Default -1 means automatic (10)
* @param b Bit rate setting. -1 = automatic, 0 = best quality, 9 = lowest quality
*/
- virtual bool compute(int n, int b)=0;
+ virtual bool compute(int64_t ts)=0;
/**
* Between frames, or before next frame, do any buffer swapping operations.
diff --git a/components/rgbd-sources/include/ftl/rgbd/group.hpp b/components/rgbd-sources/include/ftl/rgbd/group.hpp
index b339511a2..1c283ea59 100644
--- a/components/rgbd-sources/include/ftl/rgbd/group.hpp
+++ b/components/rgbd-sources/include/ftl/rgbd/group.hpp
@@ -103,6 +103,7 @@ class Group : public ftl::rgbd::Generator {
ftl::operators::Graph *pipeline_;
std::atomic<int> jobs_;
+ std::atomic<int> cjobs_;
volatile bool skip_;
ftl::timer::TimerHandle cap_id_;
ftl::timer::TimerHandle swap_id_;
@@ -111,7 +112,7 @@ class Group : public ftl::rgbd::Generator {
MUTEX mutex_;
void _retrieveJob(ftl::rgbd::Source *);
- void _computeJob(ftl::rgbd::Source *);
+ void _computeJob(ftl::rgbd::Source *, int64_t);
};
}
diff --git a/components/rgbd-sources/include/ftl/rgbd/source.hpp b/components/rgbd-sources/include/ftl/rgbd/source.hpp
index 5676c25d3..fca569366 100644
--- a/components/rgbd-sources/include/ftl/rgbd/source.hpp
+++ b/components/rgbd-sources/include/ftl/rgbd/source.hpp
@@ -99,18 +99,18 @@ class Source : public ftl::Configurable {
* may take considerable time to return, especially for sources requiring
* software stereo correspondance.
*/
- bool compute(int N=-1, int B=-1);
+ bool compute(int64_t ts);
/**
* Wrapper grab that performs capture, swap and computation steps in one.
* It is more optimal to perform capture and compute in parallel.
*/
- bool grab(int N=-1, int B=-1) {
+ /*bool grab(int N=-1, int B=-1) {
bool c = capture(0);
c = c && retrieve();
swap();
return c && compute(N,B);
- }
+ }*/
/**
* Get a copy of both colour and depth frames. Note that this does a buffer
diff --git a/components/rgbd-sources/src/group.cpp b/components/rgbd-sources/src/group.cpp
index 70c8402fc..bbd03207f 100644
--- a/components/rgbd-sources/src/group.cpp
+++ b/components/rgbd-sources/src/group.cpp
@@ -20,8 +20,11 @@ using ftl::codecs::Channels;
Group::Group() : pipeline_(nullptr) {
jobs_ = 0;
+ cjobs_ = 0;
skip_ = false;
name_ = "NoName";
+
+ builder_.setBufferSize(0);
}
Group::~Group() {
@@ -74,9 +77,9 @@ void Group::_retrieveJob(ftl::rgbd::Source *src) {
}
}
-void Group::_computeJob(ftl::rgbd::Source *src) {
+void Group::_computeJob(ftl::rgbd::Source *src, int64_t ts) {
try {
- src->compute();
+ src->compute(ts);
} catch (std::exception &ex) {
LOG(ERROR) << "Exception when computing frame";
LOG(ERROR) << ex.what();
@@ -112,13 +115,13 @@ void Group::onFrameSet(const ftl::rgbd::VideoCallback &cb) {
});
// 2. After capture, swap any internal source double buffers
- swap_id_ = ftl::timer::add(ftl::timer::kTimerSwap, [this](int64_t ts) {
+ /*swap_id_ = ftl::timer::add(ftl::timer::kTimerSwap, [this](int64_t ts) {
if (skip_) return true;
for (auto s : sources_) {
s->swap();
}
return true;
- });
+ });*/
// 3. Issue IO retrieve ad compute jobs before finding a valid
// frame at required latency to pass to callback.
@@ -128,18 +131,32 @@ void Group::onFrameSet(const ftl::rgbd::VideoCallback &cb) {
//jobs_++;
for (auto s : sources_) {
- jobs_ += 2;
+ jobs_++; // += 2;
- ftl::pool.push([this,s](int id) {
+ ftl::pool.push([this,s,ts](int id) {
_retrieveJob(s);
//if (jobs_ == 0) LOG(INFO) << "LAST JOB = Retrieve";
--jobs_;
+
+ if (cjobs_ == 0) {
+ cjobs_++;
+ s->swap();
+ ftl::pool.push([this,s,ts](int id) {
+ _computeJob(s, ts);
+ //if (jobs_ == 0) LOG(INFO) << "LAST JOB = Compute";
+
+ //LOG(INFO) << "Compute time: " << ftl::timer::get_time() - ts;
+ --cjobs_;
+ });
+ } else {
+ //LOG(WARNING) << "Frame drop";
+ }
});
- ftl::pool.push([this,s](int id) {
+ /*ftl::pool.push([this,s](int id) {
_computeJob(s);
//if (jobs_ == 0) LOG(INFO) << "LAST JOB = Compute";
--jobs_;
- });
+ });*/
}
return true;
});
diff --git a/components/rgbd-sources/src/source.cpp b/components/rgbd-sources/src/source.cpp
index ea5ff3423..25a4a01b1 100644
--- a/components/rgbd-sources/src/source.cpp
+++ b/components/rgbd-sources/src/source.cpp
@@ -253,9 +253,9 @@ bool Source::retrieve() {
else return true;
}
-bool Source::compute(int N, int B) {
+bool Source::compute(int64_t ts) {
UNIQUE_LOCK(mutex_,lk);
- return impl_ && impl_->compute(N,B);
+ return impl_ && impl_->compute(ts);
}
bool Source::setChannel(ftl::codecs::Channel c) {
diff --git a/components/rgbd-sources/src/sources/image/image.hpp b/components/rgbd-sources/src/sources/image/image.hpp
index 2e2391b7c..22cfa244d 100644
--- a/components/rgbd-sources/src/sources/image/image.hpp
+++ b/components/rgbd-sources/src/sources/image/image.hpp
@@ -16,7 +16,7 @@ class ImageSource : public ftl::rgbd::detail::Source {
bool capture(int64_t ts) { timestamp_ = ts; return true; }
bool retrieve() { return true; }
- bool compute(int n, int b) { return false; };
+ bool compute(int64_t ts) { return false; };
bool isReady() { return false; };
};
diff --git a/components/rgbd-sources/src/sources/pylon/pylon.cpp b/components/rgbd-sources/src/sources/pylon/pylon.cpp
index e9b9a2f39..76aa5f4e0 100644
--- a/components/rgbd-sources/src/sources/pylon/pylon.cpp
+++ b/components/rgbd-sources/src/sources/pylon/pylon.cpp
@@ -176,9 +176,9 @@ void PylonSource::swap() {
frames_[1] = std::move(tmp);
}
-bool PylonSource::compute(int n, int b) {
+bool PylonSource::compute(int64_t ts) {
auto &frame = frames_[1];
- host_->notify(timestamp_, frame);
+ host_->notify(ts, frame);
return true;
}
diff --git a/components/rgbd-sources/src/sources/pylon/pylon.hpp b/components/rgbd-sources/src/sources/pylon/pylon.hpp
index 868affe3b..6065aa402 100644
--- a/components/rgbd-sources/src/sources/pylon/pylon.hpp
+++ b/components/rgbd-sources/src/sources/pylon/pylon.hpp
@@ -23,7 +23,7 @@ class PylonSource : public ftl::rgbd::detail::Source {
void swap();
bool capture(int64_t ts);
bool retrieve();
- bool compute(int n=-1, int b=-1);
+ bool compute(int64_t ts);
bool isReady();
private:
diff --git a/components/rgbd-sources/src/sources/realsense/realsense_source.cpp b/components/rgbd-sources/src/sources/realsense/realsense_source.cpp
index ea061ba2c..deff49721 100644
--- a/components/rgbd-sources/src/sources/realsense/realsense_source.cpp
+++ b/components/rgbd-sources/src/sources/realsense/realsense_source.cpp
@@ -45,9 +45,10 @@ RealsenseSource::~RealsenseSource() {
}
-bool RealsenseSource::compute(int n, int b) {
- frame_.reset();
- frame_.setOrigin(&state_);
+bool RealsenseSource::retrieve() {
+ auto &frame = frames_[0];
+ frame.reset();
+ frame.setOrigin(&state_);
rs2::frameset frames;
if (!pipe_.poll_for_frames(&frames)) return false; //wait_for_frames();
@@ -66,7 +67,7 @@ bool RealsenseSource::compute(int n, int b) {
}
cv::Mat tmp_rgb(cv::Size(w, h), CV_8UC4, (void*)cframe.get_data(), cv::Mat::AUTO_STEP);
- frame_.create<GpuMat>(Channel::Colour).upload(tmp_rgb);
+ frame.create<GpuMat>(Channel::Colour).upload(tmp_rgb);
} else {
frames = align_to_depth_.process(frames);
@@ -77,15 +78,26 @@ bool RealsenseSource::compute(int n, int b) {
cv::Mat tmp_depth(cv::Size((int)w, (int)h), CV_16UC1, (void*)depth.get_data(), depth.get_stride_in_bytes());
tmp_depth.convertTo(tmp_depth, CV_32FC1, scale_);
- frame_.create<GpuMat>(Channel::Depth).upload(tmp_depth);
+ frame.create<GpuMat>(Channel::Depth).upload(tmp_depth);
cv::Mat tmp_rgb(cv::Size(w, h), CV_8UC4, (void*)rscolour_.get_data(), cv::Mat::AUTO_STEP);
- frame_.create<GpuMat>(Channel::Colour).upload(tmp_rgb);
+ frame.create<GpuMat>(Channel::Colour).upload(tmp_rgb);
}
- host_->notify(timestamp_, frame_);
+ return true;
+}
+
+bool RealsenseSource::compute(int64_t ts) {
+ auto &frame = frames_[1];
+ host_->notify(ts, frame);
return true;
}
+void RealsenseSource::swap() {
+ auto tmp = std::move(frames_[0]);
+ frames_[0] = std::move(frames_[1]);
+ frames_[1] = std::move(tmp);
+}
+
bool RealsenseSource::isReady() {
return true;
}
diff --git a/components/rgbd-sources/src/sources/realsense/realsense_source.hpp b/components/rgbd-sources/src/sources/realsense/realsense_source.hpp
index 371d305b7..bb4c701e3 100644
--- a/components/rgbd-sources/src/sources/realsense/realsense_source.hpp
+++ b/components/rgbd-sources/src/sources/realsense/realsense_source.hpp
@@ -18,9 +18,10 @@ class RealsenseSource : public ftl::rgbd::detail::Source {
~RealsenseSource();
bool capture(int64_t ts) { timestamp_ = ts; return true; }
- bool retrieve() { return true; }
- bool compute(int n=-1, int b=-1);
+ bool retrieve();
+ bool compute(int64_t ts);
bool isReady();
+ void swap();
private:
bool ready_;
@@ -28,6 +29,7 @@ class RealsenseSource : public ftl::rgbd::detail::Source {
rs2::pipeline pipe_;
rs2::align align_to_depth_;
rs2::frame rscolour_;
+ Frame frames_[2];
};
}
diff --git a/components/rgbd-sources/src/sources/screencapture/screencapture.cpp b/components/rgbd-sources/src/sources/screencapture/screencapture.cpp
index 5a6af978f..9ef0ee0a2 100644
--- a/components/rgbd-sources/src/sources/screencapture/screencapture.cpp
+++ b/components/rgbd-sources/src/sources/screencapture/screencapture.cpp
@@ -197,7 +197,7 @@ bool ScreenCapture::retrieve() {
return true;
}
-bool ScreenCapture::compute(int n, int b) {
+bool ScreenCapture::compute(int64_t ts) {
if (!ready_) return false;
cv::Mat img;
@@ -213,7 +213,7 @@ bool ScreenCapture::compute(int n, int b) {
frame_.create<cv::Mat>(Channel::Colour) = img;
}
- host_->notify(timestamp_, frame_);
+ host_->notify(ts, frame_);
return true;
}
diff --git a/components/rgbd-sources/src/sources/screencapture/screencapture.hpp b/components/rgbd-sources/src/sources/screencapture/screencapture.hpp
index 8480359e6..942db7d56 100644
--- a/components/rgbd-sources/src/sources/screencapture/screencapture.hpp
+++ b/components/rgbd-sources/src/sources/screencapture/screencapture.hpp
@@ -25,7 +25,7 @@ class ScreenCapture : public ftl::rgbd::detail::Source {
bool capture(int64_t ts) { timestamp_ = ts; return true; };
void swap() override;
bool retrieve();
- bool compute(int n=-1, int b=-1);
+ bool compute(int64_t ts);
bool isReady();
size_t getOffsetX() const { return (offset_x_ > full_width_-params_.width) ? full_width_-params_.width : offset_x_; }
diff --git a/components/rgbd-sources/src/sources/stereovideo/stereovideo.cpp b/components/rgbd-sources/src/sources/stereovideo/stereovideo.cpp
index aeff9c431..e23203703 100644
--- a/components/rgbd-sources/src/sources/stereovideo/stereovideo.cpp
+++ b/components/rgbd-sources/src/sources/stereovideo/stereovideo.cpp
@@ -326,7 +326,7 @@ void StereoVideoSource::swap() {
frames_[1] = std::move(tmp);
}
-bool StereoVideoSource::compute(int n, int b) {
+bool StereoVideoSource::compute(int64_t ts) {
auto &frame = frames_[1];
if (lsrc_->isStereo()) {
@@ -347,7 +347,7 @@ bool StereoVideoSource::compute(int n, int b) {
if (!frame.hasChannel(Channel::Left)) { return false; }
}
- host_->notify(capts_, frame);
+ host_->notify(ts, frame);
return true;
}
diff --git a/components/rgbd-sources/src/sources/stereovideo/stereovideo.hpp b/components/rgbd-sources/src/sources/stereovideo/stereovideo.hpp
index fba038df6..e7f2ee887 100644
--- a/components/rgbd-sources/src/sources/stereovideo/stereovideo.hpp
+++ b/components/rgbd-sources/src/sources/stereovideo/stereovideo.hpp
@@ -28,7 +28,7 @@ class StereoVideoSource : public detail::Source {
void swap();
bool capture(int64_t ts);
bool retrieve();
- bool compute(int n, int b);
+ bool compute(int64_t ts);
bool isReady();
Camera parameters(ftl::codecs::Channel chan) override;
diff --git a/components/rgbd-sources/test/source_unit.cpp b/components/rgbd-sources/test/source_unit.cpp
index 29f406272..7c1b3a0e9 100644
--- a/components/rgbd-sources/test/source_unit.cpp
+++ b/components/rgbd-sources/test/source_unit.cpp
@@ -41,7 +41,7 @@ class ImageSource : public ftl::rgbd::detail::Source {
bool capture(int64_t ts) { return true; }
bool retrieve() { return true; }
- bool compute(int n, int b) { return true; };
+ bool compute(int64_t ts) { return true; };
bool isReady() { return true; };
};
@@ -53,7 +53,7 @@ class ScreenCapture : public ftl::rgbd::detail::Source {
bool capture(int64_t ts) { return true; }
bool retrieve() { return true; }
- bool compute(int n, int b) { return true; };
+ bool compute(int64_t ts) { return true; };
bool isReady() { return true; };
};
@@ -68,7 +68,7 @@ class StereoVideoSource : public ftl::rgbd::detail::Source {
bool capture(int64_t ts) { return true; }
bool retrieve() { return true; }
- bool compute(int n, int b) { return true; };
+ bool compute(int64_t ts) { return true; };
bool isReady() { return true; };
};
@@ -80,7 +80,7 @@ class NetSource : public ftl::rgbd::detail::Source {
bool capture(int64_t ts) { return true; }
bool retrieve() { return true; }
- bool compute(int n, int b) { return true; };
+ bool compute(int64_t ts) { return true; };
bool isReady() { return true; };
};
@@ -92,7 +92,7 @@ class SnapshotSource : public ftl::rgbd::detail::Source {
bool capture(int64_t ts) { return true; }
bool retrieve() { return true; }
- bool compute(int n, int b) { return true; };
+ bool compute(int64_t ts) { return true; };
bool isReady() { return true; };
};
@@ -104,7 +104,7 @@ class FileSource : public ftl::rgbd::detail::Source {
bool capture(int64_t ts) { return true; }
bool retrieve() { return true; }
- bool compute(int n, int b) { return true; };
+ bool compute(int64_t ts) { return true; };
bool isReady() { return true; };
};
@@ -116,7 +116,7 @@ class RealsenseSource : public ftl::rgbd::detail::Source {
bool capture(int64_t ts) { return true; }
bool retrieve() { return true; }
- bool compute(int n, int b) { return true; };
+ bool compute(int64_t ts) { return true; };
bool isReady() { return true; };
};
@@ -128,7 +128,7 @@ class PylonSource : public ftl::rgbd::detail::Source {
bool capture(int64_t ts) { return true; }
bool retrieve() { return true; }
- bool compute(int n, int b) { return true; };
+ bool compute(int64_t ts) { return true; };
bool isReady() { return true; };
};
@@ -140,7 +140,7 @@ class MiddleburySource : public ftl::rgbd::detail::Source {
bool capture(int64_t ts) { return true; }
bool retrieve() { return true; }
- bool compute(int n, int b) { return true; };
+ bool compute(int64_t ts) { return true; };
bool isReady() { return true; };
};
--
GitLab