From 4f93d400cb766fdbb9553173a4773e55221b1942 Mon Sep 17 00:00:00 2001
From: Nicolas Pope <nwpope@utu.fi>
Date: Wed, 19 Aug 2020 21:41:04 +0300
Subject: [PATCH] Fix render from feed
---
applications/vision/src/main.cpp | 1 +
components/renderers/cpp/src/CUDARender.cpp | 5 +-
.../streams/include/ftl/streams/feed.hpp | 1 +
components/streams/src/feed.cpp | 85 ++++++++++---------
.../streams/src/renderers/openvr_render.cpp | 6 +-
.../streams/src/renderers/screen_render.cpp | 3 +-
6 files changed, 56 insertions(+), 45 deletions(-)
diff --git a/applications/vision/src/main.cpp b/applications/vision/src/main.cpp
index b9f893300..7bff748fc 100644
--- a/applications/vision/src/main.cpp
+++ b/applications/vision/src/main.cpp
@@ -240,6 +240,7 @@ static void run(ftl::Configurable *root) {
// Make sure upload has completed.
cudaSafeCall(cudaEventSynchronize(fs->frames[0].uploadEvent()));
+ // TODO: Try depth pipeline again here if failed first time.
fs->flush(ftl::codecs::Channel::Colour);
const_cast<ftl::data::FrameSetPtr&>(fs).reset();
diff --git a/components/renderers/cpp/src/CUDARender.cpp b/components/renderers/cpp/src/CUDARender.cpp
index 1ed37cca6..3b5a6adc4 100644
--- a/components/renderers/cpp/src/CUDARender.cpp
+++ b/components/renderers/cpp/src/CUDARender.cpp
@@ -99,7 +99,8 @@ CUDARender::CUDARender(nlohmann::json &config) : ftl::render::FSRenderer(config)
}
}
- cudaSafeCall(cudaStreamCreate(&stream_));
+ //cudaSafeCall(cudaStreamCreate(&stream_));
+ stream_ = 0;
last_frame_ = -1;
temp_.store();
@@ -510,6 +511,8 @@ void CUDARender::begin(ftl::rgbd::Frame &out, ftl::codecs::Channel chan) {
throw FTL_Error("Already rendering");
}
+ stream_ = out.stream();
+
out_ = &out;
const auto &camera = out.getLeftCamera();
cv::cuda::Stream cvstream = cv::cuda::StreamAccessor::wrapStream(stream_);
diff --git a/components/streams/include/ftl/streams/feed.hpp b/components/streams/include/ftl/streams/feed.hpp
index ac141a045..4aab46ca2 100644
--- a/components/streams/include/ftl/streams/feed.hpp
+++ b/components/streams/include/ftl/streams/feed.hpp
@@ -235,6 +235,7 @@ private:
void _createPipeline(uint32_t fsid);
ftl::operators::Graph* _addPipeline(uint32_t fsid);
+ void _dispatch(const ftl::data::FrameSetPtr &fs);
void _beginRecord(Filter *f);
void _stopRecording();
diff --git a/components/streams/src/feed.cpp b/components/streams/src/feed.cpp
index 54b35d200..4253b8026 100644
--- a/components/streams/src/feed.cpp
+++ b/components/streams/src/feed.cpp
@@ -179,55 +179,25 @@ Feed::Feed(nlohmann::json &config, ftl::net::Universe*net) :
SHARED_LOCK(mtx_, lk);
if (pre_pipelines_.count(fs->frameset()) == 1) {
- pipeline = pre_pipelines_[fs->frameset()]; //->apply(*fs, *fs, 0);
+ pipeline = pre_pipelines_[fs->frameset()];
}
lk.unlock();
bool did_pipe = false;
- if (pipeline) did_pipe = pipeline->apply(*fs, *fs, [this,fs]() {
- SHARED_LOCK(mtx_, lk);
+ if (pipeline) {
+ bool did_pipe = pipeline->apply(*fs, *fs, [this,fs]() {
+ _dispatch(fs);
+ });
- std::atomic_store(&latest_.at(fs->frameset()), fs);
-
- if (fs->hasAnyChanged(Channel::Thumbnail)) {
- _saveThumbnail(fs);
- }
-
- for (auto* filter : filters_) {
- // TODO: smarter update (update only when changed) instead of
- // filter->channels_available_ = fs->channels();
-
- if (filter->sources().empty()) {
- //filter->channels_available_ = fs->channels();
- filter->handler_.triggerParallel(fs);
- }
- else {
- // TODO: process partial/complete sets here (drop), that is
- // intersection filter->sources() and fs->sources() is
- // same as filter->sources().
-
- // TODO: reverse map source ids required here?
- for (const auto& src : filter->sources()) {
- //if (fs->hasFrame(src)) {
- if (fs->frameset() == src) {
- //filter->channels_available_ = fs->channels();
- filter->handler_.triggerParallel(fs);
- break;
- }
- }
- }
+ if (!did_pipe) {
+ LOG(WARNING) << "GUI Pipeline dropped";
}
- });
- //cudaSafeCall(cudaStreamSynchronize(fs->frames[0].stream()));
-
- //lk.lock();
-
- if (!did_pipe) {
- LOG(WARNING) << "GUI Pipeline dropped";
+ } else {
+ _dispatch(fs);
}
-
+
return true;
});
@@ -279,6 +249,41 @@ Feed::~Feed() {
}
}
+void Feed::_dispatch(const ftl::data::FrameSetPtr &fs) {
+ SHARED_LOCK(mtx_, lk);
+
+ std::atomic_store(&latest_.at(fs->frameset()), fs);
+
+ if (fs->hasAnyChanged(Channel::Thumbnail)) {
+ _saveThumbnail(fs);
+ }
+
+ for (auto* filter : filters_) {
+ // TODO: smarter update (update only when changed) instead of
+ // filter->channels_available_ = fs->channels();
+
+ if (filter->sources().empty()) {
+ //filter->channels_available_ = fs->channels();
+ filter->handler_.triggerParallel(fs);
+ }
+ else {
+ // TODO: process partial/complete sets here (drop), that is
+ // intersection filter->sources() and fs->sources() is
+ // same as filter->sources().
+
+ // TODO: reverse map source ids required here?
+ for (const auto& src : filter->sources()) {
+ //if (fs->hasFrame(src)) {
+ if (fs->frameset() == src) {
+ //filter->channels_available_ = fs->channels();
+ filter->handler_.triggerParallel(fs);
+ break;
+ }
+ }
+ }
+ }
+}
+
void Feed::_saveThumbnail(const ftl::data::FrameSetPtr& fs) {
// TODO: Put thumb somewhere here...
}
diff --git a/components/streams/src/renderers/openvr_render.cpp b/components/streams/src/renderers/openvr_render.cpp
index 6c76a6bd7..a448206e2 100644
--- a/components/streams/src/renderers/openvr_render.cpp
+++ b/components/streams/src/renderers/openvr_render.cpp
@@ -361,8 +361,8 @@ bool OpenVRRender::retrieve(ftl::data::Frame &frame_out) {
texture1_.make(width, height, ftl::utility::GLTexture::Type::BGRA);
texture2_.make(width, height, ftl::utility::GLTexture::Type::BGRA);
- rgbdframe.create<cv::cuda::GpuMat>(Channel::Colour) = texture1_.map(renderer_->getCUDAStream());
- rgbdframe.create<cv::cuda::GpuMat>(Channel::Colour2) = texture2_.map(renderer_->getCUDAStream());
+ rgbdframe.create<cv::cuda::GpuMat>(Channel::Colour) = texture1_.map(rgbdframe.stream());
+ rgbdframe.create<cv::cuda::GpuMat>(Channel::Colour2) = texture2_.map(rgbdframe.stream());
rgbdframe.create<cv::cuda::GpuMat>(Channel::Depth).create(height, width, CV_32F);
rgbdframe.createTexture<float>(Channel::Depth);
@@ -521,7 +521,7 @@ bool OpenVRRender::retrieve(ftl::data::Frame &frame_out) {
post_pipe_->append<ftl::operators::GTAnalysis>("gtanalyse");
}
- post_pipe_->apply(rgbdframe, rgbdframe, nullptr, renderer_->getCUDAStream());
+ post_pipe_->apply(rgbdframe, rgbdframe);
if (host_->value("enable_touch", false)) {
ftl::render::collision2touch(rgbdframe, renderer_->getCollisions(), sets, my_id_, host_->value("touch_min", 0.01f), host_->value("touch_max", 0.05f));
diff --git a/components/streams/src/renderers/screen_render.cpp b/components/streams/src/renderers/screen_render.cpp
index 2e2d20532..4556cfaec 100644
--- a/components/streams/src/renderers/screen_render.cpp
+++ b/components/streams/src/renderers/screen_render.cpp
@@ -225,7 +225,8 @@ bool ScreenRender::retrieve(ftl::data::Frame &frame_out) {
post_pipe_->append<ftl::operators::GTAnalysis>("gtanalyse");
}
- post_pipe_->apply(rgbdframe, rgbdframe, 0);
+ post_pipe_->apply(rgbdframe, rgbdframe);
+ cudaSafeCall(cudaStreamSynchronize(rgbdframe.stream()));
if (host_->value("enable_touch", false)) {
ftl::render::collision2touch(rgbdframe, renderer_->getCollisions(), sets, my_id_, host_->value("touch_min", 0.01f), host_->value("touch_max", 0.05f));
--
GitLab