From 402f49c5ade8f7cf77f4b2c8aad915ab4245868a Mon Sep 17 00:00:00 2001
From: Nicolas Pope <nicolas.pope@utu.fi>
Date: Mon, 24 Feb 2020 15:07:13 +0200
Subject: [PATCH] Resolve VR performance problems
---
applications/gui/src/camera.cpp | 80 ++++++++++++++++++++++------
applications/gui/src/camera.hpp | 6 ++-
applications/gui/src/main.cpp | 36 ++++++++++++-
applications/gui/src/media_panel.cpp | 4 +-
applications/gui/src/screen.cpp | 40 +++++++-------
applications/gui/src/screen.hpp | 2 +
applications/gui/src/src_window.hpp | 2 +
7 files changed, 131 insertions(+), 39 deletions(-)
diff --git a/applications/gui/src/camera.cpp b/applications/gui/src/camera.cpp
index 8895db1b2..debe91bc6 100644
--- a/applications/gui/src/camera.cpp
+++ b/applications/gui/src/camera.cpp
@@ -78,6 +78,9 @@ ftl::gui::Camera::Camera(ftl::gui::Screen *screen, int fsmask, int fid, ftl::cod
record_stream_ = nullptr;
transform_ix_ = -1;
stereo_ = false;
+ rx_ = 0;
+ ry_ = 0;
+ framesets_ = nullptr;
colouriser_ = ftl::create<ftl::render::Colouriser>(screen->root(), "colouriser");
@@ -248,6 +251,14 @@ void ftl::gui::Camera::setStereo(bool v) {
}
}
+static ftl::codecs::Channel mapToSecondChannel(ftl::codecs::Channel c) {
+ switch (c) {
+ case Channel::Depth : return Channel::Depth2;
+ case Channel::Normals : return Channel::Normals2;
+ default: return c;
+ }
+}
+
void ftl::gui::Camera::_draw(std::vector<ftl::rgbd::FrameSet*> &fss) {
frame_.reset();
frame_.setOrigin(&state_);
@@ -273,6 +284,9 @@ void ftl::gui::Camera::_draw(std::vector<ftl::rgbd::FrameSet*> &fss) {
if (channel_ != Channel::Left && channel_ != Channel::Right && channel_ != Channel::None) {
renderer_->blend(0.5f, channel_);
+ if (isStereo()) {
+ renderer2_->blend(0.5f, mapToSecondChannel(channel_));
+ }
}
renderer_->end();
@@ -352,6 +366,8 @@ void ftl::gui::Camera::_downloadFrames(ftl::cuda::TextureObject<uchar4> &a, ftl:
if (im2_.cols != im1_.cols || im2_.rows != im1_.rows) {
throw FTL_Error("Left and right images are different sizes");
}
+
+ new_frame_.clear();
}
void ftl::gui::Camera::_downloadFrame(ftl::cuda::TextureObject<uchar4> &a) {
@@ -365,6 +381,8 @@ void ftl::gui::Camera::_downloadFrame(ftl::cuda::TextureObject<uchar4> &a) {
height_ = im1_.rows;
im2_ = cv::Mat();
+
+ new_frame_.clear();
}
void ftl::gui::Camera::update(int fsid, const ftl::codecs::Channels<0> &c) {
@@ -379,6 +397,8 @@ void ftl::gui::Camera::update(int fsid, const ftl::codecs::Channels<0> &c) {
void ftl::gui::Camera::update(std::vector<ftl::rgbd::FrameSet*> &fss) {
UNIQUE_LOCK(mutex_, lk);
+ framesets_ = &fss;
+
//if (fss.size() <= fsid_) return;
if (fid_ == 255) {
name_ = "Virtual Camera";
@@ -443,14 +463,17 @@ void ftl::gui::Camera::mouseMovement(int rx, int ry, int button) {
//if (!src_->hasCapabilities(ftl::rgbd::kCapMovable)) return;
if (fid_ < 255) return;
if (button == 1) {
- float rrx = ((float)ry * 0.2f * delta_);
+ rx_ += rx;
+ ry_ += ry;
+
+ /*float rrx = ((float)ry * 0.2f * delta_);
//orientation_[2] += std::cos(orientation_[1])*((float)rel[1] * 0.2f * delta_);
float rry = (float)rx * 0.2f * delta_;
float rrz = 0.0;
Eigen::Affine3d r = create_rotation_matrix(rrx, -rry, rrz);
- rotmat_ = rotmat_ * r.matrix();
+ rotmat_ = rotmat_ * r.matrix();*/
}
}
@@ -578,18 +601,21 @@ void ftl::gui::Camera::active(bool a) {
}
}
-const GLTexture &ftl::gui::Camera::captureFrame() {
+const void ftl::gui::Camera::captureFrame() {
float now = (float)glfwGetTime();
+ if (!screen_->isVR() && (now - ftime_) < 0.04f) return;
+
delta_ = now - ftime_;
ftime_ = now;
+ LOG(INFO) << "Frame delta: " << delta_;
+
//if (src_ && src_->isReady()) {
if (width_ > 0 && height_ > 0) {
- UNIQUE_LOCK(mutex_, lk);
-
if (screen_->isVR()) {
#ifdef HAVE_OPENVR
+ vr::VRCompositor()->SetTrackingSpace(vr::TrackingUniverseStanding);
vr::VRCompositor()->WaitGetPoses(rTrackedDevicePose_, vr::k_unMaxTrackedDeviceCount, NULL, 0 );
if (isStereo() && rTrackedDevicePose_[vr::k_unTrackedDeviceIndex_Hmd].bPoseIsValid )
@@ -627,6 +653,19 @@ const GLTexture &ftl::gui::Camera::captureFrame() {
//LOG(ERROR) << "No VR Pose";
}
#endif
+ } else {
+ // Use mouse to move camera
+
+ float rrx = ((float)ry_ * 0.2f * delta_);
+ float rry = (float)rx_ * 0.2f * delta_;
+ float rrz = 0.0;
+
+
+ Eigen::Affine3d r = create_rotation_matrix(rrx, -rry, rrz);
+ rotmat_ = rotmat_ * r.matrix();
+
+ rx_ = 0;
+ ry_ = 0;
}
eye_[0] += (neye_[0] - eye_[0]) * lerpSpeed_ * delta_;
@@ -637,25 +676,32 @@ const GLTexture &ftl::gui::Camera::captureFrame() {
Eigen::Affine3d t(trans);
Eigen::Matrix4d viewPose = t.matrix() * rotmat_;
- if (isVirtual()) {
- if (transform_ix_ == -1) {
- state_.setPose(viewPose);
- } else if (transform_ix_ >= 0) {
- transforms_[transform_ix_] = viewPose;
+ {
+ UNIQUE_LOCK(mutex_, lk);
+
+ if (isVirtual()) {
+ if (transform_ix_ == -1) {
+ state_.setPose(viewPose);
+ } else if (transform_ix_ >= 0) {
+ transforms_[transform_ix_] = viewPose;
+ }
}
}
- cv::Mat tmp;
+ if (framesets_) draw(*framesets_);
- if (im1_.rows != 0) {
- texture1_.update(im1_);
- }
- if (isStereo() && im2_.rows != 0) {
- texture2_.update(im2_);
+ {
+ //UNIQUE_LOCK(mutex_, lk);
+ if (im1_.rows != 0) {
+ texture1_.update(im1_);
+ }
+ if (isStereo() && im2_.rows != 0) {
+ texture2_.update(im2_);
+ }
}
}
- return texture1_;
+ //return texture1_;
}
void ftl::gui::Camera::snapshot(const std::string &filename) {
diff --git a/applications/gui/src/camera.hpp b/applications/gui/src/camera.hpp
index ce764be86..7003f3c96 100644
--- a/applications/gui/src/camera.hpp
+++ b/applications/gui/src/camera.hpp
@@ -74,7 +74,7 @@ class Camera {
*/
void active(bool);
- const GLTexture &captureFrame();
+ const void captureFrame();
const GLTexture &getLeft() const { return texture1_; }
const GLTexture &getRight() const { return texture2_; }
@@ -129,6 +129,10 @@ class Camera {
cv::Mat im1_; // first channel (left)
cv::Mat im2_; // second channel ("right")
bool stereo_;
+ std::atomic_flag new_frame_;
+ int rx_;
+ int ry_;
+ std::vector<ftl::rgbd::FrameSet*> *framesets_;
ftl::render::CUDARender *renderer_;
ftl::render::CUDARender *renderer2_;
diff --git a/applications/gui/src/main.cpp b/applications/gui/src/main.cpp
index 6b1867b06..71afa31ed 100644
--- a/applications/gui/src/main.cpp
+++ b/applications/gui/src/main.cpp
@@ -40,7 +40,41 @@ int main(int argc, char **argv) {
nanogui::ref<ftl::gui::Screen> app = new ftl::gui::Screen(root, net, controller);
app->drawAll();
app->setVisible(true);
- nanogui::mainloop();
+ //nanogui::mainloop(20);
+
+ float last_draw_time = 0.0f;
+ float last_vr_time = 0.0f;
+
+ while (ftl::running) {
+ nanogui::Screen *screen = app;
+ if (!app->visible()) {
+ ftl::running = false;
+ } else if (glfwWindowShouldClose(app->glfwWindow())) {
+ app->setVisible(false);
+ ftl::running = false;
+ } else {
+ float now = (float)glfwGetTime();
+ float delta = now - last_draw_time;
+
+ // Generate poses and render and virtual frame here
+ // at full FPS (25 without VR and 90 with VR currently)
+ app->drawFast();
+ last_vr_time = now;
+
+ // Only draw the GUI at 25fps
+ if (delta >= 0.04f) {
+ last_draw_time = now;
+ app->drawAll();
+ }
+ }
+
+ /* Wait for mouse/keyboard or empty refresh events */
+ //glfwWaitEvents();
+ glfwPollEvents();
+ }
+
+ /* Process events once more */
+ glfwPollEvents();
LOG(INFO) << "Stopping...";
ftl::timer::stop(false);
diff --git a/applications/gui/src/media_panel.cpp b/applications/gui/src/media_panel.cpp
index e93a4affe..7197ef47a 100644
--- a/applications/gui/src/media_panel.cpp
+++ b/applications/gui/src/media_panel.cpp
@@ -118,13 +118,13 @@ MediaPanel::MediaPanel(ftl::gui::Screen *screen, ftl::gui::SourceWindow *sourceW
if (!screen_->isVR()) {
if (screen_->switchVR(true) == true) {
button_vr->setTextColor(nanogui::Color(0.5f,0.5f,1.0f,1.0f));
- this->button_channels_->setEnabled(false);
+ //this->button_channels_->setEnabled(false);
}
}
else {
if (screen_->switchVR(false) == false) {
button_vr->setTextColor(nanogui::Color(1.0f,1.0f,1.0f,1.0f));
- this->button_channels_->setEnabled(true);
+ //this->button_channels_->setEnabled(true);
}
}
});
diff --git a/applications/gui/src/screen.cpp b/applications/gui/src/screen.cpp
index 39a9b65dc..e2f8a7f16 100644
--- a/applications/gui/src/screen.cpp
+++ b/applications/gui/src/screen.cpp
@@ -520,29 +520,12 @@ void ftl::gui::Screen::draw(NVGcontext *ctx) {
if (camera_) {
imageSize = {camera_->width(), camera_->height()};
- mImageID = camera_->captureFrame().texture();
+ mImageID = camera_->getLeft().texture();
leftEye_ = mImageID;
rightEye_ = camera_->getRight().texture();
//if (camera_->getChannel() != ftl::codecs::Channel::Left) { mImageID = rightEye_; }
- #ifdef HAVE_OPENVR
- if (isVR() && imageSize[0] > 0 && camera_->getLeft().isValid() && camera_->getRight().isValid()) {
-
- glBindTexture(GL_TEXTURE_2D, leftEye_);
- vr::Texture_t leftEyeTexture = {(void*)(uintptr_t)leftEye_, vr::TextureType_OpenGL, vr::ColorSpace_Gamma };
- vr::VRCompositor()->Submit(vr::Eye_Left, &leftEyeTexture );
-
- glBindTexture(GL_TEXTURE_2D, rightEye_);
- vr::Texture_t rightEyeTexture = {(void*)(uintptr_t)rightEye_, vr::TextureType_OpenGL, vr::ColorSpace_Gamma };
- vr::VRCompositor()->Submit(vr::Eye_Right, &rightEyeTexture );
-
- glFlush();
-
- mImageID = leftEye_;
- }
- #endif
-
if (mImageID < std::numeric_limits<unsigned int>::max() && imageSize[0] > 0) {
auto mScale = (screenSize.cwiseQuotient(imageSize).minCoeff()) * zoom_;
Vector2f scaleFactor = mScale * imageSize.cwiseQuotient(screenSize);
@@ -577,3 +560,24 @@ void ftl::gui::Screen::draw(NVGcontext *ctx) {
screen()->performLayout(ctx);
nanogui::Screen::draw(ctx);
}
+
+void ftl::gui::Screen::drawFast() {
+ if (camera_) {
+ camera_->captureFrame();
+
+ #ifdef HAVE_OPENVR
+ if (isVR() && camera_->width() > 0 && camera_->getLeft().isValid() && camera_->getRight().isValid()) {
+
+ //glBindTexture(GL_TEXTURE_2D, leftEye_);
+ vr::Texture_t leftEyeTexture = {(void*)(uintptr_t)leftEye_, vr::TextureType_OpenGL, vr::ColorSpace_Gamma };
+ vr::VRCompositor()->Submit(vr::Eye_Left, &leftEyeTexture );
+
+ //glBindTexture(GL_TEXTURE_2D, rightEye_);
+ vr::Texture_t rightEyeTexture = {(void*)(uintptr_t)rightEye_, vr::TextureType_OpenGL, vr::ColorSpace_Gamma };
+ vr::VRCompositor()->Submit(vr::Eye_Right, &rightEyeTexture );
+
+ glFlush();
+ }
+ #endif
+ }
+}
diff --git a/applications/gui/src/screen.hpp b/applications/gui/src/screen.hpp
index 90cd519bb..92c619551 100644
--- a/applications/gui/src/screen.hpp
+++ b/applications/gui/src/screen.hpp
@@ -37,6 +37,8 @@ class Screen : public nanogui::Screen {
virtual void draw(NVGcontext *ctx);
+ void drawFast();
+
ftl::Configurable *root() { return root_; }
ftl::net::Universe *net() { return net_; }
ftl::ctrl::Master *control() { return ctrl_; }
diff --git a/applications/gui/src/src_window.hpp b/applications/gui/src/src_window.hpp
index da001950e..165143c14 100644
--- a/applications/gui/src/src_window.hpp
+++ b/applications/gui/src/src_window.hpp
@@ -48,6 +48,8 @@ class SourceWindow : public nanogui::Window {
void recordVideo(const std::string &filename);
void stopRecordingVideo();
+ inline std::vector<ftl::rgbd::FrameSet*> &getFramesets() { return framesets_; }
+
inline void paused(bool p) { paused_ = p; }
private:
--
GitLab