Resolve VR performance problems

applications/gui/src/camera.cpp

@@ -78,6 +78,9 @@ ftl::gui::Camera::Camera(ftl::gui::Screen *screen, int fsmask, int fid, ftl::cod
 	record_stream_ = nullptr;
 	transform_ix_ = -1;
 	stereo_ = false;
+	rx_ = 0;
+	ry_ = 0;
+	framesets_ = nullptr;
 
 	colouriser_ = ftl::create<ftl::render::Colouriser>(screen->root(), "colouriser");
 
@@ -248,6 +251,14 @@ void ftl::gui::Camera::setStereo(bool v) {
 	}
 }
 
+static ftl::codecs::Channel mapToSecondChannel(ftl::codecs::Channel c) {
+	switch (c) {
+		case Channel::Depth		: return Channel::Depth2;
+		case Channel::Normals	: return Channel::Normals2;
+		default: return c;
+	}
+}
+
 void ftl::gui::Camera::_draw(std::vector<ftl::rgbd::FrameSet*> &fss) {
 	frame_.reset();
 	frame_.setOrigin(&state_);
@@ -273,6 +284,9 @@ void ftl::gui::Camera::_draw(std::vector<ftl::rgbd::FrameSet*> &fss) {
 
 			if (channel_ != Channel::Left && channel_ != Channel::Right && channel_ != Channel::None) {
 				renderer_->blend(0.5f, channel_);
+				if (isStereo()) {
+					renderer2_->blend(0.5f, mapToSecondChannel(channel_));
+				}
 			}
 
 			renderer_->end();
@@ -352,6 +366,8 @@ void ftl::gui::Camera::_downloadFrames(ftl::cuda::TextureObject<uchar4> &a, ftl:
 	if (im2_.cols != im1_.cols || im2_.rows != im1_.rows) {
 		throw FTL_Error("Left and right images are different sizes");
 	}
+
+	new_frame_.clear();
 }
 
 void ftl::gui::Camera::_downloadFrame(ftl::cuda::TextureObject<uchar4> &a) {
@@ -365,6 +381,8 @@ void ftl::gui::Camera::_downloadFrame(ftl::cuda::TextureObject<uchar4> &a) {
 	height_ = im1_.rows;
 
 	im2_ = cv::Mat();
+
+	new_frame_.clear();
 }
 
 void ftl::gui::Camera::update(int fsid, const ftl::codecs::Channels<0> &c) {
@@ -379,6 +397,8 @@ void ftl::gui::Camera::update(int fsid, const ftl::codecs::Channels<0> &c) {
 void ftl::gui::Camera::update(std::vector<ftl::rgbd::FrameSet*> &fss) {
 	UNIQUE_LOCK(mutex_, lk);
 
+	framesets_ = &fss;
+
 	//if (fss.size() <= fsid_) return;
 	if (fid_ == 255) {
 		name_ = "Virtual Camera";
@@ -443,14 +463,17 @@ void ftl::gui::Camera::mouseMovement(int rx, int ry, int button) {
 	//if (!src_->hasCapabilities(ftl::rgbd::kCapMovable)) return;
 	if (fid_ < 255) return;
 	if (button == 1) {
-		float rrx = ((float)ry * 0.2f * delta_);
+		rx_ += rx;
+		ry_ += ry;
+
+		/*float rrx = ((float)ry * 0.2f * delta_);
 		//orientation_[2] += std::cos(orientation_[1])*((float)rel[1] * 0.2f * delta_);
 		float rry = (float)rx * 0.2f * delta_;
 		float rrz = 0.0;
 
 
 		Eigen::Affine3d r = create_rotation_matrix(rrx, -rry, rrz);
-		rotmat_ = rotmat_ * r.matrix();
+		rotmat_ = rotmat_ * r.matrix();*/
 	}
 }
 
@@ -578,18 +601,21 @@ void ftl::gui::Camera::active(bool a) {
 	}
 }
 
-const GLTexture &ftl::gui::Camera::captureFrame() {
+const void ftl::gui::Camera::captureFrame() {
 	float now = (float)glfwGetTime();
+	if (!screen_->isVR() && (now - ftime_) < 0.04f) return;
+
 	delta_ = now - ftime_;
 	ftime_ = now;
 
+	LOG(INFO) << "Frame delta: " << delta_;
+
 	//if (src_ && src_->isReady()) {
 	if (width_ > 0 && height_ > 0) {
-		UNIQUE_LOCK(mutex_, lk);
-
 		if (screen_->isVR()) {
 			#ifdef HAVE_OPENVR
 			
+			vr::VRCompositor()->SetTrackingSpace(vr::TrackingUniverseStanding);
 			vr::VRCompositor()->WaitGetPoses(rTrackedDevicePose_, vr::k_unMaxTrackedDeviceCount, NULL, 0 );
 
 			if (isStereo() && rTrackedDevicePose_[vr::k_unTrackedDeviceIndex_Hmd].bPoseIsValid )
@@ -627,6 +653,19 @@ const GLTexture &ftl::gui::Camera::captureFrame() {
 				//LOG(ERROR) << "No VR Pose";
 			}
 			#endif
+		} else {
+			// Use mouse to move camera
+
+			float rrx = ((float)ry_ * 0.2f * delta_);
+			float rry = (float)rx_ * 0.2f * delta_;
+			float rrz = 0.0;
+
+
+			Eigen::Affine3d r = create_rotation_matrix(rrx, -rry, rrz);
+			rotmat_ = rotmat_ * r.matrix();
+
+			rx_ = 0;
+			ry_ = 0;
 		}
 
 		eye_[0] += (neye_[0] - eye_[0]) * lerpSpeed_ * delta_;
@@ -637,25 +676,32 @@ const GLTexture &ftl::gui::Camera::captureFrame() {
 		Eigen::Affine3d t(trans);
 		Eigen::Matrix4d viewPose = t.matrix() * rotmat_;
 
-		if (isVirtual()) {
-			if (transform_ix_ == -1) {
-				state_.setPose(viewPose);
-			} else if (transform_ix_ >= 0) {
-				transforms_[transform_ix_] = viewPose;
+		{
+			UNIQUE_LOCK(mutex_, lk);
+
+			if (isVirtual()) {
+				if (transform_ix_ == -1) {
+					state_.setPose(viewPose);
+				} else if (transform_ix_ >= 0) {
+					transforms_[transform_ix_] = viewPose;
+				}
 			}
 		}
 
-		cv::Mat tmp;
+		if (framesets_) draw(*framesets_);
 
-		if (im1_.rows != 0) {
-			texture1_.update(im1_);
-		}
-		if (isStereo() && im2_.rows != 0) {
-			texture2_.update(im2_);
+		{
+			//UNIQUE_LOCK(mutex_, lk);
+			if (im1_.rows != 0) {
+				texture1_.update(im1_);
+			}
+			if (isStereo() && im2_.rows != 0) {
+				texture2_.update(im2_);
+			}
 		}
 	}
 
-	return texture1_;
+	//return texture1_;
 }
 
 void ftl::gui::Camera::snapshot(const std::string &filename) {

applications/gui/src/camera.hpp

@@ -74,7 +74,7 @@ class Camera {
 	 */
 	void active(bool);
 
-	const GLTexture &captureFrame();
+	const void captureFrame();
 	const GLTexture &getLeft() const { return texture1_; }
 	const GLTexture &getRight() const { return texture2_; }
 
@@ -129,6 +129,10 @@ class Camera {
 	cv::Mat im1_; // first channel (left)
 	cv::Mat im2_; // second channel ("right")
 	bool stereo_;
+	std::atomic_flag new_frame_;
+	int rx_;
+	int ry_;
+	std::vector<ftl::rgbd::FrameSet*> *framesets_;
 
 	ftl::render::CUDARender *renderer_;
 	ftl::render::CUDARender *renderer2_;

applications/gui/src/main.cpp

@@ -40,7 +40,41 @@ int main(int argc, char **argv) {
 			nanogui::ref<ftl::gui::Screen> app = new ftl::gui::Screen(root, net, controller);
 			app->drawAll();
 			app->setVisible(true);
-			nanogui::mainloop();
+			//nanogui::mainloop(20);
+
+			float last_draw_time = 0.0f;
+			float last_vr_time = 0.0f;
+
+			while (ftl::running) {
+				nanogui::Screen *screen = app;
+				if (!app->visible()) {
+					ftl::running = false;
+				} else if (glfwWindowShouldClose(app->glfwWindow())) {
+					app->setVisible(false);
+					ftl::running = false;
+				} else {
+					float now = (float)glfwGetTime();
+					float delta = now - last_draw_time;
+
+					// Generate poses and render and virtual frame here
+					// at full FPS (25 without VR and 90 with VR currently)
+					app->drawFast();
+					last_vr_time = now;
+
+					// Only draw the GUI at 25fps
+					if (delta >= 0.04f) {
+						last_draw_time = now;
+						app->drawAll();
+					}
+				}
+
+				/* Wait for mouse/keyboard or empty refresh events */
+				//glfwWaitEvents();
+				glfwPollEvents();
+			}
+
+        	/* Process events once more */
+        	glfwPollEvents();
 
 			LOG(INFO) << "Stopping...";
 			ftl::timer::stop(false);

applications/gui/src/media_panel.cpp

@@ -118,13 +118,13 @@ MediaPanel::MediaPanel(ftl::gui::Screen *screen, ftl::gui::SourceWindow *sourceW
 			if (!screen_->isVR()) {
 				if (screen_->switchVR(true) == true) {
 					button_vr->setTextColor(nanogui::Color(0.5f,0.5f,1.0f,1.0f));
-					this->button_channels_->setEnabled(false);
+					//this->button_channels_->setEnabled(false);
 				}
 			}
 			else {
 				if (screen_->switchVR(false) == false) {
 					button_vr->setTextColor(nanogui::Color(1.0f,1.0f,1.0f,1.0f));
-					this->button_channels_->setEnabled(true);
+					//this->button_channels_->setEnabled(true);
 				}
 			}
 		});

applications/gui/src/screen.cpp

@@ -520,29 +520,12 @@ void ftl::gui::Screen::draw(NVGcontext *ctx) {
 	if (camera_) {
 		imageSize = {camera_->width(), camera_->height()};
 
-		mImageID = camera_->captureFrame().texture();
+		mImageID = camera_->getLeft().texture();
 		leftEye_ = mImageID;
 		rightEye_ = camera_->getRight().texture();
 
 		//if (camera_->getChannel() != ftl::codecs::Channel::Left) { mImageID = rightEye_; }
 
-		#ifdef HAVE_OPENVR
-		if (isVR() && imageSize[0] > 0 && camera_->getLeft().isValid() && camera_->getRight().isValid()) {
-			
-			glBindTexture(GL_TEXTURE_2D, leftEye_);
-			vr::Texture_t leftEyeTexture = {(void*)(uintptr_t)leftEye_, vr::TextureType_OpenGL, vr::ColorSpace_Gamma };
-			vr::VRCompositor()->Submit(vr::Eye_Left, &leftEyeTexture );
-
-			glBindTexture(GL_TEXTURE_2D, rightEye_);
-			vr::Texture_t rightEyeTexture = {(void*)(uintptr_t)rightEye_, vr::TextureType_OpenGL, vr::ColorSpace_Gamma };
-			vr::VRCompositor()->Submit(vr::Eye_Right, &rightEyeTexture );
-
-			glFlush();
-			
-			mImageID = leftEye_;
-		}
-		#endif
-
 		if (mImageID < std::numeric_limits<unsigned int>::max() && imageSize[0] > 0) {
 			auto mScale = (screenSize.cwiseQuotient(imageSize).minCoeff()) * zoom_;
 			Vector2f scaleFactor = mScale * imageSize.cwiseQuotient(screenSize);
@@ -577,3 +560,24 @@ void ftl::gui::Screen::draw(NVGcontext *ctx) {
 	screen()->performLayout(ctx);
 	nanogui::Screen::draw(ctx);
 }
+
+void ftl::gui::Screen::drawFast() {
+	if (camera_) {
+		camera_->captureFrame();
+
+		#ifdef HAVE_OPENVR
+		if (isVR() && camera_->width() > 0 && camera_->getLeft().isValid() && camera_->getRight().isValid()) {
+			
+			//glBindTexture(GL_TEXTURE_2D, leftEye_);
+			vr::Texture_t leftEyeTexture = {(void*)(uintptr_t)leftEye_, vr::TextureType_OpenGL, vr::ColorSpace_Gamma };
+			vr::VRCompositor()->Submit(vr::Eye_Left, &leftEyeTexture );
+
+			//glBindTexture(GL_TEXTURE_2D, rightEye_);
+			vr::Texture_t rightEyeTexture = {(void*)(uintptr_t)rightEye_, vr::TextureType_OpenGL, vr::ColorSpace_Gamma };
+			vr::VRCompositor()->Submit(vr::Eye_Right, &rightEyeTexture );
+
+			glFlush();
+		}
+		#endif
+	}
+}

applications/gui/src/screen.hpp

@@ -37,6 +37,8 @@ class Screen : public nanogui::Screen {
 
 	virtual void draw(NVGcontext *ctx);
 
+	void drawFast();
+
 	ftl::Configurable *root() { return root_; }
 	ftl::net::Universe *net() { return net_; }
 	ftl::ctrl::Master *control() { return ctrl_; }

applications/gui/src/src_window.hpp

@@ -48,6 +48,8 @@ class SourceWindow : public nanogui::Window {
 	void recordVideo(const std::string &filename);
 	void stopRecordingVideo();
 
+	inline std::vector<ftl::rgbd::FrameSet*> &getFramesets() { return framesets_; }
+
 	inline void paused(bool p) { paused_ = p; }
 
 	private: