From 8bca96759cff41ab3d54f7737850bdca9a584330 Mon Sep 17 00:00:00 2001
From: Nicolas Pope <nicolas.pope@utu.fi>
Date: Fri, 28 Jun 2019 12:59:29 +0300
Subject: [PATCH] Refactor GUI components
---
applications/gui/CMakeLists.txt | 4 +
applications/gui/src/camera.cpp | 295 +++++++++
applications/gui/src/camera.hpp | 71 +++
applications/gui/src/gltexture.cpp | 33 +
applications/gui/src/gltexture.hpp | 24 +
applications/gui/src/main.cpp | 590 +-----------------
applications/gui/src/media_panel.cpp | 139 +++++
applications/gui/src/media_panel.hpp | 24 +
applications/gui/src/pose_window.cpp | 114 +++-
applications/gui/src/pose_window.hpp | 7 +-
applications/gui/src/screen.cpp | 308 +++++++++
applications/gui/src/screen.hpp | 74 +++
applications/gui/src/src_window.cpp | 116 ++--
applications/gui/src/src_window.hpp | 22 +-
applications/reconstruct/src/registration.cpp | 2 +-
.../reconstruct/src/scene_rep_hash_sdf.cu | 2 +-
.../common/cpp/include/ftl/configurable.hpp | 19 +-
.../common/cpp/include/ftl/configuration.hpp | 11 +-
.../common/cpp/include/nlohmann/json.hpp | 9 +-
components/common/cpp/src/configurable.cpp | 9 +-
.../net/cpp/include/ftl/net/universe.hpp | 2 +-
components/net/cpp/src/universe.cpp | 4 +-
.../rgbd-sources/include/ftl/rgbd/source.hpp | 20 +
components/rgbd-sources/src/net.cpp | 28 -
24 files changed, 1170 insertions(+), 757 deletions(-)
create mode 100644 applications/gui/src/camera.cpp
create mode 100644 applications/gui/src/camera.hpp
create mode 100644 applications/gui/src/gltexture.cpp
create mode 100644 applications/gui/src/gltexture.hpp
create mode 100644 applications/gui/src/media_panel.cpp
create mode 100644 applications/gui/src/media_panel.hpp
create mode 100644 applications/gui/src/screen.cpp
create mode 100644 applications/gui/src/screen.hpp
diff --git a/applications/gui/CMakeLists.txt b/applications/gui/CMakeLists.txt
index 6b0d3944c..cda2b89d2 100644
--- a/applications/gui/CMakeLists.txt
+++ b/applications/gui/CMakeLists.txt
@@ -8,6 +8,10 @@ set(GUISRC
src/src_window.cpp
src/config_window.cpp
src/pose_window.cpp
+ src/screen.cpp
+ src/gltexture.cpp
+ src/camera.cpp
+ src/media_panel.cpp
)
add_executable(ftl-gui ${GUISRC})
diff --git a/applications/gui/src/camera.cpp b/applications/gui/src/camera.cpp
new file mode 100644
index 000000000..bb715383b
--- /dev/null
+++ b/applications/gui/src/camera.cpp
@@ -0,0 +1,295 @@
+#include "camera.hpp"
+#include "pose_window.hpp"
+#include "screen.hpp"
+
+#include <nanogui/glutil.h>
+
+using ftl::rgbd::isValidDepth;
+using ftl::gui::GLTexture;
+using ftl::gui::PoseWindow;
+
+// TODO(Nick) MOVE
+class StatisticsImage {
+private:
+ std::vector<float> data_;
+ cv::Size size_;
+ int n_;
+
+public:
+ StatisticsImage(cv::Size size) {
+ size_ = size;
+ data_ = std::vector<float>(size.width * size.height * 2, 0.0f);
+ n_ = 0;
+ }
+
+ void update(const cv::Mat &in);
+ void getStdDev(cv::Mat &out);
+ void getMean(cv::Mat &out);
+};
+
+void StatisticsImage::update(const cv::Mat &in) {
+ DCHECK(in.type() == CV_32F);
+ DCHECK(in.size() == size_);
+ // Welford's Method
+
+ n_++;
+ for (int i = 0; i < size_.width * size_.height; i++) {
+ float x = ((float*) in.data)[i];
+ if (!isValidDepth(x)) { continue; } // invalid value
+ float &m = data_[2*i];
+ float &S = data_[2*i+1];
+ float m_prev = m;
+ m = m + (x - m) / n_;
+ S = S + (x - m) * (x - m_prev);
+ }
+}
+
+void StatisticsImage::getStdDev(cv::Mat &in) {
+ in = cv::Mat(size_, CV_32F, 0.0f);
+
+ for (int i = 0; i < size_.width * size_.height; i++) {
+ float &m = data_[2*i];
+ float &S = data_[2*i+1];
+ ((float*) in.data)[i] = sqrt(S / n_);
+ }
+}
+
+void StatisticsImage::getMean(cv::Mat &in) {
+ in = cv::Mat(size_, CV_32F, 0.0f);
+
+ for (int i = 0; i < size_.width * size_.height; i++) {
+ ((float*) in.data)[i] = data_[2*i];
+ }
+}
+
+class StatisticsImageNSamples {
+private:
+ std::vector<cv::Mat> samples_;
+ cv::Size size_;
+ int i_;
+ int n_;
+
+public:
+ StatisticsImageNSamples(cv::Size size, int n) {
+ size_ = size;
+ samples_ = std::vector<cv::Mat>(n);
+ i_ = 0;
+ n_ = n;
+ }
+
+ void update(const cv::Mat &in);
+ void getStdDev(cv::Mat &out);
+ void getVariance(cv::Mat &out);
+ void getMean(cv::Mat &out);
+};
+
+void StatisticsImageNSamples::update(const cv::Mat &in) {
+ DCHECK(in.type() == CV_32F);
+ DCHECK(in.size() == size_);
+
+ i_ = (i_ + 1) % n_;
+ in.copyTo(samples_[i_]);
+}
+
+void StatisticsImageNSamples::getStdDev(cv::Mat &out) {
+ cv::Mat var;
+ getVariance(var);
+ cv::sqrt(var, out);
+}
+
+void StatisticsImageNSamples::getVariance(cv::Mat &out) {
+ // Welford's Method
+ cv::Mat mat_m(size_, CV_32F, 0.0f);
+ cv::Mat mat_S(size_, CV_32F, 0.0f);
+
+ float n = 0.0f;
+ for (int i_sample = (i_ + 1) % n_; i_sample != i_; i_sample = (i_sample + 1) % n_) {
+ if (samples_[i_sample].size() != size_) continue;
+ n += 1.0f;
+ for (int i = 0; i < size_.width * size_.height; i++) {
+ float &x = ((float*) samples_[i_sample].data)[i];
+ float &m = ((float*) mat_m.data)[i];
+ float &S = ((float*) mat_S.data)[i];
+ float m_prev = m;
+
+ if (!isValidDepth(x)) continue;
+
+ m = m + (x - m) / n;
+ S = S + (x - m) * (x - m_prev);
+ }
+ }
+
+ mat_S.copyTo(out);
+}
+
+void StatisticsImageNSamples::getMean(cv::Mat &in) {}
+
+static Eigen::Affine3d create_rotation_matrix(float ax, float ay, float az) {
+ Eigen::Affine3d rx =
+ Eigen::Affine3d(Eigen::AngleAxisd(ax, Eigen::Vector3d(1, 0, 0)));
+ Eigen::Affine3d ry =
+ Eigen::Affine3d(Eigen::AngleAxisd(ay, Eigen::Vector3d(0, 1, 0)));
+ Eigen::Affine3d rz =
+ Eigen::Affine3d(Eigen::AngleAxisd(az, Eigen::Vector3d(0, 0, 1)));
+ return rz * rx * ry;
+}
+
+ftl::gui::Camera::Camera(ftl::gui::Screen *screen, ftl::rgbd::Source *src) : screen_(screen), src_(src) {
+ eye_ = Eigen::Vector3d(0.0f, 0.0f, 0.0f);
+ neye_ = Eigen::Vector4d(0.0f, 0.0f, 0.0f, 0.0f);
+ rotmat_.setIdentity();
+ //up_ = Eigen::Vector3f(0,1.0f,0);
+ lerpSpeed_ = 0.999f;
+ depth_ = false;
+ ftime_ = (float)glfwGetTime();
+ pause_ = false;
+
+ channel_ = ftl::rgbd::kChanLeft;
+
+ channels_.push_back(ftl::rgbd::kChanLeft);
+ channels_.push_back(ftl::rgbd::kChanDepth);
+
+ // Create pose window...
+ posewin_ = new PoseWindow(screen, src_->getURI());
+ posewin_->setTheme(screen->windowtheme);
+ posewin_->setVisible(false);
+}
+
+ftl::gui::Camera::~Camera() {
+
+}
+
+ftl::rgbd::Source *ftl::gui::Camera::source() {
+ return src_;
+}
+
+void ftl::gui::Camera::setPose(const Eigen::Matrix4d &p) {
+ eye_[0] = p(0,3);
+ eye_[1] = p(1,3);
+ eye_[2] = p(2,3);
+
+ double sx = Eigen::Vector3d(p(0,0), p(1,0), p(2,0)).norm();
+ double sy = Eigen::Vector3d(p(0,1), p(1,1), p(2,1)).norm();
+ double sz = Eigen::Vector3d(p(0,2), p(1,2), p(2,2)).norm();
+
+ Eigen::Matrix4d rot = p;
+ rot(0,3) = 0.0;
+ rot(1,3) = 0.0;
+ rot(2,3) = 0.0;
+ rot(0,0) = rot(0,0) / sx;
+ rot(1,0) = rot(1,0) / sx;
+ rot(2,0) = rot(2,0) / sx;
+ rot(0,1) = rot(0,1) / sy;
+ rot(1,1) = rot(1,1) / sy;
+ rot(2,1) = rot(2,1) / sy;
+ rot(0,2) = rot(0,2) / sz;
+ rot(1,2) = rot(1,2) / sz;
+ rot(2,2) = rot(2,2) / sz;
+ rotmat_ = rot;
+}
+
+void ftl::gui::Camera::mouseMovement(int rx, int ry, int button) {
+ if (button == 1) {
+ float rrx = ((float)ry * 0.2f * delta_);
+ //orientation_[2] += std::cos(orientation_[1])*((float)rel[1] * 0.2f * delta_);
+ float rry = (float)rx * 0.2f * delta_;
+ float rrz = 0.0;
+
+
+ Eigen::Affine3d r = create_rotation_matrix(rrx, -rry, rrz);
+ rotmat_ = rotmat_ * r.matrix();
+ }
+}
+
+void ftl::gui::Camera::keyMovement(int key, int modifiers) {
+ if (key == 263 || key == 262) {
+ float mag = (modifiers & 0x1) ? 0.01f : 0.1f;
+ float scalar = (key == 263) ? -mag : mag;
+ neye_ += rotmat_*Eigen::Vector4d(scalar,0.0,0.0,1.0);
+ return;
+ } else if (key == 264 || key == 265) {
+ float mag = (modifiers & 0x1) ? 0.01f : 0.1f;
+ float scalar = (key == 264) ? -mag : mag;
+ neye_ += rotmat_*Eigen::Vector4d(0.0,0.0,scalar,1.0);
+ return;
+ } else if (key == 266 || key == 267) {
+ float mag = (modifiers & 0x1) ? 0.01f : 0.1f;
+ float scalar = (key == 266) ? -mag : mag;
+ neye_ += rotmat_*Eigen::Vector4d(0.0,scalar,0.0,1.0);
+ return;
+ }
+}
+
+void ftl::gui::Camera::showPoseWindow() {
+ posewin_->setVisible(true);
+}
+
+void ftl::gui::Camera::showSettings() {
+
+}
+
+const GLTexture &ftl::gui::Camera::thumbnail() {
+
+}
+
+const GLTexture &ftl::gui::Camera::captureFrame() {
+ float now = (float)glfwGetTime();
+ delta_ = now - ftime_;
+ ftime_ = now;
+
+ if (src_ && src_->isReady()) {
+ cv::Mat rgb, depth;
+
+ // Lerp the Eye
+ eye_[0] += (neye_[0] - eye_[0]) * lerpSpeed_ * delta_;
+ eye_[1] += (neye_[1] - eye_[1]) * lerpSpeed_ * delta_;
+ eye_[2] += (neye_[2] - eye_[2]) * lerpSpeed_ * delta_;
+
+ Eigen::Translation3d trans(eye_);
+ Eigen::Affine3d t(trans);
+ Eigen::Matrix4d viewPose = t.matrix() * rotmat_;
+
+ src_->setPose(viewPose);
+ src_->grab();
+ src_->getFrames(rgb, depth);
+
+ if (!stats_ && depth.rows > 0) {
+ stats_ = new StatisticsImageNSamples(depth.size(), 25);
+ }
+
+ if (stats_ && depth.rows > 0) { stats_->update(depth); }
+
+ cv::Mat tmp;
+
+ switch(channel_) {
+ case ftl::rgbd::kChanDepth:
+ if (depth.rows == 0) { break; }
+ //imageSize = Vector2f(depth.cols,depth.rows);
+ depth.convertTo(tmp, CV_8U, 255.0f / 5.0f);
+ tmp = 255 - tmp;
+ applyColorMap(tmp, tmp, cv::COLORMAP_JET);
+ texture_.update(tmp);
+ break;
+
+ case ftl::rgbd::kChanDeviation:
+ if (depth.rows == 0) { break; }
+ //imageSize = Vector2f(depth.cols, depth.rows);
+ stats_->getStdDev(tmp);
+ tmp.convertTo(tmp, CV_8U, 50.0);
+ applyColorMap(tmp, tmp, cv::COLORMAP_HOT);
+ texture_.update(tmp);
+ break;
+
+ default:
+ if (rgb.rows == 0) { break; }
+ //imageSize = Vector2f(rgb.cols,rgb.rows);
+ texture_.update(rgb);
+ }
+ }
+
+ return texture_;
+}
+
+nlohmann::json ftl::gui::Camera::getMetaData() {
+
+}
diff --git a/applications/gui/src/camera.hpp b/applications/gui/src/camera.hpp
new file mode 100644
index 000000000..590bdbcee
--- /dev/null
+++ b/applications/gui/src/camera.hpp
@@ -0,0 +1,71 @@
+#ifndef _FTL_GUI_CAMERA_HPP_
+#define _FTL_GUI_CAMERA_HPP_
+
+#include <ftl/rgbd/source.hpp>
+#include "gltexture.hpp"
+
+#include <string>
+
+class StatisticsImageNSamples;
+
+namespace ftl {
+namespace gui {
+
+class Screen;
+class PoseWindow;
+
+class Camera {
+ public:
+ Camera(ftl::gui::Screen *screen, ftl::rgbd::Source *src);
+ ~Camera();
+
+ ftl::rgbd::Source *source();
+
+ int width() { return (src_) ? src_->parameters().width : 0; }
+ int height() { return (src_) ? src_->parameters().height : 0; }
+
+ void setPose(const Eigen::Matrix4d &p);
+
+ void mouseMovement(int rx, int ry, int button);
+ void keyMovement(int key, int modifiers);
+
+ void showPoseWindow();
+ void showSettings();
+
+ void setChannel(ftl::rgbd::channel_t c) { channel_ = c; };
+
+ void togglePause();
+ void isPaused();
+ const std::vector<ftl::rgbd::channel_t> &availableChannels();
+
+ const GLTexture &thumbnail();
+ const GLTexture &captureFrame();
+
+ nlohmann::json getMetaData();
+
+ StatisticsImageNSamples *stats_ = nullptr;
+
+ private:
+ Screen *screen_;
+ ftl::rgbd::Source *src_;
+ GLTexture thumb_;
+ GLTexture texture_;
+ ftl::gui::PoseWindow *posewin_;
+ nlohmann::json meta_;
+ Eigen::Vector4d neye_;
+ Eigen::Vector3d eye_;
+ //Eigen::Vector3f orientation_;
+ Eigen::Matrix4d rotmat_;
+ float ftime_;
+ float delta_;
+ float lerpSpeed_;
+ bool depth_;
+ bool pause_;
+ ftl::rgbd::channel_t channel_;
+ std::vector<ftl::rgbd::channel_t> channels_;
+};
+
+}
+}
+
+#endif // _FTL_GUI_CAMERA_HPP_
diff --git a/applications/gui/src/gltexture.cpp b/applications/gui/src/gltexture.cpp
new file mode 100644
index 000000000..752894a3c
--- /dev/null
+++ b/applications/gui/src/gltexture.cpp
@@ -0,0 +1,33 @@
+#include "gltexture.hpp"
+
+#include <nanogui/opengl.h>
+#include <loguru.hpp>
+
+using ftl::gui::GLTexture;
+
+GLTexture::GLTexture() {
+ glid_ = std::numeric_limits<unsigned int>::max();
+}
+
+GLTexture::~GLTexture() {
+ //glDeleteTextures(1, &glid_);
+}
+
+void GLTexture::update(cv::Mat &m) {
+ if (glid_ == std::numeric_limits<unsigned int>::max()) {
+ glGenTextures(1, &glid_);
+ glBindTexture(GL_TEXTURE_2D, glid_);
+ cv::Mat m(cv::Size(100,100), CV_8UC3);
+ glTexImage2D(GL_TEXTURE_2D, 0, GL_RGB8, m.cols, m.rows, 0, GL_RGB, GL_UNSIGNED_BYTE, m.data);
+ glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
+ glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
+ glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_NEAREST);
+ glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR);
+ }
+ if (m.rows == 0) return;
+ glBindTexture(GL_TEXTURE_2D, glid_);
+ // TODO Allow for other formats
+ glTexImage2D(GL_TEXTURE_2D, 0, GL_RGB8, m.cols, m.rows, 0, GL_BGR, GL_UNSIGNED_BYTE, m.data);
+ auto err = glGetError();
+ if (err != 0) LOG(ERROR) << "OpenGL Texture error: " << err;
+}
diff --git a/applications/gui/src/gltexture.hpp b/applications/gui/src/gltexture.hpp
new file mode 100644
index 000000000..4fb0bdaef
--- /dev/null
+++ b/applications/gui/src/gltexture.hpp
@@ -0,0 +1,24 @@
+#ifndef _FTL_GUI_GLTEXTURE_HPP_
+#define _FTL_GUI_GLTEXTURE_HPP_
+
+#include <opencv2/opencv.hpp>
+
+namespace ftl {
+namespace gui {
+
+class GLTexture {
+ public:
+ GLTexture();
+ ~GLTexture();
+
+ void update(cv::Mat &m);
+ unsigned int texture() const { return glid_; }
+
+ private:
+ unsigned int glid_;
+};
+
+}
+}
+
+#endif // _FTL_GUI_GLTEXTURE_HPP_
diff --git a/applications/gui/src/main.cpp b/applications/gui/src/main.cpp
index 69512cc07..974afff14 100644
--- a/applications/gui/src/main.cpp
+++ b/applications/gui/src/main.cpp
@@ -5,594 +5,8 @@
#include <loguru.hpp>
-#include <opencv2/opencv.hpp>
+#include "screen.hpp"
-#include <nanogui/opengl.h>
-#include <nanogui/glutil.h>
-#include <nanogui/screen.h>
-#include <nanogui/window.h>
-#include <nanogui/layout.h>
-#include <nanogui/imageview.h>
-#include <nanogui/combobox.h>
-#include <nanogui/label.h>
-#include <nanogui/toolbutton.h>
-
-#include "ctrl_window.hpp"
-#include "src_window.hpp"
-
-using std::string;
-using ftl::rgbd::Source;
-
-using ftl::rgbd::isValidDepth;
-
-/*struct SourceViews {
- ftl::rgbd::RGBDSource *source;
- GLTexture texture;
- nanogui::ImageView *view;
-};*/
-
-class StatisticsImage {
-private:
- std::vector<float> data_;
- cv::Size size_;
- int n_;
-
-public:
- StatisticsImage(cv::Size size) {
- size_ = size;
- data_ = std::vector<float>(size.width * size.height * 2, 0.0f);
- n_ = 0;
- }
-
- void update(const cv::Mat &in);
- void getStdDev(cv::Mat &out);
- void getMean(cv::Mat &out);
-};
-
-void StatisticsImage::update(const cv::Mat &in) {
- DCHECK(in.type() == CV_32F);
- DCHECK(in.size() == size_);
- // Welford's Method
-
- n_++;
- for (int i = 0; i < size_.width * size_.height; i++) {
- float x = ((float*) in.data)[i];
- if (!isValidDepth(x)) { continue; } // invalid value
- float &m = data_[2*i];
- float &S = data_[2*i+1];
- float m_prev = m;
- m = m + (x - m) / n_;
- S = S + (x - m) * (x - m_prev);
- }
-}
-
-void StatisticsImage::getStdDev(cv::Mat &in) {
- in = cv::Mat(size_, CV_32F, 0.0f);
-
- for (int i = 0; i < size_.width * size_.height; i++) {
- float &m = data_[2*i];
- float &S = data_[2*i+1];
- ((float*) in.data)[i] = sqrt(S / n_);
- }
-}
-
-void StatisticsImage::getMean(cv::Mat &in) {
- in = cv::Mat(size_, CV_32F, 0.0f);
-
- for (int i = 0; i < size_.width * size_.height; i++) {
- ((float*) in.data)[i] = data_[2*i];
- }
-}
-
-class StatisticsImageNSamples {
-private:
- std::vector<cv::Mat> samples_;
- cv::Size size_;
- int i_;
- int n_;
-
-public:
- StatisticsImageNSamples(cv::Size size, int n) {
- size_ = size;
- samples_ = std::vector<cv::Mat>(n);
- i_ = 0;
- n_ = n;
- }
-
- void update(const cv::Mat &in);
- void getStdDev(cv::Mat &out);
- void getVariance(cv::Mat &out);
- void getMean(cv::Mat &out);
-};
-
-void StatisticsImageNSamples::update(const cv::Mat &in) {
- DCHECK(in.type() == CV_32F);
- DCHECK(in.size() == size_);
-
- i_ = (i_ + 1) % n_;
- in.copyTo(samples_[i_]);
-}
-
-void StatisticsImageNSamples::getStdDev(cv::Mat &out) {
- cv::Mat var;
- getVariance(var);
- cv::sqrt(var, out);
-}
-
-void StatisticsImageNSamples::getVariance(cv::Mat &out) {
- // Welford's Method
- cv::Mat mat_m(size_, CV_32F, 0.0f);
- cv::Mat mat_S(size_, CV_32F, 0.0f);
-
- float n = 0.0f;
- for (int i_sample = (i_ + 1) % n_; i_sample != i_; i_sample = (i_sample + 1) % n_) {
- if (samples_[i_sample].size() != size_) continue;
- n += 1.0f;
- for (int i = 0; i < size_.width * size_.height; i++) {
- float &x = ((float*) samples_[i_sample].data)[i];
- float &m = ((float*) mat_m.data)[i];
- float &S = ((float*) mat_S.data)[i];
- float m_prev = m;
-
- if (!isValidDepth(x)) continue;
-
- m = m + (x - m) / n;
- S = S + (x - m) * (x - m_prev);
- }
- }
-
- mat_S.copyTo(out);
-}
-
-void StatisticsImageNSamples::getMean(cv::Mat &in) {}
-
-namespace {
- constexpr char const *const defaultImageViewVertexShader =
- R"(#version 330
- uniform vec2 scaleFactor;
- uniform vec2 position;
- in vec2 vertex;
- out vec2 uv;
- void main() {
- uv = vertex;
- vec2 scaledVertex = (vertex * scaleFactor) + position;
- gl_Position = vec4(2.0*scaledVertex.x - 1.0,
- 1.0 - 2.0*scaledVertex.y,
- 0.0, 1.0);
- })";
-
- constexpr char const *const defaultImageViewFragmentShader =
- R"(#version 330
- uniform sampler2D image;
- out vec4 color;
- in vec2 uv;
- void main() {
- color = texture(image, uv);
- })";
-
-
- class GLTexture {
- public:
- GLTexture() {
- glGenTextures(1, &glid_);
- glBindTexture(GL_TEXTURE_2D, glid_);
- cv::Mat m(cv::Size(100,100), CV_8UC3);
- glTexImage2D(GL_TEXTURE_2D, 0, GL_RGB8, m.cols, m.rows, 0, GL_RGB, GL_UNSIGNED_BYTE, m.data);
- glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
- glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
- glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_NEAREST);
- glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR);
- }
- ~GLTexture() {
- glDeleteTextures(1, &glid_);
- }
-
- void update(cv::Mat &m) {
- if (m.rows == 0) return;
- glBindTexture(GL_TEXTURE_2D, glid_);
- // TODO Allow for other formats
- glTexImage2D(GL_TEXTURE_2D, 0, GL_RGB8, m.cols, m.rows, 0, GL_BGR, GL_UNSIGNED_BYTE, m.data);
- auto err = glGetError();
- if (err != 0) LOG(ERROR) << "OpenGL Texture error: " << err;
- }
-
- unsigned int texture() const { return glid_; }
-
- private:
- unsigned int glid_;
- };
-
- template<class T>
- Eigen::Matrix<T,4,4> lookAt
- (
- Eigen::Matrix<T,3,1> const & eye,
- Eigen::Matrix<T,3,1> const & center,
- Eigen::Matrix<T,3,1> const & up
- )
- {
- typedef Eigen::Matrix<T,4,4> Matrix4;
- typedef Eigen::Matrix<T,3,1> Vector3;
-
- Vector3 f = (center - eye).normalized();
- Vector3 u = up.normalized();
- Vector3 s = f.cross(u).normalized();
- u = s.cross(f);
-
- Matrix4 res;
- res << s.x(),s.y(),s.z(),-s.dot(eye),
- u.x(),u.y(),u.z(),-u.dot(eye),
- -f.x(),-f.y(),-f.z(),f.dot(eye),
- 0,0,0,1;
-
- return res;
- }
-}
-
-
-static Eigen::Affine3f create_rotation_matrix(float ax, float ay, float az) {
- Eigen::Affine3f rx =
- Eigen::Affine3f(Eigen::AngleAxisf(ax, Eigen::Vector3f(1, 0, 0)));
- Eigen::Affine3f ry =
- Eigen::Affine3f(Eigen::AngleAxisf(ay, Eigen::Vector3f(0, 1, 0)));
- Eigen::Affine3f rz =
- Eigen::Affine3f(Eigen::AngleAxisf(az, Eigen::Vector3f(0, 0, 1)));
- return ry * rz * rx;
-}
-
-
-class FTLApplication : public nanogui::Screen {
- public:
- explicit FTLApplication(ftl::Configurable *root, ftl::net::Universe *net, ftl::ctrl::Master *controller) : nanogui::Screen(Eigen::Vector2i(1024, 768), "FT-Lab Remote Presence") {
- using namespace nanogui;
- net_ = net;
- ctrl_ = controller;
-
- status_ = "FT-Lab Remote Presence System";
-
- setSize(Vector2i(1280,720));
-
- Theme *toolbuttheme = new Theme(*theme());
- toolbuttheme->mBorderDark = nanogui::Color(0,0);
- toolbuttheme->mBorderLight = nanogui::Color(0,0);
- toolbuttheme->mButtonGradientBotFocused = nanogui::Color(60,255);
- toolbuttheme->mButtonGradientBotUnfocused = nanogui::Color(0,0);
- toolbuttheme->mButtonGradientTopFocused = nanogui::Color(60,255);
- toolbuttheme->mButtonGradientTopUnfocused = nanogui::Color(0,0);
- toolbuttheme->mButtonGradientTopPushed = nanogui::Color(60,180);
- toolbuttheme->mButtonGradientBotPushed = nanogui::Color(60,180);
-
- Theme *windowtheme = new Theme(*theme());
- windowtheme->mWindowFillFocused = nanogui::Color(220, 200);
- windowtheme->mWindowFillUnfocused = nanogui::Color(220, 200);
- windowtheme->mWindowHeaderGradientBot = nanogui::Color(60,230);
- windowtheme->mWindowHeaderGradientTop = nanogui::Color(60,230);
- windowtheme->mTextColor = nanogui::Color(20,255);
- windowtheme->mWindowCornerRadius = 2;
- windowtheme->mButtonGradientBotFocused = nanogui::Color(210,255);
- windowtheme->mButtonGradientBotUnfocused = nanogui::Color(190,255);
- windowtheme->mButtonGradientTopFocused = nanogui::Color(230,255);
- windowtheme->mButtonGradientTopUnfocused = nanogui::Color(230,255);
- windowtheme->mButtonGradientTopPushed = nanogui::Color(170,255);
- windowtheme->mButtonGradientBotPushed = nanogui::Color(210,255);
- windowtheme->mBorderDark = nanogui::Color(150,255);
- windowtheme->mBorderMedium = nanogui::Color(165,255);
- windowtheme->mBorderLight = nanogui::Color(230,255);
- windowtheme->mButtonFontSize = 16;
- windowtheme->mTextColorShadow = nanogui::Color(0,0);
- windowtheme->mWindowTitleUnfocused = windowtheme->mWindowTitleFocused;
- windowtheme->mWindowTitleFocused = nanogui::Color(240,255);
- windowtheme->mIconScale = 0.85f;
-
- auto toolbar = new Window(this, "");
- toolbar->setPosition(Vector2i(0,0));
- toolbar->setFixedWidth(50);
- toolbar->setFixedHeight(height());
- //toolbar->setLayout(new BoxLayout(Orientation::Vertical,
- // Alignment::Middle, 0, 10));
-
- setResizeCallback([this,toolbar](Vector2i s) {
- toolbar->setFixedHeight(s[1]);
- });
-
- auto innertool = new Widget(toolbar);
- innertool->setLayout(new BoxLayout(Orientation::Vertical,
- Alignment::Middle, 0, 10));
- innertool->setPosition(Vector2i(5,10));
-
- // Padding widget
- //auto w = new Widget(innertool);
- //w->setHeight(10);
-
- auto button = new ToolButton(innertool, ENTYPO_ICON_HOME);
- button->setIconExtraScale(1.5f);
- button->setTheme(toolbuttheme);
- button->setTooltip("Home");
- button->setFixedSize(Vector2i(40,40));
- button->setCallback([this]() {
- //swindow_->setVisible(true);
- });
-
- button = new ToolButton(innertool, ENTYPO_ICON_EDIT);
- button->setIconExtraScale(1.5f);
- button->setTheme(toolbuttheme);
- button->setTooltip("Edit Scene");
- button->setFixedSize(Vector2i(40,40));
- button->setCallback([this]() {
- //swindow_->setVisible(true);
- });
-
- button = new ToolButton(innertool, ENTYPO_ICON_CAMERA);
- button->setIconExtraScale(1.5f);
- button->setTheme(toolbuttheme);
- button->setTooltip("Camera Sources");
- button->setFixedSize(Vector2i(40,40));
- button->setCallback([this]() {
- swindow_->setVisible(true);
- });
-
- button = new ToolButton(innertool, ENTYPO_ICON_SIGNAL);
- button->setIconExtraScale(1.5f);
- button->setTheme(toolbuttheme);
- button->setTooltip("Connections");
- button->setFixedSize(Vector2i(40,40));
- button->setCallback([this]() {
- cwindow_->setVisible(true);
- });
-
- button = new ToolButton(toolbar, ENTYPO_ICON_COG);
- button->setIconExtraScale(1.5f);
- button->setTheme(toolbuttheme);
- button->setTooltip("Settings");
- button->setFixedSize(Vector2i(40,40));
- button->setPosition(Vector2i(5,height()-50));
-
- //configwindow_ = new ConfigWindow(parent, ctrl_);
- cwindow_ = new ftl::gui::ControlWindow(this, controller);
- swindow_ = new ftl::gui::SourceWindow(this, controller);
-
- cwindow_->setPosition(Eigen::Vector2i(80, 20));
- swindow_->setPosition(Eigen::Vector2i(80, 400));
- cwindow_->setVisible(false);
- swindow_->setVisible(false);
- cwindow_->setTheme(windowtheme);
- swindow_->setTheme(windowtheme);
-
- //src_ = nullptr;
- eye_ = Eigen::Vector3f(0.0f, 0.0f, 0.0f);
- neye_ = Eigen::Vector4f(0.0f, 0.0f, 0.0f, 0.0f);
- orientation_ = Eigen::Vector3f(0.0f, 0.0f, 0.0f);
- up_ = Eigen::Vector3f(0,1.0f,0);
- //lookPoint_ = Eigen::Vector3f(0.0f,0.0f,-4.0f);
- lerpSpeed_ = 0.999f;
- depth_ = false;
- ftime_ = (float)glfwGetTime();
-
- mShader.init("RGBDShader", defaultImageViewVertexShader,
- defaultImageViewFragmentShader);
-
- MatrixXu indices(3, 2);
- indices.col(0) << 0, 1, 2;
- indices.col(1) << 2, 3, 1;
-
- MatrixXf vertices(2, 4);
- vertices.col(0) << 0, 0;
- vertices.col(1) << 1, 0;
- vertices.col(2) << 0, 1;
- vertices.col(3) << 1, 1;
-
- mShader.bind();
- mShader.uploadIndices(indices);
- mShader.uploadAttrib("vertex", vertices);
-
- setVisible(true);
- performLayout();
- }
-
- ~FTLApplication() {
- mShader.free();
- }
-
- bool mouseMotionEvent(const Eigen::Vector2i &p, const Eigen::Vector2i &rel, int button, int modifiers) {
- if (Screen::mouseMotionEvent(p, rel, button, modifiers)) {
- return true;
- } else {
- if (button == 1) {
- orientation_[0] += ((float)rel[1] * 0.2f * delta_);
- //orientation_[2] += std::cos(orientation_[1])*((float)rel[1] * 0.2f * delta_);
- orientation_[1] -= (float)rel[0] * 0.2f * delta_;
- }
- }
- }
-
- bool mouseButtonEvent(const nanogui::Vector2i &p, int button, bool down, int modifiers) {
- if (Screen::mouseButtonEvent(p, button, down, modifiers)) {
- return true;
- } else {
- auto src_ = swindow_->getSource();
- if (src_ && src_->isReady() && down) {
- Eigen::Vector2f screenSize = size().cast<float>();
- auto mScale = (screenSize.cwiseQuotient(imageSize).minCoeff());
- Eigen::Vector2f scaleFactor = mScale * imageSize.cwiseQuotient(screenSize);
- Eigen::Vector2f positionInScreen(0.0f, 0.0f);
- auto mOffset = (screenSize - (screenSize.cwiseProduct(scaleFactor))) / 2;
- Eigen::Vector2f positionAfterOffset = positionInScreen + mOffset;
-
- float sx = ((float)p[0] - positionAfterOffset[0]) / mScale;
- float sy = ((float)p[1] - positionAfterOffset[1]) / mScale;
-
- Eigen::Vector4f camPos;
-
- try {
- camPos = src_->point(sx,sy).cast<float>();
- } catch(...) {
- return true;
- }
-
- camPos *= -1.0f;
- Eigen::Vector4f worldPos = src_->getPose().cast<float>() * camPos;
- //lookPoint_ = Eigen::Vector3f(worldPos[0],worldPos[1],worldPos[2]);
- LOG(INFO) << "Depth at click = " << -camPos[2];
- return true;
- }
- return false;
- }
- }
-
- bool keyboardEvent(int key, int scancode, int action, int modifiers) {
- using namespace Eigen;
- if (Screen::keyboardEvent(key, scancode, action, modifiers)) {
- return true;
- } else {
- LOG(INFO) << "Key press " << key << " - " << action << " - " << modifiers;
- if (key == 263 || key == 262) {
- float mag = (modifiers & 0x1) ? 0.01f : 0.1f;
- float scalar = (key == 263) ? -mag : mag;
- Eigen::Affine3f r = create_rotation_matrix(orientation_[0], orientation_[1], orientation_[2]);
- neye_ += r.matrix()*Vector4f(scalar,0.0,0.0,1.0);
- return true;
- } else if (key == 264 || key == 265) {
- float mag = (modifiers & 0x1) ? 0.01f : 0.1f;
- float scalar = (key == 264) ? -mag : mag;
- Eigen::Affine3f r = create_rotation_matrix(orientation_[0], orientation_[1], orientation_[2]);
- neye_ += r.matrix()*Vector4f(0.0,0.0,scalar,1.0);
- return true;
- } else if (key == 266 || key == 267) {
- float mag = (modifiers & 0x1) ? 0.01f : 0.1f;
- float scalar = (key == 266) ? -mag : mag;
- Eigen::Affine3f r = create_rotation_matrix(orientation_[0], orientation_[1], orientation_[2]);
- neye_ += r.matrix()*Vector4f(0.0,scalar,0.0,1.0);
- return true;
- } else if (action == 1 && key == 'H') {
- swindow_->setVisible(false);
- cwindow_->setVisible(false);
- } else if (action == 1 && key == 32) {
- ctrl_->pause();
- }
- return false;
- }
- }
-
- StatisticsImageNSamples *stats_ = nullptr;
-
- virtual void draw(NVGcontext *ctx) {
- using namespace Eigen;
-
- auto src_ = swindow_->getSource();
- imageSize = {0, 0};
-
- float now = (float)glfwGetTime();
- delta_ = now - ftime_;
- ftime_ = now;
-
- if (src_) {
- cv::Mat rgb, depth;
-
- // Lerp the Eye
- eye_[0] += (neye_[0] - eye_[0]) * lerpSpeed_ * delta_;
- eye_[1] += (neye_[1] - eye_[1]) * lerpSpeed_ * delta_;
- eye_[2] += (neye_[2] - eye_[2]) * lerpSpeed_ * delta_;
-
- Eigen::Affine3f r = create_rotation_matrix(orientation_[0], orientation_[1], orientation_[2]);
- Eigen::Translation3f trans(eye_);
- Eigen::Affine3f t(trans);
- Eigen::Matrix4f viewPose = (t * r).matrix();
-
- src_->setPose(viewPose.cast<double>());
- src_->grab();
- src_->getFrames(rgb, depth);
-
- if (!stats_ && depth.rows > 0) {
- stats_ = new StatisticsImageNSamples(depth.size(), 25);
- }
-
- if (stats_ && depth.rows > 0) { stats_->update(depth); }
-
- cv::Mat tmp;
-
- using ftl::gui::SourceWindow;
- switch(swindow_->getMode()) {
- case SourceWindow::Mode::depth:
- if (depth.rows == 0) { break; }
- imageSize = Vector2f(depth.cols,depth.rows);
- depth.convertTo(tmp, CV_8U, 255.0f / 5.0f);
- tmp = 255 - tmp;
- applyColorMap(tmp, tmp, cv::COLORMAP_JET);
- texture_.update(tmp);
- mImageID = texture_.texture();
- break;
-
- case SourceWindow::Mode::stddev:
- if (depth.rows == 0) { break; }
- imageSize = Vector2f(depth.cols, depth.rows);
- stats_->getStdDev(tmp);
- tmp.convertTo(tmp, CV_8U, 50.0);
- applyColorMap(tmp, tmp, cv::COLORMAP_HOT);
- texture_.update(tmp);
- mImageID = texture_.texture();
- break;
-
- default:
- if (rgb.rows == 0) { break; }
- imageSize = Vector2f(rgb.cols,rgb.rows);
- texture_.update(rgb);
- mImageID = texture_.texture();
- }
- }
-
- Vector2f screenSize = size().cast<float>();
-
- if (imageSize[0] > 0) {
- auto mScale = (screenSize.cwiseQuotient(imageSize).minCoeff());
- Vector2f scaleFactor = mScale * imageSize.cwiseQuotient(screenSize);
- Vector2f positionInScreen(0.0f, 0.0f);
- auto mOffset = (screenSize - (screenSize.cwiseProduct(scaleFactor))) / 2;
- Vector2f positionAfterOffset = positionInScreen + mOffset;
- Vector2f imagePosition = positionAfterOffset.cwiseQuotient(screenSize);
- //glEnable(GL_SCISSOR_TEST);
- //float r = screen->pixelRatio();
- /* glScissor(positionInScreen.x() * r,
- (screenSize.y() - positionInScreen.y() - size().y()) * r,
- size().x() * r, size().y() * r);*/
- mShader.bind();
- glActiveTexture(GL_TEXTURE0);
- glBindTexture(GL_TEXTURE_2D, mImageID);
- mShader.setUniform("image", 0);
- mShader.setUniform("scaleFactor", scaleFactor);
- mShader.setUniform("position", imagePosition);
- mShader.drawIndexed(GL_TRIANGLES, 0, 2);
- //glDisable(GL_SCISSOR_TEST);
- }
-
- nvgTextAlign(ctx, NVG_ALIGN_RIGHT);
- nvgText(ctx, screenSize[0]-10, screenSize[1]-20, status_.c_str(), NULL);
-
- /* Draw the user interface */
- screen()->performLayout(ctx);
- Screen::draw(ctx);
- }
-
- private:
- ftl::gui::SourceWindow *swindow_;
- ftl::gui::ControlWindow *cwindow_;
- //std::vector<SourceViews> sources_;
- ftl::net::Universe *net_;
- nanogui::GLShader mShader;
- GLuint mImageID;
- //Source *src_;
- GLTexture texture_;
- Eigen::Vector3f eye_;
- Eigen::Vector4f neye_;
- Eigen::Vector3f orientation_;
- Eigen::Vector3f up_;
- //Eigen::Vector3f lookPoint_;
- float lerpSpeed_;
- bool depth_;
- float ftime_;
- float delta_;
- Eigen::Vector2f imageSize;
- ftl::ctrl::Master *ctrl_;
- std::string status_;
-};
int main(int argc, char **argv) {
auto root = ftl::configure(argc, argv, "gui_default");
@@ -620,7 +34,7 @@ int main(int argc, char **argv) {
nanogui::init();
/* scoped variables */ {
- nanogui::ref<FTLApplication> app = new FTLApplication(root, net, controller);
+ nanogui::ref<ftl::gui::Screen> app = new ftl::gui::Screen(root, net, controller);
app->drawAll();
app->setVisible(true);
nanogui::mainloop();
diff --git a/applications/gui/src/media_panel.cpp b/applications/gui/src/media_panel.cpp
new file mode 100644
index 000000000..757259aef
--- /dev/null
+++ b/applications/gui/src/media_panel.cpp
@@ -0,0 +1,139 @@
+#include "media_panel.hpp"
+#include "screen.hpp"
+#include "camera.hpp"
+
+#include <nanogui/layout.h>
+#include <nanogui/button.h>
+#include <nanogui/popupbutton.h>
+#include <nanogui/entypo.h>
+
+#ifdef HAVE_LIBARCHIVE
+#include "ftl/rgbd/snapshot.hpp"
+#endif
+
+using ftl::gui::MediaPanel;
+
+MediaPanel::MediaPanel(ftl::gui::Screen *screen) : nanogui::Window(screen, ""), screen_(screen) {
+ using namespace nanogui;
+
+ paused_ = false;
+
+ setLayout(new BoxLayout(Orientation::Horizontal,
+ Alignment::Middle, 5, 10));
+
+ auto size = Vector2i(400, 60);
+ //setFixedSize(size);
+ setPosition(Vector2i(screen->width() / 2 - size[0]/2, screen->height() - 30 - size[1]));
+
+ Theme *mediatheme = new Theme(*theme());
+ mediatheme->mIconScale = 1.2f;
+ mediatheme->mWindowDropShadowSize = 0;
+ mediatheme->mWindowFillFocused = nanogui::Color(45, 150);
+ mediatheme->mWindowFillUnfocused = nanogui::Color(45, 80);
+ mediatheme->mButtonGradientTopUnfocused = nanogui::Color(0,0);
+ mediatheme->mButtonGradientBotUnfocused = nanogui::Color(0,0);
+ mediatheme->mButtonGradientTopFocused = nanogui::Color(80,230);
+ mediatheme->mButtonGradientBotFocused = nanogui::Color(80,230);
+ mediatheme->mIconColor = nanogui::Color(255,255);
+ mediatheme->mBorderDark = nanogui::Color(0,0);
+ mediatheme->mBorderMedium = nanogui::Color(0,0);
+ mediatheme->mBorderLight = nanogui::Color(0,0);
+ mediatheme->mDropShadow = nanogui::Color(0,0);
+ mediatheme->mButtonFontSize = 30;
+
+ setTheme(mediatheme);
+
+ auto button = new Button(this, "", ENTYPO_ICON_EDIT);
+ button->setTooltip("Edit camera properties");
+ button->setCallback([this]() {
+ auto *cam = screen_->activeCamera();
+ if (cam) cam->showPoseWindow();
+ });
+
+ button = new Button(this, "", ENTYPO_ICON_CONTROLLER_STOP);
+ button->setCallback([this]() {
+ screen_->setActiveCamera(nullptr);
+ });
+
+ button = new Button(this, "", ENTYPO_ICON_CONTROLLER_PAUS);
+ button->setCallback([this,button]() {
+ paused_ = !paused_;
+ screen_->control()->pause();
+ if (paused_) {
+ button->setIcon(ENTYPO_ICON_CONTROLLER_PLAY);
+ } else {
+ button->setIcon(ENTYPO_ICON_CONTROLLER_PAUS);
+ }
+ });
+
+ //button = new Button(this, "", ENTYPO_ICON_CONTROLLER_RECORD);
+
+ #ifdef HAVE_LIBARCHIVE
+ auto button_snapshot = new Button(this, "", ENTYPO_ICON_IMAGES);
+ button_snapshot->setTooltip("Screen capture");
+ button_snapshot->setCallback([this] {
+ ftl::gui::Camera *cam = screen_->activeCamera();
+ if (!cam) return;
+
+ try {
+ char timestamp[18];
+ std::time_t t=std::time(NULL);
+ std::strftime(timestamp, sizeof(timestamp), "%F-%H%M%S", std::localtime(&t));
+ auto writer = ftl::rgbd::SnapshotWriter(std::string(timestamp) + ".tar.gz");
+ cv::Mat rgb, depth;
+ cam->source()->getFrames(rgb, depth);
+ if (!writer.addCameraRGBD(
+ "0", // TODO
+ rgb,
+ depth,
+ cam->source()->getPose(),
+ cam->source()->parameters()
+ )) {
+ LOG(ERROR) << "Snapshot failed";
+ }
+ }
+ catch(std::runtime_error) {
+ LOG(ERROR) << "Snapshot failed (file error)";
+ }
+ });
+#endif
+
+ auto popbutton = new PopupButton(this, "", ENTYPO_ICON_LAYERS);
+ popbutton->setSide(Popup::Side::Right);
+ popbutton->setChevronIcon(ENTYPO_ICON_CHEVRON_SMALL_RIGHT);
+ Popup *popup = popbutton->popup();
+ popup->setLayout(new GroupLayout());
+ popup->setAnchorHeight(100);
+
+ button = new Button(popup, "Left");
+ button->setFlags(Button::RadioButton);
+ button->setPushed(true);
+ button->setCallback([this]() {
+ ftl::gui::Camera *cam = screen_->activeCamera();
+ if (cam) {
+ cam->setChannel(ftl::rgbd::kChanLeft);
+ }
+ });
+
+ button = new Button(popup, "Depth");
+ button->setFlags(Button::RadioButton);
+ button->setCallback([this]() {
+ ftl::gui::Camera *cam = screen_->activeCamera();
+ if (cam) {
+ cam->setChannel(ftl::rgbd::kChanDepth);
+ }
+ });
+
+ button = new Button(popup, "Deviation");
+ button->setFlags(Button::RadioButton);
+ button->setCallback([this]() {
+ ftl::gui::Camera *cam = screen_->activeCamera();
+ if (cam) {
+ cam->setChannel(ftl::rgbd::kChanDeviation);
+ }
+ });
+}
+
+MediaPanel::~MediaPanel() {
+
+}
diff --git a/applications/gui/src/media_panel.hpp b/applications/gui/src/media_panel.hpp
new file mode 100644
index 000000000..0f17bd340
--- /dev/null
+++ b/applications/gui/src/media_panel.hpp
@@ -0,0 +1,24 @@
+#ifndef _FTL_GUI_MEDIAPANEL_HPP_
+#define _FTL_GUI_MEDIAPANEL_HPP_
+
+#include <nanogui/window.h>
+
+namespace ftl {
+namespace gui {
+
+class Screen;
+
+class MediaPanel : public nanogui::Window {
+ public:
+ MediaPanel(ftl::gui::Screen *);
+ ~MediaPanel();
+
+ private:
+ ftl::gui::Screen *screen_;
+ bool paused_;
+};
+
+}
+}
+
+#endif // _FTL_GUI_MEDIAPANEL_HPP_
diff --git a/applications/gui/src/pose_window.cpp b/applications/gui/src/pose_window.cpp
index 4c3df6e9b..d1afdb739 100644
--- a/applications/gui/src/pose_window.cpp
+++ b/applications/gui/src/pose_window.cpp
@@ -1,4 +1,6 @@
#include "pose_window.hpp"
+#include "screen.hpp"
+#include "camera.hpp"
#include <nanogui/combobox.h>
#include <nanogui/label.h>
@@ -6,10 +8,21 @@
#include <nanogui/button.h>
using ftl::gui::PoseWindow;
+using ftl::gui::Screen;
using std::string;
-PoseWindow::PoseWindow(nanogui::Widget *parent, ftl::ctrl::Master *ctrl, const std::string &src)
- : nanogui::Window(parent, "Pose Adjust"), ctrl_(ctrl), src_(src) {
+static Eigen::Affine3d create_rotation_matrix(float ax, float ay, float az) {
+ Eigen::Affine3d rx =
+ Eigen::Affine3d(Eigen::AngleAxisd(ax, Eigen::Vector3d(1, 0, 0)));
+ Eigen::Affine3d ry =
+ Eigen::Affine3d(Eigen::AngleAxisd(ay, Eigen::Vector3d(0, 1, 0)));
+ Eigen::Affine3d rz =
+ Eigen::Affine3d(Eigen::AngleAxisd(az, Eigen::Vector3d(0, 0, 1)));
+ return ry * rz * rx;
+}
+
+PoseWindow::PoseWindow(ftl::gui::Screen *screen, const std::string &src)
+ : nanogui::Window(screen, "Pose Adjust"), screen_(screen), src_(src) {
using namespace nanogui;
//setLayout(new nanogui::GroupLayout());
@@ -19,7 +32,7 @@ PoseWindow::PoseWindow(nanogui::Widget *parent, ftl::ctrl::Master *ctrl, const s
pose_param_ = kPoseTranslation;
pose_precision_ = 0.1;
- pose_ = ctrl_->getPose(src_);
+ pose_ = screen_->control()->getPose(src_);
//Widget *tools = new Widget(this);
// tools->setLayout(new BoxLayout(Orientation::Horizontal,
@@ -29,16 +42,16 @@ PoseWindow::PoseWindow(nanogui::Widget *parent, ftl::ctrl::Master *ctrl, const s
grouping->setLayout(new GroupLayout());
new Label(grouping, "Select source","sans-bold");
- available_ = ctrl->getNet()->findAll<string>("list_streams");
+ available_ = screen_->net()->findAll<string>("list_streams");
auto select = new ComboBox(grouping, available_);
select->setSelectedIndex(std::distance(available_.begin(), std::find(available_.begin(), available_.end(), src_)));
select->setCallback([this,select](int ix) {
src_ = available_[ix];
- pose_ = ctrl_->getPose(src_);
+ pose_ = screen_->control()->getPose(src_);
});
- ctrl->getNet()->onConnect([this,select](ftl::net::Peer *p) {
- available_ = ctrl_->getNet()->findAll<string>("list_streams");
+ screen_->net()->onConnect([this,select](ftl::net::Peer *p) {
+ available_ = screen_->control()->getNet()->findAll<string>("list_streams");
select->setItems(available_);
});
@@ -52,7 +65,9 @@ PoseWindow::PoseWindow(nanogui::Widget *parent, ftl::ctrl::Master *ctrl, const s
button_opt->setTooltip("Virtual view to this pose");
//button_opt->setFlags(Button::ToggleButton);
//button_opt->setPushed(false);
- button_opt->setCallback([this]() { });
+ button_opt->setCallback([this]() {
+ screen_->activeCamera()->setPose(pose_);
+ });
button_opt = new Button(tools, "", ENTYPO_ICON_LINK);
button_opt->setTooltip("Link virtual current pose to this pose");
@@ -68,11 +83,11 @@ PoseWindow::PoseWindow(nanogui::Widget *parent, ftl::ctrl::Master *ctrl, const s
button_rgb->setTooltip("Adjust camera location");
button_rgb->setFlags(Button::RadioButton);
button_rgb->setPushed(true);
- button_rgb->setChangeCallback([this](bool state) { pose_param_ = kPoseTranslation; });
+ button_rgb->setCallback([this]() { pose_param_ = kPoseTranslation; });
auto button_depth = new Button(tools, "Rotation");
button_depth->setFlags(Button::RadioButton);
- button_depth->setChangeCallback([this](bool state) { pose_param_ = kPoseRotation; });
+ button_depth->setCallback([this]() { pose_param_ = kPoseRotation; });
auto button_stddev = new Button(tools, "Raw");
button_stddev->setTooltip("Edit the numbers directly");
@@ -110,49 +125,80 @@ PoseWindow::PoseWindow(nanogui::Widget *parent, ftl::ctrl::Master *ctrl, const s
button = new Button(tools, "Up");
button->setCallback([this]() {
- Eigen::Affine3d transform(Eigen::Translation3d(0.0,-pose_precision_,0.0));
- Eigen::Matrix4d matrix = transform.matrix();
- pose_ *= matrix;
- ctrl_->setPose(src_, pose_);
+ if (pose_param_ == kPoseTranslation) {
+ Eigen::Affine3d transform(Eigen::Translation3d(0.0,-pose_precision_,0.0));
+ Eigen::Matrix4d matrix = transform.matrix();
+ pose_ *= matrix;
+ } else if (pose_param_ == kPoseRotation) {
+ Eigen::Affine3d r = create_rotation_matrix(pose_precision_, 0.0, 0.0);
+ pose_ = r.matrix() * pose_;
+ }
+
+ screen_->control()->setPose(src_, pose_);
});
button = new Button(tools, "", ENTYPO_ICON_CHEVRON_UP);
button->setCallback([this]() {
- Eigen::Affine3d transform(Eigen::Translation3d(0.0,0.0,-pose_precision_));
- Eigen::Matrix4d matrix = transform.matrix();
- pose_ *= matrix;
- ctrl_->setPose(src_, pose_);
+ if (pose_param_ == kPoseTranslation) {
+ Eigen::Affine3d transform(Eigen::Translation3d(0.0,0.0,-pose_precision_));
+ Eigen::Matrix4d matrix = transform.matrix();
+ pose_ *= matrix;
+ } else if (pose_param_ == kPoseRotation) {
+ Eigen::Affine3d r = create_rotation_matrix(0.0, 0.0, pose_precision_);
+ pose_ = r.matrix() * pose_;
+ }
+ screen_->control()->setPose(src_, pose_);
});
button = new Button(tools, "Down");
button->setCallback([this]() {
- Eigen::Affine3d transform(Eigen::Translation3d(0.0,pose_precision_,0.0));
- Eigen::Matrix4d matrix = transform.matrix();
- pose_ *= matrix;
- ctrl_->setPose(src_, pose_);
+ if (pose_param_ == kPoseTranslation) {
+ Eigen::Affine3d transform(Eigen::Translation3d(0.0,pose_precision_,0.0));
+ Eigen::Matrix4d matrix = transform.matrix();
+ pose_ *= matrix;
+ } else if (pose_param_ == kPoseRotation) {
+ Eigen::Affine3d r = create_rotation_matrix(-pose_precision_, 0.0, 0.0);
+ pose_ = r.matrix() * pose_;
+ }
+ screen_->control()->setPose(src_, pose_);
});
button = new Button(tools, "", ENTYPO_ICON_CHEVRON_LEFT);
button->setCallback([this]() {
- Eigen::Affine3d transform(Eigen::Translation3d(-pose_precision_,0.0,0.0));
- Eigen::Matrix4d matrix = transform.matrix();
- pose_ *= matrix;
- ctrl_->setPose(src_, pose_);
+ if (pose_param_ == kPoseTranslation) {
+ Eigen::Affine3d transform(Eigen::Translation3d(-pose_precision_,0.0,0.0));
+ Eigen::Matrix4d matrix = transform.matrix();
+ pose_ *= matrix;
+ } else if (pose_param_ == kPoseRotation) {
+ Eigen::Affine3d r = create_rotation_matrix(0.0, pose_precision_, 0.0);
+ pose_ = r.matrix() * pose_;
+ }
+ screen_->control()->setPose(src_, pose_);
});
new Widget(tools);
button = new Button(tools, "", ENTYPO_ICON_CHEVRON_RIGHT);
button->setCallback([this]() {
- Eigen::Affine3d transform(Eigen::Translation3d(pose_precision_,0.0,0.0));
- Eigen::Matrix4d matrix = transform.matrix();
- pose_ *= matrix;
- ctrl_->setPose(src_, pose_);
+ if (pose_param_ == kPoseTranslation) {
+ Eigen::Affine3d transform(Eigen::Translation3d(pose_precision_,0.0,0.0));
+ Eigen::Matrix4d matrix = transform.matrix();
+ pose_ *= matrix;
+ } else if (pose_param_ == kPoseRotation) {
+ Eigen::Affine3d r = create_rotation_matrix(0.0, -pose_precision_, 0.0);
+ pose_ = r.matrix() * pose_;
+ }
+ screen_->control()->setPose(src_, pose_);
});
new Widget(tools);
button = new Button(tools, "", ENTYPO_ICON_CHEVRON_DOWN);
button->setCallback([this]() {
- Eigen::Affine3d transform(Eigen::Translation3d(0.0,0.0,pose_precision_));
- Eigen::Matrix4d matrix = transform.matrix();
- pose_ *= matrix;
- ctrl_->setPose(src_, pose_);
+ if (pose_param_ == kPoseTranslation) {
+ Eigen::Affine3d transform(Eigen::Translation3d(0.0,0.0,pose_precision_));
+ Eigen::Matrix4d matrix = transform.matrix();
+ pose_ *= matrix;
+ } else if (pose_param_ == kPoseRotation) {
+ Eigen::Affine3d r = create_rotation_matrix(0.0, 0.0, -pose_precision_);
+ pose_ = r.matrix() * pose_;
+ }
+ screen_->control()->setPose(src_, pose_);
});
}
diff --git a/applications/gui/src/pose_window.hpp b/applications/gui/src/pose_window.hpp
index cfe03adff..bbd041414 100644
--- a/applications/gui/src/pose_window.hpp
+++ b/applications/gui/src/pose_window.hpp
@@ -8,16 +8,17 @@
namespace ftl {
namespace gui {
+class Screen;
+
/**
* Manage connected nodes and add new connections.
*/
class PoseWindow : public nanogui::Window {
public:
- PoseWindow(nanogui::Widget *parent, ftl::ctrl::Master *ctrl, const std::string &src);
+ PoseWindow(ftl::gui::Screen *screen, const std::string &src);
~PoseWindow();
private:
- ftl::ctrl::Master *ctrl_;
std::vector<std::string> available_;
std::string src_;
@@ -30,6 +31,8 @@ class PoseWindow : public nanogui::Window {
poseparameter_t pose_param_;
float pose_precision_;
Eigen::Matrix4d pose_;
+ ftl::gui::Screen *screen_;
+ bool poselink_;
};
}
diff --git a/applications/gui/src/screen.cpp b/applications/gui/src/screen.cpp
new file mode 100644
index 000000000..4175e5084
--- /dev/null
+++ b/applications/gui/src/screen.cpp
@@ -0,0 +1,308 @@
+#include "screen.hpp"
+
+#include <nanogui/opengl.h>
+#include <nanogui/glutil.h>
+#include <nanogui/screen.h>
+#include <nanogui/window.h>
+#include <nanogui/layout.h>
+#include <nanogui/imageview.h>
+#include <nanogui/combobox.h>
+#include <nanogui/label.h>
+#include <nanogui/toolbutton.h>
+
+#include <opencv2/opencv.hpp>
+
+#include <loguru.hpp>
+
+#include "ctrl_window.hpp"
+#include "src_window.hpp"
+#include "camera.hpp"
+#include "media_panel.hpp"
+
+using ftl::gui::Screen;
+using ftl::gui::Camera;
+using std::string;
+using ftl::rgbd::Source;
+using ftl::rgbd::isValidDepth;
+
+namespace {
+ constexpr char const *const defaultImageViewVertexShader =
+ R"(#version 330
+ uniform vec2 scaleFactor;
+ uniform vec2 position;
+ in vec2 vertex;
+ out vec2 uv;
+ void main() {
+ uv = vertex;
+ vec2 scaledVertex = (vertex * scaleFactor) + position;
+ gl_Position = vec4(2.0*scaledVertex.x - 1.0,
+ 1.0 - 2.0*scaledVertex.y,
+ 0.0, 1.0);
+ })";
+
+ constexpr char const *const defaultImageViewFragmentShader =
+ R"(#version 330
+ uniform sampler2D image;
+ out vec4 color;
+ in vec2 uv;
+ void main() {
+ color = texture(image, uv);
+ })";
+}
+
+ftl::gui::Screen::Screen(ftl::Configurable *proot, ftl::net::Universe *pnet, ftl::ctrl::Master *controller) : nanogui::Screen(Eigen::Vector2i(1024, 768), "FT-Lab Remote Presence") {
+ using namespace nanogui;
+ net_ = pnet;
+ ctrl_ = controller;
+ root_ = proot;
+ camera_ = nullptr;
+
+ status_ = "FT-Lab Remote Presence System";
+
+ setSize(Vector2i(1280,720));
+
+ Theme *toolbuttheme = new Theme(*theme());
+ toolbuttheme->mBorderDark = nanogui::Color(0,0);
+ toolbuttheme->mBorderLight = nanogui::Color(0,0);
+ toolbuttheme->mButtonGradientBotFocused = nanogui::Color(60,255);
+ toolbuttheme->mButtonGradientBotUnfocused = nanogui::Color(0,0);
+ toolbuttheme->mButtonGradientTopFocused = nanogui::Color(60,255);
+ toolbuttheme->mButtonGradientTopUnfocused = nanogui::Color(0,0);
+ toolbuttheme->mButtonGradientTopPushed = nanogui::Color(60,180);
+ toolbuttheme->mButtonGradientBotPushed = nanogui::Color(60,180);
+
+ windowtheme = new Theme(*theme());
+ windowtheme->mWindowFillFocused = nanogui::Color(220, 200);
+ windowtheme->mWindowFillUnfocused = nanogui::Color(220, 200);
+ windowtheme->mWindowHeaderGradientBot = nanogui::Color(60,230);
+ windowtheme->mWindowHeaderGradientTop = nanogui::Color(60,230);
+ windowtheme->mTextColor = nanogui::Color(20,255);
+ windowtheme->mWindowCornerRadius = 2;
+ windowtheme->mButtonGradientBotFocused = nanogui::Color(210,255);
+ windowtheme->mButtonGradientBotUnfocused = nanogui::Color(190,255);
+ windowtheme->mButtonGradientTopFocused = nanogui::Color(230,255);
+ windowtheme->mButtonGradientTopUnfocused = nanogui::Color(230,255);
+ windowtheme->mButtonGradientTopPushed = nanogui::Color(170,255);
+ windowtheme->mButtonGradientBotPushed = nanogui::Color(210,255);
+ windowtheme->mBorderDark = nanogui::Color(150,255);
+ windowtheme->mBorderMedium = nanogui::Color(165,255);
+ windowtheme->mBorderLight = nanogui::Color(230,255);
+ windowtheme->mButtonFontSize = 16;
+ windowtheme->mTextColorShadow = nanogui::Color(0,0);
+ windowtheme->mWindowTitleUnfocused = windowtheme->mWindowTitleFocused;
+ windowtheme->mWindowTitleFocused = nanogui::Color(240,255);
+ windowtheme->mIconScale = 0.85f;
+
+ auto toolbar = new Window(this, "");
+ toolbar->setPosition(Vector2i(0,0));
+ toolbar->setFixedWidth(50);
+ toolbar->setFixedHeight(height());
+ //toolbar->setLayout(new BoxLayout(Orientation::Vertical,
+ // Alignment::Middle, 0, 10));
+
+ setResizeCallback([this,toolbar](Vector2i s) {
+ toolbar->setFixedHeight(s[1]);
+ mwindow_->setPosition(Vector2i(s[0] / 2 - mwindow_->width()/2, s[1] - 30 - mwindow_->height()));
+ });
+
+ auto innertool = new Widget(toolbar);
+ innertool->setLayout(new BoxLayout(Orientation::Vertical,
+ Alignment::Middle, 0, 10));
+ innertool->setPosition(Vector2i(5,10));
+
+ // Padding widget
+ //auto w = new Widget(innertool);
+ //w->setHeight(10);
+
+ auto button = new ToolButton(innertool, ENTYPO_ICON_HOME);
+ button->setIconExtraScale(1.5f);
+ button->setTheme(toolbuttheme);
+ button->setTooltip("Home");
+ button->setFixedSize(Vector2i(40,40));
+ button->setCallback([this]() {
+ //swindow_->setVisible(true);
+ });
+
+ /*button = new ToolButton(innertool, ENTYPO_ICON_PLUS);
+ button->setIconExtraScale(1.5f);
+ button->setTheme(toolbuttheme);
+ button->setTooltip("Add new");
+ button->setFixedSize(Vector2i(40,40));
+ button->setCallback([this]() {
+ //swindow_->setVisible(true);
+ });*/
+
+ button = new ToolButton(innertool, ENTYPO_ICON_PLUS);
+ button->setIconExtraScale(1.5f);
+ button->setTheme(toolbuttheme);
+ button->setTooltip("Camera Sources");
+ button->setFixedSize(Vector2i(40,40));
+ button->setCallback([this]() {
+ swindow_->setVisible(true);
+ });
+
+ button = new ToolButton(innertool, ENTYPO_ICON_TOOLS);
+ button->setIconExtraScale(1.5f);
+ button->setTheme(toolbuttheme);
+ button->setTooltip("Connections");
+ button->setFixedSize(Vector2i(40,40));
+ button->setCallback([this]() {
+ cwindow_->setVisible(true);
+ });
+
+ button = new ToolButton(toolbar, ENTYPO_ICON_COG);
+ button->setIconExtraScale(1.5f);
+ button->setTheme(toolbuttheme);
+ button->setTooltip("Settings");
+ button->setFixedSize(Vector2i(40,40));
+ button->setPosition(Vector2i(5,height()-50));
+
+ //configwindow_ = new ConfigWindow(parent, ctrl_);
+ cwindow_ = new ftl::gui::ControlWindow(this, controller);
+ swindow_ = new ftl::gui::SourceWindow(this);
+ mwindow_ = new ftl::gui::MediaPanel(this);
+ mwindow_->setVisible(false);
+
+ cwindow_->setPosition(Eigen::Vector2i(80, 20));
+ swindow_->setPosition(Eigen::Vector2i(80, 400));
+ cwindow_->setVisible(false);
+ swindow_->setVisible(false);
+ cwindow_->setTheme(windowtheme);
+ swindow_->setTheme(windowtheme);
+
+ mShader.init("RGBDShader", defaultImageViewVertexShader,
+ defaultImageViewFragmentShader);
+
+ MatrixXu indices(3, 2);
+ indices.col(0) << 0, 1, 2;
+ indices.col(1) << 2, 3, 1;
+
+ MatrixXf vertices(2, 4);
+ vertices.col(0) << 0, 0;
+ vertices.col(1) << 1, 0;
+ vertices.col(2) << 0, 1;
+ vertices.col(3) << 1, 1;
+
+ mShader.bind();
+ mShader.uploadIndices(indices);
+ mShader.uploadAttrib("vertex", vertices);
+
+ setVisible(true);
+ performLayout();
+}
+
+ftl::gui::Screen::~Screen() {
+ mShader.free();
+}
+
+void ftl::gui::Screen::setActiveCamera(ftl::gui::Camera *cam) {
+ camera_ = cam;
+
+ if (cam) {
+ status_ = cam->source()->getURI();
+ mwindow_->setVisible(true);
+ } else {
+ mwindow_->setVisible(false);
+ status_ = "No camera...";
+ }
+}
+
+bool ftl::gui::Screen::mouseMotionEvent(const Eigen::Vector2i &p, const Eigen::Vector2i &rel, int button, int modifiers) {
+ if (nanogui::Screen::mouseMotionEvent(p, rel, button, modifiers)) {
+ return true;
+ } else {
+ if (camera_) camera_->mouseMovement(rel[0], rel[1], button);
+ }
+}
+
+bool ftl::gui::Screen::mouseButtonEvent(const nanogui::Vector2i &p, int button, bool down, int modifiers) {
+ if (nanogui::Screen::mouseButtonEvent(p, button, down, modifiers)) {
+ return true;
+ } else {
+ if (camera_ && down) {
+ Eigen::Vector2f screenSize = size().cast<float>();
+ auto mScale = (screenSize.cwiseQuotient(imageSize).minCoeff());
+ Eigen::Vector2f scaleFactor = mScale * imageSize.cwiseQuotient(screenSize);
+ Eigen::Vector2f positionInScreen(0.0f, 0.0f);
+ auto mOffset = (screenSize - (screenSize.cwiseProduct(scaleFactor))) / 2;
+ Eigen::Vector2f positionAfterOffset = positionInScreen + mOffset;
+
+ float sx = ((float)p[0] - positionAfterOffset[0]) / mScale;
+ float sy = ((float)p[1] - positionAfterOffset[1]) / mScale;
+
+ Eigen::Vector4f camPos;
+
+ try {
+ camPos = camera_->source()->point(sx,sy).cast<float>();
+ } catch(...) {
+ return true;
+ }
+
+ camPos *= -1.0f;
+ Eigen::Vector4f worldPos = camera_->source()->getPose().cast<float>() * camPos;
+ //lookPoint_ = Eigen::Vector3f(worldPos[0],worldPos[1],worldPos[2]);
+ LOG(INFO) << "Depth at click = " << -camPos[2];
+ return true;
+ }
+ return false;
+ }
+}
+
+bool ftl::gui::Screen::keyboardEvent(int key, int scancode, int action, int modifiers) {
+ using namespace Eigen;
+ if (nanogui::Screen::keyboardEvent(key, scancode, action, modifiers)) {
+ return true;
+ } else {
+ LOG(INFO) << "Key press " << key << " - " << action << " - " << modifiers;
+
+ if (key >= 262 && key <= 267) {
+ if (camera_) camera_->keyMovement(key, modifiers);
+ return true;
+ } else if (action == 1 && key == 32) {
+ ctrl_->pause();
+ return true;
+ }
+ return false;
+ }
+}
+
+void ftl::gui::Screen::draw(NVGcontext *ctx) {
+ using namespace Eigen;
+
+ Vector2f screenSize = size().cast<float>();
+
+ if (camera_) {
+ imageSize = {camera_->width(), camera_->height()};
+
+ mImageID = camera_->captureFrame().texture();
+
+ if (imageSize[0] > 0) {
+ auto mScale = (screenSize.cwiseQuotient(imageSize).minCoeff());
+ Vector2f scaleFactor = mScale * imageSize.cwiseQuotient(screenSize);
+ Vector2f positionInScreen(0.0f, 0.0f);
+ auto mOffset = (screenSize - (screenSize.cwiseProduct(scaleFactor))) / 2;
+ Vector2f positionAfterOffset = positionInScreen + mOffset;
+ Vector2f imagePosition = positionAfterOffset.cwiseQuotient(screenSize);
+ //glEnable(GL_SCISSOR_TEST);
+ //float r = screen->pixelRatio();
+ /* glScissor(positionInScreen.x() * r,
+ (screenSize.y() - positionInScreen.y() - size().y()) * r,
+ size().x() * r, size().y() * r);*/
+ mShader.bind();
+ glActiveTexture(GL_TEXTURE0);
+ glBindTexture(GL_TEXTURE_2D, mImageID);
+ mShader.setUniform("image", 0);
+ mShader.setUniform("scaleFactor", scaleFactor);
+ mShader.setUniform("position", imagePosition);
+ mShader.drawIndexed(GL_TRIANGLES, 0, 2);
+ //glDisable(GL_SCISSOR_TEST);
+ }
+ }
+
+ nvgTextAlign(ctx, NVG_ALIGN_RIGHT);
+ nvgText(ctx, screenSize[0]-10, screenSize[1]-20, status_.c_str(), NULL);
+
+ /* Draw the user interface */
+ screen()->performLayout(ctx);
+ nanogui::Screen::draw(ctx);
+}
\ No newline at end of file
diff --git a/applications/gui/src/screen.hpp b/applications/gui/src/screen.hpp
new file mode 100644
index 000000000..7746aa49e
--- /dev/null
+++ b/applications/gui/src/screen.hpp
@@ -0,0 +1,74 @@
+#ifndef _FTL_GUI_SCREEN_HPP_
+#define _FTL_GUI_SCREEN_HPP_
+
+#include <nanogui/screen.h>
+#include <nanogui/glutil.h>
+#include <ftl/master.hpp>
+#include <ftl/net/universe.hpp>
+#include <ftl/configuration.hpp>
+
+#include "ctrl_window.hpp"
+#include "src_window.hpp"
+#include "gltexture.hpp"
+
+class StatisticsImageNSamples;
+
+namespace ftl {
+namespace gui {
+
+class Camera;
+class MediaPanel;
+
+class Screen : public nanogui::Screen {
+ public:
+ explicit Screen(ftl::Configurable *root, ftl::net::Universe *net, ftl::ctrl::Master *controller);
+ ~Screen();
+
+ bool mouseMotionEvent(const Eigen::Vector2i &p, const Eigen::Vector2i &rel, int button, int modifiers);
+ bool mouseButtonEvent(const nanogui::Vector2i &p, int button, bool down, int modifiers);
+ bool keyboardEvent(int key, int scancode, int action, int modifiers);
+
+ void setActivePose(const Eigen::Matrix4d &p);
+
+ virtual void draw(NVGcontext *ctx);
+
+ ftl::Configurable *root() { return root_; }
+ ftl::net::Universe *net() { return net_; }
+ ftl::ctrl::Master *control() { return ctrl_; }
+
+ void setActiveCamera(ftl::gui::Camera*);
+ ftl::gui::Camera *activeCamera() { return camera_; }
+
+ nanogui::Theme *windowtheme;
+ nanogui::Theme *specialtheme;
+
+ private:
+ ftl::gui::SourceWindow *swindow_;
+ ftl::gui::ControlWindow *cwindow_;
+ ftl::gui::MediaPanel *mwindow_;
+ //std::vector<SourceViews> sources_;
+ ftl::net::Universe *net_;
+ nanogui::GLShader mShader;
+ GLuint mImageID;
+ //Source *src_;
+ GLTexture texture_;
+ Eigen::Vector3f eye_;
+ Eigen::Vector4f neye_;
+ Eigen::Vector3f orientation_;
+ Eigen::Vector3f up_;
+ //Eigen::Vector3f lookPoint_;
+ float lerpSpeed_;
+ bool depth_;
+ float ftime_;
+ float delta_;
+ Eigen::Vector2f imageSize;
+ ftl::ctrl::Master *ctrl_;
+ ftl::Configurable *root_;
+ std::string status_;
+ ftl::gui::Camera *camera_;
+};
+
+}
+}
+
+#endif // _FTL_GUI_SCREEN_HPP_
diff --git a/applications/gui/src/src_window.cpp b/applications/gui/src/src_window.cpp
index b5e70e898..8a0f93da2 100644
--- a/applications/gui/src/src_window.cpp
+++ b/applications/gui/src/src_window.cpp
@@ -1,6 +1,7 @@
#include "src_window.hpp"
-#include "pose_window.hpp"
+#include "screen.hpp"
+#include "camera.hpp"
#include <nanogui/imageview.h>
#include <nanogui/textbox.h>
@@ -17,92 +18,46 @@
#endif
using ftl::gui::SourceWindow;
+using ftl::gui::Screen;
using ftl::rgbd::Source;
using std::string;
+using ftl::config::json_t;
-class GLTexture {
- public:
- GLTexture() {
- glGenTextures(1, &glid_);
- glBindTexture(GL_TEXTURE_2D, glid_);
- cv::Mat m(cv::Size(100,100), CV_8UC3);
- glTexImage2D(GL_TEXTURE_2D, 0, GL_RGB8, m.cols, m.rows, 0, GL_RGB, GL_UNSIGNED_BYTE, m.data);
- glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
- glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
- glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_NEAREST);
- glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR);
- }
- ~GLTexture() {
- glDeleteTextures(1, &glid_);
- }
-
- void update(cv::Mat &m) {
- if (m.rows == 0) return;
- glBindTexture(GL_TEXTURE_2D, glid_);
- // TODO Allow for other formats
- glTexImage2D(GL_TEXTURE_2D, 0, GL_RGB8, m.cols, m.rows, 0, GL_BGR, GL_UNSIGNED_BYTE, m.data);
- auto err = glGetError();
- if (err != 0) LOG(ERROR) << "OpenGL Texture error: " << err;
- }
-
- unsigned int texture() const { return glid_; }
-
- private:
- unsigned int glid_;
-};
-
-template<class T>
-Eigen::Matrix<T,4,4> lookAt
-(
- Eigen::Matrix<T,3,1> const & eye,
- Eigen::Matrix<T,3,1> const & center,
- Eigen::Matrix<T,3,1> const & up
-)
-{
- typedef Eigen::Matrix<T,4,4> Matrix4;
- typedef Eigen::Matrix<T,3,1> Vector3;
-
- Vector3 f = (center - eye).normalized();
- Vector3 u = up.normalized();
- Vector3 s = f.cross(u).normalized();
- u = s.cross(f);
-
- Matrix4 res;
- res << s.x(),s.y(),s.z(),-s.dot(eye),
- u.x(),u.y(),u.z(),-u.dot(eye),
- -f.x(),-f.y(),-f.z(),f.dot(eye),
- 0,0,0,1;
-
- return res;
-}
-
-SourceWindow::SourceWindow(nanogui::Widget *parent, ftl::ctrl::Master *ctrl)
- : nanogui::Window(parent, "Source View"), ctrl_(ctrl) {
+SourceWindow::SourceWindow(ftl::gui::Screen *screen)
+ : nanogui::Window(screen, ""), screen_(screen) {
setLayout(new nanogui::GroupLayout());
using namespace nanogui;
- mode_ = Mode::rgb;
- src_ = ftl::create<Source>(ctrl->getRoot(), "source", ctrl->getNet());
+ //if (!screen->root()->get<json_t>("sources")) {
+ // screen->root()->getConfig()["sources"] = json_t::array();
+ //}
+
+ //src_ = ftl::create<Source>(ctrl->getRoot(), "source", ctrl->getNet());
//Widget *tools = new Widget(this);
// tools->setLayout(new BoxLayout(Orientation::Horizontal,
// Alignment::Middle, 0, 6));
new Label(this, "Select source","sans-bold");
- available_ = ctrl->getNet()->findAll<string>("list_streams");
+ available_ = screen_->control()->getNet()->findAll<string>("list_streams");
auto select = new ComboBox(this, available_);
select->setCallback([this,select](int ix) {
+ //src_->set("uri", available_[ix]);
+ // TODO(Nick) Check camera exists first
+ screen_->setActiveCamera(cameras_[available_[ix]]);
LOG(INFO) << "Change source: " << ix;
- src_->set("uri", available_[ix]);
});
- ctrl->getNet()->onConnect([this,select](ftl::net::Peer *p) {
- available_ = ctrl_->getNet()->findAll<string>("list_streams");
+ _updateCameras();
+
+ screen->net()->onConnect([this,select](ftl::net::Peer *p) {
+ available_ = screen_->net()->findAll<string>("list_streams");
select->setItems(available_);
+ _updateCameras();
});
- new Label(this, "Source Options","sans-bold");
+ /*new Label(this, "Source Options","sans-bold");
auto tools = new Widget(this);
tools->setLayout(new BoxLayout(Orientation::Horizontal,
@@ -123,11 +78,11 @@ SourceWindow::SourceWindow(nanogui::Widget *parent, ftl::ctrl::Master *ctrl)
button_stddev->setFlags(Button::RadioButton);
button_stddev->setChangeCallback([this](bool state) { mode_ = Mode::stddev; });
- auto button_pose = new Button(this, "Adjust Pose", ENTYPO_ICON_COMPASS);
- button_pose->setCallback([this]() {
- auto posewin = new PoseWindow(screen(), ctrl_, src_->getURI());
- posewin->setTheme(theme());
- });
+ //auto button_pose = new Button(this, "Adjust Pose", ENTYPO_ICON_COMPASS);
+ //button_pose->setCallback([this]() {
+ // auto posewin = new PoseWindow(screen_, screen_->control(), src_->getURI());
+ // posewin->setTheme(theme());
+ //});
#ifdef HAVE_LIBARCHIVE
auto button_snapshot = new Button(this, "Snapshot", ENTYPO_ICON_IMAGES);
@@ -159,7 +114,24 @@ SourceWindow::SourceWindow(nanogui::Widget *parent, ftl::ctrl::Master *ctrl)
//cam.view = imageView;
//imageView->setGridThreshold(20);
//imageView->setSource(src_);
- //image_ = imageView;
+ //image_ = imageView;*/
+}
+
+void SourceWindow::_updateCameras() {
+ for (auto s : available_) {
+ if (cameras_.find(s) == cameras_.end()) {
+ json_t srcjson;
+ srcjson["uri"] = s;
+ screen_->root()->getConfig()["sources"].push_back(srcjson);
+ std::vector<ftl::rgbd::Source*> srcs = ftl::createArray<ftl::rgbd::Source>(screen_->root(), "sources", screen_->net());
+ auto *src = srcs[srcs.size()-1];
+
+ auto *cam = new ftl::gui::Camera(screen_, src);
+ cameras_[s] = cam;
+ } else {
+ LOG(INFO) << "Camera already exists: " << s;
+ }
+ }
}
SourceWindow::~SourceWindow() {
diff --git a/applications/gui/src/src_window.hpp b/applications/gui/src/src_window.hpp
index e7aa19213..77f529c8c 100644
--- a/applications/gui/src/src_window.hpp
+++ b/applications/gui/src/src_window.hpp
@@ -6,6 +6,7 @@
#include <ftl/uuid.hpp>
#include <ftl/rgbd/source.hpp>
#include <vector>
+#include <map>
#include <string>
class VirtualCameraView;
@@ -13,26 +14,23 @@ class VirtualCameraView;
namespace ftl {
namespace gui {
-/**
- * Manage connected nodes and add new connections.
- */
+class Screen;
+class Camera;
+
class SourceWindow : public nanogui::Window {
public:
- enum class Mode { rgb, depth, stddev };
- SourceWindow(nanogui::Widget *parent, ftl::ctrl::Master *ctrl);
+ SourceWindow(ftl::gui::Screen *screen);
~SourceWindow();
- ftl::rgbd::Source *getSource() const { return src_; }
- bool getDepth() const { return mode_ == Mode::depth; }
- Mode getMode() { return mode_; }
+ const std::vector<ftl::gui::Camera*> &getCameras();
private:
- ftl::ctrl::Master *ctrl_;
- ftl::rgbd::Source *src_;
- Mode mode_;
- VirtualCameraView *image_;
+ ftl::gui::Screen *screen_;
+ std::map<std::string, ftl::gui::Camera*> cameras_;
std::vector<std::string> available_;
+ void _updateCameras();
+
};
}
diff --git a/applications/reconstruct/src/registration.cpp b/applications/reconstruct/src/registration.cpp
index ad8c1e8cd..5b828242e 100644
--- a/applications/reconstruct/src/registration.cpp
+++ b/applications/reconstruct/src/registration.cpp
@@ -551,7 +551,7 @@ bool ChessboardRegistrationChain::findTransformations(vector<Matrix4f> &data) {
<< getSource(edge.second)->getID() << " to source"
<< getSource(edge.first)->getID();
- nlohmann::json conf(config_);
+ nlohmann::json conf(getConfig());
conf["targetsource"] = getSource(edge.first)->getID();
conf["chain"] = false;
diff --git a/applications/reconstruct/src/scene_rep_hash_sdf.cu b/applications/reconstruct/src/scene_rep_hash_sdf.cu
index 521a34601..c29e66b16 100644
--- a/applications/reconstruct/src/scene_rep_hash_sdf.cu
+++ b/applications/reconstruct/src/scene_rep_hash_sdf.cu
@@ -623,7 +623,7 @@ __global__ void starveVoxelsKernel(HashData hashData) {
//is typically exectued only every n'th frame
int weight = hashData.d_SDFBlocks[entry.ptr + threadIdx.x].weight;
- weight = max(0, weight-1);
+ weight = max(0, weight-2);
hashData.d_SDFBlocks[entry.ptr + threadIdx.x].weight = weight; //CHECK Remove to totally clear previous frame (Nick)
}
diff --git a/components/common/cpp/include/ftl/configurable.hpp b/components/common/cpp/include/ftl/configurable.hpp
index 7999bcfa8..03197ce1e 100644
--- a/components/common/cpp/include/ftl/configurable.hpp
+++ b/components/common/cpp/include/ftl/configurable.hpp
@@ -50,7 +50,7 @@ class Configurable {
/**
* Return raw JSON entity for this Configurable.
*/
- nlohmann::json &getConfig() { return config_; }
+ nlohmann::json &getConfig() { return *config_; }
/**
* Get a configuration property from the json object. Returns an optional
@@ -77,7 +77,7 @@ class Configurable {
*/
template <typename T>
void set(const std::string &name, T value) {
- config_[name] = value;
+ (*config_)[name] = value;
_trigger(name);
}
@@ -96,8 +96,10 @@ class Configurable {
*/
void on(const std::string &prop, std::function<void(const config::Event&)>);
+ void patchPtr(nlohmann::json &newcfg) { config_ = &newcfg; }
+
protected:
- nlohmann::json &config_;
+ nlohmann::json *config_; // FIXME(Nick) Should be a reference but this can be invalidated...
private:
std::map<std::string, std::list<std::function<void(const config::Event&)>>> observers_;
@@ -120,16 +122,17 @@ void Configurable::set<const std::string&>(const std::string &name, const std::s
template <typename T>
std::optional<T> ftl::Configurable::get(const std::string &name) {
- if (!config_[name].is_null()) {
+ if (!config_->is_object() && !config_->is_null()) LOG(FATAL) << "Config is not an object";
+ if (!(*config_)[name].is_null()) {
try {
- return config_[name].get<T>();
+ return (*config_)[name].get<T>();
} catch (...) {
return {};
}
- } else if (config_["$ref"].is_string()) {
+ } else if ((*config_)["$ref"].is_string()) {
// TODO(Nick) Add # if missing
// TODO(Nick) Cache result of ref loopkup
- std::string res_uri = config_["$ref"].get<std::string>()+"/"+name;
+ std::string res_uri = (*config_)["$ref"].get<std::string>()+"/"+name;
auto &r = ftl::config::resolve(res_uri);
DLOG(2) << "GET: " << res_uri << " = " << r;
@@ -137,7 +140,7 @@ std::optional<T> ftl::Configurable::get(const std::string &name) {
try {
return r.get<T>();
} catch (...) {
- LOG(ERROR) << "Missing: " << config_["$id"].get<std::string>()+"/"+name;
+ LOG(ERROR) << "Missing: " << (*config_)["$id"].get<std::string>()+"/"+name;
return {};
}
} else {
diff --git a/components/common/cpp/include/ftl/configuration.hpp b/components/common/cpp/include/ftl/configuration.hpp
index db6cb1438..3333ed576 100644
--- a/components/common/cpp/include/ftl/configuration.hpp
+++ b/components/common/cpp/include/ftl/configuration.hpp
@@ -125,11 +125,14 @@ T *ftl::config::create(json_t &link, ARGS ...args) {
ftl::Configurable *cfg = ftl::config::find(link["$id"].get<std::string>());
if (!cfg) {
- // try {
+ try {
cfg = new T(link, args...);
- //} catch(...) {
- // LOG(FATAL) << "Could not construct " << link;
- //}
+ } catch(...) {
+ LOG(FATAL) << "Could not construct " << link;
+ }
+ } else {
+ // Make sure configurable has newest object pointer
+ cfg->patchPtr(link);
}
try {
diff --git a/components/common/cpp/include/nlohmann/json.hpp b/components/common/cpp/include/nlohmann/json.hpp
index 68b715f04..93a5e9a2a 100644
--- a/components/common/cpp/include/nlohmann/json.hpp
+++ b/components/common/cpp/include/nlohmann/json.hpp
@@ -1,3 +1,10 @@
+/*
+ * Modified by: Nicolas Pope
+ * Changes:
+ * * Allow // comments
+ * * Extend exception messages to include location info
+ */
+
/*
__ _____ _____ _____
__| | __| | | | JSON for Modern C++
@@ -15592,7 +15599,7 @@ class basic_json
return m_value.object->operator[](key);
}
- JSON_THROW(type_error::create(305, "cannot use operator[] with a string argument with " + std::string(type_name())));
+ JSON_THROW(type_error::create(305, "cannot use operator[] with a string argument with " + std::string(type_name()) + std::string(" [key = ") + key + std::string("]")));
}
/*!
diff --git a/components/common/cpp/src/configurable.cpp b/components/common/cpp/src/configurable.cpp
index 87f2a454f..7cea5adeb 100644
--- a/components/common/cpp/src/configurable.cpp
+++ b/components/common/cpp/src/configurable.cpp
@@ -9,12 +9,15 @@ using ftl::config::json_t;
extern nlohmann::json null_json;
-Configurable::Configurable() : config_(null_json) {
+Configurable::Configurable() : config_(&null_json) {
ftl::config::registerConfigurable(this);
}
-Configurable::Configurable(nlohmann::json &config) : config_(config) {
- //if (config["uri"].is_string()) __changeURI(config["uri"].get<std::string>(), this);
+Configurable::Configurable(nlohmann::json &config) : config_(&config) {
+ if (!config.is_object()) {
+ LOG(FATAL) << "Configurable json is not an object: " << config;
+ }
+
ftl::config::registerConfigurable(this);
}
diff --git a/components/net/cpp/include/ftl/net/universe.hpp b/components/net/cpp/include/ftl/net/universe.hpp
index 0c71db7fe..6afdd6107 100644
--- a/components/net/cpp/include/ftl/net/universe.hpp
+++ b/components/net/cpp/include/ftl/net/universe.hpp
@@ -204,7 +204,7 @@ class Universe : public ftl::Configurable {
bool active_;
ftl::UUID this_peer;
std::shared_mutex net_mutex_;
- std::shared_mutex handler_mutex_;
+ std::recursive_mutex handler_mutex_;
fd_set sfderror_;
fd_set sfdread_;
std::vector<ftl::net::Listener*> listeners_;
diff --git a/components/net/cpp/src/universe.cpp b/components/net/cpp/src/universe.cpp
index 76b3e546a..488a2acde 100644
--- a/components/net/cpp/src/universe.cpp
+++ b/components/net/cpp/src/universe.cpp
@@ -436,7 +436,7 @@ void Universe::removeCallback(callback_t cbid) {
}
void Universe::_notifyConnect(Peer *p) {
- SHARED_LOCK(handler_mutex_,lk);
+ UNIQUE_LOCK(handler_mutex_,lk);
peer_ids_[p->id()] = p;
for (auto &i : on_connect_) {
@@ -451,7 +451,7 @@ void Universe::_notifyConnect(Peer *p) {
void Universe::_notifyDisconnect(Peer *p) {
// In all cases, should already be locked outside this function call
//unique_lock<mutex> lk(net_mutex_);
- SHARED_LOCK(handler_mutex_,lk);
+ UNIQUE_LOCK(handler_mutex_,lk);
for (auto &i : on_disconnect_) {
try {
i.h(p);
diff --git a/components/rgbd-sources/include/ftl/rgbd/source.hpp b/components/rgbd-sources/include/ftl/rgbd/source.hpp
index d90385f50..c43525ae4 100644
--- a/components/rgbd-sources/include/ftl/rgbd/source.hpp
+++ b/components/rgbd-sources/include/ftl/rgbd/source.hpp
@@ -32,6 +32,16 @@ enum capability_t {
kCapDisparity // Raw disparity is available
};
+typedef unsigned int channel_t;
+
+static const unsigned int kChanLeft = 1;
+static const unsigned int kChanDepth = 2;
+static const unsigned int kChanRight = 3;
+static const unsigned int kChanDisparity = 4;
+static const unsigned int kChanDeviation = 5;
+
+static const unsigned int kChanOverlay1 = 100;
+
/**
* RGBD Generic data source configurable entity. This class hides the
* internal implementation of an RGBD source by providing accessor functions
@@ -150,6 +160,14 @@ class Source : public ftl::Configurable {
*/
void reset();
+ void pause(bool);
+ bool isPaused() { return paused_; }
+
+ void bullet(bool);
+ bool isBullet() { return bullet_; }
+
+ bool thumbnail(cv::Mat &t);
+
ftl::net::Universe *getNet() const { return net_; }
std::string getURI() { return value("uri", std::string("")); }
@@ -166,6 +184,8 @@ class Source : public ftl::Configurable {
Eigen::Matrix4d pose_;
ftl::net::Universe *net_;
std::shared_mutex mutex_;
+ bool paused_;
+ bool bullet_;
detail::Source *_createImplementation();
detail::Source *_createFileImpl(const ftl::URI &uri);
diff --git a/components/rgbd-sources/src/net.cpp b/components/rgbd-sources/src/net.cpp
index b71d18e08..330fc907e 100644
--- a/components/rgbd-sources/src/net.cpp
+++ b/components/rgbd-sources/src/net.cpp
@@ -93,29 +93,11 @@ NetSource::~NetSource() {
host_->getNet()->removeCallback(h_);
}
-void NetSource::_recv(const vector<unsigned char> &jpg, const vector<unsigned char> &d) {
- cv::Mat tmp_rgb, tmp_depth;
-
- // Decode in temporary buffers to prevent long locks
- cv::imdecode(jpg, cv::IMREAD_COLOR, &tmp_rgb);
- cv::imdecode(d, cv::IMREAD_UNCHANGED, &tmp_depth);
-
- // Lock host to prevent grab
- UNIQUE_LOCK(host_->mutex(),lk);
- rgb_ = tmp_rgb;
- tmp_depth.convertTo(depth_, CV_32FC1, 1.0f/(16.0f*100.0f));
- N_--;
- //lk.unlock();
-}
-
void NetSource::_recvChunk(int frame, int chunk, bool delta, const vector<unsigned char> &jpg, const vector<unsigned char> &d) {
cv::Mat tmp_rgb, tmp_depth;
if (!active_) return;
- //LOG(INFO) << "Received chunk " << (int)chunk;
-
- //try {
// Decode in temporary buffers to prevent long locks
cv::imdecode(jpg, cv::IMREAD_COLOR, &tmp_rgb);
cv::imdecode(d, cv::IMREAD_UNCHANGED, &tmp_depth);
@@ -132,20 +114,11 @@ void NetSource::_recvChunk(int frame, int chunk, bool delta, const vector<unsign
cv::Rect roi(cx,cy,chunk_width_,chunk_height_);
cv::Mat chunkRGB = rgb_(roi);
- //cv::Mat ichunkDepth = idepth_(roi);
cv::Mat chunkDepth = depth_(roi);
tmp_rgb.copyTo(chunkRGB);
- //tmp_depth.convertTo(tmp_depth, CV_16UC1);
- //if (delta) ichunkDepth = tmp_depth - ichunkDepth;
- //tmp_depth.copyTo(ichunkDepth);
tmp_depth.convertTo(chunkDepth, CV_32FC1, 1.0f/(16.0f*10.0f));
if (chunk == 0) N_--;
- //lk.unlock();
- //} catch(...) {
- // LOG(ERROR) << "Decode exception";
- // return;
- //}
}
void NetSource::setPose(const Eigen::Matrix4d &pose) {
@@ -167,7 +140,6 @@ void NetSource::_updateURI() {
active_ = false;
auto uri = host_->get<string>("uri");
- // TODO(Nick) If URI changes then must unbind + rebind.
if (uri_.size() > 0) {
host_->getNet()->unbind(uri_);
}
--
GitLab