#include "screen.hpp" #include <nanogui/opengl.h> #include <nanogui/glutil.h> #include <nanogui/screen.h> #include <nanogui/window.h> #include <nanogui/layout.h> #include <nanogui/imageview.h> #include <nanogui/label.h> #include <nanogui/toolbutton.h> #include <nanogui/popupbutton.h> #include <nlohmann/json.hpp> #include <loguru.hpp> #include "ctrl_window.hpp" #include "src_window.hpp" #include "config_window.hpp" #include "camera.hpp" #include "media_panel.hpp" #ifdef HAVE_OPENVR #include "vr.hpp" #endif using ftl::gui::Screen; using ftl::gui::Camera; using std::string; using ftl::rgbd::Source; using ftl::rgbd::isValidDepth; namespace { constexpr char const *const defaultImageViewVertexShader = R"(#version 330 uniform vec2 scaleFactor; uniform vec2 position; in vec2 vertex; out vec2 uv; void main() { uv = vertex; vec2 scaledVertex = (vertex * scaleFactor) + position; gl_Position = vec4(2.0*scaledVertex.x - 1.0, 2.0*scaledVertex.y - 1.0, 0.0, 1.0); })"; constexpr char const *const defaultImageViewFragmentShader = R"(#version 330 uniform sampler2D image1; uniform sampler2D image2; uniform float blendAmount; out vec4 color; in vec2 uv; void main() { color = blendAmount * texture(image1, uv) + (1.0 - blendAmount) * texture(image2, uv); color.w = 1.0f; })"; } ftl::gui::Screen::Screen(ftl::Configurable *proot, ftl::net::Universe *pnet, ftl::ctrl::Master *controller) : nanogui::Screen(Eigen::Vector2i(1024, 768), "FT-Lab Remote Presence"), status_("FT-Lab Remote Presence System") { using namespace nanogui; net_ = pnet; ctrl_ = controller; root_ = proot; camera_ = nullptr; #ifdef HAVE_OPENVR HMD_ = nullptr; #endif zoom_ = root_->value("zoom", 1.0f); root_->on("zoom", [this](const ftl::config::Event &e) { zoom_ = root_->value("zoom", 1.0f); }); pos_x_ = root_->value("position_x", 0.0f); root_->on("position_x", [this](const ftl::config::Event &e) { pos_x_ = root_->value("position_x", 0.0f); }); pos_y_ = root_->value("position_y", 0.0f); root_->on("position_y", [this](const ftl::config::Event &e) { pos_y_ = root_->value("position_y", 0.0f); }); shortcuts_ = ftl::create<ftl::Configurable>(root_, "shortcuts"); setSize(Vector2i(1280,720)); toolbuttheme = new Theme(*theme()); toolbuttheme->mBorderDark = nanogui::Color(0,0); toolbuttheme->mBorderLight = nanogui::Color(0,0); toolbuttheme->mButtonGradientBotFocused = nanogui::Color(60,255); toolbuttheme->mButtonGradientBotUnfocused = nanogui::Color(0,0); toolbuttheme->mButtonGradientTopFocused = nanogui::Color(60,255); toolbuttheme->mButtonGradientTopUnfocused = nanogui::Color(0,0); toolbuttheme->mButtonGradientTopPushed = nanogui::Color(60,180); toolbuttheme->mButtonGradientBotPushed = nanogui::Color(60,180); toolbuttheme->mTextColor = nanogui::Color(0.9f,0.9f,0.9f,0.9f); mediatheme = new Theme(*theme()); mediatheme->mIconScale = 1.2f; mediatheme->mWindowDropShadowSize = 0; mediatheme->mWindowFillFocused = nanogui::Color(45, 150); mediatheme->mWindowFillUnfocused = nanogui::Color(45, 80); mediatheme->mButtonGradientTopUnfocused = nanogui::Color(0,0); mediatheme->mButtonGradientBotUnfocused = nanogui::Color(0,0); mediatheme->mButtonGradientTopFocused = nanogui::Color(80,230); mediatheme->mButtonGradientBotFocused = nanogui::Color(80,230); mediatheme->mIconColor = nanogui::Color(255,255); mediatheme->mTextColor = nanogui::Color(1.0f,1.0f,1.0f,1.0f); mediatheme->mBorderDark = nanogui::Color(0,0); mediatheme->mBorderMedium = nanogui::Color(0,0); mediatheme->mBorderLight = nanogui::Color(0,0); mediatheme->mDropShadow = nanogui::Color(0,0); mediatheme->mButtonFontSize = 30; mediatheme->mStandardFontSize = 20; windowtheme = new Theme(*theme()); windowtheme->mWindowFillFocused = nanogui::Color(220, 200); windowtheme->mWindowFillUnfocused = nanogui::Color(220, 200); windowtheme->mWindowHeaderGradientBot = nanogui::Color(60,230); windowtheme->mWindowHeaderGradientTop = nanogui::Color(60,230); windowtheme->mTextColor = nanogui::Color(20,255); windowtheme->mWindowCornerRadius = 2; windowtheme->mButtonGradientBotFocused = nanogui::Color(210,255); windowtheme->mButtonGradientBotUnfocused = nanogui::Color(190,255); windowtheme->mButtonGradientTopFocused = nanogui::Color(230,255); windowtheme->mButtonGradientTopUnfocused = nanogui::Color(230,255); windowtheme->mButtonGradientTopPushed = nanogui::Color(170,255); windowtheme->mButtonGradientBotPushed = nanogui::Color(210,255); windowtheme->mBorderDark = nanogui::Color(150,255); windowtheme->mBorderMedium = nanogui::Color(165,255); windowtheme->mBorderLight = nanogui::Color(230,255); windowtheme->mButtonFontSize = 16; windowtheme->mTextColorShadow = nanogui::Color(0,0); windowtheme->mWindowTitleUnfocused = windowtheme->mWindowTitleFocused; windowtheme->mWindowTitleFocused = nanogui::Color(240,255); windowtheme->mIconScale = 0.85f; auto toolbar = new Window(this, ""); toolbar->setPosition(Vector2i(0,0)); toolbar->setFixedWidth(50); toolbar->setFixedHeight(height()); //toolbar->setLayout(new BoxLayout(Orientation::Vertical, // Alignment::Middle, 0, 10)); setResizeCallback([this,toolbar](Vector2i s) { toolbar->setFixedHeight(s[1]); mwindow_->setPosition(Vector2i(s[0] / 2 - mwindow_->width()/2, s[1] - 30 - mwindow_->height())); }); auto innertool = new Widget(toolbar); innertool->setLayout(new BoxLayout(Orientation::Vertical, Alignment::Middle, 0, 10)); innertool->setPosition(Vector2i(5,10)); // Padding widget //auto w = new Widget(innertool); //w->setHeight(10); auto button = new ToolButton(innertool, ENTYPO_ICON_HOME); button->setIconExtraScale(1.5f); button->setTheme(toolbuttheme); button->setTooltip("Home"); button->setFixedSize(Vector2i(40,40)); button->setCallback([this]() { //swindow_->setVisible(true); setActiveCamera(nullptr); }); /*button = new ToolButton(innertool, ENTYPO_ICON_PLUS); button->setIconExtraScale(1.5f); button->setTheme(toolbuttheme); button->setTooltip("Add new"); button->setFixedSize(Vector2i(40,40)); button->setCallback([this]() { //swindow_->setVisible(true); });*/ auto popbutton = new PopupButton(innertool, "", ENTYPO_ICON_PLUS); popbutton->setIconExtraScale(1.5f); popbutton->setTheme(toolbuttheme); popbutton->setTooltip("Add"); popbutton->setFixedSize(Vector2i(40,40)); popbutton->setSide(Popup::Side::Right); popbutton->setChevronIcon(0); Popup *popup = popbutton->popup(); popup->setLayout(new GroupLayout()); popup->setTheme(toolbuttheme); //popup->setAnchorHeight(100); auto itembutton = new Button(popup, "Add Camera", ENTYPO_ICON_CAMERA); itembutton->setCallback([this,popup]() { swindow_->setVisible(true); popup->setVisible(false); }); itembutton = new Button(popup, "Add Node", ENTYPO_ICON_LAPTOP); itembutton->setCallback([this,popup]() { cwindow_->setVisible(true); popup->setVisible(false); }); popbutton = new PopupButton(innertool, "", ENTYPO_ICON_TOOLS); popbutton->setIconExtraScale(1.5f); popbutton->setTheme(toolbuttheme); popbutton->setTooltip("Tools"); popbutton->setFixedSize(Vector2i(40,40)); popbutton->setSide(Popup::Side::Right); popbutton->setChevronIcon(0); popup = popbutton->popup(); popup->setLayout(new GroupLayout()); popup->setTheme(toolbuttheme); //popbutton->setCallback([this]() { // cwindow_->setVisible(true); //}); itembutton = new Button(popup, "Connections"); itembutton->setCallback([this,popup]() { cwindow_->setVisible(true); popup->setVisible(false); }); itembutton = new Button(popup, "Manual Registration"); itembutton->setCallback([this,popup]() { // Show pose win... popup->setVisible(false); }); itembutton = new Button(innertool, "", ENTYPO_ICON_COG); itembutton->setIconExtraScale(1.5f); itembutton->setTheme(toolbuttheme); itembutton->setTooltip("Settings"); itembutton->setFixedSize(Vector2i(40,40)); itembutton->setCallback([this]() { auto config_window = new ConfigWindow(this, ctrl_); config_window->setTheme(windowtheme); }); /* //net_->onConnect([this,popup](ftl::net::Peer *p) { { LOG(INFO) << "NET CONNECT"; auto node_details = ctrl_->getControllers(); for (auto &d : node_details) { LOG(INFO) << "ADDING TITLE: " << d.dump(); auto peer = ftl::UUID(d["id"].get<std::string>()); auto itembutton = new Button(popup, d["title"].get<std::string>()); itembutton->setCallback([this,popup,peer]() { auto config_window = new ConfigWindow(this, ctrl_); config_window->setTheme(windowtheme); }); } } //}); itembutton = new Button(popup, "Local"); itembutton->setCallback([this,popup]() { auto config_window = new ConfigWindow(this, ctrl_); config_window->setTheme(windowtheme); }); */ //configwindow_ = new ConfigWindow(parent, ctrl_); //cwindow_ = new ftl::gui::ControlWindow(this, controller); swindow_ = new ftl::gui::SourceWindow(this); mwindow_ = new ftl::gui::MediaPanel(this, swindow_); mwindow_->setVisible(false); mwindow_->setTheme(mediatheme); //cwindow_->setPosition(Eigen::Vector2i(80, 20)); //swindow_->setPosition(Eigen::Vector2i(80, 400)); //cwindow_->setVisible(false); swindow_->setVisible(true); swindow_->center(); //cwindow_->setTheme(windowtheme); swindow_->setTheme(mediatheme); mShader.init("RGBDShader", defaultImageViewVertexShader, defaultImageViewFragmentShader); MatrixXu indices(3, 2); indices.col(0) << 0, 1, 2; indices.col(1) << 2, 3, 1; MatrixXf vertices(2, 4); vertices.col(0) << 0, 0; vertices.col(1) << 1, 0; vertices.col(2) << 0, 1; vertices.col(3) << 1, 1; mShader.bind(); mShader.uploadIndices(indices); mShader.uploadAttrib("vertex", vertices); setVisible(true); performLayout(); } #ifdef HAVE_OPENVR bool ftl::gui::Screen::initVR() { if (!vr::VR_IsHmdPresent()) { return false; } vr::EVRInitError eError = vr::VRInitError_None; HMD_ = vr::VR_Init( &eError, vr::VRApplication_Scene ); if (eError != vr::VRInitError_None) { HMD_ = nullptr; LOG(ERROR) << "Unable to init VR runtime: " << vr::VR_GetVRInitErrorAsEnglishDescription(eError); return false; } return true; } bool ftl::gui::Screen::isVR() { auto *cam = activeCamera(); if (HMD_ == nullptr || cam == nullptr) { return false; } return cam->isVR(); } bool ftl::gui::Screen::switchVR(bool on) { if (isVR() == on) { return on; } if (on && (HMD_ == nullptr) && !initVR()) { return false; } if (on) { activeCamera()->setVR(true); } else { activeCamera()->setVR(false); } return isVR(); } bool ftl::gui::Screen::isHmdPresent() { return vr::VR_IsHmdPresent(); } #endif ftl::gui::Screen::~Screen() { mShader.free(); #ifdef HAVE_OPENVR if (HMD_ != nullptr) { vr::VR_Shutdown(); } #endif } void ftl::gui::Screen::setActiveCamera(ftl::gui::Camera *cam) { if (camera_) camera_->active(false); camera_ = cam; if (cam) { status_ = cam->name(); mwindow_->setVisible(true); mwindow_->cameraChanged(); swindow_->setVisible(false); cam->active(true); } else { mwindow_->setVisible(false); swindow_->setVisible(true); status_ = "[No camera]"; } } bool ftl::gui::Screen::scrollEvent(const Eigen::Vector2i &p, const Eigen::Vector2f &rel) { if (nanogui::Screen::scrollEvent(p, rel)) { return true; } else { zoom_ += zoom_ * 0.1f * rel[1]; return true; } } bool ftl::gui::Screen::mouseMotionEvent(const Eigen::Vector2i &p, const Eigen::Vector2i &rel, int button, int modifiers) { if (nanogui::Screen::mouseMotionEvent(p, rel, button, modifiers)) { return true; } else { if (camera_) { if (button == 1) { camera_->mouseMovement(rel[0], rel[1], button); } else if (button == 2) { pos_x_ += rel[0]; pos_y_ += -rel[1]; } } } return true; } bool ftl::gui::Screen::mouseButtonEvent(const nanogui::Vector2i &p, int button, bool down, int modifiers) { if (nanogui::Screen::mouseButtonEvent(p, button, down, modifiers)) { return true; } else { if (!camera_) return false; //ol movable = camera_->source()->hasCapabilities(ftl::rgbd::kCapMovable); if (button == 0 && down) { Eigen::Vector2f screenSize = size().cast<float>(); auto mScale = (screenSize.cwiseQuotient(imageSize).minCoeff()); Eigen::Vector2f scaleFactor = mScale * imageSize.cwiseQuotient(screenSize); Eigen::Vector2f positionInScreen(0.0f, 0.0f); auto mOffset = (screenSize - (screenSize.cwiseProduct(scaleFactor))) / 2; Eigen::Vector2f positionAfterOffset = positionInScreen + mOffset; float sx = ((float)p[0] - positionAfterOffset[0]) / mScale; float sy = ((float)p[1] - positionAfterOffset[1]) / mScale; //Eigen::Vector4f camPos; //try { //camPos = camera_->source()->point(sx,sy).cast<float>(); //} catch(...) { // return true; //} //camPos *= -1.0f; //Eigen::Vector4f worldPos = camera_->source()->getPose().cast<float>() * camPos; //lookPoint_ = Eigen::Vector3f(worldPos[0],worldPos[1],worldPos[2]); //LOG(INFO) << "Depth at click = " << -camPos[2]; return true; } else { } return false; } } static std::string generateKeyComboStr(int key, int modifiers) { std::string res = ""; switch(modifiers) { case 1: res += "Shift+"; break; case 2: res += "Ctrl+"; break; case 3: res += "Ctrl+Shift+"; break; case 4: res += "Alt+"; break; default: break; } if (key < 127 && key >= 32) { char buf[2] = { (char)key, 0 }; return res + std::string(buf); } else { return ""; } } bool ftl::gui::Screen::keyboardEvent(int key, int scancode, int action, int modifiers) { using namespace Eigen; if (nanogui::Screen::keyboardEvent(key, scancode, action, modifiers)) { return true; } else { //LOG(INFO) << "Key press " << key << " - " << action << " - " << modifiers; if ((key >= 262 && key <= 267) || (key >= '0' && key <= '9')) { if (camera_) camera_->keyMovement(key, modifiers); return true; } else if (action == 1 && key == 'H') { swindow_->setVisible(false); //cwindow_->setVisible(false); } else if (action == 1) { std::string combo = generateKeyComboStr(key, modifiers); if (combo.size() > 0) { LOG(INFO) << "Key combo = " << combo; auto s = shortcuts_->get<nlohmann::json>(combo); if (s) { //LOG(INFO) << "FOUND KEYBOARD SHORTCUT"; std::string op = (*s).value("op",std::string("=")); std::string uri = (*s).value("uri",std::string("")); if (op == "toggle") { auto v = ftl::config::get(uri); if (v.is_boolean()) { ftl::config::update(uri, !v.get<bool>()); } } else if (op == "+=") { auto v = ftl::config::get(uri); if (v.is_number_float()) { ftl::config::update(uri, v.get<float>() + (*s).value("value",0.0f)); } else if (v.is_number_integer()) { ftl::config::update(uri, v.get<int>() + (*s).value("value",0)); } } else if (op == "-=") { auto v = ftl::config::get(uri); if (v.is_number_float()) { ftl::config::update(uri, v.get<float>() - (*s).value("value",0.0f)); } else if (v.is_number_integer()) { ftl::config::update(uri, v.get<int>() - (*s).value("value",0)); } } else if (op == "=") { ftl::config::update(uri, (*s)["value"]); } } } } return false; } } void ftl::gui::Screen::draw(NVGcontext *ctx) { using namespace Eigen; Vector2f screenSize = size().cast<float>(); if (camera_) { imageSize = {camera_->width(), camera_->height()}; mImageID = camera_->getLeft().texture(); leftEye_ = mImageID; rightEye_ = camera_->getRight().texture(); //if (camera_->getChannel() != ftl::codecs::Channel::Left) { mImageID = rightEye_; } if (mImageID < std::numeric_limits<unsigned int>::max() && imageSize[0] > 0) { auto mScale = (screenSize.cwiseQuotient(imageSize).minCoeff()) * zoom_; Vector2f scaleFactor = mScale * imageSize.cwiseQuotient(screenSize); Vector2f positionInScreen(pos_x_, pos_y_); auto mOffset = (screenSize - (screenSize.cwiseProduct(scaleFactor))) / 2; Vector2f positionAfterOffset = positionInScreen + mOffset; Vector2f imagePosition = positionAfterOffset.cwiseQuotient(screenSize); //glEnable(GL_SCISSOR_TEST); //float r = screen->pixelRatio(); /* glScissor(positionInScreen.x() * r, (screenSize.y() - positionInScreen.y() - size().y()) * r, size().x() * r, size().y() * r);*/ mShader.bind(); glActiveTexture(GL_TEXTURE0); glBindTexture(GL_TEXTURE_2D, leftEye_); glActiveTexture(GL_TEXTURE1); glBindTexture(GL_TEXTURE_2D, (camera_->isStereo() && camera_->getRight().isValid()) ? rightEye_ : leftEye_); mShader.setUniform("image1", 0); mShader.setUniform("image2", 1); mShader.setUniform("blendAmount", (camera_->isStereo()) ? root_->value("blending", 0.5f) : 1.0f); mShader.setUniform("scaleFactor", scaleFactor); mShader.setUniform("position", imagePosition); mShader.drawIndexed(GL_TRIANGLES, 0, 2); //glDisable(GL_SCISSOR_TEST); } } nvgTextAlign(ctx, NVG_ALIGN_RIGHT); nvgText(ctx, screenSize[0]-10, screenSize[1]-20, status_.c_str(), NULL); /* Draw the user interface */ screen()->performLayout(ctx); nanogui::Screen::draw(ctx); } void ftl::gui::Screen::drawFast() { if (camera_) { camera_->captureFrame(); #ifdef HAVE_OPENVR if (isVR() && camera_->width() > 0 && camera_->getLeft().isValid() && camera_->getRight().isValid()) { //glBindTexture(GL_TEXTURE_2D, leftEye_); vr::Texture_t leftEyeTexture = {(void*)(uintptr_t)leftEye_, vr::TextureType_OpenGL, vr::ColorSpace_Gamma }; vr::VRCompositor()->Submit(vr::Eye_Left, &leftEyeTexture ); //glBindTexture(GL_TEXTURE_2D, rightEye_); vr::Texture_t rightEyeTexture = {(void*)(uintptr_t)rightEye_, vr::TextureType_OpenGL, vr::ColorSpace_Gamma }; vr::VRCompositor()->Submit(vr::Eye_Right, &rightEyeTexture ); glFlush(); } #endif } }