From 76e46ab54c0a75fe2c7e5514ae7b60c05bb89263 Mon Sep 17 00:00:00 2001 From: Nicolas Pope <nwpope@utu.fi> Date: Tue, 28 Jul 2020 11:00:53 +0300 Subject: [PATCH] Remove old GUI --- applications/gui/CMakeLists.txt | 48 -- applications/gui/src/camera.cpp | 923 ------------------------- applications/gui/src/camera.hpp | 210 ------ applications/gui/src/config_window.cpp | 232 ------- applications/gui/src/config_window.hpp | 33 - applications/gui/src/ctrl_window.cpp | 139 ---- applications/gui/src/ctrl_window.hpp | 35 - applications/gui/src/frameset_mgr.cpp | 9 - applications/gui/src/frameset_mgr.hpp | 17 - applications/gui/src/gltexture.cpp | 158 ----- applications/gui/src/gltexture.hpp | 52 -- applications/gui/src/main.cpp | 117 ---- applications/gui/src/media_panel.cpp | 272 -------- applications/gui/src/media_panel.hpp | 81 --- applications/gui/src/pose_window.cpp | 207 ------ applications/gui/src/pose_window.hpp | 41 -- applications/gui/src/record_window.cpp | 122 ---- applications/gui/src/record_window.hpp | 20 - applications/gui/src/scene.hpp | 25 - applications/gui/src/screen.cpp | 687 ------------------ applications/gui/src/screen.hpp | 129 ---- applications/gui/src/src_window.cpp | 479 ------------- applications/gui/src/src_window.hpp | 95 --- applications/gui/src/statsimage.cpp | 72 -- applications/gui/src/statsimage.hpp | 47 -- applications/gui/src/thumbview.cpp | 35 - applications/gui/src/thumbview.hpp | 29 - applications/gui/src/vr.cpp | 36 - applications/gui/src/vr.hpp | 57 -- applications/gui/test/CMakeLists.txt | 0 applications/gui/test/tests.cpp | 2 - 31 files changed, 4409 deletions(-) delete mode 100644 applications/gui/CMakeLists.txt delete mode 100644 applications/gui/src/camera.cpp delete mode 100644 applications/gui/src/camera.hpp delete mode 100644 applications/gui/src/config_window.cpp delete mode 100644 applications/gui/src/config_window.hpp delete mode 100644 applications/gui/src/ctrl_window.cpp delete mode 100644 applications/gui/src/ctrl_window.hpp delete mode 100644 applications/gui/src/frameset_mgr.cpp delete mode 100644 applications/gui/src/frameset_mgr.hpp delete mode 100644 applications/gui/src/gltexture.cpp delete mode 100644 applications/gui/src/gltexture.hpp delete mode 100644 applications/gui/src/main.cpp delete mode 100644 applications/gui/src/media_panel.cpp delete mode 100644 applications/gui/src/media_panel.hpp delete mode 100644 applications/gui/src/pose_window.cpp delete mode 100644 applications/gui/src/pose_window.hpp delete mode 100644 applications/gui/src/record_window.cpp delete mode 100644 applications/gui/src/record_window.hpp delete mode 100644 applications/gui/src/scene.hpp delete mode 100644 applications/gui/src/screen.cpp delete mode 100644 applications/gui/src/screen.hpp delete mode 100644 applications/gui/src/src_window.cpp delete mode 100644 applications/gui/src/src_window.hpp delete mode 100644 applications/gui/src/statsimage.cpp delete mode 100644 applications/gui/src/statsimage.hpp delete mode 100644 applications/gui/src/thumbview.cpp delete mode 100644 applications/gui/src/thumbview.hpp delete mode 100644 applications/gui/src/vr.cpp delete mode 100644 applications/gui/src/vr.hpp delete mode 100644 applications/gui/test/CMakeLists.txt delete mode 100644 applications/gui/test/tests.cpp diff --git a/applications/gui/CMakeLists.txt b/applications/gui/CMakeLists.txt deleted file mode 100644 index 8b552cb93..000000000 --- a/applications/gui/CMakeLists.txt +++ /dev/null @@ -1,48 +0,0 @@ -# Need to include staged files and libs -#include_directories(${PROJECT_SOURCE_DIR}/reconstruct/include) -#include_directories(${PROJECT_BINARY_DIR}) - -set(GUISRC - src/main.cpp - #src/ctrl_window.cpp - src/src_window.cpp - src/config_window.cpp - src/pose_window.cpp - src/screen.cpp - src/gltexture.cpp - src/camera.cpp - src/media_panel.cpp - src/thumbview.cpp - src/record_window.cpp - src/frameset_mgr.cpp -) - -if (HAVE_OPENVR) - list(APPEND GUISRC "src/vr.cpp") -endif() - -# Various preprocessor definitions have been generated by NanoGUI -add_definitions(${NANOGUI_EXTRA_DEFS}) - -# On top of adding the path to nanogui/include, you may need extras -include_directories(${NANOGUI_EXTRA_INCS}) - -add_executable(ftl-gui ${GUISRC}) -install(TARGETS ftl-gui DESTINATION bin COMPONENT gui) - -target_include_directories(ftl-gui PUBLIC - $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/include> - $<BUILD_INTERFACE:${PROJECT_SOURCE_DIR}/ext/nanogui/include> - $<INSTALL_INTERFACE:include> - PRIVATE src) - -#if (CUDA_FOUND) -#set_property(TARGET ftl-gui PROPERTY CUDA_SEPARABLE_COMPILATION ON) -#endif() - -#target_include_directories(cv-node PUBLIC ${PROJECT_SOURCE_DIR}/include) -target_link_libraries(ftl-gui ftlcommon ftlctrl ftlrgbd ftlstreams ftlrender Threads::Threads ${OpenCV_LIBS} openvr ftlnet nanogui ${NANOGUI_EXTRA_LIBS}) - -if (BUILD_TESTS) - add_subdirectory(test) -endif() diff --git a/applications/gui/src/camera.cpp b/applications/gui/src/camera.cpp deleted file mode 100644 index 5d3cb2acc..000000000 --- a/applications/gui/src/camera.cpp +++ /dev/null @@ -1,923 +0,0 @@ -#include "camera.hpp" -#include "pose_window.hpp" -#include "screen.hpp" -#include <nanogui/glutil.h> - -#include <ftl/profiler.hpp> - -#include <opencv2/imgproc.hpp> -#include <opencv2/imgcodecs.hpp> -#include <opencv2/cudaarithm.hpp> - -#include <ftl/operators/antialiasing.hpp> -#include <ftl/cuda/normals.hpp> -#include <ftl/render/colouriser.hpp> -#include <ftl/cuda/transform.hpp> -#include <ftl/operators/gt_analysis.hpp> -#include <ftl/operators/poser.hpp> -#include <ftl/cuda/colour_cuda.hpp> -#include <ftl/streams/parsers.hpp> - -#include <ftl/render/overlay.hpp> -#include "statsimage.hpp" - -#define LOGURU_REPLACE_GLOG 1 -#include <loguru.hpp> - -#include <fstream> - -#ifdef HAVE_OPENVR -#include "vr.hpp" -#endif - -using ftl::rgbd::isValidDepth; -using ftl::gui::GLTexture; -using ftl::gui::PoseWindow; -using ftl::codecs::Channel; -using ftl::codecs::Channels; -using cv::cuda::GpuMat; - - -static int vcamcount = 0; - -static Eigen::Affine3d create_rotation_matrix(float ax, float ay, float az) { - Eigen::Affine3d rx = - Eigen::Affine3d(Eigen::AngleAxisd(ax, Eigen::Vector3d(1, 0, 0))); - Eigen::Affine3d ry = - Eigen::Affine3d(Eigen::AngleAxisd(ay, Eigen::Vector3d(0, 1, 0))); - Eigen::Affine3d rz = - Eigen::Affine3d(Eigen::AngleAxisd(az, Eigen::Vector3d(0, 0, 1))); - return rz * rx * ry; -} - -ftl::gui::Camera::Camera(ftl::gui::Screen *screen, int fsmask, int fid, ftl::codecs::Channel c) - : screen_(screen), fsmask_(fsmask), fid_(fid), texture1_(GLTexture::Type::BGRA), texture2_(GLTexture::Type::BGRA), depth1_(GLTexture::Type::Float), channel_(c),channels_(0u) { - - eye_ = Eigen::Vector3d::Zero(); - neye_ = Eigen::Vector4d::Zero(); - rotmat_.setIdentity(); - - //up_ = Eigen::Vector3f(0,1.0f,0); - lerpSpeed_ = 0.999f; - sdepth_ = false; - ftime_ = (float)glfwGetTime(); - pause_ = false; - -#ifdef HAVE_OPENVR - vr_mode_ = false; -#endif - - //channel_ = Channel::Left; - - channels_ += c; - //channels_ += Channel::Depth; - width_ = 0; - height_ = 0; - - // Create pose window... - //posewin_ = new PoseWindow(screen, src_->getURI()); - //posewin_->setTheme(screen->windowtheme); - //posewin_->setVisible(false); - posewin_ = nullptr; - renderer_ = nullptr; - renderer2_ = nullptr; - post_pipe_ = nullptr; - record_stream_ = nullptr; - transform_ix_ = -1; - stereo_ = false; - rx_ = 0; - ry_ = 0; - framesets_ = nullptr; - - colouriser_ = ftl::create<ftl::render::Colouriser>(screen->root(), "colouriser"); - overlayer_ = ftl::create<ftl::overlay::Overlay>(screen->root(), "overlay"); - - // Is virtual camera? - if (fid == 255) { - renderer_ = ftl::create<ftl::render::CUDARender>(screen_->root(), std::string("vcam")+std::to_string(vcamcount++)); - // Allow mask to be changed - fsmask_ = renderer_->value("fsmask", fsmask_); - renderer_->on("fsmask", [this](const ftl::config::Event &e) { - fsmask_ = renderer_->value("fsmask", fsmask_); - }); - - // Allow Pose origin to be changed - pose_source_ = renderer_->value("pose_source", pose_source_); - renderer_->on("pose_source", [this](const ftl::config::Event &e) { - pose_source_ = renderer_->value("pose_source", pose_source_); - }); - - intrinsics_ = ftl::create<ftl::Configurable>(renderer_, "intrinsics"); - - state_.getLeft() = ftl::rgbd::Camera::from(intrinsics_); - state_.getRight() = state_.getLeft(); - - intrinsics_->on("width", [this](const ftl::config::Event &e) { - state_.getLeft() = ftl::rgbd::Camera::from(intrinsics_); - state_.getRight() = state_.getLeft(); - }); - - intrinsics_->on("focal", [this](const ftl::config::Event &e) { - state_.getLeft() = ftl::rgbd::Camera::from(intrinsics_); - state_.getRight() = state_.getLeft(); - }); - - { - Eigen::Matrix4d pose; - pose.setIdentity(); - state_.setPose(pose); - - for (auto &t : transforms_) { - t.setIdentity(); - } - } - { - double camera_initial_x = intrinsics_->value("camera_x", 0.0); - double camera_initial_y = intrinsics_->value("camera_y", -1.75); - double camera_initial_z = intrinsics_->value("camera_z", 0.0); - - double lookat_initial_x = intrinsics_->value("lookat_x", 1.0); - double lookat_initial_y = intrinsics_->value("lookat_y", 0.0); - double lookat_initial_z = intrinsics_->value("lookat_z", 0.0); - - Eigen::Vector3f head(camera_initial_x, camera_initial_y, camera_initial_z); - Eigen::Vector3f lookat(lookat_initial_x, lookat_initial_y, lookat_initial_z); - // TODO up vector - Eigen::Matrix4f pose = nanogui::lookAt(head, head+lookat, Eigen::Vector3f(0.0f, 1.0f, 0.0f)); - - eye_ = Eigen::Vector3d(camera_initial_x, camera_initial_y, camera_initial_z); - neye_ = Eigen::Vector4d(eye_(0), eye_(1), eye_(2), 0.0); - rotmat_ = pose.cast<double>(); - rotmat_.block(0, 3, 3, 1).setZero(); - } - } -} - -ftl::gui::Camera::~Camera() { - //delete writer_; - //delete fileout_; -} - -void ftl::gui::Camera::drawUpdated(std::vector<ftl::rgbd::FrameSet*> &fss) { - // Only draw if frameset updated. - if (!stale_frame_.test_and_set()) { - draw(fss); - } -} - -void ftl::gui::Camera::draw(std::vector<ftl::rgbd::FrameSet*> &fss) { - if (fid_ != 255) { - for (auto *fs : fss) { - if (!usesFrameset(fs->id)) continue; - UNIQUE_LOCK(fs->mtx, lk); - - ftl::rgbd::Frame *frame = nullptr; - - if ((size_t)fid_ >= fs->frames.size()) return; - if (!fs->hasFrame(fid_)) return; - - frame = &fs->frames[fid_]; - if (!frame->hasChannel(channel_)) return; - - auto &buf = colouriser_->colourise(*frame, channel_, 0); - auto &buf2 = frame->getTexture<uchar4>(Channel::Colour); - ftl::cuda::compositeInverse(buf2, buf, 0); - - // For non-virtual cameras, copy the CUDA texture into the opengl - // texture device-to-device. - texture1_.make(buf.width(), buf.height()); - auto dst1 = texture1_.map(0); - cudaMemcpy2D(dst1.data, dst1.step1(), buf.devicePtr(), buf.pitch(), buf.width()*4, buf.height(), cudaMemcpyDeviceToDevice); - ftl::cuda::flip<uchar4>(dst1, 0); - texture1_.unmap(0); - - depth1_.make(buf.width(), buf.height()); - dst1 = depth1_.map(0); - dst1.setTo(cv::Scalar(0.5f)); - depth1_.unmap(0); - - width_ = texture1_.width(); - height_ = texture1_.height(); - return; - } - } - //if (fsid_ >= fss.size()) return; - - //auto &fs = *fss[fsid_]; - - _applyPoseEffects(fss); - - UNIQUE_LOCK(mutex_, lk2); - //state_.getLeft().fx = intrinsics_->value("focal", 700.0f); - //state_.getLeft().fy = state_.getLeft().fx; - _draw(fss); -} - -std::pair<const ftl::rgbd::Frame *, const ftl::codecs::Face *> ftl::gui::Camera::_selectFace(std::vector<ftl::rgbd::FrameSet*> &fss) { - for (auto *fset : fss) { - for (const auto &f : fset->frames) { - if (f.hasChannel(Channel::Faces)) { - std::vector<ftl::codecs::Face> data; - f.get(Channel::Faces, data); - - if (data.size() > 0) { - return {&f,&(*data.rbegin())}; - } - } - } - } - return {nullptr, nullptr}; -} - -void ftl::gui::Camera::_generateWindow(const ftl::rgbd::Frame &f, const ftl::codecs::Face &face, Eigen::Matrix4d &pose_adjust, ftl::render::ViewPort &vp) { - auto cam = ftl::rgbd::Camera::from(intrinsics_); - auto d = face; - - float screenWidth = intrinsics_->value("screen_size", 0.6f); // In meters - float screenHeight = (9.0f/16.0f) * screenWidth; - - float screenDistance = (d.depth > cam.minDepth && d.depth < cam.maxDepth) ? d.depth : intrinsics_->value("screen_dist_default", 0.5f); // Face distance from screen in meters - - auto pos = f.getLeft().screenToCam(float(d.box.x+(d.box.width/2)), float(d.box.y+(d.box.height/2)), screenDistance); - Eigen::Vector3f eye; - eye[0] = -pos.x; - eye[1] = pos.y; - eye[2] = -pos.z; - //eye[3] = 0; - - Eigen::Translation3f trans(eye); - Eigen::Affine3f t(trans); - Eigen::Matrix4f viewPose = t.matrix(); - - // Calculate where the screen is within current camera space - Eigen::Vector4f p1 = viewPose.cast<float>() * (Eigen::Vector4f(screenWidth/2.0, screenHeight/2.0, 0, 1)); - Eigen::Vector4f p2 = viewPose.cast<float>() * (Eigen::Vector4f(screenWidth/2.0, -screenHeight/2.0, 0, 1)); - Eigen::Vector4f p3 = viewPose.cast<float>() * (Eigen::Vector4f(-screenWidth/2.0, screenHeight/2.0, 0, 1)); - Eigen::Vector4f p4 = viewPose.cast<float>() * (Eigen::Vector4f(-screenWidth/2.0, -screenHeight/2.0, 0, 1)); - p1 = p1 / p1[3]; - p2 = p2 / p2[3]; - p3 = p3 / p3[3]; - p4 = p4 / p4[3]; - float2 p1screen = cam.camToScreen<float2>(make_float3(p1[0],p1[1],p1[2])); - float2 p2screen = cam.camToScreen<float2>(make_float3(p2[0],p2[1],p2[2])); - float2 p3screen = cam.camToScreen<float2>(make_float3(p3[0],p3[1],p3[2])); - float2 p4screen = cam.camToScreen<float2>(make_float3(p4[0],p4[1],p4[2])); - - std::vector<cv::Point2f> quad_pts; - std::vector<cv::Point2f> squre_pts; - quad_pts.push_back(cv::Point2f(p1screen.x,p1screen.y)); - quad_pts.push_back(cv::Point2f(p2screen.x,p2screen.y)); - quad_pts.push_back(cv::Point2f(p3screen.x,p3screen.y)); - quad_pts.push_back(cv::Point2f(p4screen.x,p4screen.y)); - squre_pts.push_back(cv::Point2f(0,0)); - squre_pts.push_back(cv::Point2f(0,cam.height)); - squre_pts.push_back(cv::Point2f(cam.width,0)); - squre_pts.push_back(cv::Point2f(cam.width,cam.height)); - - cv::Mat transmtx = cv::getPerspectiveTransform(quad_pts,squre_pts); - //cv::Mat transformed = cv::Mat::zeros(overlay_.rows, overlay_.cols, CV_8UC4); - //cv::warpPerspective(im1_, im1_, transmtx, im1_.size()); - - // TODO: Use the transmtx above for perspective distortion.. - - //ftl::render::ViewPort vp; - vp.x = std::min(p4screen.x, std::min(p3screen.x, std::min(p1screen.x,p2screen.x))); - vp.y = std::min(p4screen.y, std::min(p3screen.y, std::min(p1screen.y,p2screen.y))); - vp.width = std::max(p4screen.x, std::max(p3screen.x, std::max(p1screen.x,p2screen.x))) - vp.x; - vp.height = std::max(p4screen.y, std::max(p3screen.y, std::max(p1screen.y,p2screen.y))) - vp.y; - /*vp.warpMatrix.entries[0] = transmtx.at<float>(0,0); - vp.warpMatrix.entries[1] = transmtx.at<float>(1,0); - vp.warpMatrix.entries[2] = transmtx.at<float>(2,0); - vp.warpMatrix.entries[3] = transmtx.at<float>(0,1); - vp.warpMatrix.entries[4] = transmtx.at<float>(1,1); - vp.warpMatrix.entries[5] = transmtx.at<float>(2,1); - vp.warpMatrix.entries[6] = transmtx.at<float>(0,2); - vp.warpMatrix.entries[7] = transmtx.at<float>(1,2); - vp.warpMatrix.entries[8] = transmtx.at<float>(2,2); - vp.warpMatrix = vp.warpMatrix.getInverse(); //.getInverse();*/ - //renderer_->setViewPort(ftl::render::ViewPortMode::Warping, vp); - - pose_adjust = viewPose.cast<double>(); -} - -void ftl::gui::Camera::_applyPoseEffects(std::vector<ftl::rgbd::FrameSet*> &fss) { - if (renderer_->value("window_effect", false)) { - auto [frame,face] = _selectFace(fss); - if (face) { - Eigen::Matrix4d windowPose; - ftl::render::ViewPort windowViewPort; - _generateWindow(*frame, *face, windowPose, windowViewPort); - - // Apply the window effect - renderer_->setViewPort(ftl::render::ViewPortMode::Stretch, windowViewPort); - state_.getPose() = windowPose * state_.getPose(); - } - } -} - -void ftl::gui::Camera::setStereo(bool v) { - UNIQUE_LOCK(mutex_, lk); - - if (isVirtual()) { - stereo_ = v; - } else if (v && availableChannels().has(Channel::Right)) { - stereo_ = true; - } else { - stereo_ = false; - } -} - -static ftl::codecs::Channel mapToSecondChannel(ftl::codecs::Channel c) { - switch (c) { - case Channel::Depth : return Channel::Depth2; - case Channel::Normals : return Channel::Normals2; - default: return c; - } -} - -void ftl::gui::Camera::_draw(std::vector<ftl::rgbd::FrameSet*> &fss) { - frame_.reset(); - frame_.setOrigin(&state_); - - // Make sure an OpenGL pixel buffer exists - texture1_.make(state_.getLeft().width, state_.getLeft().height); - depth1_.make(state_.getLeft().width, state_.getLeft().height); - if (isStereo()) texture2_.make(state_.getRight().width, state_.getRight().height); - - // Map the GL pixel buffer to a GpuMat - frame_.create<cv::cuda::GpuMat>(Channel::Colour) = texture1_.map(renderer_->getCUDAStream()); - frame_.create<cv::cuda::GpuMat>(Channel::Depth) = depth1_.map(renderer_->getCUDAStream()); - frame_.createTexture<float>(Channel::Depth); - if (isStereo()) frame_.create<cv::cuda::GpuMat>(Channel::Colour2) = texture2_.map((renderer2_) ? renderer2_->getCUDAStream() : 0); - - // TODO: Remove; - overlay_.create(state_.getLeft().height, state_.getLeft().width, CV_8UC4); - //frame_.create<cv::Mat>(Channel::Overlay) = overlay_; - - //overlay_.setTo(cv::Scalar(0,0,0,0)); - //bool enable_overlay = overlayer_->value("enabled", false); - - { - FTL_Profile("Render",0.034); - renderer_->begin(frame_, Channel::Colour); - if (isStereo()) { - if (!renderer2_) { - renderer2_ = ftl::create<ftl::render::CUDARender>(screen_->root(), std::string("vcam")+std::to_string(vcamcount++)); - } - renderer2_->begin(frame_, Channel::Colour2); - } - - try { - for (auto *fs : fss) { - if (!usesFrameset(fs->id)) continue; - - fs->mtx.lock(); - renderer_->submit(fs, ftl::codecs::Channels<0>(Channel::Colour), transforms_[fs->id]); - if (isStereo()) renderer2_->submit(fs, ftl::codecs::Channels<0>(Channel::Colour), transforms_[fs->id]); - - //if (enable_overlay) { - // Generate and upload an overlay image. - // overlayer_->apply(*fs, overlay_, state_); - // frame_.upload(Channel::Overlay, renderer_->getCUDAStream()); - //} - } - - renderer_->render(); - if (isStereo()) renderer2_->render(); - - if (channel_ != Channel::Left && channel_ != Channel::Right && channel_ != Channel::None) { - renderer_->blend(channel_); - if (isStereo()) { - renderer2_->blend(mapToSecondChannel(channel_)); - } - } - - //if (enable_overlay) { - // renderer_->blend(Channel::Overlay); - //} - - renderer_->end(); - if (isStereo()) renderer2_->end(); - } catch(std::exception &e) { - LOG(ERROR) << "Exception in render: " << e.what(); - } - - for (auto *fs : fss) { - if (!usesFrameset(fs->id)) continue; - fs->mtx.unlock(); - } - } - - if (!post_pipe_) { - post_pipe_ = ftl::config::create<ftl::operators::Graph>(screen_->root(), "post_filters"); - post_pipe_->append<ftl::operators::FXAA>("fxaa"); - post_pipe_->append<ftl::operators::GTAnalysis>("gtanalyse"); - } - - post_pipe_->apply(frame_, frame_, 0); - - channels_ = frame_.getChannels(); - - frame_.get<cv::cuda::GpuMat>(Channel::Depth).download(im_depth_); - cv::flip(im_depth_, im_depth_, 0); - - //frame_.get<cv::cuda::GpuMat>(Channel::Normals).download(im_normals_); - //im_normals_.createMatHeader().convertTo(im_normals_f_, CV_32FC4); - //cv::flip(im_normals_f_, im_normals_f_, 0); - - // Normalize depth map - frame_.get<cv::cuda::GpuMat>(Channel::Depth).convertTo(frame_.get<cv::cuda::GpuMat>(Channel::Depth), CV_32F, 1.0/8.0); - - width_ = texture1_.width(); - height_ = texture1_.height(); - - if (record_stream_ && record_stream_->active()) { - // TODO: Allow custom channel selection - ftl::rgbd::FrameSet fs2; - auto &f = fs2.frames.emplace_back(); - fs2.count = 1; - fs2.mask = 1; - //fs2.stale = false; - fs2.set(ftl::data::FSFlag::STALE); - if (frame_.hasChannel(Channel::Colour2)) { - frame_.swapTo(Channels<0>(Channel::Colour) | Channel::Colour2, f); - ftl::cuda::flip(f.getTexture<uchar4>(Channel::Colour), 0); - ftl::cuda::flip(f.getTexture<uchar4>(Channel::Colour2), 0); - } else { - frame_.swapTo(Channels<0>(Channel::Colour), f); // Channel::Colour + Channel::Depth - ftl::cuda::flip(f.getTexture<uchar4>(Channel::Colour), 0); - } - - fs2.timestamp = ftl::timer::get_time(); - fs2.id = 0; - record_sender_->post(fs2); - record_stream_->select(0, Channels<0>(Channel::Colour)); - // Reverse the flip - if (f.hasChannel(Channel::Colour2)) { - ftl::cuda::flip(f.getTexture<uchar4>(Channel::Colour), 0); - ftl::cuda::flip(f.getTexture<uchar4>(Channel::Colour2), 0); - } else { - ftl::cuda::flip(f.getTexture<uchar4>(Channel::Colour), 0); - } - f.swapTo(Channels<0>(Channel::Colour), frame_); - } else if (do_snapshot_) { - do_snapshot_ = false; - cv::Mat flipped; - cv::Mat im1; - - frame_.get<cv::cuda::GpuMat>(Channel::Colour).download(im1); - - { - //UNIQUE_LOCK(mutex_, lk); - cv::flip(im1, flipped, 0); - } - cv::cvtColor(flipped, flipped, cv::COLOR_BGRA2BGR); - cv::imwrite(snapshot_filename_, flipped); - } - - // Unmap GL buffer from CUDA and finish updating GL texture - texture1_.unmap(renderer_->getCUDAStream()); - depth1_.unmap(renderer_->getCUDAStream()); - if (isStereo()) texture2_.unmap(renderer2_->getCUDAStream()); -} - -void ftl::gui::Camera::update(int fsid, const ftl::codecs::Channels<0> &c) { - if (!isVirtual() && ((1 << fsid) & fsmask_)) { - channels_ += c; - //if (c.has(Channel::Depth)) { - //channels_ += Channel::ColourNormals; - //} - } -} - -void ftl::gui::Camera::update(std::vector<ftl::rgbd::FrameSet*> &fss) { - UNIQUE_LOCK(mutex_, lk); - - framesets_ = &fss; - stale_frame_.clear(); - - if (screen_->activeCamera() == this) { - for (auto *fs : fss) { - if (!usesFrameset(fs->id)) continue; - - for (auto &f : fs->frames) { - //if (f.hasChanged(Channel::Pose)) { - f.patchPose(T_); - //} - } - } - } - - //if (fss.size() <= fsid_) return; - if (fid_ == 255) { - name_ = "Virtual Camera"; - } else { - for (auto *fs : fss) { - if (!usesFrameset(fs->id)) continue; - - ftl::rgbd::Frame *frame = nullptr; - - if ((size_t)fid_ >= fs->frames.size()) return; - frame = &fs->frames[fid_]; - channels_ = frame->getChannels(); - - if (frame->hasChannel(Channel::Messages)) { - msgs_.clear(); - frame->get(Channel::Messages, msgs_); - } - - auto n = frame->get<std::string>("name"); - if (n) { - name_ = *n; - } else { - name_ = "No name"; - } - state_.getLeft() = frame->getLeftCamera(); - return; - } - } -} - -void ftl::gui::Camera::setPose(const Eigen::Matrix4d &p) { - eye_[0] = p(0,3); - eye_[1] = p(1,3); - eye_[2] = p(2,3); - - double sx = Eigen::Vector3d(p(0,0), p(1,0), p(2,0)).norm(); - double sy = Eigen::Vector3d(p(0,1), p(1,1), p(2,1)).norm(); - double sz = Eigen::Vector3d(p(0,2), p(1,2), p(2,2)).norm(); - - Eigen::Matrix4d rot = p; - rot(0,3) = 0.0; - rot(1,3) = 0.0; - rot(2,3) = 0.0; - rot(0,0) = rot(0,0) / sx; - rot(1,0) = rot(1,0) / sx; - rot(2,0) = rot(2,0) / sx; - rot(0,1) = rot(0,1) / sy; - rot(1,1) = rot(1,1) / sy; - rot(2,1) = rot(2,1) / sy; - rot(0,2) = rot(0,2) / sz; - rot(1,2) = rot(1,2) / sz; - rot(2,2) = rot(2,2) / sz; - rotmat_ = rot; -} - -void ftl::gui::Camera::mouseMovement(int rx, int ry, int button) { - //if (!src_->hasCapabilities(ftl::rgbd::kCapMovable)) return; - if (fid_ < 255) return; - if (button == 1) { - rx_ += rx; - ry_ += ry; - - /*float rrx = ((float)ry * 0.2f * delta_); - //orientation_[2] += std::cos(orientation_[1])*((float)rel[1] * 0.2f * delta_); - float rry = (float)rx * 0.2f * delta_; - float rrz = 0.0; - - - Eigen::Affine3d r = create_rotation_matrix(rrx, -rry, rrz); - rotmat_ = rotmat_ * r.matrix();*/ - } -} - -void ftl::gui::Camera::keyMovement(int key, int modifiers) { - //if (!src_->hasCapabilities(ftl::rgbd::kCapMovable)) return; - if (fid_ < 255) return; - if (key == 263 || key == 262) { - float mag = (modifiers & 0x1) ? 0.01f : 0.1f; - float scalar = (key == 263) ? -mag : mag; - neye_ += rotmat_*Eigen::Vector4d(scalar,0.0,0.0,1.0); - return; - } else if (key == 264 || key == 265) { - float mag = (modifiers & 0x1) ? 0.01f : 0.1f; - float scalar = (key == 264) ? -mag : mag; - neye_ += rotmat_*Eigen::Vector4d(0.0,0.0,scalar,1.0); - return; - } else if (key == 266 || key == 267) { - float mag = (modifiers & 0x1) ? 0.01f : 0.1f; - float scalar = (key == 266) ? -mag : mag; - neye_ += rotmat_*Eigen::Vector4d(0.0,scalar,0.0,1.0); - return; - } else if (key >= '0' && key <= '5' && modifiers == 2) { // Ctrl+NUMBER - int ix = key - (int)('0'); - transform_ix_ = ix-1; - return; - } -} - -void ftl::gui::Camera::showPoseWindow() { - posewin_->setVisible(true); -} - -void ftl::gui::Camera::showSettings() { - -} - -#ifdef HAVE_OPENVR -bool ftl::gui::Camera::setVR(bool on) { - if (on == vr_mode_) { - LOG(WARNING) << "VR mode already enabled"; - return on; - } - vr_mode_ = on; - - if (on) { - setStereo(true); - - UNIQUE_LOCK(mutex_, lk); - //src_->set("baseline", baseline_); - state_.getLeft().baseline = baseline_; - - Eigen::Matrix3d intrinsic; - - unsigned int size_x, size_y; - screen_->getVR()->GetRecommendedRenderTargetSize(&size_x, &size_y); - state_.getLeft().width = size_x; - state_.getLeft().height = size_y; - state_.getRight().width = size_x; - state_.getRight().height = size_y; - - intrinsic = getCameraMatrix(screen_->getVR(), vr::Eye_Left); - CHECK(intrinsic(0, 2) < 0 && intrinsic(1, 2) < 0); - state_.getLeft().fx = intrinsic(0,0); - state_.getLeft().fy = intrinsic(0,0); - state_.getLeft().cx = intrinsic(0,2); - state_.getLeft().cy = intrinsic(1,2); - - intrinsic = getCameraMatrix(screen_->getVR(), vr::Eye_Right); - CHECK(intrinsic(0, 2) < 0 && intrinsic(1, 2) < 0); - state_.getRight().fx = intrinsic(0,0); - state_.getRight().fy = intrinsic(0,0); - state_.getRight().cx = intrinsic(0,2); - state_.getRight().cy = intrinsic(1,2); - - vr_mode_ = true; - } - else { - vr_mode_ = false; - setStereo(false); - - UNIQUE_LOCK(mutex_, lk); - state_.getLeft() = ftl::rgbd::Camera::from(intrinsics_); - state_.getRight() = state_.getLeft(); - } - - return vr_mode_; -} -#endif - -void ftl::gui::Camera::setChannel(Channel c) { - UNIQUE_LOCK(mutex_, lk); - channel_ = c; -} - -/*static void drawEdges( const cv::Mat &in, cv::Mat &out, - const int ksize = 3, double weight = -1.0, const int threshold = 32, - const int threshold_type = cv::THRESH_TOZERO) -{ - cv::Mat edges; - cv::Laplacian(in, edges, 8, ksize); - cv::threshold(edges, edges, threshold, 255, threshold_type); - - cv::Mat edges_color(in.size(), CV_8UC4); - cv::addWeighted(edges, weight, out, 1.0, 0.0, out, CV_8UC4); -}*/ - - -void ftl::gui::Camera::active(bool a) { - if (a) { - - } else { - neye_[0] = eye_[0]; - neye_[1] = eye_[1]; - neye_[2] = eye_[2]; - } -} - -void ftl::gui::Camera::drawOverlay(const Eigen::Vector2f &s) { - if (!framesets_) return; - //UNIQUE_LOCK(mutex_,lk); - for (auto *fs : *framesets_) { - if (!usesFrameset(fs->id)) continue; - - // Generate and upload an overlay image. - overlayer_->draw(*fs, state_, s); - } -} - -const void ftl::gui::Camera::captureFrame() { - float now = (float)glfwGetTime(); - if (!screen_->isVR() && (now - ftime_) < 0.04f) return; - - delta_ = now - ftime_; - ftime_ = now; - - //LOG(INFO) << "Frame delta: " << delta_; - - //if (src_ && src_->isReady()) { - if (width_ > 0 && height_ > 0) { - Eigen::Matrix4d viewPose; - - if (screen_->isVR()) { - #ifdef HAVE_OPENVR - - vr::VRCompositor()->SetTrackingSpace(vr::TrackingUniverseStanding); - vr::VRCompositor()->WaitGetPoses(rTrackedDevicePose_, vr::k_unMaxTrackedDeviceCount, NULL, 0 ); - - if (isStereo() && rTrackedDevicePose_[vr::k_unTrackedDeviceIndex_Hmd].bPoseIsValid ) - { - Eigen::Matrix4d eye_l = ConvertSteamVRMatrixToMatrix4( - vr::VRSystem()->GetEyeToHeadTransform(vr::Eye_Left)); - - //Eigen::Matrix4d eye_r = ConvertSteamVRMatrixToMatrix4( - // vr::VRSystem()->GetEyeToHeadTransform(vr::Eye_Left)); - - float baseline_in = 2.0 * eye_l(0, 3); - - if (baseline_in != baseline_) { - baseline_ = baseline_in; - //src_->set("baseline", baseline_); - state_.getLeft().baseline = baseline_; - state_.getRight().baseline = baseline_; - } - Eigen::Matrix4d pose = ConvertSteamVRMatrixToMatrix4(rTrackedDevicePose_[vr::k_unTrackedDeviceIndex_Hmd].mDeviceToAbsoluteTracking); - Eigen::Vector3d ea = pose.block<3, 3>(0, 0).eulerAngles(0, 1, 2); - - Eigen::Vector3d vreye; - vreye[0] = pose(0, 3); - vreye[1] = -pose(1, 3); - vreye[2] = -pose(2, 3); - - // NOTE: If modified, should be verified with VR headset! - Eigen::Matrix3d R; - R = Eigen::AngleAxisd(ea[0], Eigen::Vector3d::UnitX()) * - Eigen::AngleAxisd(-ea[1], Eigen::Vector3d::UnitY()) * - Eigen::AngleAxisd(-ea[2], Eigen::Vector3d::UnitZ()); - - //double rd = 180.0 / 3.141592; - //LOG(INFO) << "rotation x: " << ea[0] *rd << ", y: " << ea[1] * rd << ", z: " << ea[2] * rd; - // pose.block<3, 3>(0, 0) = R; - - rotmat_.block(0, 0, 3, 3) = R; - - // TODO: Apply a rotation to orient also - - eye_[0] += (neye_[0] - eye_[0]) * lerpSpeed_ * delta_; - eye_[1] += (neye_[1] - eye_[1]) * lerpSpeed_ * delta_; - eye_[2] += (neye_[2] - eye_[2]) * lerpSpeed_ * delta_; - - Eigen::Translation3d trans(eye_ + vreye); - Eigen::Affine3d t(trans); - viewPose = t.matrix() * rotmat_; - - } else { - //LOG(ERROR) << "No VR Pose"; - } - #endif - } else { - if (pose_source_.size() == 0) { - // Use mouse to move camera - - float rrx = ((float)ry_ * 0.2f * delta_); - float rry = (float)rx_ * 0.2f * delta_; - float rrz = 0.0; - - Eigen::Affine3d r = create_rotation_matrix(rrx, -rry, rrz); - rotmat_ = rotmat_ * r.matrix(); - - rx_ = 0; - ry_ = 0; - - eye_[0] += (neye_[0] - eye_[0]) * lerpSpeed_ * delta_; - eye_[1] += (neye_[1] - eye_[1]) * lerpSpeed_ * delta_; - eye_[2] += (neye_[2] - eye_[2]) * lerpSpeed_ * delta_; - - Eigen::Translation3d trans(eye_); - Eigen::Affine3d t(trans); - viewPose = t.matrix() * rotmat_; - } else { - // Use some other pose source. - if (!ftl::operators::Poser::get(pose_source_, viewPose)) { - LOG(ERROR) << "Missing pose: " << pose_source_; - } - } - } - - { - UNIQUE_LOCK(mutex_, lk); - - if (isVirtual()) { - if (transform_ix_ == -1) { - state_.setPose(viewPose); - } else if (transform_ix_ >= 0) { - transforms_[transform_ix_] = viewPose; - } - } - } - - if (framesets_) draw(*framesets_); - } - - //return texture1_; -} - -void ftl::gui::Camera::snapshot(const std::string &filename) { - /*cv::Mat flipped; - - { - UNIQUE_LOCK(mutex_, lk); - //cv::flip(im1_, flipped, 0); - } - cv::cvtColor(flipped, flipped, cv::COLOR_BGRA2BGR); - cv::imwrite(filename, flipped);*/ - snapshot_filename_ = filename; - do_snapshot_ = true; -} - -void ftl::gui::Camera::startVideoRecording(const std::string &filename, const std::string &uri) { - if (!record_stream_) { - file_stream_ = ftl::create<ftl::stream::File>(screen_->root(), "video2d"); - file_stream_->setMode(ftl::stream::File::Mode::Write); - net_stream_ = ftl::create<ftl::stream::Net>(screen_->root(), "liveStream", screen_->net()); - - record_stream_ = ftl::create<ftl::stream::Broadcast>(screen_->root(), "recordStream"); - //record_stream_->add(file_stream_); - //record_stream_->add(net_stream_); - - record_sender_ = ftl::create<ftl::stream::Sender>(screen_->root(), "videoEncode"); - record_sender_->value("codec", 2); // Default H264 - record_sender_->set("iframes", 50); // Add iframes by default - record_sender_->value("stereo", false); // If both channels, then default to stereo - - record_sender_->onRequest([this](const ftl::codecs::StreamPacket &spkt, const ftl::codecs::Packet &pkt) { - if (spkt.channel == ftl::codecs::Channel::Pose) { - auto pose = ftl::stream::parsePose(pkt); - ftl::operators::Poser::set(std::string("live"), pose); - } - }); - } - - if (record_stream_->active()) return; - - record_stream_->clear(); - - if (filename.size() > 0) { - file_stream_->set("filename", filename); - record_stream_->add(file_stream_); - } - - if (uri.size() > 0) { - net_stream_->set("uri", uri); - record_stream_->add(net_stream_); - } - - record_sender_->setStream(record_stream_); - - LOG(INFO) << "About to record"; - if (record_stream_->begin()) LOG(INFO) << "Recording started..."; -} - -void ftl::gui::Camera::stopVideoRecording() { - if (record_stream_ && record_stream_->active()) record_stream_->end(); -} - -float ftl::gui::Camera::getDepth(int x, int y) { - if (x < 0 || y < 0) { return NAN; } - UNIQUE_LOCK(mutex_, lk); - if (x >= im_depth_.cols || y >= im_depth_.rows) { return NAN; } - LOG(INFO) << y << ", " << x; - return im_depth_.createMatHeader().at<float>(y, x); -} - -cv::Point3f ftl::gui::Camera::getPoint(int x, int y) { - if (x < 0 || y < 0) { return cv::Point3f(); } - UNIQUE_LOCK(mutex_, lk); - LOG(INFO) << y << ", " << x; - if (x >= im_depth_.cols || y >= im_depth_.rows) { return cv::Point3f(); } - float d = im_depth_.createMatHeader().at<float>(y, x); - - auto point = frame_.getLeftCamera().screenToCam(x, y, d); - Eigen::Vector4d p(point.x, point.y, point.z, 1.0f); - Eigen::Matrix4d pose = frame_.getPose(); - Eigen::Vector4d point_eigen = pose * p; - return cv::Point3f(point_eigen(0), point_eigen(1), point_eigen(2)); -} - -/* -cv::Point3f ftl::gui::Camera::getNormal(int x, int y) { - UNIQUE_LOCK(mutex_, lk); - LOG(INFO) << y << ", " << x; - if (x >= im_normals_.cols || y >= im_normals_.rows) { return cv::Point3f(); } - auto n = im_normals_f_.at<cv::Vec4f>(y, x); - return cv::Point3f(n[0], n[1], n[2]); -} -*/ - -void ftl::gui::Camera::setTransform(const Eigen::Matrix4d &T) { - T_ = T * T_; -} - -Eigen::Matrix4d ftl::gui::Camera::getTransform() const { - return T_; -} diff --git a/applications/gui/src/camera.hpp b/applications/gui/src/camera.hpp deleted file mode 100644 index 730ba380d..000000000 --- a/applications/gui/src/camera.hpp +++ /dev/null @@ -1,210 +0,0 @@ -#ifndef _FTL_GUI_CAMERA_HPP_ -#define _FTL_GUI_CAMERA_HPP_ - -#include <ftl/rgbd/frameset.hpp> -#include <ftl/render/CUDARender.hpp> -#include <ftl/render/overlay.hpp> -#include <ftl/codecs/writer.hpp> -#include "gltexture.hpp" - -#include <ftl/streams/filestream.hpp> -#include <ftl/streams/netstream.hpp> -#include <ftl/streams/sender.hpp> -#include <ftl/codecs/faces.hpp> - -#include <string> -#include <array> - -#ifdef HAVE_OPENVR -#include <openvr/openvr.h> -#endif - -class StatisticsImage; - -namespace ftl { -namespace gui { - -class Screen; -class PoseWindow; - -class Camera { - public: - Camera(ftl::gui::Screen *screen, int fsmask, int fid, ftl::codecs::Channel chan=ftl::codecs::Channel::Colour); - ~Camera(); - - Camera(const Camera &)=delete; - - int width() const { return width_; } - int height() const { return height_; } - - int getFramesetMask() const { return fsmask_; } - - bool usesFrameset(int id) const { return fsmask_ & (1 << id); } - - void setPose(const Eigen::Matrix4d &p); - - void mouseMovement(int rx, int ry, int button); - void keyMovement(int key, int modifiers); - - void showPoseWindow(); - void showSettings(); - - void setChannel(ftl::codecs::Channel c); - const ftl::codecs::Channel getChannel() { return channel_; } - - void togglePause(); - void isPaused(); - inline bool isVirtual() const { return fid_ == 255; } - const ftl::codecs::Channels<0> &availableChannels() { return channels_; } - inline bool isStereo() const { return stereo_; } - - void setStereo(bool v); - - /** - * Main function to obtain latest frames. - */ - void update(std::vector<ftl::rgbd::FrameSet *> &fss); - - /** - * Update the available channels. - */ - void update(int fsid, const ftl::codecs::Channels<0> &c); - - /** - * Draw virtual camera only if the frameset has been updated since last - * draw. - */ - void drawUpdated(std::vector<ftl::rgbd::FrameSet*> &fss); - - void draw(std::vector<ftl::rgbd::FrameSet*> &fss); - - void drawOverlay(const Eigen::Vector2f &); - - inline int64_t getFrameTimeMS() const { return int64_t(delta_ * 1000.0f); } - - const ftl::rgbd::Camera &getIntrinsics() const { return state_.getLeft(); } - const Eigen::Matrix4d getPose() const { UNIQUE_LOCK(mutex_, lk); return state_.getPose(); } - - /** - * @internal. Used to inform the camera if it is the active camera or not. - */ - void active(bool); - - const void captureFrame(); - const GLTexture &getLeft() const { return texture1_; } - const GLTexture &getRight() const { return texture2_; } - const GLTexture &getDepth() const { return depth1_; } - - void snapshot(const std::string &filename); - - void startVideoRecording(const std::string &filename, const std::string &uri); - - void stopVideoRecording(); - - //nlohmann::json getMetaData(); - - const std::string &name() const { return name_; } - - StatisticsImage *stats_ = nullptr; - - float getDepth(int x, int y); - float getDepth(float x, float y) { return getDepth((int) round(x), (int) round(y)); } - cv::Point3f getPoint(int x, int y); - cv::Point3f getPoint(float x, float y) { return getPoint((int) round(x), (int) round(y)); } - //cv::Point3f getNormal(int x, int y); - //cv::Point3f getNormal(float x, float y) { return getNormal((int) round(x), (int) round(y)); } - void setTransform(const Eigen::Matrix4d &T); - Eigen::Matrix4d getTransform() const; - const std::vector<std::string> &getMessages() const { return msgs_; } - -#ifdef HAVE_OPENVR - bool isVR() { return vr_mode_; } - bool setVR(bool on); -#else - bool isVR() { return false; } -#endif - - private: - - Screen *screen_; - unsigned int fsmask_; // Frameset Mask - int fid_; - - int width_; - int height_; - - GLTexture texture1_; // first channel (always left at the moment) - GLTexture texture2_; // second channel ("right") - GLTexture depth1_; - - ftl::gui::PoseWindow *posewin_; - //nlohmann::json meta_; - Eigen::Vector4d neye_; - Eigen::Vector3d eye_; - //Eigen::Vector3f orientation_; - Eigen::Matrix4d rotmat_; - float ftime_; - float delta_; - float lerpSpeed_; - bool sdepth_; - bool pause_; - bool do_snapshot_ = false; - std::string pose_source_; - std::string snapshot_filename_; - ftl::codecs::Channel channel_; - ftl::codecs::Channels<0> channels_; - - cv::cuda::HostMem im_depth_; - //cv::cuda::HostMem im_normals_; - cv::Mat im_normals_f_; - - cv::Mat overlay_; // first channel (left) - bool stereo_; - std::atomic_flag stale_frame_; - int rx_; - int ry_; - std::vector<ftl::rgbd::FrameSet*> *framesets_; - - ftl::render::CUDARender *renderer_; - ftl::render::CUDARender *renderer2_; - ftl::render::Colouriser *colouriser_; - ftl::overlay::Overlay *overlayer_; - - ftl::Configurable *intrinsics_; - ftl::operators::Graph *post_pipe_; - ftl::rgbd::Frame frame_; - ftl::rgbd::FrameState state_; - ftl::stream::File *file_stream_; - ftl::stream::Net *net_stream_; - ftl::stream::Broadcast *record_stream_; - ftl::stream::Sender *record_sender_; - - std::string name_; - - std::vector<std::string> msgs_; - - int transform_ix_; - std::array<Eigen::Matrix4d,ftl::stream::kMaxStreams> transforms_; // Frameset transforms for virtual cam - Eigen::Matrix4d T_ = Eigen::Matrix4d::Identity(); - - mutable MUTEX mutex_; - - #ifdef HAVE_OPENVR - vr::TrackedDevicePose_t rTrackedDevicePose_[ vr::k_unMaxTrackedDeviceCount ]; - bool vr_mode_; - float baseline_; - #endif - - void _downloadFrames(ftl::cuda::TextureObject<uchar4> &, ftl::cuda::TextureObject<uchar4> &); - void _downloadFrames(ftl::cuda::TextureObject<uchar4> &); - void _downloadFrames(); - void _draw(std::vector<ftl::rgbd::FrameSet*> &fss); - void _applyPoseEffects(std::vector<ftl::rgbd::FrameSet*> &fss); - std::pair<const ftl::rgbd::Frame *, const ftl::codecs::Face *> _selectFace(std::vector<ftl::rgbd::FrameSet*> &fss); - void _generateWindow(const ftl::rgbd::Frame &, const ftl::codecs::Face &face, Eigen::Matrix4d &pose_adjust, ftl::render::ViewPort &vp); -}; - -} -} - -#endif // _FTL_GUI_CAMERA_HPP_ diff --git a/applications/gui/src/config_window.cpp b/applications/gui/src/config_window.cpp deleted file mode 100644 index 91dfc19ce..000000000 --- a/applications/gui/src/config_window.cpp +++ /dev/null @@ -1,232 +0,0 @@ -#include "config_window.hpp" -#include "../screen.hpp" - -#include <nanogui/layout.h> -#include <nanogui/label.h> -#include <nanogui/button.h> -#include <nanogui/entypo.h> -#include <nanogui/formhelper.h> -#include <nanogui/vscrollpanel.h> -#include <nanogui/opengl.h> - -#include <nlohmann/json.hpp> - -#include <vector> -#include <string> - -using ftl::gui::ConfigWindow; -using std::string; -using std::vector; -using ftl::config::json_t; - -class SearchBox : public nanogui::TextBox { -private: - std::vector<std::string> configurables_; - Widget *buttons_; - std::string previous; - - void _setVisible(const std::string &str) { - // Check whether the search string has changed to prevent - // unnecessary searching. - if (str != previous) { - for (int i = configurables_.size()-1; i >= 0; --i) { - if (configurables_[i].find(mValueTemp) != std::string::npos) { - buttons_->childAt(i)->setVisible(true); - } else { - buttons_->childAt(i)->setVisible(false); - } - } - previous = str; - } - } - -public: - SearchBox(Widget *parent, std::vector<std::string> &configurables) : nanogui::TextBox(parent, ""), configurables_(configurables) { - setAlignment(TextBox::Alignment::Left); - setEditable(true); - setPlaceholder("Search"); - } - - ~SearchBox() { - } - - bool keyboardEvent(int key, int scancode, int action, int modifier) { - TextBox::keyboardEvent(key, scancode, action, modifier); - _setVisible(mValueTemp); - return true; - } - - void setButtons(Widget *buttons) { - buttons_ = buttons; - } -}; - -static std::string titleForURI(const ftl::URI &uri) { - auto *cfg = ftl::config::find(uri.getBaseURI()); - if (cfg && cfg->get<std::string>("title")) { - return *cfg->get<std::string>("title"); - } else if (uri.getPath().size() > 0) { - return uri.getPathSegment(-1); - } else { - return uri.getHost(); - } -} - -ConfigWindow::ConfigWindow(nanogui::Widget *parent, ftl::ctrl::Master *ctrl) - : nanogui::Window(parent, "Settings"), ctrl_(ctrl) { - using namespace nanogui; - - setLayout(new GroupLayout()); - setPosition(Vector2i(parent->width()/2.0f - 100.0f, parent->height()/2.0f - 100.0f)); - //setModal(true); - - auto configurables = ftl::config::list(); - const auto size = configurables.size(); - - new Label(this, "Select Configurable","sans-bold"); - - auto searchBox = new SearchBox(this, configurables); - - auto vscroll = new VScrollPanel(this); - vscroll->setFixedHeight(300); - auto buttons = new Widget(vscroll); - buttons->setLayout(new BoxLayout(Orientation::Vertical, Alignment::Fill)); - - searchBox->setButtons(buttons); - - std::vector<std::string> configurable_titles(size); - for (size_t i = 0; i < size; ++i) { - ftl::URI uri(configurables[i]); - std::string label = uri.getFragment(); - - size_t pos = label.find_last_of("/"); - if (pos != std::string::npos) label = label.substr(pos+1); - - std::string parentName = configurables[i]; - size_t pos2 = parentName.find_last_of("/"); - if (pos2 != std::string::npos) parentName = parentName.substr(0,pos2); - - // FIXME: Does not indicated parent indentation ... needs sorting? - - if (i > 0 && parentName == configurables[i-1]) { - ftl::URI uri(configurables[i-1]); - configurable_titles[i-1] = std::string("[") + titleForURI(uri) + std::string("] ") + uri.getFragment(); - - auto *prev = dynamic_cast<Button*>(buttons->childAt(buttons->childCount()-1)); - prev->setCaption(configurable_titles[i-1]); - prev->setBackgroundColor(nanogui::Color(0.3f,0.3f,0.3f,1.0f)); - prev->setTextColor(nanogui::Color(1.0f,1.0f,1.0f,1.0f)); - prev->setIconPosition(Button::IconPosition::Left); - prev->setIcon(ENTYPO_ICON_FOLDER); - } - - configurable_titles[i] = label; - - auto itembutton = new nanogui::Button(buttons, configurable_titles[i]); - std::string c = configurables[i]; - itembutton->setTooltip(c); - itembutton->setBackgroundColor(nanogui::Color(0.9f,0.9f,0.9f,0.9f)); - itembutton->setCallback([this,c]() { - //LOG(INFO) << "Change configurable: " << c; - _buildForm(c); - setVisible(false); - //this->parent()->removeChild(this); - //delete this; - //screen()->removeChild(this); - }); - } -} - -ConfigWindow::~ConfigWindow() { - -} - -void ConfigWindow::_addElements(nanogui::FormHelper *form, const std::string &suri) { - using namespace nanogui; - - Configurable *configurable = ftl::config::find(suri); - ftl::config::json_t data; - if (configurable) { - configurable->refresh(); - data = configurable->getConfig(); - } - - for (auto i=data.begin(); i!=data.end(); ++i) { - if (i.key() == "$id") continue; - - if (i.key() == "$ref" && i.value().is_string()) { - //LOG(INFO) << "Follow $ref: " << i.value(); - const std::string suri = std::string(i.value().get<string>()); - _addElements(form, suri); - continue; - } - - if (i.value().is_boolean()) { - string key = i.key(); - form->addVariable<bool>(i.key(), [this,data,key,suri](const bool &b){ - ftl::config::update(suri+"/"+key, b); - }, [data,key]() -> bool { - return data[key].get<bool>(); - }); - } else if (i.value().is_number_integer()) { - string key = i.key(); - form->addVariable<int>(i.key(), [this,data,key,suri](const int &f){ - ftl::config::update(suri+"/"+key, f); - }, [data,key]() -> int { - return data[key].get<int>(); - }); - } else if (i.value().is_number_float()) { - string key = i.key(); - form->addVariable<float>(i.key(), [this,data,key,suri](const float &f){ - ftl::config::update(suri+"/"+key, f); - }, [data,key]() -> float { - return data[key].get<float>(); - }); - } else if (i.value().is_string()) { - string key = i.key(); - form->addVariable<string>(i.key(), [this,data,key,suri](const string &f){ - ftl::config::update(suri+"/"+key, f); - }, [data,key]() -> string { - return data[key].get<string>(); - }); - } else if (i.value().is_object()) { - string key = i.key(); - - // Checking the URI with exists() prevents unloaded local configurations from being shown. - if (suri.find('#') != string::npos && exists(suri+string("/")+key)) { - form->addButton(key, [this,suri,key]() { - _buildForm(suri+string("/")+key); - })->setIcon(ENTYPO_ICON_FOLDER); - } else if (exists(suri+string("#")+key)) { - form->addButton(key, [this,suri,key]() { - _buildForm(suri+string("#")+key); - })->setIcon(ENTYPO_ICON_FOLDER); - } - } - } -} - -void ConfigWindow::_buildForm(const std::string &suri) { - using namespace nanogui; - - ftl::URI uri(suri); - - FormHelper *form = new FormHelper(this->screen()); - //form->setWindow(this); - form->addWindow(Vector2i(100,50), uri.getFragment()); - auto* theme = dynamic_cast<ftl::gui2::Screen*>(screen())->getTheme("window_light"); - form->window()->setTheme(theme); - - _addElements(form, suri); - - auto closebutton = form->addButton("Close", [this,form]() { - form->window()->setVisible(false); - delete form; - }); - closebutton->setIcon(ENTYPO_ICON_CROSS); -} - -bool ConfigWindow::exists(const std::string &uri) { - return ftl::config::find(uri) != nullptr; -} - diff --git a/applications/gui/src/config_window.hpp b/applications/gui/src/config_window.hpp deleted file mode 100644 index 06f989e0b..000000000 --- a/applications/gui/src/config_window.hpp +++ /dev/null @@ -1,33 +0,0 @@ -#ifndef _FTL_GUI_CFGWINDOW_HPP_ -#define _FTL_GUI_CFGWINDOW_HPP_ - -#include <nanogui/window.h> -#include <nanogui/formhelper.h> - -#include <ftl/master.hpp> -#include <ftl/uuid.hpp> -#include <ftl/net_configurable.hpp> - -namespace ftl { -namespace gui { - -/** - * Allow configurable editing. - */ -class ConfigWindow : public nanogui::Window { - public: - ConfigWindow(nanogui::Widget *parent, ftl::ctrl::Master *ctrl); - ~ConfigWindow(); - - private: - ftl::ctrl::Master *ctrl_; - - void _buildForm(const std::string &uri); - void _addElements(nanogui::FormHelper *form, const std::string &suri); - bool exists(const std::string &uri); -}; - -} -} - -#endif // _FTL_GUI_CFGWINDOW_HPP_ diff --git a/applications/gui/src/ctrl_window.cpp b/applications/gui/src/ctrl_window.cpp deleted file mode 100644 index 37d311964..000000000 --- a/applications/gui/src/ctrl_window.cpp +++ /dev/null @@ -1,139 +0,0 @@ -#include "ctrl_window.hpp" - -#include "config_window.hpp" - -#include <nanogui/layout.h> -#include <nanogui/label.h> -#include <nanogui/combobox.h> -#include <nanogui/button.h> -#include <nanogui/entypo.h> - -#include <vector> -#include <string> - -using ftl::gui::ControlWindow; -using ftl::gui::ConfigWindow; -using std::string; -using std::vector; - - -ControlWindow::ControlWindow(nanogui::Widget *parent, ftl::ctrl::Master *ctrl) - : nanogui::Window(parent, "Network Connections"), ctrl_(ctrl) { - setLayout(new nanogui::GroupLayout()); - - using namespace nanogui; - - _updateDetails(); - - auto tools = new Widget(this); - tools->setLayout(new BoxLayout( Orientation::Horizontal, - Alignment::Middle, 0, 6)); - - auto button = new Button(tools, "", ENTYPO_ICON_PLUS); - button->setCallback([this] { - // Show new connection dialog - _addNode(); - }); - button->setTooltip("Add new node"); - - // commented-out buttons not working/useful - /* - button = new Button(tools, "", ENTYPO_ICON_CYCLE); - button->setCallback([this] { - ctrl_->restart(); - }); - button = new Button(tools, "", ENTYPO_ICON_CONTROLLER_PAUS); - button->setCallback([this] { - ctrl_->pause(); - }); - button->setTooltip("Pause all nodes");*/ - - new Label(this, "Select Node","sans-bold"); - auto select = new ComboBox(this, node_titles_); - select->setCallback([this](int ix) { - //LOG(INFO) << "Change node: " << ix; - _changeActive(ix); - }); - - new Label(this, "Node Options","sans-bold"); - - tools = new Widget(this); - tools->setLayout(new BoxLayout( Orientation::Horizontal, - Alignment::Middle, 0, 6)); - - /*button = new Button(tools, "", ENTYPO_ICON_INFO); - button->setCallback([this] { - - }); - button->setTooltip("Node status information");*/ - - button = new Button(tools, "", ENTYPO_ICON_COG); - button->setCallback([this,parent] { - auto cfgwin = new ConfigWindow(parent, ctrl_); - cfgwin->setTheme(theme()); - }); - button->setTooltip("Edit node configuration"); - - /*button = new Button(tools, "", ENTYPO_ICON_CYCLE); - button->setCallback([this] { - ctrl_->restart(_getActiveID()); - }); - button->setTooltip("Restart this node");*/ - - /*button = new Button(tools, "", ENTYPO_ICON_CONTROLLER_PAUS); - button->setCallback([this] { - ctrl_->pause(_getActiveID()); - }); - button->setTooltip("Pause node processing");*/ - - ctrl->getNet()->onConnect([this,select](ftl::net::Peer *p) { - _updateDetails(); - select->setItems(node_titles_); - }); - - _changeActive(0); -} - -ControlWindow::~ControlWindow() { - -} - -void ControlWindow::_addNode() { - using namespace nanogui; - - FormHelper *form = new FormHelper(this->screen()); - form->addWindow(Vector2i(100,100), "Add Node"); - - auto var = form->addVariable("URI", add_node_uri_); - var->setValue("tcp://localhost:9001"); - var->setFixedWidth(200); - - form->addButton("Add", [this,form](){ - ctrl_->getNet()->connect(add_node_uri_); - form->window()->setVisible(false); - delete form; - })->setIcon(ENTYPO_ICON_PLUS); - - form->addButton("Close", [form]() { - form->window()->setVisible(false); - delete form; - })->setIcon(ENTYPO_ICON_CROSS); -} - -void ControlWindow::_updateDetails() { - node_details_ = ctrl_->getControllers(); - - node_titles_.clear(); - for (auto &d : node_details_) { - node_titles_.push_back(d["title"].get<string>()); - } -} - -void ControlWindow::_changeActive(int ix) { - active_ix_ = ix; -} - -ftl::UUID ControlWindow::_getActiveID() { - return ftl::UUID(node_details_[active_ix_]["id"].get<string>()); -} - diff --git a/applications/gui/src/ctrl_window.hpp b/applications/gui/src/ctrl_window.hpp deleted file mode 100644 index 93a45a47e..000000000 --- a/applications/gui/src/ctrl_window.hpp +++ /dev/null @@ -1,35 +0,0 @@ -#ifndef _FTL_GUI_CTRLWINDOW_HPP_ -#define _FTL_GUI_CTRLWINDOW_HPP_ - -#include <nanogui/window.h> -#include <ftl/master.hpp> -#include <ftl/uuid.hpp> - -namespace ftl { -namespace gui { - -/** - * Manage connected nodes and add new connections. - */ -class ControlWindow : public nanogui::Window { - public: - ControlWindow(nanogui::Widget *parent, ftl::ctrl::Master *ctrl); - ~ControlWindow(); - - private: - ftl::ctrl::Master *ctrl_; - std::vector<ftl::config::json_t> node_details_; - std::vector<std::string> node_titles_; - int active_ix_; - std::string add_node_uri_; - - void _updateDetails(); - void _changeActive(int); - ftl::UUID _getActiveID(); - void _addNode(); -}; - -} -} - -#endif // _FTL_GUI_CTRLWINDOW_HPP_ diff --git a/applications/gui/src/frameset_mgr.cpp b/applications/gui/src/frameset_mgr.cpp deleted file mode 100644 index 479f224d9..000000000 --- a/applications/gui/src/frameset_mgr.cpp +++ /dev/null @@ -1,9 +0,0 @@ -#include "frameset_mgr.hpp" -#include <ftl/uri.hpp> - -static int frameset_counter = 0; - -int ftl::gui::mapToFrameset(const std::string &uri) { - //ftl::URI u(uri); - return frameset_counter++; -} diff --git a/applications/gui/src/frameset_mgr.hpp b/applications/gui/src/frameset_mgr.hpp deleted file mode 100644 index c5412fd1d..000000000 --- a/applications/gui/src/frameset_mgr.hpp +++ /dev/null @@ -1,17 +0,0 @@ -#ifndef _FTL_GUI_FRAMESET_MANAGER_HPP_ -#define _FTL_GUI_FRAMESET_MANAGER_HPP_ - -#include <string> - -namespace ftl { -namespace gui { - -/** - * Given a stream URI, allocate a frameset number to that stream. - */ -int mapToFrameset(const std::string &uri); - -} -} - -#endif diff --git a/applications/gui/src/gltexture.cpp b/applications/gui/src/gltexture.cpp deleted file mode 100644 index ae3c3a05d..000000000 --- a/applications/gui/src/gltexture.cpp +++ /dev/null @@ -1,158 +0,0 @@ -#include "gltexture.hpp" - -#include <nanogui/opengl.h> -#include <loguru.hpp> - -#include <ftl/cuda_common.hpp> -#include <cuda_gl_interop.h> - -#include <ftl/exception.hpp> - -using ftl::gui::GLTexture; - -GLTexture::GLTexture(GLTexture::Type type) { - glid_ = std::numeric_limits<unsigned int>::max(); - glbuf_ = std::numeric_limits<unsigned int>::max(); - cuda_res_ = nullptr; - width_ = 0; - height_ = 0; - changed_ = true; - type_ = type; -} - -GLTexture::~GLTexture() { - //glDeleteTextures(1, &glid_); -} - -void GLTexture::update(cv::Mat &m) { - LOG(INFO) << "DEPRECATED"; - if (m.rows == 0) return; - if (glid_ == std::numeric_limits<unsigned int>::max()) { - glGenTextures(1, &glid_); - glBindTexture(GL_TEXTURE_2D, glid_); - //cv::Mat m(cv::Size(100,100), CV_8UC3); - if (type_ == Type::BGRA) { - glTexImage2D(GL_TEXTURE_2D, 0, GL_RGBA8, m.cols, m.rows, 0, GL_BGRA, GL_UNSIGNED_BYTE, m.data); - } else if (type_ == Type::Float) { - glTexImage2D(GL_TEXTURE_2D, 0, GL_R32F, m.cols, m.rows, 0, GL_RED, GL_FLOAT, m.data); - } - glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE); - glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE); - glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_NEAREST); - glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR); - } else { - //glBindTexture(GL_TEXTURE_2D, glid_); - // TODO Allow for other formats - //glTexImage2D(GL_TEXTURE_2D, 0, GL_RGBA8, m.cols, m.rows, 0, GL_BGRA, GL_UNSIGNED_BYTE, m.data); - } - auto err = glGetError(); - if (err != 0) LOG(ERROR) << "OpenGL Texture error: " << err; -} - -void GLTexture::make(int width, int height) { - if (width != width_ || height != height_) { - free(); - } - - static constexpr int ALIGNMENT = 128; - - width_ = width; - height_ = height; - stride_ = ((width*4) % ALIGNMENT != 0) ? ((width*4) + (ALIGNMENT - ((width*4) % ALIGNMENT))) / 4 : width; - - if (width == 0 || height == 0) { - throw FTL_Error("Invalid texture size"); - } - - if (glid_ == std::numeric_limits<unsigned int>::max()) { - glGenTextures(1, &glid_); - glBindTexture(GL_TEXTURE_2D, glid_); - - glPixelStorei(GL_UNPACK_ROW_LENGTH, stride_); - - //cv::Mat m(cv::Size(100,100), CV_8UC3); - //glTexImage2D(GL_TEXTURE_2D, 0, GL_RGBA8, width, height, 0, GL_BGRA, GL_UNSIGNED_BYTE, nullptr); - if (type_ == Type::BGRA) { - glTexImage2D(GL_TEXTURE_2D, 0, GL_RGBA8, width, height, 0, GL_BGRA, GL_UNSIGNED_BYTE, nullptr); - } else if (type_ == Type::Float) { - glTexImage2D(GL_TEXTURE_2D, 0, GL_R32F, width, height, 0, GL_RED, GL_FLOAT, nullptr); - } - glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE); - glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE); - glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_NEAREST); - glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR); - auto err = glGetError(); - if (err != 0) LOG(ERROR) << "OpenGL Texture error: " << err; - - glPixelStorei(GL_UNPACK_ROW_LENGTH, 0); - - glBindTexture(GL_TEXTURE_2D, 0); - - glGenBuffers(1, &glbuf_); - // Make this the current UNPACK buffer (OpenGL is state-based) - glBindBuffer(GL_PIXEL_UNPACK_BUFFER, glbuf_); - // Allocate data for the buffer. 4-channel 8-bit image - glBufferData(GL_PIXEL_UNPACK_BUFFER, stride_ * height * 4, NULL, GL_DYNAMIC_COPY); - - cudaSafeCall(cudaGraphicsGLRegisterBuffer(&cuda_res_, glbuf_, cudaGraphicsRegisterFlagsWriteDiscard)); - glBindBuffer(GL_PIXEL_UNPACK_BUFFER, 0); - } -} - -void GLTexture::free() { - if (glid_ != std::numeric_limits<unsigned int>::max()) { - glDeleteTextures(1, &glid_); - glid_ = std::numeric_limits<unsigned int>::max(); - } - - if (glbuf_ != std::numeric_limits<unsigned int>::max()) { - cudaSafeCall(cudaGraphicsUnregisterResource( cuda_res_ )); - cuda_res_ = nullptr; - glDeleteBuffers(1, &glbuf_); - glbuf_ = std::numeric_limits<unsigned int>::max(); - } -} - -cv::cuda::GpuMat GLTexture::map(cudaStream_t stream) { - void *devptr; - size_t size; - cudaSafeCall(cudaGraphicsMapResources(1, &cuda_res_, stream)); - cudaSafeCall(cudaGraphicsResourceGetMappedPointer(&devptr, &size, cuda_res_)); - return cv::cuda::GpuMat(height_, width_, (type_ == Type::BGRA) ? CV_8UC4 : CV_32F, devptr, stride_*4); -} - -void GLTexture::unmap(cudaStream_t stream) { - cudaSafeCall(cudaGraphicsUnmapResources(1, &cuda_res_, stream)); - changed_ = true; - - //glActiveTexture(GL_TEXTURE0); - glBindBuffer( GL_PIXEL_UNPACK_BUFFER, glbuf_); - // Select the appropriate texture - glBindTexture( GL_TEXTURE_2D, glid_); - glPixelStorei(GL_UNPACK_ROW_LENGTH, stride_); - // Make a texture from the buffer - if (type_ == Type::BGRA) { - glTexSubImage2D( GL_TEXTURE_2D, 0, 0, 0, width_, height_, GL_BGRA, GL_UNSIGNED_BYTE, NULL); - } else { - glTexSubImage2D( GL_TEXTURE_2D, 0, 0, 0, width_, height_, GL_RED, GL_FLOAT, NULL); - } - glPixelStorei(GL_UNPACK_ROW_LENGTH, 0); - glBindTexture(GL_TEXTURE_2D, 0); - glBindBuffer( GL_PIXEL_UNPACK_BUFFER, 0); -} - -unsigned int GLTexture::texture() const { - if (glbuf_ < std::numeric_limits<unsigned int>::max()) { - /*//glActiveTexture(GL_TEXTURE0); - glBindBuffer( GL_PIXEL_UNPACK_BUFFER, glbuf_); - // Select the appropriate texture - glBindTexture( GL_TEXTURE_2D, glid_); - // Make a texture from the buffer - glTexSubImage2D( GL_TEXTURE_2D, 0, 0, 0, width_, height_, GL_BGRA, GL_UNSIGNED_BYTE, NULL); - glBindBuffer( GL_PIXEL_UNPACK_BUFFER, 0);*/ - - return glid_; - } else { - return glid_; - } -} diff --git a/applications/gui/src/gltexture.hpp b/applications/gui/src/gltexture.hpp deleted file mode 100644 index 759f349cf..000000000 --- a/applications/gui/src/gltexture.hpp +++ /dev/null @@ -1,52 +0,0 @@ -#ifndef _FTL_GUI_GLTEXTURE_HPP_ -#define _FTL_GUI_GLTEXTURE_HPP_ - -#include <opencv2/core/mat.hpp> - -#include <cuda_runtime.h> - -struct cudaGraphicsResource; - -namespace ftl { -namespace gui { - -class GLTexture { - public: - enum class Type { - RGBA, - BGRA, - Float - }; - - explicit GLTexture(Type); - ~GLTexture(); - - void update(cv::Mat &m); - void make(int width, int height); - unsigned int texture() const; - bool isValid() const { return glid_ != std::numeric_limits<unsigned int>::max(); } - - cv::cuda::GpuMat map(cudaStream_t stream); - void unmap(cudaStream_t stream); - - void free(); - - int width() const { return width_; } - int height() const { return height_; } - - private: - unsigned int glid_; - unsigned int glbuf_; - int width_; - int height_; - int stride_; - bool changed_; - Type type_; - - cudaGraphicsResource *cuda_res_; -}; - -} -} - -#endif // _FTL_GUI_GLTEXTURE_HPP_ diff --git a/applications/gui/src/main.cpp b/applications/gui/src/main.cpp deleted file mode 100644 index 182faf06e..000000000 --- a/applications/gui/src/main.cpp +++ /dev/null @@ -1,117 +0,0 @@ -#include <ftl/configuration.hpp> -#include <ftl/net/universe.hpp> -#include <ftl/rgbd.hpp> -#include <ftl/master.hpp> -#include <ftl/net_configurable.hpp> - -#include <loguru.hpp> - -#include "screen.hpp" - -#include <cuda_gl_interop.h> - -#ifdef HAVE_PYLON -#include <pylon/PylonIncludes.h> -#endif - - -int main(int argc, char **argv) { -#ifdef HAVE_PYLON - Pylon::PylonAutoInitTerm autoInitTerm; -#endif - - auto root = ftl::configure(argc, argv, "gui_default"); - ftl::net::Universe *net = ftl::create<ftl::net::Universe>(root, "net"); - - int cuda_device; - cudaSafeCall(cudaGetDevice(&cuda_device)); - //cudaSafeCall(cudaGLSetGLDevice(cuda_device)); - - ftl::ctrl::Master *controller = new ftl::ctrl::Master(root, net); - controller->onLog([](const ftl::ctrl::LogEvent &e){ - const int v = e.verbosity; - switch (v) { - case -2: LOG(ERROR) << "Remote log: " << e.message; break; - case -1: LOG(WARNING) << "Remote log: " << e.message; break; - case 0: LOG(INFO) << "Remote log: " << e.message; break; - } - }); - - net->start(); - net->waitConnections(); - - /*auto available = net.findAll<string>("list_streams"); - for (auto &a : available) { - std::cout << " -- " << a << std::endl; - }*/ - - ftl::timer::start(); - - try { - nanogui::init(); - - { - nanogui::ref<ftl::gui::Screen> app = new ftl::gui::Screen(root, net, controller); - app->drawAll(); - app->setVisible(true); - //nanogui::mainloop(20); - - float last_draw_time = 0.0f; - - while (ftl::running) { - if (!app->visible()) { - ftl::running = false; - } else if (glfwWindowShouldClose(app->glfwWindow())) { - app->setVisible(false); - ftl::running = false; - } else { - float now = (float)glfwGetTime(); - float delta = now - last_draw_time; - - // Generate poses and render and virtual frame here - // at full FPS (25 without VR and 90 with VR currently) - app->drawFast(); - - // Only draw the GUI at 25fps - if (delta >= 0.04f) { - last_draw_time = now; - app->drawAll(); - } - } - - /* Wait for mouse/keyboard or empty refresh events */ - //glfwWaitEvents(); - glfwPollEvents(); - } - - /* Process events once more */ - glfwPollEvents(); - - LOG(INFO) << "Stopping..."; - ftl::timer::stop(false); - ftl::pool.stop(true); - LOG(INFO) << "All threads stopped."; - } - - nanogui::shutdown(); - } catch (const ftl::exception &e) { - LOG(ERROR) << "Fatal error: " << e.what(); - LOG(ERROR) << e.trace(); - } catch (const std::runtime_error &e) { - std::string error_msg = std::string("Caught a fatal error: ") + std::string(e.what()); - #if defined(_WIN32) - MessageBoxA(nullptr, error_msg.c_str(), NULL, MB_ICONERROR | MB_OK); - #else - LOG(ERROR) << error_msg; - #endif - return -1; - } - - - net->shutdown(); - delete controller; - delete net; - delete root; - - return 0; -} diff --git a/applications/gui/src/media_panel.cpp b/applications/gui/src/media_panel.cpp deleted file mode 100644 index 4677cb7e0..000000000 --- a/applications/gui/src/media_panel.cpp +++ /dev/null @@ -1,272 +0,0 @@ -#include "media_panel.hpp" -#include "screen.hpp" -#include "record_window.hpp" - -#include <nanogui/layout.h> -#include <nanogui/button.h> -#include <nanogui/popupbutton.h> -#include <nanogui/entypo.h> - -#ifdef HAVE_LIBARCHIVE -#include "ftl/rgbd/snapshot.hpp" -#endif - -using ftl::gui::MediaPanel; -using ftl::codecs::Channel; - -MediaPanel::MediaPanel(ftl::gui::Screen *screen, ftl::gui::SourceWindow *sourceWindow) : nanogui::Window(screen, ""), screen_(screen), sourceWindow_(sourceWindow) { - using namespace nanogui; - - paused_ = false; - disable_switch_channels_ = false; - record_mode_ = RecordMode::None; - - setLayout(new BoxLayout(Orientation::Horizontal, - Alignment::Middle, 5, 10)); - - auto size = Vector2i(400, 60); - //setFixedSize(size); - setPosition(Vector2i(screen->width() / 2 - size[0]/2, screen->height() - 30 - size[1])); - - auto button = new Button(this, "", ENTYPO_ICON_EDIT); - button->setTooltip("Edit camera properties"); - button->setCallback([this]() { - auto *cam = screen_->activeCamera(); - if (cam) cam->showPoseWindow(); - }); - - recordbutton_ = new PopupButton(this, "", ENTYPO_ICON_CONTROLLER_RECORD); - recordbutton_->setTooltip("Record"); - recordbutton_->setSide(Popup::Side::Right); - recordbutton_->setChevronIcon(0); - auto recordpopup = recordbutton_->popup(); - recordpopup->setLayout(new GroupLayout()); - recordpopup->setTheme(screen->toolbuttheme); - recordpopup->setAnchorHeight(180); - auto itembutton = new Button(recordpopup, "2D snapshot (.png)"); - itembutton->setCallback([this]() { - _startRecording(RecordMode::Snapshot2D); - recordbutton_->setPushed(false); - }); - itembutton = new Button(recordpopup, "Virtual camera recording (.ftl)"); - itembutton->setCallback([this]() { - _startRecording(RecordMode::Video2D); - recordbutton_->setTextColor(nanogui::Color(1.0f,0.1f,0.1f,1.0f)); - recordbutton_->setPushed(false); - }); - itembutton = new Button(recordpopup, "Virtual camera Live"); - itembutton->setCallback([this]() { - _startRecording(RecordMode::Live2D); - recordbutton_->setTextColor(nanogui::Color(1.0f,0.1f,0.1f,1.0f)); - recordbutton_->setPushed(false); - }); - itembutton = new Button(recordpopup, "3D scene snapshot (.ftl)"); - itembutton->setCallback([this]() { - _startRecording(RecordMode::Snapshot3D); - recordbutton_->setPushed(false); - }); - itembutton = new Button(recordpopup, "3D scene recording (.ftl)"); - itembutton->setCallback([this]() { - _startRecording(RecordMode::Video3D); - recordbutton_->setTextColor(nanogui::Color(1.0f,0.1f,0.1f,1.0f)); - recordbutton_->setPushed(false); - }); - itembutton = new Button(recordpopup, "Detailed recording options"); - itembutton->setCallback([this,sourceWindow] { - auto record_window = new RecordWindow(screen_, screen_, sourceWindow->getCameras(), this); - record_window->setTheme(screen_->windowtheme); - recordbutton_->setPushed(false); - recordbutton_->setEnabled(false); - }); - - recordbutton_->setCallback([this](){ - if (record_mode_ != RecordMode::None) { - _stopRecording(); - recordbutton_->setTextColor(nanogui::Color(1.0f,1.0f,1.0f,1.0f)); - recordbutton_->setPushed(false); - } - }); - - button = new Button(this, "", ENTYPO_ICON_CONTROLLER_STOP); - button->setCallback([this]() { - screen_->setActiveCamera(nullptr); - }); - - button = new Button(this, "", ENTYPO_ICON_CONTROLLER_PAUS); - button->setCallback([this,button,sourceWindow]() { - //paused_ = !paused_; - //paused_ = !(bool)ftl::config::get("[reconstruction]/controls/paused"); - //ftl::config::update("[reconstruction]/controls/paused", paused_); - - paused_ = !paused_; - sourceWindow->paused(paused_); - - if (paused_) { - button->setIcon(ENTYPO_ICON_CONTROLLER_PLAY); - } else { - button->setIcon(ENTYPO_ICON_CONTROLLER_PAUS); - } - }); - - // not very useful (l/r) - - /*auto button_dual = new Button(this, "", ENTYPO_ICON_MAP); - button_dual->setCallback([this]() { - screen_->setDualView(!screen_->getDualView()); - }); - */ - -#ifdef HAVE_OPENVR - if (this->screen_->isHmdPresent()) { - auto button_vr = new Button(this, "VR"); - button_vr->setFlags(Button::ToggleButton); - button_vr->setChangeCallback([this, button_vr](bool state) { - if (!screen_->isVR()) { - if (screen_->switchVR(true) == true) { - button_vr->setTextColor(nanogui::Color(0.5f,0.5f,1.0f,1.0f)); - //this->button_channels_->setEnabled(false); - } - } - else { - if (screen_->switchVR(false) == false) { - button_vr->setTextColor(nanogui::Color(1.0f,1.0f,1.0f,1.0f)); - //this->button_channels_->setEnabled(true); - } - } - }); - } -#endif - - button_channels_ = new PopupButton(this, "", ENTYPO_ICON_LAYERS); - button_channels_->setSide(Popup::Side::Right); - button_channels_->setChevronIcon(ENTYPO_ICON_CHEVRON_SMALL_RIGHT); - Popup *popup = button_channels_->popup(); - popup->setLayout(new GroupLayout()); - popup->setTheme(screen->toolbuttheme); - popup->setAnchorHeight(150); - - for (int i=0; i<=2; ++i) { - ftl::codecs::Channel c = static_cast<ftl::codecs::Channel>(i); - button = new Button(popup, ftl::codecs::name(c)); - button->setFlags(Button::RadioButton); - //button->setPushed(true); - button->setVisible(false); - button->setCallback([this,c]() { - ftl::gui::Camera *cam = screen_->activeCamera(); - if (cam) { - cam->setChannel(c); - } - }); - channel_buttons_[i] = button; - } - - auto *stereobut = new Button(popup, "Stereo On"); - stereobut->setCallback([this,stereobut]() { - ftl::gui::Camera *cam = screen_->activeCamera(); - if (cam) { - cam->setStereo(!cam->isStereo()); - if (cam->isStereo()) { - stereobut->setCaption("Stereo Off"); - } else { - stereobut->setCaption("Stereo On"); - } - } - }); - - - auto *popbutton = new PopupButton(popup, "More"); - popbutton->setSide(Popup::Side::Right); - popbutton->setChevronIcon(ENTYPO_ICON_CHEVRON_SMALL_RIGHT); - popup = popbutton->popup(); - popup->setLayout(new GroupLayout()); - popup->setTheme(screen->toolbuttheme); - //popup->setAnchorHeight(150); - more_button_ = popup; - - for (int i=3; i<32; ++i) { - ftl::codecs::Channel c = static_cast<ftl::codecs::Channel>(i); - button = new Button(popup, ftl::codecs::name(c)); - button->setFlags(Button::RadioButton); - //button->setPushed(true); - button->setVisible(false); - button->setCallback([this,c]() { - ftl::gui::Camera *cam = screen_->activeCamera(); - if (cam) { - cam->setChannel(c); - } - }); - channel_buttons_[i] = button; - } -} - -MediaPanel::~MediaPanel() { - -} - -void MediaPanel::_startRecording(MediaPanel::RecordMode mode) { - char timestamp[18]; - std::time_t t=std::time(NULL); - std::strftime(timestamp, sizeof(timestamp), "%F-%H%M%S", std::localtime(&t)); - - std::string filename(timestamp); - switch(mode) { - case RecordMode::Snapshot2D : filename += ".png"; break; - case RecordMode::Snapshot3D : - case RecordMode::Video3D : filename += ".ftl"; break; - case RecordMode::Video2D : filename += ".ftl"; break; - case RecordMode::Live2D : break; - default: return; - } - - if (mode == RecordMode::Video3D) { - record_mode_ = mode; - sourceWindow_->recordVideo(filename); - } else if (mode == RecordMode::Snapshot2D) { - screen_->activeCamera()->snapshot(filename); - } else if (mode == RecordMode::Video2D) { - record_mode_ = mode; - screen_->activeCamera()->startVideoRecording(filename, ""); - } else if (mode == RecordMode::Live2D) { - record_mode_ = mode; - screen_->activeCamera()->startVideoRecording("", "ftl://live.utu.fi"); - } -} - -void MediaPanel::_stopRecording() { - if (record_mode_ == RecordMode::Video3D) { - sourceWindow_->stopRecordingVideo(); - } else if (record_mode_ == RecordMode::Video2D || record_mode_ == RecordMode::Live2D) { - screen_->activeCamera()->stopVideoRecording(); - } - record_mode_ = RecordMode::None; -} - -// Update button enabled status -void MediaPanel::cameraChanged() { - ftl::gui::Camera *cam = screen_->activeCamera(); - if (cam) { - auto channels = cam->availableChannels(); - for (int i=0; i<32; ++i) { - if (channels.has(static_cast<ftl::codecs::Channel>(i))) { - channel_buttons_[i]->setVisible(true); - } else { - channel_buttons_[i]->setVisible(false); - } - - if (cam->getChannel() == static_cast<ftl::codecs::Channel>(i)) { - channel_buttons_[i]->setPushed(true); - } else { - channel_buttons_[i]->setPushed(false); - } - } - } -} - -void MediaPanel::performLayout(NVGcontext *ctx) { - nanogui::Window::performLayout(ctx); - more_button_->setAnchorHeight(more_button_->height()-20); -} - -void MediaPanel::recordWindowClosed() { - recordbutton_->setEnabled(true); -} \ No newline at end of file diff --git a/applications/gui/src/media_panel.hpp b/applications/gui/src/media_panel.hpp deleted file mode 100644 index f615be1f6..000000000 --- a/applications/gui/src/media_panel.hpp +++ /dev/null @@ -1,81 +0,0 @@ -#ifndef _FTL_GUI_MEDIAPANEL_HPP_ -#define _FTL_GUI_MEDIAPANEL_HPP_ - -#include "camera.hpp" - -#include <nanogui/window.h> - -#include "src_window.hpp" - -#include <array> - -namespace ftl { - -namespace rgbd { -class SnapshotStreamWriter; -} - -namespace gui { - -class Screen; - -class MediaPanel : public nanogui::Window { - public: - explicit MediaPanel(ftl::gui::Screen *, ftl::gui::SourceWindow *); - ~MediaPanel(); - - void cameraChanged(); - - //void startRecording2D(ftl::gui::Camera *camera, const std::string &filename); - - //void snapshot3D(ftl::gui::Camera *camera, const std::string &filename); - - //void startRecording3D(ftl::gui::Camera *camera, const std::string &filename); - - void recordWindowClosed(); - - void performLayout(NVGcontext *ctx) override; - - private: - ftl::gui::Screen *screen_; - ftl::gui::SourceWindow *sourceWindow_; - - bool paused_; - bool disable_switch_channels_; - - ftl::rgbd::SnapshotStreamWriter *writer_; - nanogui::PopupButton *button_channels_; - //nanogui::Button *right_button_; - //nanogui::Button *depth_button_; - nanogui::Popup *more_button_; - nanogui::PopupButton *recordbutton_; - std::array<nanogui::Button*,32> channel_buttons_={}; - - enum class RecordMode { - None, - Snapshot2D, - Snapshot3D, - Video2D, - Video3D, - Live2D, - Live3D - }; - RecordMode record_mode_; - - void _startRecording(RecordMode mode); - void _stopRecording(); - - /** - * These members indicate which type of recording is active, if any. - * They also include a pointer to an object which is used - * to end the recording. Only one of these members should have a value - * at any given time. - */ - //std::optional<ftl::gui::Camera*> virtualCameraRecording_; - //std::optional<ftl::Configurable*> sceneRecording_; -}; - -} -} - -#endif // _FTL_GUI_MEDIAPANEL_HPP_ diff --git a/applications/gui/src/pose_window.cpp b/applications/gui/src/pose_window.cpp deleted file mode 100644 index 2277c27c5..000000000 --- a/applications/gui/src/pose_window.cpp +++ /dev/null @@ -1,207 +0,0 @@ -#include "pose_window.hpp" -#include "screen.hpp" -#include "camera.hpp" - -#include <nanogui/combobox.h> -#include <nanogui/label.h> -#include <nanogui/layout.h> -#include <nanogui/button.h> - -using ftl::gui::PoseWindow; -using ftl::gui::Screen; -using std::string; - -static Eigen::Affine3d create_rotation_matrix(float ax, float ay, float az) { - Eigen::Affine3d rx = - Eigen::Affine3d(Eigen::AngleAxisd(ax, Eigen::Vector3d(1, 0, 0))); - Eigen::Affine3d ry = - Eigen::Affine3d(Eigen::AngleAxisd(ay, Eigen::Vector3d(0, 1, 0))); - Eigen::Affine3d rz = - Eigen::Affine3d(Eigen::AngleAxisd(az, Eigen::Vector3d(0, 0, 1))); - return ry * rz * rx; -} - -PoseWindow::PoseWindow(ftl::gui::Screen *screen, const std::string &src) - : nanogui::Window(screen, "Pose Adjust"), src_(src), screen_(screen) { - using namespace nanogui; - - //setLayout(new nanogui::GroupLayout()); - setLayout(new BoxLayout(Orientation::Vertical, - Alignment::Middle, 0, 6)); - - pose_param_ = kPoseTranslation; - pose_precision_ = 0.1; - - pose_ = screen_->control()->getPose(src_); - - //Widget *tools = new Widget(this); - // tools->setLayout(new BoxLayout(Orientation::Horizontal, - // Alignment::Middle, 0, 6)); - - auto grouping = new Widget(this); - grouping->setLayout(new GroupLayout()); - - new Label(grouping, "Select source","sans-bold"); - available_ = screen_->net()->findAll<string>("list_streams"); - auto select = new ComboBox(grouping, available_); - select->setSelectedIndex(std::distance(available_.begin(), std::find(available_.begin(), available_.end(), src_))); - select->setCallback([this,select](int ix) { - src_ = available_[ix]; - pose_ = screen_->control()->getPose(src_); - }); - - screen_->net()->onConnect([this,select](ftl::net::Peer *p) { - available_ = screen_->control()->getNet()->findAll<string>("list_streams"); - select->setItems(available_); - }); - - new Label(grouping, "Pose Options","sans-bold"); - - auto tools = new Widget(grouping); - tools->setLayout(new BoxLayout(Orientation::Horizontal, - Alignment::Middle, 0, 6)); - - auto button_opt = new Button(tools, "", ENTYPO_ICON_EYE); - button_opt->setTooltip("Virtual view to this pose"); - //button_opt->setFlags(Button::ToggleButton); - //button_opt->setPushed(false); - button_opt->setCallback([this]() { - screen_->activeCamera()->setPose(pose_); - }); - - button_opt = new Button(tools, "", ENTYPO_ICON_LINK); - button_opt->setTooltip("Link virtual current pose to this pose"); - button_opt->setFlags(Button::ToggleButton); - button_opt->setPushed(false); - button_opt->setChangeCallback([this](bool state) { }); - - tools = new Widget(grouping); - tools->setLayout(new BoxLayout(Orientation::Horizontal, - Alignment::Middle, 0, 6)); - - auto button_rgb = new Button(tools, "Translation"); - button_rgb->setTooltip("Adjust camera location"); - button_rgb->setFlags(Button::RadioButton); - button_rgb->setPushed(true); - button_rgb->setCallback([this]() { pose_param_ = kPoseTranslation; }); - - auto button_depth = new Button(tools, "Rotation"); - button_depth->setFlags(Button::RadioButton); - button_depth->setCallback([this]() { pose_param_ = kPoseRotation; }); - - auto button_stddev = new Button(tools, "Raw"); - button_stddev->setTooltip("Edit the numbers directly"); - button_stddev->setFlags(Button::RadioButton); - button_stddev->setChangeCallback([this](bool state) { pose_param_ = kPoseRaw; }); - - tools = new Widget(grouping); - tools->setLayout(new BoxLayout(Orientation::Horizontal, - Alignment::Middle, 0, 6)); - - auto button = new Button(tools, "0.1m"); - button->setFlags(Button::RadioButton); - button->setPushed(true); - button->setCallback([this](){ - pose_precision_ = 0.1f; - }); - - button = new Button(tools, "0.01m"); - button->setFlags(Button::RadioButton); - button->setCallback([this](){ - pose_precision_ = 0.01f; - }); - - button = new Button(tools, "0.001m"); - button->setFlags(Button::RadioButton); - button->setCallback([this](){ - pose_precision_ = 0.001f; - }); - - tools = new Widget(this); - auto grid = new GridLayout(Orientation::Horizontal, 3, Alignment::Middle, 5, 4); - tools->setLayout(grid); - tools->setFixedWidth(150); - - - button = new Button(tools, "Up"); - button->setCallback([this]() { - if (pose_param_ == kPoseTranslation) { - Eigen::Affine3d transform(Eigen::Translation3d(0.0,-pose_precision_,0.0)); - Eigen::Matrix4d matrix = transform.matrix(); - pose_ *= matrix; - } else if (pose_param_ == kPoseRotation) { - Eigen::Affine3d r = create_rotation_matrix(pose_precision_, 0.0, 0.0); - pose_ = r.matrix() * pose_; - } - - screen_->control()->setPose(src_, pose_); - }); - button = new Button(tools, "", ENTYPO_ICON_CHEVRON_UP); - button->setCallback([this]() { - if (pose_param_ == kPoseTranslation) { - Eigen::Affine3d transform(Eigen::Translation3d(0.0,0.0,-pose_precision_)); - Eigen::Matrix4d matrix = transform.matrix(); - pose_ *= matrix; - } else if (pose_param_ == kPoseRotation) { - Eigen::Affine3d r = create_rotation_matrix(0.0, 0.0, pose_precision_); - pose_ = r.matrix() * pose_; - } - screen_->control()->setPose(src_, pose_); - }); - button = new Button(tools, "Down"); - button->setCallback([this]() { - if (pose_param_ == kPoseTranslation) { - Eigen::Affine3d transform(Eigen::Translation3d(0.0,pose_precision_,0.0)); - Eigen::Matrix4d matrix = transform.matrix(); - pose_ *= matrix; - } else if (pose_param_ == kPoseRotation) { - Eigen::Affine3d r = create_rotation_matrix(-pose_precision_, 0.0, 0.0); - pose_ = r.matrix() * pose_; - } - screen_->control()->setPose(src_, pose_); - }); - - button = new Button(tools, "", ENTYPO_ICON_CHEVRON_LEFT); - button->setCallback([this]() { - if (pose_param_ == kPoseTranslation) { - Eigen::Affine3d transform(Eigen::Translation3d(-pose_precision_,0.0,0.0)); - Eigen::Matrix4d matrix = transform.matrix(); - pose_ *= matrix; - } else if (pose_param_ == kPoseRotation) { - Eigen::Affine3d r = create_rotation_matrix(0.0, pose_precision_, 0.0); - pose_ = r.matrix() * pose_; - } - screen_->control()->setPose(src_, pose_); - }); - new Widget(tools); - button = new Button(tools, "", ENTYPO_ICON_CHEVRON_RIGHT); - button->setCallback([this]() { - if (pose_param_ == kPoseTranslation) { - Eigen::Affine3d transform(Eigen::Translation3d(pose_precision_,0.0,0.0)); - Eigen::Matrix4d matrix = transform.matrix(); - pose_ *= matrix; - } else if (pose_param_ == kPoseRotation) { - Eigen::Affine3d r = create_rotation_matrix(0.0, -pose_precision_, 0.0); - pose_ = r.matrix() * pose_; - } - screen_->control()->setPose(src_, pose_); - }); - - new Widget(tools); - button = new Button(tools, "", ENTYPO_ICON_CHEVRON_DOWN); - button->setCallback([this]() { - if (pose_param_ == kPoseTranslation) { - Eigen::Affine3d transform(Eigen::Translation3d(0.0,0.0,pose_precision_)); - Eigen::Matrix4d matrix = transform.matrix(); - pose_ *= matrix; - } else if (pose_param_ == kPoseRotation) { - Eigen::Affine3d r = create_rotation_matrix(0.0, 0.0, -pose_precision_); - pose_ = r.matrix() * pose_; - } - screen_->control()->setPose(src_, pose_); - }); -} - -PoseWindow::~PoseWindow() { - -} diff --git a/applications/gui/src/pose_window.hpp b/applications/gui/src/pose_window.hpp deleted file mode 100644 index bbd041414..000000000 --- a/applications/gui/src/pose_window.hpp +++ /dev/null @@ -1,41 +0,0 @@ -#ifndef _FTL_GUI_POSEWINDOW_HPP_ -#define _FTL_GUI_POSEWINDOW_HPP_ - -#include <nanogui/window.h> -#include <ftl/master.hpp> -#include <ftl/uuid.hpp> - -namespace ftl { -namespace gui { - -class Screen; - -/** - * Manage connected nodes and add new connections. - */ -class PoseWindow : public nanogui::Window { - public: - PoseWindow(ftl::gui::Screen *screen, const std::string &src); - ~PoseWindow(); - - private: - std::vector<std::string> available_; - std::string src_; - - enum poseparameter_t { - kPoseTranslation, - kPoseRotation, - kPoseRaw - }; - - poseparameter_t pose_param_; - float pose_precision_; - Eigen::Matrix4d pose_; - ftl::gui::Screen *screen_; - bool poselink_; -}; - -} -} - -#endif // _FTL_GUI_POSEWINDOW_HPP_ diff --git a/applications/gui/src/record_window.cpp b/applications/gui/src/record_window.cpp deleted file mode 100644 index e358206fe..000000000 --- a/applications/gui/src/record_window.cpp +++ /dev/null @@ -1,122 +0,0 @@ -#include "record_window.hpp" - -#include "screen.hpp" - -#include <ftl/codecs/channels.hpp> - -#include <nanogui/layout.h> -#include <nanogui/button.h> -#include <nanogui/combobox.h> -#include <nanogui/label.h> -#include <nanogui/textbox.h> -#include <nanogui/tabwidget.h> - -using ftl::gui::RecordWindow; - -RecordWindow::RecordWindow(nanogui::Widget *parent, ftl::gui::Screen *screen, const std::vector<ftl::gui::Camera *> &streams, ftl::gui::MediaPanel *media_panel) - : nanogui::Window(parent, "Recording options") { - using namespace nanogui; - - setLayout(new GroupLayout()); - - new Label(this, "File name", "sans-bold"); - char timestamp[18]; - std::time_t t = std::time(NULL); - std::strftime(timestamp, sizeof(timestamp), "%F-%H%M%S", std::localtime(&t)); - Widget *fileNameBox = new Widget(this); - fileNameBox->setLayout(new BoxLayout(Orientation::Horizontal, Alignment::Middle, 0, 6)); - auto fileName = new TextBox(fileNameBox, std::string(timestamp)); - fileName->setFixedWidth(350); - fileName->setEditable(true); - auto extension = new Label(fileNameBox, ".png", "sans-bold"); - new Label(this, "Select stream", "sans-bold"); - auto streamNames = std::vector<std::string>(); - streamNames.reserve(streams.size()); - std::optional<int> ix; - int i=1; - for (const auto s : streams) { - if (s == screen->activeCamera()) { - ix = std::optional<int>(streamNames.size()); - } - - streamNames.push_back(std::string("Stream")+std::to_string(i++)); - } - auto streamSelect = new ComboBox(this, streamNames); - - TabWidget *tabWidget = add<TabWidget>(); - tabWidget->setFixedWidth(400); - auto snapshot2D = tabWidget->createTab("2D snapshot"); - auto recording2D = tabWidget->createTab("2D recording"); - auto snapshot3D = tabWidget->createTab("3D snapshot"); - auto recording3D = tabWidget->createTab("3D recording"); - - snapshot2D->setLayout(new GroupLayout()); - recording2D->setLayout(new GroupLayout()); - snapshot3D->setLayout(new GroupLayout()); - recording3D->setLayout(new GroupLayout()); - - // Set the file name extension based on the type of recording chosen. - tabWidget->setCallback([tabWidget,snapshot2D,extension](int ix) { - if (tabWidget->tab(ix) == snapshot2D) { - extension->setCaption(".png"); - } else { - extension->setCaption(".ftl"); - } - }); - - tabWidget->setActiveTab(0); - - new Label(recording2D, "Select channel (in addition to Left)", "sans-bold"); - auto recordingChannel = recording2D->add<ComboBox>(); - auto streamCallback = [this,streams,recordingChannel](int ix) { - channels_ = std::vector<ftl::codecs::Channel>(); - channel_names_ = std::vector<std::string>(); - ftl::codecs::Channels availableChannels = streams[ix]->availableChannels(); - for (auto c : availableChannels) { - channels_.push_back(c); - channel_names_.push_back(ftl::codecs::name(c)); - } - recordingChannel->setItems(channel_names_); - }; - streamSelect->setCallback(streamCallback); - - // Set the selection to the active stream and set the channel list - // to be the channels available in that stream. The callback must - // be called explicitly, since setSelectedIndex() does not trigger it. - if (ix) { - streamSelect->setSelectedIndex(ix.value()); - streamCallback(ix.value()); - } - - Widget *actionButtons = new Widget(this); - actionButtons->setLayout(new BoxLayout(Orientation::Horizontal)); - auto button = new Button(actionButtons, "Start"); - button->setCallback([this,streams,streamSelect,screen,media_panel,fileName,extension,tabWidget,snapshot2D,recording2D,snapshot3D,recording3D,recordingChannel]() { - // Check the chosen stream type and channels, then record them. - std::string name = fileName->value() + extension->caption(); - auto stream = streams[streamSelect->selectedIndex()]; - auto tab = tabWidget->tab(tabWidget->activeTab()); - if (tab == snapshot2D) { - stream->snapshot(name); - } else if (tab == recording2D) { - stream->setChannel(channels_[recordingChannel->selectedIndex()]); - screen->setActiveCamera(stream); - //media_panel->startRecording2D(stream, name); - } else if (tab == snapshot3D) { - //media_panel->snapshot3D(stream, name); - } else if (tab == recording3D) { - //media_panel->startRecording3D(stream, name); - } - dispose(); - media_panel->recordWindowClosed(); - }); - button = new Button(actionButtons, "Cancel"); - button->setCallback([this,media_panel]() { - dispose(); - media_panel->recordWindowClosed(); - }); -} - -RecordWindow::~RecordWindow() { - -} diff --git a/applications/gui/src/record_window.hpp b/applications/gui/src/record_window.hpp deleted file mode 100644 index 5a9b28fef..000000000 --- a/applications/gui/src/record_window.hpp +++ /dev/null @@ -1,20 +0,0 @@ -#include <nanogui/window.h> - -#include "camera.hpp" -#include "media_panel.hpp" - -namespace ftl { -namespace gui { - -class RecordWindow : public nanogui::Window { - public: - explicit RecordWindow(nanogui::Widget *parent, ftl::gui::Screen *screen, const std::vector<ftl::gui::Camera *> &streams, ftl::gui::MediaPanel *media_panel); - ~RecordWindow(); - - private: - std::vector<ftl::codecs::Channel> channels_; - std::vector<std::string> channel_names_; -}; - -} -} \ No newline at end of file diff --git a/applications/gui/src/scene.hpp b/applications/gui/src/scene.hpp deleted file mode 100644 index 2e2b4b89f..000000000 --- a/applications/gui/src/scene.hpp +++ /dev/null @@ -1,25 +0,0 @@ -#ifndef _FTL_GUI_SCENE_HPP_ -#define _FTL_GUI_SCENE_HPP_ - -#include <ftl/streams/receiver.hpp> - -namespace ftl { -namespace gui { - -class Camera; - -class Scene { - public: - explicit Scene(ftl::stream::Receiver *); - ~Scene(); - - inline const std::vector<ftl::gui::Camera*> cameras() const { return cameras_; }; - - private: - std::vector<ftl::gui::Camera*> cameras_; -}; - -} -} - -#endif // _FTL_GUI_SCENE_HPP_ diff --git a/applications/gui/src/screen.cpp b/applications/gui/src/screen.cpp deleted file mode 100644 index d00d9df82..000000000 --- a/applications/gui/src/screen.cpp +++ /dev/null @@ -1,687 +0,0 @@ -#include "screen.hpp" - -#include <ftl/streams/netstream.hpp> -#include <ftl/rgbd/frameset.hpp> - -#include <nanogui/opengl.h> -#include <nanogui/glutil.h> -#include <nanogui/screen.h> -#include <nanogui/window.h> -#include <nanogui/layout.h> -#include <nanogui/imageview.h> -#include <nanogui/label.h> -#include <nanogui/toolbutton.h> -#include <nanogui/popupbutton.h> - -#include <sstream> - -#include <nlohmann/json.hpp> - -#include <loguru.hpp> - -#include <opencv2/core/eigen.hpp> - -#include "ctrl_window.hpp" -#include "src_window.hpp" -#include "config_window.hpp" -#include "camera.hpp" -#include "media_panel.hpp" - -#ifdef HAVE_OPENVR -#include "vr.hpp" -#endif - -using ftl::gui::Screen; -using ftl::gui::Camera; -using std::string; -using ftl::rgbd::Source; -using ftl::rgbd::isValidDepth; - -namespace { - constexpr char const *const defaultImageViewVertexShader = - R"(#version 330 - uniform vec2 scaleFactor; - uniform vec2 position; - in vec2 vertex; - out vec2 uv; - void main() { - uv = vec2(vertex.x, vertex.y); - vec2 scaledVertex = (vertex * scaleFactor) + position; - gl_Position = vec4(2.0*scaledVertex.x - 1.0, - 2.0*scaledVertex.y - 1.0, - 0.0, 1.0); - })"; - - constexpr char const *const defaultImageViewFragmentShader = - R"(#version 330 - uniform sampler2D image1; - uniform sampler2D image2; - uniform sampler2D depthImage; - uniform float blendAmount; - out vec4 color; - in vec2 uv; - void main() { - color = blendAmount * texture(image1, uv) + (1.0 - blendAmount) * texture(image2, uv); - color.w = 1.0f; - gl_FragDepth = texture(depthImage, uv).r; - })"; -} - -template <typename T> -std::string to_string_with_precision(const T a_value, const int n = 6) { - std::ostringstream out; - out.precision(n); - out << std::fixed << a_value; - return out.str(); -} - -ftl::gui::Screen::Screen(ftl::Configurable *proot, ftl::net::Universe *pnet, ftl::ctrl::Master *controller) : - nanogui::Screen(Eigen::Vector2i(1024, 768), "FT-Lab Remote Presence"), - status_("FT-Lab Remote Presence System") { - using namespace nanogui; - net_ = pnet; - ctrl_ = controller; - root_ = proot; - camera_ = nullptr; - last_stats_count_ = 0; - - #ifdef HAVE_OPENVR - HMD_ = nullptr; - #endif - - zoom_ = root_->value("zoom", 1.0f); - root_->on("zoom", [this](const ftl::config::Event &e) { - zoom_ = root_->value("zoom", 1.0f); - }); - - pos_x_ = root_->value("position_x", 0.0f); - root_->on("position_x", [this](const ftl::config::Event &e) { - pos_x_ = root_->value("position_x", 0.0f); - }); - pos_y_ = root_->value("position_y", 0.0f); - root_->on("position_y", [this](const ftl::config::Event &e) { - pos_y_ = root_->value("position_y", 0.0f); - }); - - shortcuts_ = ftl::create<ftl::Configurable>(root_, "shortcuts"); - - setSize(Vector2i(1280,720)); - - toolbuttheme = new Theme(*theme()); - toolbuttheme->mBorderDark = nanogui::Color(0,0); - toolbuttheme->mBorderLight = nanogui::Color(0,0); - toolbuttheme->mButtonGradientBotFocused = nanogui::Color(60,255); - toolbuttheme->mButtonGradientBotUnfocused = nanogui::Color(0,0); - toolbuttheme->mButtonGradientTopFocused = nanogui::Color(60,255); - toolbuttheme->mButtonGradientTopUnfocused = nanogui::Color(0,0); - toolbuttheme->mButtonGradientTopPushed = nanogui::Color(60,180); - toolbuttheme->mButtonGradientBotPushed = nanogui::Color(60,180); - toolbuttheme->mTextColor = nanogui::Color(0.9f,0.9f,0.9f,0.9f); - - mediatheme = new Theme(*theme()); - mediatheme->mIconScale = 1.2f; - mediatheme->mWindowDropShadowSize = 0; - mediatheme->mWindowFillFocused = nanogui::Color(45, 150); - mediatheme->mWindowFillUnfocused = nanogui::Color(45, 80); - mediatheme->mButtonGradientTopUnfocused = nanogui::Color(0,0); - mediatheme->mButtonGradientBotUnfocused = nanogui::Color(0,0); - mediatheme->mButtonGradientTopFocused = nanogui::Color(80,230); - mediatheme->mButtonGradientBotFocused = nanogui::Color(80,230); - mediatheme->mIconColor = nanogui::Color(255,255); - mediatheme->mTextColor = nanogui::Color(1.0f,1.0f,1.0f,1.0f); - mediatheme->mBorderDark = nanogui::Color(0,0); - mediatheme->mBorderMedium = nanogui::Color(0,0); - mediatheme->mBorderLight = nanogui::Color(0,0); - mediatheme->mDropShadow = nanogui::Color(0,0); - mediatheme->mButtonFontSize = 30; - mediatheme->mStandardFontSize = 20; - - windowtheme = new Theme(*theme()); - windowtheme->mWindowFillFocused = nanogui::Color(220, 200); - windowtheme->mWindowFillUnfocused = nanogui::Color(220, 200); - windowtheme->mWindowHeaderGradientBot = nanogui::Color(60,230); - windowtheme->mWindowHeaderGradientTop = nanogui::Color(60,230); - windowtheme->mTextColor = nanogui::Color(20,255); - windowtheme->mWindowCornerRadius = 2; - windowtheme->mButtonGradientBotFocused = nanogui::Color(210,255); - windowtheme->mButtonGradientBotUnfocused = nanogui::Color(190,255); - windowtheme->mButtonGradientTopFocused = nanogui::Color(230,255); - windowtheme->mButtonGradientTopUnfocused = nanogui::Color(230,255); - windowtheme->mButtonGradientTopPushed = nanogui::Color(170,255); - windowtheme->mButtonGradientBotPushed = nanogui::Color(210,255); - windowtheme->mBorderDark = nanogui::Color(150,255); - windowtheme->mBorderMedium = nanogui::Color(165,255); - windowtheme->mBorderLight = nanogui::Color(230,255); - windowtheme->mButtonFontSize = 16; - windowtheme->mTextColorShadow = nanogui::Color(0,0); - windowtheme->mWindowTitleUnfocused = windowtheme->mWindowTitleFocused; - windowtheme->mWindowTitleFocused = nanogui::Color(240,255); - windowtheme->mIconScale = 0.85f; - - auto toolbar = new Window(this, ""); - toolbar->setPosition(Vector2i(0,0)); - toolbar->setFixedWidth(50); - toolbar->setFixedHeight(height()); - //toolbar->setLayout(new BoxLayout(Orientation::Vertical, - // Alignment::Middle, 0, 10)); - - setResizeCallback([this,toolbar](Vector2i s) { - toolbar->setFixedHeight(s[1]); - mwindow_->setPosition(Vector2i(s[0] / 2 - mwindow_->width()/2, s[1] - 30 - mwindow_->height())); - }); - - auto innertool = new Widget(toolbar); - innertool->setLayout(new BoxLayout(Orientation::Vertical, - Alignment::Middle, 0, 10)); - innertool->setPosition(Vector2i(5,10)); - - // Padding widget - //auto w = new Widget(innertool); - //w->setHeight(10); - - auto button = new ToolButton(innertool, ENTYPO_ICON_HOME); - button->setIconExtraScale(1.5f); - button->setTheme(toolbuttheme); - button->setTooltip("Home"); - button->setFixedSize(Vector2i(40,40)); - button->setCallback([this]() { - //swindow_->setVisible(true); - setActiveCamera(nullptr); - }); - - /*button = new ToolButton(innertool, ENTYPO_ICON_PLUS); - button->setIconExtraScale(1.5f); - button->setTheme(toolbuttheme); - button->setTooltip("Add new"); - button->setFixedSize(Vector2i(40,40)); - button->setCallback([this]() { - //swindow_->setVisible(true); - });*/ - - auto popbutton = new PopupButton(innertool, "", ENTYPO_ICON_PLUS); - popbutton->setIconExtraScale(1.5f); - popbutton->setTheme(toolbuttheme); - popbutton->setTooltip("Add"); - popbutton->setFixedSize(Vector2i(40,40)); - popbutton->setSide(Popup::Side::Right); - popbutton->setChevronIcon(0); - Popup *popup = popbutton->popup(); - popup->setLayout(new GroupLayout()); - popup->setTheme(toolbuttheme); - //popup->setAnchorHeight(100); - - auto itembutton = new Button(popup, "Add Camera", ENTYPO_ICON_CAMERA); - itembutton->setCallback([this,popup]() { - swindow_->setVisible(true); - popup->setVisible(false); - }); - - itembutton = new Button(popup, "Add Node", ENTYPO_ICON_LAPTOP); - itembutton->setCallback([this,popup]() { - cwindow_->setVisible(true); - popup->setVisible(false); - }); - - popbutton = new PopupButton(innertool, "", ENTYPO_ICON_TOOLS); - popbutton->setIconExtraScale(1.5f); - popbutton->setTheme(toolbuttheme); - popbutton->setTooltip("Tools"); - popbutton->setFixedSize(Vector2i(40,40)); - popbutton->setSide(Popup::Side::Right); - popbutton->setChevronIcon(0); - popup = popbutton->popup(); - popup->setLayout(new GroupLayout()); - popup->setTheme(toolbuttheme); - //popbutton->setCallback([this]() { - // cwindow_->setVisible(true); - //}); - - itembutton = new Button(popup, "Connections"); - itembutton->setCallback([this,popup]() { - cwindow_->setVisible(true); - popup->setVisible(false); - }); - - itembutton = new Button(popup, "Manual Registration"); - itembutton->setCallback([this,popup]() { - // Show pose win... - popup->setVisible(false); - }); - - itembutton = new Button(innertool, "", ENTYPO_ICON_COG); - itembutton->setIconExtraScale(1.5f); - itembutton->setTheme(toolbuttheme); - itembutton->setTooltip("Settings"); - itembutton->setFixedSize(Vector2i(40,40)); - - itembutton->setCallback([this]() { - auto config_window = new ConfigWindow(this, ctrl_); - config_window->setTheme(windowtheme); - }); - - /* - //net_->onConnect([this,popup](ftl::net::Peer *p) { - { - LOG(INFO) << "NET CONNECT"; - auto node_details = ctrl_->getControllers(); - - for (auto &d : node_details) { - LOG(INFO) << "ADDING TITLE: " << d.dump(); - auto peer = ftl::UUID(d["id"].get<std::string>()); - auto itembutton = new Button(popup, d["title"].get<std::string>()); - itembutton->setCallback([this,popup,peer]() { - auto config_window = new ConfigWindow(this, ctrl_); - config_window->setTheme(windowtheme); - }); - } - } - //}); - - itembutton = new Button(popup, "Local"); - itembutton->setCallback([this,popup]() { - auto config_window = new ConfigWindow(this, ctrl_); - config_window->setTheme(windowtheme); - }); - */ - - //configwindow_ = new ConfigWindow(parent, ctrl_); - //cwindow_ = new ftl::gui::ControlWindow(this, controller); - swindow_ = new ftl::gui::SourceWindow(this); - mwindow_ = new ftl::gui::MediaPanel(this, swindow_); - mwindow_->setVisible(false); - mwindow_->setTheme(mediatheme); - - //cwindow_->setPosition(Eigen::Vector2i(80, 20)); - //swindow_->setPosition(Eigen::Vector2i(80, 400)); - //cwindow_->setVisible(false); - swindow_->setVisible(true); - swindow_->center(); - //cwindow_->setTheme(windowtheme); - swindow_->setTheme(mediatheme); - - mShader.init("RGBDShader", defaultImageViewVertexShader, - defaultImageViewFragmentShader); - - MatrixXu indices(3, 2); - indices.col(0) << 0, 1, 2; - indices.col(1) << 2, 3, 1; - - MatrixXf vertices(2, 4); - vertices.col(0) << 0, 0; - vertices.col(1) << 1, 0; - vertices.col(2) << 0, 1; - vertices.col(3) << 1, 1; - - mShader.bind(); - mShader.uploadIndices(indices); - mShader.uploadAttrib("vertex", vertices); - - setVisible(true); - performLayout(); -} - -#ifdef HAVE_OPENVR -bool ftl::gui::Screen::initVR() { - if (!vr::VR_IsHmdPresent()) { - return false; - } - - vr::EVRInitError eError = vr::VRInitError_None; - HMD_ = vr::VR_Init( &eError, vr::VRApplication_Scene ); - - if (eError != vr::VRInitError_None) - { - HMD_ = nullptr; - LOG(ERROR) << "Unable to init VR runtime: " << vr::VR_GetVRInitErrorAsEnglishDescription(eError); - return false; - } - - return true; -} - -bool ftl::gui::Screen::isVR() { - auto *cam = activeCamera(); - if (HMD_ == nullptr || cam == nullptr) { return false; } - return cam->isVR(); -} - -bool ftl::gui::Screen::switchVR(bool on) { - if (isVR() == on) { return on; } - - if (on && (HMD_ == nullptr) && !initVR()) { - return false; - } - - if (on) { - activeCamera()->setVR(true); - } else { - activeCamera()->setVR(false); - } - - return isVR(); -} - -bool ftl::gui::Screen::isHmdPresent() { - return vr::VR_IsHmdPresent(); -} - -#endif - -ftl::gui::Screen::~Screen() { - mShader.free(); - - #ifdef HAVE_OPENVR - if (HMD_ != nullptr) { - vr::VR_Shutdown(); - } - #endif -} - -void ftl::gui::Screen::setActiveCamera(ftl::gui::Camera *cam) { - if (camera_) camera_->active(false); - camera_ = cam; - - if (cam) { - status_ = cam->name(); - mwindow_->setVisible(true); - mwindow_->cameraChanged(); - swindow_->setVisible(false); - cam->active(true); - } else { - mwindow_->setVisible(false); - swindow_->setVisible(true); - status_ = "[No camera]"; - } -} - -bool ftl::gui::Screen::scrollEvent(const Eigen::Vector2i &p, const Eigen::Vector2f &rel) { - if (nanogui::Screen::scrollEvent(p, rel)) { - return true; - } else { - zoom_ += zoom_ * 0.1f * rel[1]; - return true; - } -} - -bool ftl::gui::Screen::mouseMotionEvent(const Eigen::Vector2i &p, const Eigen::Vector2i &rel, int button, int modifiers) { - if (nanogui::Screen::mouseMotionEvent(p, rel, button, modifiers)) { - return true; - } else { - if (camera_) { - if (button == 1) { - camera_->mouseMovement(rel[0], rel[1], button); - } else if (button == 2) { - pos_x_ += rel[0]; - pos_y_ += -rel[1]; - } - } - } - return true; -} - -bool ftl::gui::Screen::mouseButtonEvent(const nanogui::Vector2i &p, int button, bool down, int modifiers) { - if (nanogui::Screen::mouseButtonEvent(p, button, down, modifiers)) { - return true; - } else { - if (!camera_) return false; - - Eigen::Vector2f screenSize = size().cast<float>(); - auto mScale = (screenSize.cwiseQuotient(imageSize).minCoeff()); - Eigen::Vector2f scaleFactor = mScale * imageSize.cwiseQuotient(screenSize); - Eigen::Vector2f positionInScreen(0.0f, 0.0f); - auto mOffset = (screenSize - (screenSize.cwiseProduct(scaleFactor))) / 2; - Eigen::Vector2f positionAfterOffset = positionInScreen + mOffset; - - float sx = ((float)p[0] - positionAfterOffset[0]) / mScale; - float sy = ((float)p[1] - positionAfterOffset[1]) / mScale; - - if (button == 1 && down) { - - auto p = camera_->getPoint(sx, sy); - points_.push_back(p); - - //auto n = camera_->getNormal(sx, sy); - - LOG(INFO) << "point: (" << p.x << ", " << p.y << ", " << p.z << ") added"; - if (points_.size() < 2) { return true; } - - auto p1 = Eigen::Vector3f(points_[0].x, points_[0].y, points_[0].z); - auto p2 = Eigen::Vector3f(points_[1].x, points_[1].y, points_[1].z); - - points_.clear(); - // TODO: check p1 and p2 valid - if (p1 == p2) { return true; } - auto T = nanogui::lookAt(p1, p2, Eigen::Vector3f(0.0,1.0,0.0)); - cv::Mat T_cv; - cv::eigen2cv(T, T_cv); - T_cv.convertTo(T_cv, CV_64FC1); - net_->broadcast("set_pose_adjustment", T_cv); - - return true; - - } - else if (button == 0 && down) { - auto p = camera_->getPoint(sx, sy); - LOG(INFO) << "point: " << (Eigen::Vector4d(p.x, p.y, p.z, -1.0f)).transpose(); - - //auto q = camera_->getNormal(sx, sy); - //LOG(INFO) << "normal: " << (Eigen::Vector4d(q.x, q.y, q.z, 1.0)).transpose(); - } - return false; - } -} - -static std::string generateKeyComboStr(int key, int modifiers) { - std::string res = ""; - - switch(modifiers) { - case 1: res += "Shift+"; break; - case 2: res += "Ctrl+"; break; - case 3: res += "Ctrl+Shift+"; break; - case 4: res += "Alt+"; break; - default: break; - } - - if (key < 127 && key >= 32) { - char buf[2] = { (char)key, 0 }; - return res + std::string(buf); - } else { - return ""; - } -} - -bool ftl::gui::Screen::keyboardEvent(int key, int scancode, int action, int modifiers) { - using namespace Eigen; - if (nanogui::Screen::keyboardEvent(key, scancode, action, modifiers)) { - return true; - } else { - //LOG(INFO) << "Key press " << key << " - " << action << " - " << modifiers; - - if ((key >= 262 && key <= 267) || (key >= '0' && key <= '9')) { - if (camera_) camera_->keyMovement(key, modifiers); - return true; - } else if (action == 1 && key == 'H') { - swindow_->setVisible(false); - //cwindow_->setVisible(false); - } else if (action == 1 && key == ' ') { - swindow_->togglePaused(); - } else if (action == 1) { - std::string combo = generateKeyComboStr(key, modifiers); - - if (combo.size() > 0) { - LOG(INFO) << "Key combo = " << combo; - - auto s = shortcuts_->get<nlohmann::json>(combo); - if (s) { - //LOG(INFO) << "FOUND KEYBOARD SHORTCUT"; - std::string op = (*s).value("op",std::string("=")); - std::string uri = (*s).value("uri",std::string("")); - - if (op == "toggle") { - auto v = ftl::config::get(uri); - if (v.is_boolean()) { - ftl::config::update(uri, !v.get<bool>()); - } - } else if (op == "+=") { - auto v = ftl::config::get(uri); - if (v.is_number_float()) { - ftl::config::update(uri, v.get<float>() + (*s).value("value",0.0f)); - } else if (v.is_number_integer()) { - ftl::config::update(uri, v.get<int>() + (*s).value("value",0)); - } - } else if (op == "-=") { - auto v = ftl::config::get(uri); - if (v.is_number_float()) { - ftl::config::update(uri, v.get<float>() - (*s).value("value",0.0f)); - } else if (v.is_number_integer()) { - ftl::config::update(uri, v.get<int>() - (*s).value("value",0)); - } - } else if (op == "=") { - ftl::config::update(uri, (*s)["value"]); - } - } - } - } - return false; - } -} - -void ftl::gui::Screen::draw(NVGcontext *ctx) { - using namespace Eigen; - - Vector2f screenSize = size().cast<float>(); - - if (camera_) { - imageSize = {camera_->width(), camera_->height()}; - - //if (camera_->getChannel() != ftl::codecs::Channel::Left) { mImageID = rightEye_; } - - if (camera_->getLeft().isValid() && imageSize[0] > 0) { - auto mScale = (screenSize.cwiseQuotient(imageSize).minCoeff()) * zoom_; - Vector2f scaleFactor = mScale * imageSize.cwiseQuotient(screenSize); - Vector2f positionInScreen(pos_x_, pos_y_); - auto mOffset = (screenSize - (screenSize.cwiseProduct(scaleFactor))) / 2; - Vector2f positionAfterOffset = positionInScreen + mOffset; - Vector2f imagePosition = positionAfterOffset.cwiseQuotient(screenSize); - //glEnable(GL_SCISSOR_TEST); - //float r = screen->pixelRatio(); - /* glScissor(positionInScreen.x() * r, - (screenSize.y() - positionInScreen.y() - size().y()) * r, - size().x() * r, size().y() * r);*/ - mShader.bind(); - glActiveTexture(GL_TEXTURE0); - glBindTexture(GL_TEXTURE_2D, leftEye_); - //camera_->getLeft().texture(); - glActiveTexture(GL_TEXTURE1); - glBindTexture(GL_TEXTURE_2D, (camera_->isStereo() && camera_->getRight().isValid()) ? rightEye_ : leftEye_); - glActiveTexture(GL_TEXTURE2); - glBindTexture(GL_TEXTURE_2D, camera_->getDepth().texture()); - //(camera_->isStereo() && camera_->getRight().isValid()) ? camera_->getRight().texture() : camera_->getLeft().texture(); - mShader.setUniform("image1", 0); - mShader.setUniform("image2", 1); - mShader.setUniform("depthImage", 2); - mShader.setUniform("blendAmount", (camera_->isStereo()) ? root_->value("blending", 0.5f) : 1.0f); - mShader.setUniform("scaleFactor", scaleFactor); - mShader.setUniform("position", imagePosition); - - glEnable(GL_DEPTH_TEST); - glDepthMask(GL_TRUE); - glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT); - - mShader.drawIndexed(GL_TRIANGLES, 0, 2); - //glDisable(GL_SCISSOR_TEST); - glBindBuffer(GL_PIXEL_UNPACK_BUFFER, 0); - - camera_->drawOverlay(screenSize); - - glDisable(GL_DEPTH_TEST); - } - } else { - // Must periodically render the cameras here to update any thumbnails. - auto cams = swindow_->getCameras(); - for (auto *c : cams) { - c->drawUpdated(swindow_->getFramesets()); - } - } - - nvgTextAlign(ctx, NVG_ALIGN_RIGHT); - - int offset_top = 20; - - if (root()->value("show_information", true)) { - string msg; - - auto &stats = getStatistics(); - - msg = string("Frame rate: ") + std::to_string((int)stats.fps); - nvgText(ctx, screenSize[0]-10, 20, msg.c_str(), NULL); - msg = string("Latency: ") + std::to_string((int)stats.latency) + string("ms"); - //nvgText(ctx, screenSize[0]-10, 40, msg.c_str(), NULL); - - msg = string("Bitrate: ") + to_string_with_precision(stats.bitrate, 2) + string("Mbps"); - nvgText(ctx, screenSize[0]-10, 60, msg.c_str(), NULL); - - if (camera_) { - auto intrin = camera_->getIntrinsics(); - msg = string("Resolution: ") + std::to_string(intrin.width) + string("x") + std::to_string(intrin.height); - nvgText(ctx, screenSize[0]-10, 80, msg.c_str(), NULL); - msg = string("Focal: ") + to_string_with_precision(intrin.fx, 2); - nvgText(ctx, screenSize[0]-10, 100, msg.c_str(), NULL); - - offset_top = 120; - } else { - offset_top = 80; - } - } - - if (camera_) { - auto &msgs = camera_->getMessages(); - for (auto &m : msgs) { - nvgText(ctx, screenSize[0]-10, offset_top, m.c_str(), NULL); - offset_top += 20; - } - } - - nvgText(ctx, screenSize[0]-10, screenSize[1]-20, status_.c_str(), NULL); - - /* Draw the user interface */ - screen()->performLayout(ctx); - nanogui::Screen::draw(ctx); -} - -const ftl::gui::Statistics &ftl::gui::Screen::getStatistics() { - if (--last_stats_count_ <= 0) { - auto [fps,latency] = ftl::rgbd::Builder::getStatistics(); - stats_.fps = fps; - stats_.latency = latency; - stats_.bitrate = ftl::stream::Net::getRequiredBitrate(); - last_stats_count_ = 20; - } - return stats_; -} - -void ftl::gui::Screen::drawFast() { - if (camera_) { - camera_->captureFrame(); - - glActiveTexture(GL_TEXTURE0); - mImageID = camera_->getLeft().texture(); - leftEye_ = mImageID; - rightEye_ = camera_->getRight().texture(); - - #ifdef HAVE_OPENVR - if (isVR() && camera_->width() > 0 && camera_->getLeft().isValid() && camera_->getRight().isValid()) { - - //glBindTexture(GL_TEXTURE_2D, leftEye_); - vr::Texture_t leftEyeTexture = {(void*)(uintptr_t)leftEye_, vr::TextureType_OpenGL, vr::ColorSpace_Gamma }; - vr::VRCompositor()->Submit(vr::Eye_Left, &leftEyeTexture ); - - //glBindTexture(GL_TEXTURE_2D, rightEye_); - vr::Texture_t rightEyeTexture = {(void*)(uintptr_t)rightEye_, vr::TextureType_OpenGL, vr::ColorSpace_Gamma }; - vr::VRCompositor()->Submit(vr::Eye_Right, &rightEyeTexture ); - - glFlush(); - } - #endif - } -} diff --git a/applications/gui/src/screen.hpp b/applications/gui/src/screen.hpp deleted file mode 100644 index 7af733bd7..000000000 --- a/applications/gui/src/screen.hpp +++ /dev/null @@ -1,129 +0,0 @@ -#ifndef _FTL_GUI_SCREEN_HPP_ -#define _FTL_GUI_SCREEN_HPP_ - -#include <nanogui/screen.h> -#include <nanogui/glutil.h> -#include <ftl/master.hpp> -#include <ftl/net/universe.hpp> -#include <ftl/configuration.hpp> - -#include "ctrl_window.hpp" -#include "src_window.hpp" -#include "gltexture.hpp" - -#ifdef HAVE_OPENVR -#include <openvr/openvr.h> -#endif - -class StatisticsImageNSamples; - -namespace ftl { -namespace gui { - -class Camera; -class MediaPanel; - -struct Statistics { - float fps; - float latency; - float bitrate; -}; - -class Screen : public nanogui::Screen { - public: - explicit Screen(ftl::Configurable *root, ftl::net::Universe *net, ftl::ctrl::Master *controller); - ~Screen(); - - bool mouseMotionEvent(const Eigen::Vector2i &p, const Eigen::Vector2i &rel, int button, int modifiers); - bool scrollEvent(const Eigen::Vector2i &p, const Eigen::Vector2f &rel); - bool mouseButtonEvent(const nanogui::Vector2i &p, int button, bool down, int modifiers); - bool keyboardEvent(int key, int scancode, int action, int modifiers); - - void setActivePose(const Eigen::Matrix4d &p); - - virtual void draw(NVGcontext *ctx); - - void drawFast(); - - ftl::Configurable *root() { return root_; } - ftl::net::Universe *net() { return net_; } - ftl::ctrl::Master *control() { return ctrl_; } - - void setActiveCamera(ftl::gui::Camera*); - ftl::gui::Camera *activeCamera() { return camera_; } - - const ftl::gui::Statistics &getStatistics(); - -#ifdef HAVE_OPENVR - // initialize OpenVR - bool initVR(); - - // is VR mode on/off - bool isVR(); - - // toggle VR on/off - bool switchVR(bool mode); - - bool isHmdPresent(); - - vr::IVRSystem* getVR() { return HMD_; } - -#else - bool isVR() { return false; } -#endif - - nanogui::Theme *windowtheme; - nanogui::Theme *specialtheme; - nanogui::Theme *mediatheme; - nanogui::Theme *toolbuttheme; - - private: - ftl::gui::SourceWindow *swindow_; - ftl::gui::ControlWindow *cwindow_; - ftl::gui::MediaPanel *mwindow_; - - //std::vector<SourceViews> sources_; - ftl::net::Universe *net_; - nanogui::GLShader mShader; - GLuint mImageID; - //Source *src_; - //GLTexture texture_; - Eigen::Vector3f eye_; - Eigen::Vector4f neye_; - Eigen::Vector3f orientation_; - Eigen::Vector3f up_; - //Eigen::Vector3f lookPoint_; - float lerpSpeed_; - bool depth_; - float ftime_; - float delta_; - Eigen::Vector2f imageSize; - ftl::ctrl::Master *ctrl_; - ftl::Configurable *root_; - std::string status_; - ftl::gui::Camera *camera_; - float zoom_; - float pos_x_; - float pos_y_; - - GLuint leftEye_; - GLuint rightEye_; - - std::vector<cv::Point3d> points_; - - bool show_two_images_ = false; - - ftl::Configurable *shortcuts_; - - #ifdef HAVE_OPENVR - vr::IVRSystem *HMD_; - #endif - - ftl::gui::Statistics stats_; - int last_stats_count_; -}; - -} -} - -#endif // _FTL_GUI_SCREEN_HPP_ diff --git a/applications/gui/src/src_window.cpp b/applications/gui/src/src_window.cpp deleted file mode 100644 index 6406d914a..000000000 --- a/applications/gui/src/src_window.cpp +++ /dev/null @@ -1,479 +0,0 @@ -#include "src_window.hpp" - -#include "screen.hpp" -#include "camera.hpp" -#include "scene.hpp" -#include "frameset_mgr.hpp" - -#include <ftl/profiler.hpp> -#include <ftl/codecs/shapes.hpp> -#include <ftl/utility/vectorbuffer.hpp> - -#include <nanogui/imageview.h> -#include <nanogui/textbox.h> -#include <nanogui/slider.h> -#include <nanogui/combobox.h> -#include <nanogui/label.h> -#include <nanogui/opengl.h> -#include <nanogui/glutil.h> -#include <nanogui/screen.h> -#include <nanogui/layout.h> -#include <nanogui/vscrollpanel.h> - -#define LOGURU_REPLACE_GLOG 1 -#include <loguru.hpp> - -#include <ftl/streams/netstream.hpp> - -#include "ftl/operators/colours.hpp" -#include "ftl/operators/segmentation.hpp" -#include "ftl/operators/mask.hpp" -#include "ftl/operators/antialiasing.hpp" -#include <ftl/operators/smoothing.hpp> -#include <ftl/operators/disparity.hpp> -#include <ftl/operators/detectandtrack.hpp> -#include <ftl/operators/weighting.hpp> -#include <ftl/operators/mvmls.hpp> -#include <ftl/operators/clipping.hpp> -#include <ftl/operators/poser.hpp> -#include <ftl/operators/gt_analysis.hpp> - -#include <nlohmann/json.hpp> - -#ifdef HAVE_LIBARCHIVE -#include "ftl/rgbd/snapshot.hpp" -#endif - -#include "thumbview.hpp" - -using ftl::gui::SourceWindow; -using ftl::gui::Screen; -using ftl::gui::Scene; -using ftl::rgbd::Source; -using ftl::codecs::Channel; -using ftl::codecs::Channels; -using std::string; -using std::vector; -using ftl::config::json_t; - -static ftl::rgbd::Generator *createSourceGenerator(ftl::Configurable *root, const std::vector<ftl::rgbd::Source*> &srcs) { - - auto *grp = new ftl::rgbd::Group(); - /*auto pipeline = ftl::config::create<ftl::operators::Graph>(root, "pipeline"); - pipeline->append<ftl::operators::DetectAndTrack>("facedetection")->value("enabled", false); - pipeline->append<ftl::operators::ArUco>("aruco")->value("enabled", false); - pipeline->append<ftl::operators::DepthChannel>("depth"); // Ensure there is a depth channel - grp->addPipeline(pipeline);*/ - - for (auto s : srcs) { - s->setChannel(Channel::Depth); - grp->addSource(s); - } - return grp; -} - -SourceWindow::SourceWindow(ftl::gui::Screen *screen) - : nanogui::Window(screen, ""), screen_(screen) { - setLayout(new nanogui::BoxLayout(nanogui::Orientation::Vertical, nanogui::Alignment::Fill, 20, 5)); - - using namespace nanogui; - - new Label(this, "Select Camera","sans-bold",20); - - // FIXME: Reallocating the vector may currently causes thread issues since - // it might be in use elsewhere. A safer mechanism is needed for sharing - // framesets. Temporary solution: preallocate enough slots. - pre_pipelines_.reserve(5); - framesets_.reserve(5); - - auto vscroll = new VScrollPanel(this); - ipanel_ = new Widget(vscroll); - ipanel_->setLayout(new GridLayout(nanogui::Orientation::Horizontal, 3, - nanogui::Alignment::Middle, 0, 5)); - - screen->net()->onConnect([this](ftl::net::Peer *p) { - ftl::pool.push([this](int id) { - // FIXME: Find better option that waiting here. - // Wait to make sure streams have started properly. - std::this_thread::sleep_for(std::chrono::milliseconds(100)); - UNIQUE_LOCK(mutex_, lk); - _updateCameras(screen_->net()->findAll<string>("list_streams")); - }); - }); - - UNIQUE_LOCK(mutex_, lk); - stream_ = ftl::create<ftl::stream::Muxer>(screen->root(), "muxer"); - interceptor_ = ftl::create<ftl::stream::Intercept>(screen->root(), "intercept"); - interceptor_->setStream(stream_); - receiver_ = ftl::create<ftl::stream::Receiver>(screen->root(), "receiver"); - receiver_->setStream(interceptor_); - - // Create a recorder - recorder_ = ftl::create<ftl::stream::File>(screen->root(), "recorder"); - recorder_->setMode(ftl::stream::File::Mode::Write); - - interceptor_->onIntercept([this] (const ftl::codecs::StreamPacket &spkt, const ftl::codecs::Packet &pkt) { - //LOG(INFO) << (std::string)spkt; - if (recorder_->active() && pkt.data.size() > 0) { - - if (spkt.channel == Channel::Shapes3D && spkt.frame_number == 255) { - // Decode the shapes channel to insert the virtual camera... - std::vector<ftl::codecs::Shape3D> shapes; - auto unpacked = msgpack::unpack((const char*)pkt.data.data(), pkt.data.size()); - unpacked.get().convert(shapes); - - auto *cam = screen_->activeCamera(); - - if (cam) { - // Modify shapes - auto &s = shapes.emplace_back(); - s.id = shapes.size(); - s.label = std::string("virtual-")+std::to_string(shapes.size()); - s.type = ftl::codecs::Shape3DType::CAMERA; - s.pose = cam->getPose().cast<float>(); - //LOG(INFO) << "Inject virtual : " << shapes.size(); - } - - auto npkt = pkt; - npkt.data.resize(0); - ftl::util::FTLVectorBuffer buf(npkt.data); - msgpack::pack(buf, shapes); - - recorder_->post(spkt, npkt); - } else { - recorder_->post(spkt, pkt); - } - } - }); - - paused_ = false; - cycle_ = 0; - receiver_->onFrameSet([this](ftl::rgbd::FrameSet &fs) { - return _processFrameset(fs, true); - }); - - speaker_ = ftl::create<ftl::audio::Speaker>(screen_->root(), "speaker_test"); - - receiver_->onAudio([this](ftl::audio::FrameSet &fs) { - if (framesets_.size() == 0) return true; - auto *c = screen_->activeCamera(); - int64_t renddelay = (c) ? c->getFrameTimeMS() : 0; - speaker_->setDelay(fs.timestamp - framesets_[0]->timestamp + renddelay); // Add Xms for local render time - speaker_->queue(fs.timestamp, fs.frames[0]); - - //LOG(INFO) << "Audio delay = " << (fs.timestamp - framesets_[0]->timestamp + renddelay); - return true; - }); - - /*ftl::timer::add(ftl::timer::kTimerMain, [this](int64_t ts) { - auto *c = screen_->activeCamera(); - // Only offer full framerate render on active camera. - if (c) { - c->draw(framesets_); - } - return true; - });*/ - - // Add network sources - _updateCameras(screen_->control()->getNet()->findAll<string>("list_streams")); - - // Also check for a file on command line. - // Check paths for FTL files to load. - auto paths = (*screen->root()->get<nlohmann::json>("paths")); - - for (auto &x : paths.items()) { - std::string path = x.value().get<std::string>(); - auto eix = path.find_last_of('.'); - auto ext = path.substr(eix+1); - - // Command line path is ftl file - if (ext == "ftl") { - LOG(INFO) << "Found FTL file: " << path; - int fsid = ftl::gui::mapToFrameset(path); - auto *fstream = ftl::create<ftl::stream::File>(screen->root(), std::string("ftlfile-")+std::to_string(fsid)); - fstream->set("filename", path); - available_[path] = fstream; - stream_->add(fstream, fsid); - } else if (path.rfind("device:", 0) == 0) { - ftl::URI uri(path); - uri.to_json(screen->root()->getConfig()["sources"].emplace_back()); - } else { - ftl::URI uri(path); - if (uri.getScheme() == ftl::URI::SCHEME_TCP || uri.getScheme() == ftl::URI::SCHEME_WS) { - screen->net()->connect(path); - } - } - } - - // Finally, check for any device sources configured - std::vector<Source*> devices; - // Create a vector of all input RGB-Depth sources - if (screen->root()->getConfig()["sources"].size() > 0) { - devices = ftl::createArray<Source>(screen->root(), "sources", screen->control()->getNet()); - auto *gen = createSourceGenerator(screen->root(), devices); - int fsid = ftl::gui::mapToFrameset(screen->root()->getID()); - - gen->onFrameSet([this, fsid](ftl::rgbd::FrameSet &fs) { - fs.id = fsid; // Set a frameset id to something unique. - return _processFrameset(fs, false); - }); - } - - stream_->begin(); -} - -bool SourceWindow::_processFrameset(ftl::rgbd::FrameSet &fs, bool fromstream) { - // Request the channels required by current camera configuration - if (fromstream) { - auto cs = _aggregateChannels(fs.id); - - auto avail = static_cast<const ftl::stream::Stream*>(interceptor_)->available(fs.id); - if (cs.has(Channel::Depth) && !avail.has(Channel::Depth) && avail.has(Channel::Right)) { - cs -= Channel::Depth; - cs += Channel::Right; - } - interceptor_->select(fs.id, cs); - } - - // Make sure there are enough framesets allocated - { - UNIQUE_LOCK(mutex_, lk); - _checkFrameSets(fs.id); - } - - if (!paused_) { - if (!fs.test(ftl::data::FSFlag::PARTIAL) || !screen_->root()->value("drop_partial_framesets", false)) { - // Enforce interpolated colour and GPU upload - for (size_t i=0; i<fs.frames.size(); ++i) { - if (!fs.hasFrame(i)) continue; - fs.frames[i].createTexture<uchar4>(Channel::Colour, true); - - // TODO: Do all channels. This is a fix for screen capture sources. - if (!fs.frames[i].isGPU(Channel::Colour)) fs.frames[i].upload(Channels<0>(Channel::Colour), pre_pipelines_[fs.id]->getStream()); - } - - fs.mask &= pre_pipelines_[fs.id]->value("frame_mask", 0xFFFF); - - { - FTL_Profile("Prepipe",0.020); - pre_pipelines_[fs.id]->apply(fs, fs, 0); - } - - fs.swapTo(*framesets_[fs.id]); - } else { - LOG(WARNING) << "Dropping frameset: " << fs.timestamp; - } - } - - const auto *cstream = interceptor_; - { - UNIQUE_LOCK(mutex_, lk); - _createDefaultCameras(*framesets_[fs.id], true); // cstream->available(fs.id).has(Channel::Depth) - } - - //LOG(INFO) << "Channels = " << (unsigned int)cstream->available(fs.id); - - size_t i=0; - for (auto cam : cameras_) { - // Only update the camera periodically unless the active camera - if (screen_->activeCamera() == cam.second.camera || - (screen_->activeCamera() == nullptr && cycle_ % cameras_.size() == i++)) cam.second.camera->update(framesets_); - - ftl::codecs::Channels<0> channels; - if (fromstream) channels = cstream->available(fs.id); - //if ((*framesets_[fs.id]).frames.size() > 0) channels += (*framesets_[fs.id]).frames[0].getChannels(); - cam.second.camera->update(fs.id, channels); - } - ++cycle_; - - return true; -} - -void SourceWindow::_checkFrameSets(size_t id) { - while (framesets_.size() <= id) { - auto *p = ftl::config::create<ftl::operators::Graph>(screen_->root(), std::string("pre_filters") + std::to_string(framesets_.size())); - p->append<ftl::operators::DepthChannel>("depth")->value("enabled", false); - p->append<ftl::operators::ClipScene>("clipping")->value("enabled", false); - //p->append<ftl::operators::ColourChannels>("colour"); // Convert BGR to BGRA - p->append<ftl::operators::DetectAndTrack>("facedetection")->value("enabled", false); - p->append<ftl::operators::ArUco>("aruco")->value("enabled", false); - //p->append<ftl::operators::HFSmoother>("hfnoise"); - p->append<ftl::operators::CrossSupport>("cross"); - p->append<ftl::operators::PixelWeights>("weights"); - p->append<ftl::operators::CullWeight>("remove_weights")->value("enabled", false); - p->append<ftl::operators::DegradeWeight>("degrade"); - p->append<ftl::operators::VisCrossSupport>("viscross")->set("enabled", false); - p->append<ftl::operators::BorderMask>("border_mask"); - p->append<ftl::operators::CullDiscontinuity>("remove_discontinuity"); - p->append<ftl::operators::MultiViewMLS>("mvmls")->value("enabled", false); - p->append<ftl::operators::Poser>("poser")->value("enabled", true); - p->append<ftl::operators::GTAnalysis>("gtanalyse"); - - pre_pipelines_.push_back(p); - framesets_.push_back(new ftl::rgbd::FrameSet); - } -} - -void SourceWindow::recordVideo(const std::string &filename) { - if (!recorder_->active()) { - recorder_->set("filename", filename); - recorder_->begin(); - LOG(INFO) << "Recording started: " << filename; - - // TODO: Inject pose and calibrations - stream_->reset(); - } -} - -void SourceWindow::stopRecordingVideo() { - if (recorder_->active()) { - recorder_->end(); - LOG(INFO) << "Recording stopped."; - } -} - -ftl::codecs::Channels<0> SourceWindow::_aggregateChannels(int id) { - ftl::codecs::Channels<0> cs = ftl::codecs::Channels<0>(Channel::Colour); - for (auto cam : cameras_) { - if (cam.second.camera->usesFrameset(id)) { - if (cam.second.camera->isVirtual()) { - cs += Channel::Depth; - } else { - if (cam.second.camera->getChannel() != Channel::None) { - cs += cam.second.camera->getChannel(); - } - } - } - } - - return cs; -} - -void SourceWindow::_createDefaultCameras(ftl::rgbd::FrameSet &fs, bool makevirtual) { - for (size_t i=0; i<fs.frames.size(); ++i) { - size_t id = (fs.id << 8) + i; - if (cameras_.find(id) == cameras_.end()) { - auto *cam = new ftl::gui::Camera(screen_, 1 << fs.id, i); - cameras_[id] = { - cam, - nullptr - }; - } - } - - if (makevirtual && cameras_.find((fs.id << 8) + 255) == cameras_.end()) { - auto *cam = new ftl::gui::Camera(screen_, 1 << fs.id, 255); - cameras_[(fs.id << 8) + 255] = { - cam, - nullptr - }; - } -} - -std::vector<ftl::gui::Camera*> SourceWindow::getCameras() { - auto cameras = std::vector<ftl::gui::Camera*>(); - cameras.reserve(cameras_.size()); - - for (auto cam : cameras_) { - cameras.push_back(cam.second.camera); - } - return cameras; -} - -void SourceWindow::_updateCameras(const vector<string> &netcams) { - if (netcams.size() == 0) return; - - int ncount = 0; - for (auto s : netcams) { - if (available_.count(s) == 0) { - auto *stream = ftl::create<ftl::stream::Net>(screen_->root(), string("netstream")+std::to_string(available_.size()), screen_->net()); - available_[s] = stream; - stream->set("uri", s); - int fsid = ftl::gui::mapToFrameset(s); - stream_->add(stream, fsid); - - LOG(INFO) << "Add Stream: " << stream->value("uri", std::string("NONE")) << " (" << fsid << ")"; - ++ncount; - } else { - LOG(INFO) << "Stream exists: " << s; - } - - // FIXME: Check for already existing... - //if (streams_.find(s) == cameras_.end()) { - //available_.push_back(s); - //json_t srcjson; - //srcjson["uri"] = s; - //screen_->root()->getConfig()["streams"].push_back(srcjson); - - //screen_->root()->getConfig()["receivers"].push_back(json_t{}); - //} - } - - //stream_->reset(); - if (ncount > 0) stream_->begin(); - - //std::vector<ftl::stream::Net*> strms = ftl::createArray<ftl::stream::Net>(screen_->root(), "streams", screen_->net()); - - /*for (int i=0; i<strms.size(); ++i) { - auto *stream = strms[i]; - bool isspecial = (stream->get<std::string>("uri") == screen_->root()->value("data_stream",std::string(""))); - if (isspecial) LOG(INFO) << "Adding special stream"; - stream_->add(stream, (isspecial) ? 1 : 0); - - LOG(INFO) << "Add Stream: " << stream->value("uri", std::string("NONE")); - - //Scene *scene = new Scene(receiver); - //scenes_.push_back(scene); - - - }*/ - - //refresh_thumbs_ = true; - //if (thumbs_.size() != available_.size()) { - // thumbs_.resize(available_.size()); - //} -} - -SourceWindow::~SourceWindow() { - -} - -void SourceWindow::draw(NVGcontext *ctx) { - //if (refresh_thumbs_) { - UNIQUE_LOCK(mutex_, lk); - //refresh_thumbs_ = false; - - //if (thumbs_.size() < cameras_.size()) thumbs_.resize(cameras_.size()); - - //for (size_t i=0; i<thumbs_.size(); ++i) { - int i = 0; - for (auto &camera : cameras_) { - cv::Mat t; - auto *cam = camera.second.camera; - //if (cam) { - //cam->draw(framesets_); - // if (cam->thumbnail(t)) { - // thumbs_[i].update(t); - // } - //} - - if (!camera.second.thumbview) camera.second.thumbview = new ftl::gui::ThumbView(ipanel_, screen_, cam); - camera.second.thumbview->setFixedSize(nanogui::Vector2i(320,180)); - - auto *iv = dynamic_cast<nanogui::ImageView*>(camera.second.thumbview); - - /*if ((size_t)ipanel_->childCount() < i+1) { - new ftl::gui::ThumbView(ipanel_, screen_, cam); - }*/ - //if (thumbs_[i].isValid()) dynamic_cast<nanogui::ImageView*>(camera.second.thumbview)->bindImage(thumbs_[i].texture()); - if (cam->getLeft().isValid()) iv->bindImage(cam->getLeft().texture()); - ++i; - } - - // TODO(Nick) remove excess image views - - center(); - //} - - nanogui::Window::draw(ctx); -} diff --git a/applications/gui/src/src_window.hpp b/applications/gui/src/src_window.hpp deleted file mode 100644 index ce412c06d..000000000 --- a/applications/gui/src/src_window.hpp +++ /dev/null @@ -1,95 +0,0 @@ -#ifndef _FTL_GUI_SRCWINDOW_HPP_ -#define _FTL_GUI_SRCWINDOW_HPP_ - -#include <nanogui/window.h> -#include <nanogui/imageview.h> -#include <ftl/master.hpp> -#include <ftl/uuid.hpp> -#include <ftl/rgbd/source.hpp> -#include <ftl/threads.hpp> -#include <vector> -#include <map> -#include <unordered_map> -#include <string> -#include "gltexture.hpp" - -#include <ftl/streams/stream.hpp> -#include <ftl/streams/receiver.hpp> -#include <ftl/streams/filestream.hpp> - -#include <ftl/audio/speaker.hpp> - -class VirtualCameraView; - -namespace ftl { -namespace gui { - -class Screen; -class Scene; -class Camera; -class ThumbView; - -/** - * Main class for managing all data streams and corresponding cameras. It - * will automatically locate all available streams and generate default cameras - * for each frame of each stream found. It will also add a single default - * virtual camera. Additional cameras can be added. This class directly - * receives all frameset data and then forwards it to the individual cameras - * for drawing/rendering. - */ -class SourceWindow : public nanogui::Window { - public: - explicit SourceWindow(ftl::gui::Screen *screen); - ~SourceWindow(); - - std::vector<ftl::gui::Camera*> getCameras(); - - virtual void draw(NVGcontext *ctx); - - void recordVideo(const std::string &filename); - void stopRecordingVideo(); - - inline std::vector<ftl::rgbd::FrameSet*> &getFramesets() { return framesets_; } - - inline void paused(bool p) { paused_ = p; } - inline void togglePaused() { paused_ = !paused_; } - - private: - ftl::gui::Screen *screen_; - - struct CameraEntry { - ftl::gui::Camera *camera; - ftl::gui::ThumbView *thumbview; - //GLTexture thumb; - }; - - std::map<int, CameraEntry> cameras_; - ftl::stream::Muxer *stream_; - ftl::stream::Intercept *interceptor_; - ftl::stream::File *recorder_; - ftl::stream::Receiver *receiver_; - std::unordered_map<std::string, ftl::stream::Stream*> available_; - std::vector<GLTexture> thumbs_; - bool refresh_thumbs_; - nanogui::Widget *ipanel_; - int cycle_; - std::vector<ftl::operators::Graph*> pre_pipelines_; - MUTEX mutex_; - - ftl::audio::Speaker *speaker_; - - std::vector<ftl::rgbd::FrameSet*> framesets_; - bool paused_; - - void _updateCameras(const std::vector<std::string> &netcams); - void _createDefaultCameras(ftl::rgbd::FrameSet &fs, bool makevirtual); - ftl::codecs::Channels<0> _aggregateChannels(int id); - void _checkFrameSets(size_t id); - bool _processFrameset(ftl::rgbd::FrameSet &fs, bool); - -}; - -} -} - -#endif // _FTL_GUI_SRCWINDOW_HPP_ diff --git a/applications/gui/src/statsimage.cpp b/applications/gui/src/statsimage.cpp deleted file mode 100644 index dc8179fdc..000000000 --- a/applications/gui/src/statsimage.cpp +++ /dev/null @@ -1,72 +0,0 @@ -#include "statsimage.hpp" - -using ftl::gui::StatisticsImage; - -StatisticsImage::StatisticsImage(cv::Size size) : - StatisticsImage(size, std::numeric_limits<float>::infinity()) {} - -StatisticsImage::StatisticsImage(cv::Size size, float max_f) { - size_ = size; - n_ = 0.0f; - data_ = cv::Mat(size, CV_32FC3, cv::Scalar(0.0, 0.0, 0.0)); - - // TODO - if (!std::isinf(max_f)) { - LOG(WARNING) << "TODO: max_f_ not used. Values calculated for all samples"; - } -} - -void StatisticsImage::reset() { - n_ = 0.0f; - data_ = cv::Scalar(0.0, 0.0, 0.0); -} - -void StatisticsImage::update(const cv::Mat &in) { - DCHECK(in.type() == CV_32F); - DCHECK(in.size() == size_); - - n_ = n_ + 1.0f; - - // Welford's Method - for (int row = 0; row < in.rows; row++) { - float* ptr_data = data_.ptr<float>(row); - const float* ptr_in = in.ptr<float>(row); - - for (int col = 0; col < in.cols; col++, ptr_in++) { - float x = *ptr_in; - float &m = *ptr_data++; - float &s = *ptr_data++; - float &f = *ptr_data++; - float m_prev = m; - - if (!ftl::rgbd::isValidDepth(x)) continue; - - f = f + 1.0f; - m = m + (x - m) / f; - s = s + (x - m) * (x - m_prev); - } - } -} - -void StatisticsImage::getVariance(cv::Mat &out) { - std::vector<cv::Mat> channels(3); - cv::split(data_, channels); - cv::divide(channels[1], channels[2], out); -} - -void StatisticsImage::getStdDev(cv::Mat &out) { - getVariance(out); - cv::sqrt(out, out); -} - -void StatisticsImage::getMean(cv::Mat &out) { - std::vector<cv::Mat> channels(3); - cv::split(data_, channels); - out = channels[0]; -} - -void StatisticsImage::getValidRatio(cv::Mat &out) { - std::vector<cv::Mat> channels(3); - cv::split(data_, channels); - cv::divide(channels[2], n_, out); -} diff --git a/applications/gui/src/statsimage.hpp b/applications/gui/src/statsimage.hpp deleted file mode 100644 index c796bfb74..000000000 --- a/applications/gui/src/statsimage.hpp +++ /dev/null @@ -1,47 +0,0 @@ -#ifndef _FTL_GUI_STATISTICSIMAGE_HPP_ -#define _FTL_GUI_STATISTICSIMAGE_HPP_ - -#include <opencv2/core/mat.hpp> - -namespace ftl { -namespace gui { - -class StatisticsImage { -private: - cv::Mat data_; // CV_32FC3, channels: m, s, f - cv::Size size_; // image size - float n_; // total number of samples - -public: - explicit StatisticsImage(cv::Size size); - StatisticsImage(cv::Size size, float max_f); - - /* @brief reset all statistics to 0 - */ - void reset(); - - /* @brief update statistics with new values - */ - void update(const cv::Mat &in); - - /* @brief variance (depth) - */ - void getVariance(cv::Mat &out); - - /* @brief standard deviation (depth) - */ - void getStdDev(cv::Mat &out); - - /* @brief mean value (depth) - */ - void getMean(cv::Mat &out); - - /* @brief percent of samples having valid depth value - */ - void getValidRatio(cv::Mat &out); -}; - -} -} - -#endif // _FTL_GUI_STATISTICSIMAGE_HPP_ \ No newline at end of file diff --git a/applications/gui/src/thumbview.cpp b/applications/gui/src/thumbview.cpp deleted file mode 100644 index 5fdbd4ec4..000000000 --- a/applications/gui/src/thumbview.cpp +++ /dev/null @@ -1,35 +0,0 @@ -#include "thumbview.hpp" -#include "screen.hpp" -#include "camera.hpp" - -using ftl::gui::ThumbView; -using ftl::gui::Screen; -using ftl::gui::Camera; - -ThumbView::ThumbView(nanogui::Widget *parent, ftl::gui::Screen *screen, ftl::gui::Camera *cam) - : ImageView(parent, 0), screen_(screen), cam_(cam) { - setCursor(nanogui::Cursor::Hand); -} - -ThumbView::~ThumbView() { - -} - -bool ThumbView::mouseButtonEvent(const nanogui::Vector2i &p, int button, bool down, int modifiers) { - if (button == 0 && !down) { - screen_->setActiveCamera(cam_); - return true; - } else { - return false; - } -} - -void ThumbView::draw(NVGcontext *ctx) { - ImageView::draw(ctx); - - nvgScissor(ctx, mPos.x(), mPos.y(), mSize.x(), mSize.y()); - nvgFontSize(ctx, 14); - nvgFontFace(ctx, "sans-bold"); - //nvgText(ctx, mPos.x() + 10, mPos.y()+mSize.y() - 10, cam_->source()->getURI().c_str(), NULL); - nvgResetScissor(ctx); -} diff --git a/applications/gui/src/thumbview.hpp b/applications/gui/src/thumbview.hpp deleted file mode 100644 index 9bbac8097..000000000 --- a/applications/gui/src/thumbview.hpp +++ /dev/null @@ -1,29 +0,0 @@ -#ifndef _FTL_GUI_THUMBVIEW_HPP_ -#define _FTL_GUI_THUMBVIEW_HPP_ - -#include <nanogui/imageview.h> - -namespace ftl { -namespace gui { - -class Screen; -class Camera; - -class ThumbView : public nanogui::ImageView { - public: - ThumbView(nanogui::Widget *parent, ftl::gui::Screen *screen, ftl::gui::Camera *cam); - ~ThumbView(); - - bool mouseButtonEvent(const nanogui::Vector2i &p, int button, bool down, int modifiers); - - void draw(NVGcontext *ctx); - - private: - Screen *screen_; - Camera *cam_; -}; - -} -} - -#endif // _FTL_GUI_THUMBVIEW_HPP_ diff --git a/applications/gui/src/vr.cpp b/applications/gui/src/vr.cpp deleted file mode 100644 index b300a27b2..000000000 --- a/applications/gui/src/vr.cpp +++ /dev/null @@ -1,36 +0,0 @@ -#include "loguru.hpp" -#include "vr.hpp" - -Eigen::Matrix3d getCameraMatrix(const double tanx1, - const double tanx2, - const double tany1, - const double tany2, - const double size_x, - const double size_y) { - - Eigen::Matrix3d C = Eigen::Matrix3d::Identity(); - - CHECK(tanx1 < 0 && tanx2 > 0 && tany1 < 0 && tany2 > 0); - CHECK(size_x > 0 && size_y > 0); - - double fx = size_x / (-tanx1 + tanx2); - double fy = size_y / (-tany1 + tany2); - C(0,0) = fx; - C(1,1) = fy; - C(0,2) = tanx1 * fx; - C(1,2) = tany1 * fy; - - // safe to remove - CHECK((int) (abs(tanx1 * fx) + abs(tanx2 * fx)) == (int) size_x); - CHECK((int) (abs(tany1 * fy) + abs(tany2 * fy)) == (int) size_y); - - return C; -} - -Eigen::Matrix3d getCameraMatrix(vr::IVRSystem *vr, const vr::Hmd_Eye &eye) { - float tanx1, tanx2, tany1, tany2; - uint32_t size_x, size_y; - vr->GetProjectionRaw(eye, &tanx1, &tanx2, &tany1, &tany2); - vr->GetRecommendedRenderTargetSize(&size_x, &size_y); - return getCameraMatrix(tanx1, tanx2, tany1, tany2, size_x, size_y); -} \ No newline at end of file diff --git a/applications/gui/src/vr.hpp b/applications/gui/src/vr.hpp deleted file mode 100644 index 7e8f6314f..000000000 --- a/applications/gui/src/vr.hpp +++ /dev/null @@ -1,57 +0,0 @@ -#include <openvr/openvr.h> -#include <Eigen/Eigen> -#include <openvr/openvr.h> - -/* @brief Calculate (pinhole camera) intrinsic matrix from OpenVR parameters - * @param Tangent of left half angle (negative) from center view axis - * @param Tangent of right half angle from center view axis - * @param Tangent of top half angle (negative) from center view axis - * @param Tangent of bottom half angle from center view axis - * @param Image width - * @param Image height - * - * Parameters are provided by IVRSystem::GetProjectionRaw and - * IVRSystem::GetRecommendedRenderTargetSize. - * - * tanx1 = x1 / fx (1) - * tanx2 = x2 / fy (2) - * x1 + x2 = size_x (3) - * - * :. fx = size_x / (-tanx1 + tanx2) - * - * fy can be calculated in same way - */ -Eigen::Matrix3d getCameraMatrix(const double tanx1, - const double tanx2, - const double tany1, - const double tany2, - const double size_x, - const double size_y); - -/* - * @brief Same as above, but uses given IVRSystem and eye. - */ -Eigen::Matrix3d getCameraMatrix(vr::IVRSystem *vr, const vr::Hmd_Eye &eye); - - -static inline Eigen::Matrix4d ConvertSteamVRMatrixToMatrix4( const vr::HmdMatrix34_t &matPose ) -{ - Eigen::Matrix4d matrixObj; - matrixObj << - matPose.m[0][0], matPose.m[0][1], matPose.m[0][2], matPose.m[0][3], - matPose.m[1][0], matPose.m[1][1], matPose.m[1][2], matPose.m[1][3], - matPose.m[2][0], matPose.m[2][1], matPose.m[2][2], matPose.m[2][3], - 0.0, 0.0, 0.0, 1.0; - return matrixObj; -} - -static inline Eigen::Matrix4d ConvertSteamVRMatrixToMatrix4( const vr::HmdMatrix44_t &matPose ) -{ - Eigen::Matrix4d matrixObj; - matrixObj << - matPose.m[0][0], matPose.m[0][1], matPose.m[0][2], matPose.m[0][3], - matPose.m[1][0], matPose.m[1][1], matPose.m[1][2], matPose.m[1][3], - matPose.m[2][0], matPose.m[2][1], matPose.m[2][2], matPose.m[2][3], - matPose.m[3][0], matPose.m[3][1], matPose.m[3][2], matPose.m[3][3]; - return matrixObj; -} \ No newline at end of file diff --git a/applications/gui/test/CMakeLists.txt b/applications/gui/test/CMakeLists.txt deleted file mode 100644 index e69de29bb..000000000 diff --git a/applications/gui/test/tests.cpp b/applications/gui/test/tests.cpp deleted file mode 100644 index 0c7c351f4..000000000 --- a/applications/gui/test/tests.cpp +++ /dev/null @@ -1,2 +0,0 @@ -#define CATCH_CONFIG_MAIN -#include "catch.hpp" -- GitLab