From fc110bb761dc935548d66626e3425b4a214c0e97 Mon Sep 17 00:00:00 2001 From: Nicolas Pope <nwpope@utu.fi> Date: Sat, 11 Jul 2020 12:34:02 +0300 Subject: [PATCH] WIP Screencapture generates mouse events --- applications/gui2/src/modules/camera.cpp | 17 +++++++- applications/gui2/src/modules/camera.hpp | 4 ++ applications/gui2/src/views/camera.cpp | 12 ++++++ applications/gui2/src/views/camera.hpp | 1 + applications/gui2/src/widgets/imageview.cpp | 2 +- .../codecs/include/ftl/codecs/channels.hpp | 3 +- .../codecs/include/ftl/codecs/touch.hpp | 35 +++++++++++++++++ components/rgbd-sources/CMakeLists.txt | 2 +- components/rgbd-sources/src/source.cpp | 3 +- .../sources/screencapture/screencapture.cpp | 39 +++++++++++++++++++ .../sources/screencapture/screencapture.hpp | 2 + components/structures/src/new_frame.cpp | 2 +- 12 files changed, 116 insertions(+), 6 deletions(-) create mode 100644 components/codecs/include/ftl/codecs/touch.hpp diff --git a/applications/gui2/src/modules/camera.cpp b/applications/gui2/src/modules/camera.cpp index 24eec071b..21c070303 100644 --- a/applications/gui2/src/modules/camera.cpp +++ b/applications/gui2/src/modules/camera.cpp @@ -83,6 +83,7 @@ void Camera::activate(ftl::data::FrameID id) { if (paused) return true; std::atomic_store(¤t_fs_, fs); + std::atomic_store(&latest_, fs); // Need to notify GUI thread when first data comes if (!has_seen_frame_) { @@ -142,9 +143,23 @@ bool Camera::hasFrame() { } void Camera::sendPose(const Eigen::Matrix4d &pose) { - if (auto ptr = std::atomic_load(¤t_fs_)) { + if (auto ptr = std::atomic_load(&latest_)) { auto response = ptr->frames[frame_idx].response(); auto &rgbdresponse = response.cast<ftl::rgbd::Frame>(); rgbdresponse.setPose() = pose; } } + +void Camera::touch(int id, ftl::codecs::TouchType t, int x, int y, float d) { + // TODO: Check for touch capability first + if (auto ptr = std::atomic_load(&latest_)) { + auto response = ptr->frames[frame_idx].response(); + auto &data = response.create<std::vector<ftl::codecs::Touch>>(Channel::Touch); + auto &pt = data.emplace_back(); + pt.id = id; + pt.type = t; + pt.x = x; + pt.y = y; + pt.d = d; + } +} diff --git a/applications/gui2/src/modules/camera.hpp b/applications/gui2/src/modules/camera.hpp index 018ee20c4..e3abfac78 100644 --- a/applications/gui2/src/modules/camera.hpp +++ b/applications/gui2/src/modules/camera.hpp @@ -6,6 +6,7 @@ #include <ftl/render/colouriser.hpp> #include <ftl/render/overlay.hpp> +#include <ftl/codecs/touch.hpp> namespace ftl { namespace gui2 { @@ -27,6 +28,8 @@ public: ftl::cuda::TextureObject<uchar4>& getFrame(); bool getFrame(ftl::cuda::TextureObject<uchar4>&); + void touch(int id, ftl::codecs::TouchType t, int x, int y, float d); + /** Check if new frame is available */ bool hasFrame(); @@ -43,6 +46,7 @@ private: bool has_seen_frame_ = false; ftl::data::FrameSetPtr current_fs_; + ftl::data::FrameSetPtr latest_; ftl::cuda::TextureObject<uchar4> current_frame_; std::unique_ptr<ftl::render::Colouriser> colouriser_; diff --git a/applications/gui2/src/views/camera.cpp b/applications/gui2/src/views/camera.cpp index 806ccc33d..004b1f854 100644 --- a/applications/gui2/src/views/camera.cpp +++ b/applications/gui2/src/views/camera.cpp @@ -2,6 +2,8 @@ #include <nanogui/layout.h> #include <nanogui/button.h> +#include <ftl/codecs/touch.hpp> + #include "camera.hpp" #include "../modules/camera.hpp" @@ -134,6 +136,16 @@ CameraView::~CameraView() { //LOG(INFO) << __func__ << " (" << this << ")"; } +bool CameraView::mouseButtonEvent(const Eigen::Vector2i &p, int button, bool down, int modifiers) { + //LOG(INFO) << "mouseButtonEvent: " << p << " - " << button; + if (button == 0) { + // FIXME: Allow for zoom and offset of image... + ctrl_->touch(0, ftl::codecs::TouchType::MOUSE_LEFT, p.x()-mPos.x(), p.y()-mPos.y(), 0.0f); + return true; + } + return false; +} + void CameraView::draw(NVGcontext *ctx) { if (ctrl_->hasFrame()) { texture_.copyFrom(ctrl_->getFrame()); diff --git a/applications/gui2/src/views/camera.hpp b/applications/gui2/src/views/camera.hpp index 7230562ac..e90804508 100644 --- a/applications/gui2/src/views/camera.hpp +++ b/applications/gui2/src/views/camera.hpp @@ -40,6 +40,7 @@ public: CameraView(Screen* parent, Camera* ctrl); virtual ~CameraView(); virtual void draw(NVGcontext *ctx) override; + virtual bool mouseButtonEvent(const Eigen::Vector2i &p, int button, bool down, int modifiers) override; protected: Camera* ctrl_; diff --git a/applications/gui2/src/widgets/imageview.cpp b/applications/gui2/src/widgets/imageview.cpp index f5dff3229..2dd45d4c6 100644 --- a/applications/gui2/src/widgets/imageview.cpp +++ b/applications/gui2/src/widgets/imageview.cpp @@ -177,7 +177,7 @@ void ftl::gui2::ImageView::zoom(int amount, const Vector2f& focusPosition) { } bool ftl::gui2::ImageView::mouseDragEvent(const Vector2i& p, const Vector2i& rel, int button, int /*modifiers*/) { - if ((button & (1 << GLFW_MOUSE_BUTTON_LEFT)) != 0 && !mFixedOffset) { + if ((button & (1 << GLFW_MOUSE_BUTTON_RIGHT)) != 0 && !mFixedOffset) { setImageCoordinateAt((p + rel).cast<float>(), imageCoordinateAt(p.cast<float>())); return true; } diff --git a/components/codecs/include/ftl/codecs/channels.hpp b/components/codecs/include/ftl/codecs/channels.hpp index a2222b826..b6aba67dd 100644 --- a/components/codecs/include/ftl/codecs/channels.hpp +++ b/components/codecs/include/ftl/codecs/channels.hpp @@ -61,7 +61,8 @@ enum struct Channel : int { Faces = 2049, // Data about detected faces Transforms = 2050, // Transformation matrices for framesets Shapes3D = 2051, // Labeled 3D shapes - Messages = 2052 // Vector of Strings + Messages = 2052, // Vector of Strings + Touch = 2053 // List of touch data type (each touch point) }; inline bool isVideo(Channel c) { return (int)c < 32; }; diff --git a/components/codecs/include/ftl/codecs/touch.hpp b/components/codecs/include/ftl/codecs/touch.hpp new file mode 100644 index 000000000..d07eb32d4 --- /dev/null +++ b/components/codecs/include/ftl/codecs/touch.hpp @@ -0,0 +1,35 @@ +#ifndef _FTL_CODECS_TOUCH_HPP_ +#define _FTL_CODECS_TOUCH_HPP_ + +#include <ftl/utility/msgpack.hpp> + +namespace ftl { +namespace codecs { + +enum class TouchType { + MOUSE_LEFT=0, + MOUSE_RIGHT=1, + MOUSE_MIDDLE=2, + TOUCH_SCREEN=3, + COLLISION=16 +}; + +struct Touch { + Touch() {}; + + int id; + TouchType type; + uchar strength; + int x; + int y; + float d; + + MSGPACK_DEFINE(id, type, strength, x, y, d); +}; + +} +} + +MSGPACK_ADD_ENUM(ftl::codecs::TouchType); + +#endif diff --git a/components/rgbd-sources/CMakeLists.txt b/components/rgbd-sources/CMakeLists.txt index 8fafef644..8b26fd5bc 100644 --- a/components/rgbd-sources/CMakeLists.txt +++ b/components/rgbd-sources/CMakeLists.txt @@ -36,7 +36,7 @@ if (CUDA_FOUND) set_property(TARGET ftlrgbd PROPERTY CUDA_SEPARABLE_COMPILATION OFF) endif() -target_link_libraries(ftlrgbd ftlcalibration ftlcommon ${OpenCV_LIBS} ${LIBSGM_LIBRARIES} ${CUDA_LIBRARIES} Eigen3::Eigen realsense ftlnet ${LibArchive_LIBRARIES} ftlcodecs ftloperators ftldata ${X11_X11_LIB} ${X11_Xext_LIB} Pylon) +target_link_libraries(ftlrgbd ftlcalibration ftlcommon ${OpenCV_LIBS} ${LIBSGM_LIBRARIES} ${CUDA_LIBRARIES} Eigen3::Eigen realsense ftlnet ${LibArchive_LIBRARIES} ftlcodecs ftloperators ftldata ${X11_X11_LIB} ${X11_Xext_LIB} ${X11_Xtst_LIB} Pylon) if (BUILD_TESTS) add_subdirectory(test) diff --git a/components/rgbd-sources/src/source.cpp b/components/rgbd-sources/src/source.cpp index 02710bc87..1d8d2b2bd 100644 --- a/components/rgbd-sources/src/source.cpp +++ b/components/rgbd-sources/src/source.cpp @@ -166,7 +166,8 @@ void Source::reset() { "offset_z", "size", "focal", - "device_left" + "device_left", + "enable_touch" }); impl_ = createImplementation(*uristr, this); } diff --git a/components/rgbd-sources/src/sources/screencapture/screencapture.cpp b/components/rgbd-sources/src/sources/screencapture/screencapture.cpp index e8aa30760..595c6dfc0 100644 --- a/components/rgbd-sources/src/sources/screencapture/screencapture.cpp +++ b/components/rgbd-sources/src/sources/screencapture/screencapture.cpp @@ -7,16 +7,20 @@ #include <opencv2/calib3d.hpp> #include <Eigen/Eigen> #include <opencv2/core/eigen.hpp> +#include <ftl/rgbd/capabilities.hpp> +#include <ftl/codecs/touch.hpp> using ftl::rgbd::detail::ScreenCapture; using ftl::codecs::Channel; using cv::cuda::GpuMat; +using ftl::rgbd::Capability; #ifdef HAVE_X11 #include <X11/Xlib.h> #include <X11/Xutil.h> #include <X11/extensions/XShm.h> +#include <X11/extensions/XTest.h> #include <sys/ipc.h> #include <sys/shm.h> @@ -194,10 +198,33 @@ ScreenCapture::~ScreenCapture() { #endif } +void ScreenCapture::_mouseClick(int button, int x, int y) { + auto &s = *impl_state_; + + XTestFakeMotionEvent (s.display, 0, x, y, CurrentTime); + XSync(s.display, 0); + std::this_thread::sleep_for(std::chrono::milliseconds(10)); + XTestFakeButtonEvent (s.display, button, True, CurrentTime); + //XSync(s.display, 0); + //std::this_thread::sleep_for(std::chrono::milliseconds(10)); + XTestFakeButtonEvent (s.display, button, False, CurrentTime); +} + bool ScreenCapture::retrieve(ftl::rgbd::Frame &frame) { if (!ready_) return false; cv::Mat img; + if (host_->value("enable_touch", false)) { + if (frame.changed(Channel::Touch)) { + LOG(INFO) << "GOT TOUCH DATA"; + const auto &touches = frame.get<std::vector<ftl::codecs::Touch>>(Channel::Touch); + for (const auto &t : touches) { + LOG(INFO) << " -- " << t.x << "," << t.y; + _mouseClick(1, t.x, t.y); + } + } + } + #ifdef HAVE_X11 XShmGetImage(impl_state_->display, impl_state_->root, impl_state_->ximg, getOffsetX(), getOffsetY(), 0x00ffffff); img = cv::Mat(params_.height, params_.width, CV_8UC4, impl_state_->ximg->data); @@ -206,6 +233,18 @@ bool ScreenCapture::retrieve(ftl::rgbd::Frame &frame) { if (do_update_params_) { frame.setPose() = pose_; frame.setLeft() = params_; + + auto &meta = frame.create<std::map<std::string,std::string>>(Channel::MetaData); + meta["name"] = host_->value("name", host_->getID()); + meta["uri"] = host_->value("uri", std::string("")); + + if (!frame.has(Channel::Capabilities)) { + auto &cap = frame.create<std::unordered_set<Capability>>(Channel::Capabilities); + cap.emplace(Capability::VIDEO); + cap.emplace(Capability::LIVE); + cap.emplace(Capability::TOUCH); + } + do_update_params_ = false; } diff --git a/components/rgbd-sources/src/sources/screencapture/screencapture.hpp b/components/rgbd-sources/src/sources/screencapture/screencapture.hpp index a84dd2bc0..5bf7b9448 100644 --- a/components/rgbd-sources/src/sources/screencapture/screencapture.hpp +++ b/components/rgbd-sources/src/sources/screencapture/screencapture.hpp @@ -43,6 +43,8 @@ class ScreenCapture : public ftl::rgbd::BaseSourceImpl { bool do_update_params_ = false; ImplState *impl_state_; + + void _mouseClick(int button, int x, int y); }; } diff --git a/components/structures/src/new_frame.cpp b/components/structures/src/new_frame.cpp index ba21b33c6..46df69496 100644 --- a/components/structures/src/new_frame.cpp +++ b/components/structures/src/new_frame.cpp @@ -38,7 +38,7 @@ bool ftl::data::isPersistent(ftl::codecs::Channel c) { bool ftl::data::isAggregate(ftl::codecs::Channel c) { auto i = reg_channels.find(c); - return (i != reg_channels.end()) ? i->second.mode == StorageMode::AGGREGATE : int(c) >= 32 && int(c) < 64; + return (i != reg_channels.end()) ? i->second.mode == StorageMode::AGGREGATE : (int(c) >= 32 && int(c) < 64) || int(c) >= 4096; } size_t ftl::data::getChannelType(ftl::codecs::Channel c) { -- GitLab