diff --git a/applications/gui2/src/modules/camera.cpp b/applications/gui2/src/modules/camera.cpp index d6c0c8e927691a349fb5c45b6f36499c4b64c575..038e7e1712261b2fc938cc977c98f6ce42233d36 100644 --- a/applications/gui2/src/modules/camera.cpp +++ b/applications/gui2/src/modules/camera.cpp @@ -75,6 +75,7 @@ void Camera::activate(ftl::data::FrameID id) { frame_idx = id.source(); has_seen_frame_ = false; + point_.id = -1; std::mutex m; std::condition_variable cv; @@ -96,6 +97,15 @@ void Camera::activate(ftl::data::FrameID id) { if (!view) return true; + if (point_.id >= 0) { + auto response = fs->frames[frame_idx].response(); + auto &data = response.create<std::vector<ftl::codecs::Touch>>(Channel::Touch); + data.resize(1); + UNIQUE_LOCK(mtx_, lk); + data[0] = point_; + //point_.strength = 0; + } + // Deal with audio if (fs->frames[frame_idx].hasOwn(Channel::AudioStereo)) { speaker->queue(fs->timestamp(), fs->frames[frame_idx]); @@ -151,9 +161,17 @@ void Camera::sendPose(const Eigen::Matrix4d &pose) { } } -void Camera::touch(int id, ftl::codecs::TouchType t, int x, int y, float d) { +void Camera::touch(int id, ftl::codecs::TouchType t, int x, int y, float d, int strength) { + UNIQUE_LOCK(mtx_, lk); + point_.id = id; + point_.type = t; + point_.x = x; + point_.y = y; + point_.d = d; + point_.strength = strength; //std::max((unsigned char)strength, point_.strength); + // TODO: Check for touch capability first - if (auto ptr = std::atomic_load(&latest_)) { + /*if (auto ptr = std::atomic_load(&latest_)) { auto response = ptr->frames[frame_idx].response(); auto &data = response.create<std::vector<ftl::codecs::Touch>>(Channel::Touch); data.resize(0); @@ -163,5 +181,8 @@ void Camera::touch(int id, ftl::codecs::TouchType t, int x, int y, float d) { pt.x = x; pt.y = y; pt.d = d; - } + pt.strength = strength; + + + }*/ } diff --git a/applications/gui2/src/modules/camera.hpp b/applications/gui2/src/modules/camera.hpp index 66ce92c2d6af31724e5d301e2dafd91b88e1e1f0..c9e56d3845af69b246a3649c7c26596decf5ae0b 100644 --- a/applications/gui2/src/modules/camera.hpp +++ b/applications/gui2/src/modules/camera.hpp @@ -32,7 +32,7 @@ public: bool getFrame(ftl::cuda::TextureObject<uchar4>&); std::unordered_set<ftl::codecs::Channel> availableChannels(); - void touch(int id, ftl::codecs::TouchType t, int x, int y, float d); + void touch(int id, ftl::codecs::TouchType t, int x, int y, float d, int strength); /** Check if new frame is available */ bool hasFrame(); @@ -47,6 +47,7 @@ private: ftl::stream::Feed::Filter *filter = nullptr; std::atomic_bool paused = false; // TODO: implement in InputOutput bool has_seen_frame_ = false; + ftl::codecs::Touch point_; ftl::data::FrameSetPtr current_fs_; ftl::data::FrameSetPtr latest_; @@ -57,6 +58,8 @@ private: CameraView* view = nullptr; + MUTEX mtx_; + void initiate_(ftl::data::Frame &frame); }; diff --git a/applications/gui2/src/views/camera.cpp b/applications/gui2/src/views/camera.cpp index e8ad12861d059a3b488e9a396aa6873ee7b168db..9c0966b995ee0c4e05dca8c33ea55809bb0913ea 100644 --- a/applications/gui2/src/views/camera.cpp +++ b/applications/gui2/src/views/camera.cpp @@ -207,12 +207,23 @@ void CameraView::setPan(bool v) { } } +bool CameraView::mouseMotionEvent(const Eigen::Vector2i &p, const Eigen::Vector2i &rel, int button, int modifiers) { + //if (button == 1) { + auto pos = imview_->imageCoordinateAt((p - mPos + rel).cast<float>()); + if (pos.x() >= 0.0f && pos.y() >= 0.0f) { + ctrl_->touch(0, ftl::codecs::TouchType::MOUSE_LEFT, pos.x(), pos.y(), 0.0f, (button > 0) ? 255 : 0); + } + return true; + //} + return false; +} + bool CameraView::mouseButtonEvent(const Eigen::Vector2i &p, int button, bool down, int modifiers) { //LOG(INFO) << "mouseButtonEvent: " << p << " - " << button; - if (button == 0 && !down) { + if (button == 0) { auto pos = imview_->imageCoordinateAt((p - mPos).cast<float>()); if (pos.x() >= 0.0f && pos.y() >= 0.0f) { - ctrl_->touch(0, ftl::codecs::TouchType::MOUSE_LEFT, pos.x(), pos.y(), 0.0f); + ctrl_->touch(0, ftl::codecs::TouchType::MOUSE_LEFT, pos.x(), pos.y(), 0.0f, (down) ? 255 : 0); } return true; } diff --git a/applications/gui2/src/views/camera.hpp b/applications/gui2/src/views/camera.hpp index 1cc8dd1164587eb14d51b201d97c50c398743cd9..aff6d9e8c4185e90d86ceaa6685b5ff4ece73695 100644 --- a/applications/gui2/src/views/camera.hpp +++ b/applications/gui2/src/views/camera.hpp @@ -22,6 +22,7 @@ public: virtual void draw(NVGcontext* ctx) override; virtual void performLayout(NVGcontext* ctx) override; virtual bool mouseButtonEvent(const Eigen::Vector2i &p, int button, bool down, int modifiers) override; + virtual bool mouseMotionEvent(const Eigen::Vector2i &p, const Eigen::Vector2i &rel, int button, int modifiers) override; void refresh(); void setZoom(bool enable); diff --git a/components/codecs/include/ftl/codecs/touch.hpp b/components/codecs/include/ftl/codecs/touch.hpp index d07eb32d42e362404ea41bc680d56e3534157b64..a48ba834ad3820e355e42362ae80c1fcc0006012 100644 --- a/components/codecs/include/ftl/codecs/touch.hpp +++ b/components/codecs/include/ftl/codecs/touch.hpp @@ -19,7 +19,7 @@ struct Touch { int id; TouchType type; - uchar strength; + uint8_t strength; int x; int y; float d; diff --git a/components/rgbd-sources/src/sources/screencapture/screencapture.cpp b/components/rgbd-sources/src/sources/screencapture/screencapture.cpp index fef465c2067120c55641828632c6ee8190c2bd8c..7e46a557a2ee95fe758a0c2eeb50655539a6bd0e 100644 --- a/components/rgbd-sources/src/sources/screencapture/screencapture.cpp +++ b/components/rgbd-sources/src/sources/screencapture/screencapture.cpp @@ -10,10 +10,14 @@ #include <ftl/rgbd/capabilities.hpp> #include <ftl/codecs/touch.hpp> +#include <opencv2/imgproc.hpp> + using ftl::rgbd::detail::ScreenCapture; using ftl::codecs::Channel; using cv::cuda::GpuMat; using ftl::rgbd::Capability; +using ftl::codecs::Touch; +using ftl::codecs::TouchType; #ifdef HAVE_X11 #include <X11/Xlib.h> @@ -63,6 +67,7 @@ ScreenCapture::ScreenCapture(ftl::rgbd::Source *host) capabilities_ = kCapVideo; ready_ = false; + primary_touch_.id = -1; #ifdef HAVE_X11 @@ -198,7 +203,7 @@ ScreenCapture::~ScreenCapture() { #endif } -void ScreenCapture::_mouseClick(int button, int x, int y) { +/*void ScreenCapture::_mouseClick(int button, int x, int y) { #ifdef HAVE_X11 auto &s = *impl_state_; @@ -210,6 +215,70 @@ void ScreenCapture::_mouseClick(int button, int x, int y) { XTestFakeButtonEvent (s.display, Button1, False, CurrentTime); #endif +}*/ + +void ScreenCapture::_release() { + pressed_ = false; + #ifdef HAVE_X11 + auto &s = *impl_state_; + //XTestFakeButtonEvent (s.display, Button1, False, CurrentTime); + #endif +} + +void ScreenCapture::_press() { + pressed_ = true; + #ifdef HAVE_X11 + auto &s = *impl_state_; + //XTestFakeButtonEvent (s.display, Button1, True, CurrentTime); + #endif + + LOG(INFO) << "PRESS"; +} + +void ScreenCapture::_move(int x, int y) { + #ifdef HAVE_X11 + + auto &s = *impl_state_; + //XTestFakeMotionEvent (s.display, 0, x, y, CurrentTime); + //XSync(s.display, 0); + #endif +} + +void ScreenCapture::_noTouch() { + if (primary_touch_.id >= 0 && primary_touch_.strength > 0) { + // RELEASE BUTTON + _release(); + } + primary_touch_.id = -1; +} + +void ScreenCapture::_singleTouch(const ftl::codecs::Touch &t) { + // Ignore right clicks currently + if (t.type != TouchType::MOUSE_LEFT && t.type != TouchType::COLLISION) return; + + if ((primary_touch_.id >= 0 && primary_touch_.id != t.id) || (primary_touch_.id == t.id && primary_touch_.strength > 0 && t.strength == 0)) { + // RELEASE BUTTON + _release(); + } + + // Move mouse if no primary or ID is the same. + if (primary_touch_.id == -1 || t.id == primary_touch_.id) { + // But only if changed...? + // MOVE MOUSE + _move(t.x, t.y); + } + + // If no primary or same and intensity is > 0, then press + if ((primary_touch_.id == -1 && t.strength > 0) || (primary_touch_.id == t.id && primary_touch_.strength == 0 && t.strength > 0)) { + // PRESS EVENT + _press(); + } + + primary_touch_ = t; +} + +void ScreenCapture::_multiTouch(const std::vector<ftl::codecs::Touch> &touches) { + } bool ScreenCapture::retrieve(ftl::rgbd::Frame &frame) { @@ -218,25 +287,45 @@ bool ScreenCapture::retrieve(ftl::rgbd::Frame &frame) { // TODO: Proper, press, release and motion behaviour // Also, render the cursor location + + #ifdef HAVE_X11 + XShmGetImage(impl_state_->display, impl_state_->root, impl_state_->ximg, getOffsetX(), getOffsetY(), 0x00ffffff); + img = cv::Mat(params_.height, params_.width, CV_8UC4, impl_state_->ximg->data); + #endif + if (host_->value("enable_touch", false)) { if (frame.changed(Channel::Touch)) { const auto &touches = frame.get<std::vector<ftl::codecs::Touch>>(Channel::Touch); - LOG(INFO) << "GOT TOUCH DATA " << touches.size(); + //LOG(INFO) << "GOT TOUCH DATA " << touches.size(); - for (const auto &t : touches) { + /*for (const auto &t : touches) { LOG(INFO) << " -- " << t.x << "," << t.y; + }*/ + + if (touches.size() == 0) { + _noTouch(); + } else if (touches.size() == 1) { + //_mouseClick(1, touches[0].x, touches[0].y); + _singleTouch(touches[0]); + } else if (touches.size() == 2) { + _multiTouch(touches); + } else { + // Too many touches, not supported } - if (touches.size() > 0) { - _mouseClick(1, touches[0].x, touches[0].y); + } else { + _noTouch(); + } + + // If there is a touch, render it. + if (primary_touch_.id >= 0) { + if (pressed_) { + cv::circle(img, cv::Point(primary_touch_.x, primary_touch_.y), 5, cv::Scalar(0,0,255), 3); + } else { + cv::circle(img, cv::Point(primary_touch_.x, primary_touch_.y), 3, cv::Scalar(0,0,255)); } } } - #ifdef HAVE_X11 - XShmGetImage(impl_state_->display, impl_state_->root, impl_state_->ximg, getOffsetX(), getOffsetY(), 0x00ffffff); - img = cv::Mat(params_.height, params_.width, CV_8UC4, impl_state_->ximg->data); - #endif - if (do_update_params_) { frame.setPose() = pose_; frame.setLeft() = params_; diff --git a/components/rgbd-sources/src/sources/screencapture/screencapture.hpp b/components/rgbd-sources/src/sources/screencapture/screencapture.hpp index 5bf7b9448d8877bd137aa385ca8b6b68f2d4ee06..39b9a146e0f1ed173d3fcc3240bb3896a0a40d44 100644 --- a/components/rgbd-sources/src/sources/screencapture/screencapture.hpp +++ b/components/rgbd-sources/src/sources/screencapture/screencapture.hpp @@ -3,6 +3,7 @@ #include "../../basesource.hpp" #include <ftl/config.h> +#include <ftl/codecs/touch.hpp> namespace ftl { @@ -41,10 +42,19 @@ class ScreenCapture : public ftl::rgbd::BaseSourceImpl { size_t offset_y_; Eigen::Matrix4d pose_; bool do_update_params_ = false; + bool pressed_ = false; + ftl::codecs::Touch primary_touch_; ImplState *impl_state_; - void _mouseClick(int button, int x, int y); + //void _mouseClick(int button, int x, int y); + + void _singleTouch(const ftl::codecs::Touch &t); + void _press(); + void _release(); + void _move(int x, int y); + void _noTouch(); + void _multiTouch(const std::vector<ftl::codecs::Touch> &); }; } diff --git a/components/streams/include/ftl/streams/filestream.hpp b/components/streams/include/ftl/streams/filestream.hpp index 6805554830aabf86b3ab8de96ee167c01825381a..5b139546f69a590a2f791e6d2b8dd79df8b8abcc 100644 --- a/components/streams/include/ftl/streams/filestream.hpp +++ b/components/streams/include/ftl/streams/filestream.hpp @@ -83,6 +83,9 @@ class File : public Stream { bool _open(); bool _checkFile(); + + /* Apply version patches etc... */ + void _patchPackets(ftl::codecs::StreamPacket &spkt, ftl::codecs::Packet &pkt); }; } diff --git a/components/streams/src/filestream.cpp b/components/streams/src/filestream.cpp index 35a70f2175bfecde1746b798877be8856ae9281a..acadd13ef1e1fe333b4baa0d8d3abe0d59dd1d3a 100644 --- a/components/streams/src/filestream.cpp +++ b/components/streams/src/filestream.cpp @@ -155,24 +155,8 @@ bool File::readPacket(std::tuple<ftl::codecs::StreamPacket,ftl::codecs::Packet> return false; } - // Fix to clear flags for version 2. - if (version_ <= 2) { - std::get<1>(data).flags = 0; - } - if (version_ < 4) { - std::get<0>(data).frame_number = std::get<0>(data).streamID; - std::get<0>(data).streamID = 0; - if (isFloatChannel(std::get<0>(data).channel)) std::get<1>(data).flags |= ftl::codecs::kFlagFloat; - - auto codec = std::get<1>(data).codec; - if (codec == ftl::codecs::codec_t::HEVC) std::get<1>(data).codec = ftl::codecs::codec_t::HEVC_LOSSLESS; - } - std::get<0>(data).version = 4; - - // Fix for flags corruption - if (std::get<1>(data).data.size() == 0) { - std::get<1>(data).flags = 0; - } + // Correct for older version differences. + _patchPackets(std::get<0>(data), std::get<1>(data)); return true; } @@ -180,6 +164,27 @@ bool File::readPacket(std::tuple<ftl::codecs::StreamPacket,ftl::codecs::Packet> return false; } +void File::_patchPackets(ftl::codecs::StreamPacket &spkt, ftl::codecs::Packet &pkt) { + // Fix to clear flags for version 2. + if (version_ <= 2) { + pkt.flags = 0; + } + if (version_ < 4) { + spkt.frame_number = spkt.streamID; + spkt.streamID = 0; + if (isFloatChannel(spkt.channel)) pkt.flags |= ftl::codecs::kFlagFloat; + + auto codec = pkt.codec; + if (codec == ftl::codecs::codec_t::HEVC) pkt.codec = ftl::codecs::codec_t::HEVC_LOSSLESS; + } + spkt.version = 4; + + // Fix for flags corruption + if (pkt.data.size() == 0) { + pkt.flags = 0; + } +} + bool File::tick(int64_t ts) { if (!active_) return false; if (mode_ != Mode::Read) { diff --git a/components/structures/include/ftl/data/new_frame.hpp b/components/structures/include/ftl/data/new_frame.hpp index 1be1544f3caf13cd12161cda4d6c38012d4ca1f8..dab1a8bb266fc74b13d77f93b28c7e6badbac8e9 100644 --- a/components/structures/include/ftl/data/new_frame.hpp +++ b/components/structures/include/ftl/data/new_frame.hpp @@ -129,7 +129,7 @@ struct Aggregator { * It can be moved around but not copied since the quantity of data involved in * a frame is huge. * - * A frame does through the following stages: + * A frame goes through the following stages: * 1) Creation from reused memory in `Pool` * 2) Populate with incoming initial data/changes (from stream) * 3) Store of changes to persistent memory