From fc110bb761dc935548d66626e3425b4a214c0e97 Mon Sep 17 00:00:00 2001
From: Nicolas Pope <nwpope@utu.fi>
Date: Sat, 11 Jul 2020 12:34:02 +0300
Subject: [PATCH] WIP Screencapture generates mouse events

---
 applications/gui2/src/modules/camera.cpp      | 17 +++++++-
 applications/gui2/src/modules/camera.hpp      |  4 ++
 applications/gui2/src/views/camera.cpp        | 12 ++++++
 applications/gui2/src/views/camera.hpp        |  1 +
 applications/gui2/src/widgets/imageview.cpp   |  2 +-
 .../codecs/include/ftl/codecs/channels.hpp    |  3 +-
 .../codecs/include/ftl/codecs/touch.hpp       | 35 +++++++++++++++++
 components/rgbd-sources/CMakeLists.txt        |  2 +-
 components/rgbd-sources/src/source.cpp        |  3 +-
 .../sources/screencapture/screencapture.cpp   | 39 +++++++++++++++++++
 .../sources/screencapture/screencapture.hpp   |  2 +
 components/structures/src/new_frame.cpp       |  2 +-
 12 files changed, 116 insertions(+), 6 deletions(-)
 create mode 100644 components/codecs/include/ftl/codecs/touch.hpp

diff --git a/applications/gui2/src/modules/camera.cpp b/applications/gui2/src/modules/camera.cpp
index 24eec071b..21c070303 100644
--- a/applications/gui2/src/modules/camera.cpp
+++ b/applications/gui2/src/modules/camera.cpp
@@ -83,6 +83,7 @@ void Camera::activate(ftl::data::FrameID id) {
 			if (paused) return true;
 
 			std::atomic_store(&current_fs_, fs);
+			std::atomic_store(&latest_, fs);
 
 			// Need to notify GUI thread when first data comes
 			if (!has_seen_frame_) {
@@ -142,9 +143,23 @@ bool Camera::hasFrame() {
 }
 
 void Camera::sendPose(const Eigen::Matrix4d &pose) {
-	if (auto ptr = std::atomic_load(&current_fs_)) {
+	if (auto ptr = std::atomic_load(&latest_)) {
 		auto response = ptr->frames[frame_idx].response();
 		auto &rgbdresponse = response.cast<ftl::rgbd::Frame>();
 		rgbdresponse.setPose() = pose;
 	}
 }
+
+void Camera::touch(int id, ftl::codecs::TouchType t, int x, int y, float d) {
+	// TODO: Check for touch capability first
+	if (auto ptr = std::atomic_load(&latest_)) {
+		auto response = ptr->frames[frame_idx].response();
+		auto &data = response.create<std::vector<ftl::codecs::Touch>>(Channel::Touch);
+		auto &pt = data.emplace_back();
+		pt.id = id;
+		pt.type = t;
+		pt.x = x;
+		pt.y = y;
+		pt.d = d;
+	}
+}
diff --git a/applications/gui2/src/modules/camera.hpp b/applications/gui2/src/modules/camera.hpp
index 018ee20c4..e3abfac78 100644
--- a/applications/gui2/src/modules/camera.hpp
+++ b/applications/gui2/src/modules/camera.hpp
@@ -6,6 +6,7 @@
 
 #include <ftl/render/colouriser.hpp>
 #include <ftl/render/overlay.hpp>
+#include <ftl/codecs/touch.hpp>
 
 namespace ftl {
 namespace gui2 {
@@ -27,6 +28,8 @@ public:
 	ftl::cuda::TextureObject<uchar4>& getFrame();
 	bool getFrame(ftl::cuda::TextureObject<uchar4>&);
 
+	void touch(int id, ftl::codecs::TouchType t, int x, int y, float d);
+
 	/** Check if new frame is available */
 	bool hasFrame();
 
@@ -43,6 +46,7 @@ private:
 	bool has_seen_frame_ = false;
 
 	ftl::data::FrameSetPtr current_fs_;
+	ftl::data::FrameSetPtr latest_;
 	ftl::cuda::TextureObject<uchar4> current_frame_;
 
 	std::unique_ptr<ftl::render::Colouriser> colouriser_;
diff --git a/applications/gui2/src/views/camera.cpp b/applications/gui2/src/views/camera.cpp
index 806ccc33d..004b1f854 100644
--- a/applications/gui2/src/views/camera.cpp
+++ b/applications/gui2/src/views/camera.cpp
@@ -2,6 +2,8 @@
 #include <nanogui/layout.h>
 #include <nanogui/button.h>
 
+#include <ftl/codecs/touch.hpp>
+
 #include "camera.hpp"
 
 #include "../modules/camera.hpp"
@@ -134,6 +136,16 @@ CameraView::~CameraView() {
 	//LOG(INFO) << __func__ << " (" << this << ")";
 }
 
+bool CameraView::mouseButtonEvent(const Eigen::Vector2i &p, int button, bool down, int modifiers) {
+	//LOG(INFO) << "mouseButtonEvent: " << p << " - " << button;
+	if (button == 0) {
+		// FIXME: Allow for zoom and offset of image...
+		ctrl_->touch(0, ftl::codecs::TouchType::MOUSE_LEFT, p.x()-mPos.x(), p.y()-mPos.y(), 0.0f);
+		return true;
+	}
+	return false;
+}
+
 void CameraView::draw(NVGcontext *ctx) {
 	if (ctrl_->hasFrame()) {
 		texture_.copyFrom(ctrl_->getFrame());
diff --git a/applications/gui2/src/views/camera.hpp b/applications/gui2/src/views/camera.hpp
index 7230562ac..e90804508 100644
--- a/applications/gui2/src/views/camera.hpp
+++ b/applications/gui2/src/views/camera.hpp
@@ -40,6 +40,7 @@ public:
 	CameraView(Screen* parent, Camera* ctrl);
 	virtual ~CameraView();
 	virtual void draw(NVGcontext *ctx) override;
+	virtual bool mouseButtonEvent(const Eigen::Vector2i &p, int button, bool down, int modifiers) override;
 
 protected:
 	Camera* ctrl_;
diff --git a/applications/gui2/src/widgets/imageview.cpp b/applications/gui2/src/widgets/imageview.cpp
index f5dff3229..2dd45d4c6 100644
--- a/applications/gui2/src/widgets/imageview.cpp
+++ b/applications/gui2/src/widgets/imageview.cpp
@@ -177,7 +177,7 @@ void ftl::gui2::ImageView::zoom(int amount, const Vector2f& focusPosition) {
 }
 
 bool ftl::gui2::ImageView::mouseDragEvent(const Vector2i& p, const Vector2i& rel, int button, int /*modifiers*/) {
-	if ((button & (1 << GLFW_MOUSE_BUTTON_LEFT)) != 0 && !mFixedOffset) {
+	if ((button & (1 << GLFW_MOUSE_BUTTON_RIGHT)) != 0 && !mFixedOffset) {
 		setImageCoordinateAt((p + rel).cast<float>(), imageCoordinateAt(p.cast<float>()));
 		return true;
 	}
diff --git a/components/codecs/include/ftl/codecs/channels.hpp b/components/codecs/include/ftl/codecs/channels.hpp
index a2222b826..b6aba67dd 100644
--- a/components/codecs/include/ftl/codecs/channels.hpp
+++ b/components/codecs/include/ftl/codecs/channels.hpp
@@ -61,7 +61,8 @@ enum struct Channel : int {
 	Faces			= 2049, // Data about detected faces
 	Transforms		= 2050,	// Transformation matrices for framesets
 	Shapes3D		= 2051,	// Labeled 3D shapes
-	Messages		= 2052	// Vector of Strings
+	Messages		= 2052,	// Vector of Strings
+	Touch			= 2053  // List of touch data type (each touch point)
 };
 
 inline bool isVideo(Channel c) { return (int)c < 32; };
diff --git a/components/codecs/include/ftl/codecs/touch.hpp b/components/codecs/include/ftl/codecs/touch.hpp
new file mode 100644
index 000000000..d07eb32d4
--- /dev/null
+++ b/components/codecs/include/ftl/codecs/touch.hpp
@@ -0,0 +1,35 @@
+#ifndef _FTL_CODECS_TOUCH_HPP_
+#define _FTL_CODECS_TOUCH_HPP_
+
+#include <ftl/utility/msgpack.hpp>
+
+namespace ftl {
+namespace codecs {
+
+enum class TouchType {
+	MOUSE_LEFT=0,
+	MOUSE_RIGHT=1,
+	MOUSE_MIDDLE=2,
+	TOUCH_SCREEN=3,
+	COLLISION=16
+};
+
+struct Touch {
+	Touch() {};
+
+	int id;
+	TouchType type;
+	uchar strength;
+	int x;
+	int y;
+	float d;
+
+	MSGPACK_DEFINE(id, type, strength, x, y, d);
+};
+
+}
+}
+
+MSGPACK_ADD_ENUM(ftl::codecs::TouchType);
+
+#endif
diff --git a/components/rgbd-sources/CMakeLists.txt b/components/rgbd-sources/CMakeLists.txt
index 8fafef644..8b26fd5bc 100644
--- a/components/rgbd-sources/CMakeLists.txt
+++ b/components/rgbd-sources/CMakeLists.txt
@@ -36,7 +36,7 @@ if (CUDA_FOUND)
 set_property(TARGET ftlrgbd PROPERTY CUDA_SEPARABLE_COMPILATION OFF)
 endif()
 
-target_link_libraries(ftlrgbd ftlcalibration ftlcommon ${OpenCV_LIBS} ${LIBSGM_LIBRARIES} ${CUDA_LIBRARIES} Eigen3::Eigen realsense ftlnet ${LibArchive_LIBRARIES} ftlcodecs ftloperators ftldata ${X11_X11_LIB} ${X11_Xext_LIB} Pylon)
+target_link_libraries(ftlrgbd ftlcalibration ftlcommon ${OpenCV_LIBS} ${LIBSGM_LIBRARIES} ${CUDA_LIBRARIES} Eigen3::Eigen realsense ftlnet ${LibArchive_LIBRARIES} ftlcodecs ftloperators ftldata ${X11_X11_LIB} ${X11_Xext_LIB} ${X11_Xtst_LIB} Pylon)
 
 if (BUILD_TESTS)
 add_subdirectory(test)
diff --git a/components/rgbd-sources/src/source.cpp b/components/rgbd-sources/src/source.cpp
index 02710bc87..1d8d2b2bd 100644
--- a/components/rgbd-sources/src/source.cpp
+++ b/components/rgbd-sources/src/source.cpp
@@ -166,7 +166,8 @@ void Source::reset() {
 		"offset_z",
 		"size",
 		"focal",
-		"device_left"
+		"device_left",
+		"enable_touch"
 	});
 	impl_ = createImplementation(*uristr, this);
 }
diff --git a/components/rgbd-sources/src/sources/screencapture/screencapture.cpp b/components/rgbd-sources/src/sources/screencapture/screencapture.cpp
index e8aa30760..595c6dfc0 100644
--- a/components/rgbd-sources/src/sources/screencapture/screencapture.cpp
+++ b/components/rgbd-sources/src/sources/screencapture/screencapture.cpp
@@ -7,16 +7,20 @@
 #include <opencv2/calib3d.hpp>
 #include <Eigen/Eigen>
 #include <opencv2/core/eigen.hpp>
+#include <ftl/rgbd/capabilities.hpp>
+#include <ftl/codecs/touch.hpp>
 
 using ftl::rgbd::detail::ScreenCapture;
 using ftl::codecs::Channel;
 using cv::cuda::GpuMat;
+using ftl::rgbd::Capability;
 
 #ifdef HAVE_X11
 #include <X11/Xlib.h>
 #include <X11/Xutil.h>
 
 #include <X11/extensions/XShm.h>
+#include <X11/extensions/XTest.h>
 #include <sys/ipc.h>
 #include <sys/shm.h>
 
@@ -194,10 +198,33 @@ ScreenCapture::~ScreenCapture() {
 	#endif
 }
 
+void ScreenCapture::_mouseClick(int button, int x, int y) {
+	auto &s = *impl_state_;
+
+    XTestFakeMotionEvent (s.display, 0, x, y, CurrentTime);
+	XSync(s.display, 0);
+	std::this_thread::sleep_for(std::chrono::milliseconds(10));
+	XTestFakeButtonEvent (s.display, button, True,  CurrentTime);
+	//XSync(s.display, 0);
+	//std::this_thread::sleep_for(std::chrono::milliseconds(10));
+	XTestFakeButtonEvent (s.display, button, False, CurrentTime);
+}
+
 bool ScreenCapture::retrieve(ftl::rgbd::Frame &frame) {
 	if (!ready_) return false;
 	cv::Mat img;
 
+	if (host_->value("enable_touch", false)) {
+		if (frame.changed(Channel::Touch)) {
+			LOG(INFO) << "GOT TOUCH DATA";
+			const auto &touches = frame.get<std::vector<ftl::codecs::Touch>>(Channel::Touch);
+			for (const auto &t : touches) {
+				LOG(INFO) << " -- " << t.x << "," << t.y;
+				_mouseClick(1, t.x, t.y);
+			}
+		}
+	}
+
 	#ifdef HAVE_X11
 	XShmGetImage(impl_state_->display, impl_state_->root, impl_state_->ximg, getOffsetX(), getOffsetY(), 0x00ffffff);
     img = cv::Mat(params_.height, params_.width, CV_8UC4, impl_state_->ximg->data);
@@ -206,6 +233,18 @@ bool ScreenCapture::retrieve(ftl::rgbd::Frame &frame) {
 	if (do_update_params_) {
 		frame.setPose() = pose_;
 		frame.setLeft() = params_;
+
+		auto &meta = frame.create<std::map<std::string,std::string>>(Channel::MetaData);
+		meta["name"] = host_->value("name", host_->getID());
+		meta["uri"] = host_->value("uri", std::string(""));
+
+		if (!frame.has(Channel::Capabilities)) {
+			auto &cap = frame.create<std::unordered_set<Capability>>(Channel::Capabilities);
+			cap.emplace(Capability::VIDEO);
+			cap.emplace(Capability::LIVE);
+			cap.emplace(Capability::TOUCH);
+		}
+
 		do_update_params_ = false;
 	}
 
diff --git a/components/rgbd-sources/src/sources/screencapture/screencapture.hpp b/components/rgbd-sources/src/sources/screencapture/screencapture.hpp
index a84dd2bc0..5bf7b9448 100644
--- a/components/rgbd-sources/src/sources/screencapture/screencapture.hpp
+++ b/components/rgbd-sources/src/sources/screencapture/screencapture.hpp
@@ -43,6 +43,8 @@ class ScreenCapture : public ftl::rgbd::BaseSourceImpl {
 	bool do_update_params_ = false;
 
 	ImplState *impl_state_;
+
+	void _mouseClick(int button, int x, int y);
 };
 
 }
diff --git a/components/structures/src/new_frame.cpp b/components/structures/src/new_frame.cpp
index ba21b33c6..46df69496 100644
--- a/components/structures/src/new_frame.cpp
+++ b/components/structures/src/new_frame.cpp
@@ -38,7 +38,7 @@ bool ftl::data::isPersistent(ftl::codecs::Channel c) {
 
 bool ftl::data::isAggregate(ftl::codecs::Channel c) {
 	auto i = reg_channels.find(c);
-	return (i != reg_channels.end()) ? i->second.mode == StorageMode::AGGREGATE : int(c) >= 32 && int(c) < 64;
+	return (i != reg_channels.end()) ? i->second.mode == StorageMode::AGGREGATE : (int(c) >= 32 && int(c) < 64) || int(c) >= 4096;
 }
 
 size_t ftl::data::getChannelType(ftl::codecs::Channel c) {
-- 
GitLab