From a9ecee853484f43fdeebadcb0cc4b4dd6faf7eb7 Mon Sep 17 00:00:00 2001
From: Nicolas Pope <nwpope@utu.fi>
Date: Wed, 20 May 2020 17:11:51 +0300
Subject: [PATCH] Allow use of web pose by GUI
---
applications/gui/src/camera.cpp | 10 +++++++-
.../operators/include/ftl/operators/poser.hpp | 1 +
components/operators/src/poser.cpp | 16 ++++++++++++
components/streams/src/sender.cpp | 2 +-
web-service/public/js/bundle.js | 25 ++++++++++++++++++-
web-service/public/js/index.js | 25 ++++++++++++++++++-
6 files changed, 75 insertions(+), 4 deletions(-)
diff --git a/applications/gui/src/camera.cpp b/applications/gui/src/camera.cpp
index 0c3b58778..5d3cb2acc 100644
--- a/applications/gui/src/camera.cpp
+++ b/applications/gui/src/camera.cpp
@@ -16,6 +16,7 @@
#include <ftl/operators/gt_analysis.hpp>
#include <ftl/operators/poser.hpp>
#include <ftl/cuda/colour_cuda.hpp>
+#include <ftl/streams/parsers.hpp>
#include <ftl/render/overlay.hpp>
#include "statsimage.hpp"
@@ -847,7 +848,14 @@ void ftl::gui::Camera::startVideoRecording(const std::string &filename, const st
record_sender_ = ftl::create<ftl::stream::Sender>(screen_->root(), "videoEncode");
record_sender_->value("codec", 2); // Default H264
record_sender_->set("iframes", 50); // Add iframes by default
- record_sender_->value("stereo", true); // If both channels, then default to stereo
+ record_sender_->value("stereo", false); // If both channels, then default to stereo
+
+ record_sender_->onRequest([this](const ftl::codecs::StreamPacket &spkt, const ftl::codecs::Packet &pkt) {
+ if (spkt.channel == ftl::codecs::Channel::Pose) {
+ auto pose = ftl::stream::parsePose(pkt);
+ ftl::operators::Poser::set(std::string("live"), pose);
+ }
+ });
}
if (record_stream_->active()) return;
diff --git a/components/operators/include/ftl/operators/poser.hpp b/components/operators/include/ftl/operators/poser.hpp
index 96e6b1926..732ee47ed 100644
--- a/components/operators/include/ftl/operators/poser.hpp
+++ b/components/operators/include/ftl/operators/poser.hpp
@@ -23,6 +23,7 @@ class Poser : public ftl::operators::Operator {
bool apply(ftl::rgbd::FrameSet &in, ftl::rgbd::FrameSet &out, cudaStream_t stream) override;
static bool get(const std::string &name, Eigen::Matrix4d &pose);
+ static bool set(const std::string &name, const Eigen::Matrix4d &pose);
private:
struct PoseState {
diff --git a/components/operators/src/poser.cpp b/components/operators/src/poser.cpp
index d03ef35fe..f81c11d39 100644
--- a/components/operators/src/poser.cpp
+++ b/components/operators/src/poser.cpp
@@ -55,6 +55,22 @@ bool Poser::get(const std::string &name, Eigen::Matrix4d &pose) {
}
}
+bool Poser::set(const std::string &name, const Eigen::Matrix4d &pose) {
+ auto p = pose_db__.find(name);
+ if (p == pose_db__.end()) {
+ ftl::operators::Poser::PoseState ps;
+ ps.pose = pose;
+ ps.locked = false;
+ pose_db__.emplace(std::make_pair(name,ps));
+ LOG(INFO) << "POSE ID: " << name;
+ } else {
+ // TODO: Merge poses
+ if (!(*p).second.locked) (*p).second.pose = pose;
+ //LOG(INFO) << "POSE ID: " << idstr;
+ }
+ return true;
+}
+
bool Poser::apply(ftl::rgbd::FrameSet &in, ftl::rgbd::FrameSet &out, cudaStream_t stream) {
if (in.hasChannel(Channel::Shapes3D)) {
std::vector<ftl::codecs::Shape3D> transforms;
diff --git a/components/streams/src/sender.cpp b/components/streams/src/sender.cpp
index 8277a73d9..87bc83019 100644
--- a/components/streams/src/sender.cpp
+++ b/components/streams/src/sender.cpp
@@ -44,7 +44,7 @@ void Sender::setStream(ftl::stream::Stream*s) {
if (stream_) stream_->onPacket(nullptr);
stream_ = s;
stream_->onPacket([this](const ftl::codecs::StreamPacket &spkt, const ftl::codecs::Packet &pkt) {
- //LOG(INFO) << "SENDER REQUEST : " << (int)spkt.channel;
+ LOG(INFO) << "SENDER REQUEST : " << (int)spkt.channel;
//if (state_cb_) state_cb_(spkt.channel, spkt.streamID, spkt.frame_number);
if (reqcb_) reqcb_(spkt,pkt);
diff --git a/web-service/public/js/bundle.js b/web-service/public/js/bundle.js
index 7557e2bdf..fd03f5f40 100644
--- a/web-service/public/js/bundle.js
+++ b/web-service/public/js/bundle.js
@@ -4371,6 +4371,11 @@ function FTLStream(peer, uri, element) {
this.uri = uri;
this.peer = peer;
+ this.current = "";
+ this.current_fs = 0;
+ this.current_source = 0;
+ this.current_channel = 0;
+
//this.elements_ = {};
//this.converters_ = {};
@@ -4387,6 +4392,12 @@ function FTLStream(peer, uri, element) {
this.start(0,0,0);
}
this.outer.appendChild(this.play_button);
+
+ this.element.onclick = () => {
+ let pose = [1,0,0,0,0,1,0,0,0,0,1,0,0,0,0,1];
+ this.setPose(pose);
+ }
+
this.converter = null;
let rxcount = 0;
@@ -4422,16 +4433,28 @@ function FTLStream(peer, uri, element) {
}
}
} else if (pckg[0] === 103) {
- console.log(msgpack.decode(pckg[5]));
+ //console.log(msgpack.decode(pckg[5]));
}
});
//this.start();
}
+FTLStream.prototype.setPose = function(pose) {
+ if (pose.length != 16) {
+ console.error("Invalid pose");
+ return;
+ }
+ this.peer.send(this.uri, 0, [1, this.current_fs, this.current_source, 66],
+ [103, 7, 1, 0, 0, msgpack.encode(pose)]);
+}
+
FTLStream.prototype.start = function(fs, source, channel) {
let id = "id-"+fs+"-"+source+"-"+channel;
this.current = id;
+ this.current_fs = fs;
+ this.current_source = source;
+ this.current_channel = channel;
if (this.found) {
this.peer.send(this.uri, 0, [1,fs,255,channel],[255,7,35,0,0,Buffer.alloc(0)]);
diff --git a/web-service/public/js/index.js b/web-service/public/js/index.js
index 12956dad8..70b7c1006 100644
--- a/web-service/public/js/index.js
+++ b/web-service/public/js/index.js
@@ -125,6 +125,11 @@ function FTLStream(peer, uri, element) {
this.uri = uri;
this.peer = peer;
+ this.current = "";
+ this.current_fs = 0;
+ this.current_source = 0;
+ this.current_channel = 0;
+
//this.elements_ = {};
//this.converters_ = {};
@@ -141,6 +146,12 @@ function FTLStream(peer, uri, element) {
this.start(0,0,0);
}
this.outer.appendChild(this.play_button);
+
+ this.element.onclick = () => {
+ let pose = [1,0,0,0,0,1,0,0,0,0,1,0,0,0,0,1];
+ this.setPose(pose);
+ }
+
this.converter = null;
let rxcount = 0;
@@ -176,16 +187,28 @@ function FTLStream(peer, uri, element) {
}
}
} else if (pckg[0] === 103) {
- console.log(msgpack.decode(pckg[5]));
+ //console.log(msgpack.decode(pckg[5]));
}
});
//this.start();
}
+FTLStream.prototype.setPose = function(pose) {
+ if (pose.length != 16) {
+ console.error("Invalid pose");
+ return;
+ }
+ this.peer.send(this.uri, 0, [1, this.current_fs, this.current_source, 66],
+ [103, 7, 1, 0, 0, msgpack.encode(pose)]);
+}
+
FTLStream.prototype.start = function(fs, source, channel) {
let id = "id-"+fs+"-"+source+"-"+channel;
this.current = id;
+ this.current_fs = fs;
+ this.current_source = source;
+ this.current_channel = channel;
if (this.found) {
this.peer.send(this.uri, 0, [1,fs,255,channel],[255,7,35,0,0,Buffer.alloc(0)]);
--
GitLab