From 93b7083cf09a560d486905429521f2cdc99e7862 Mon Sep 17 00:00:00 2001
From: Iiro Rastas <iitara@utu.fi>
Date: Wed, 27 Nov 2019 16:09:41 +0200
Subject: [PATCH] Add GUI recording features
2D snapshots now have alpha blending between the channels.
3D snapshots have been added, though the feature is not yet functional.
Recording window for detailed options has been added, though it is still
missing most functionality.
---
applications/gui/CMakeLists.txt | 1 +
applications/gui/src/camera.cpp | 77 +++++++++++------
applications/gui/src/camera.hpp | 6 +-
applications/gui/src/media_panel.cpp | 39 +++++++--
applications/gui/src/media_panel.hpp | 6 +-
applications/gui/src/record_window.cpp | 83 +++++++++++++++++++
applications/gui/src/record_window.hpp | 19 +++++
applications/gui/src/screen.cpp | 2 +-
applications/gui/src/src_window.cpp | 3 +-
applications/reconstruct/src/main.cpp | 47 ++++++++++-
.../rgbd-sources/include/ftl/rgbd/source.hpp | 29 +++++++
components/rgbd-sources/src/source.cpp | 18 ++++
12 files changed, 293 insertions(+), 37 deletions(-)
create mode 100644 applications/gui/src/record_window.cpp
create mode 100644 applications/gui/src/record_window.hpp
diff --git a/applications/gui/CMakeLists.txt b/applications/gui/CMakeLists.txt
index fe553becc..9764984a7 100644
--- a/applications/gui/CMakeLists.txt
+++ b/applications/gui/CMakeLists.txt
@@ -13,6 +13,7 @@ set(GUISRC
src/camera.cpp
src/media_panel.cpp
src/thumbview.cpp
+ src/record_window.cpp
)
if (HAVE_OPENVR)
diff --git a/applications/gui/src/camera.cpp b/applications/gui/src/camera.cpp
index 0a9d04c04..38f6c5f77 100644
--- a/applications/gui/src/camera.cpp
+++ b/applications/gui/src/camera.cpp
@@ -139,7 +139,6 @@ ftl::gui::Camera::Camera(ftl::gui::Screen *screen, ftl::rgbd::Source *src) : scr
sdepth_ = false;
ftime_ = (float)glfwGetTime();
pause_ = false;
- recording_ = false;
fileout_ = new std::ofstream();
writer_ = new ftl::codecs::Writer(*fileout_);
recorder_ = std::function([this](ftl::rgbd::Source *src, const ftl::codecs::StreamPacket &spkt, const ftl::codecs::Packet &pkt) {
@@ -358,6 +357,27 @@ static void drawEdges( const cv::Mat &in, cv::Mat &out,
cv::addWeighted(edges, weight, out, 1.0, 0.0, out, CV_8UC3);
}
+cv::Mat ftl::gui::Camera::visualizeActiveChannel() {
+ cv::Mat result;
+ switch(channel_) {
+ case Channel::Smoothing:
+ case Channel::Confidence:
+ visualizeEnergy(im2_, result, 1.0);
+ break;
+ case Channel::Density:
+ case Channel::Energy:
+ visualizeEnergy(im2_, result, 10.0);
+ break;
+ case Channel::Depth:
+ visualizeDepthMap(im2_, result, 7.0);
+ if (screen_->root()->value("showEdgesInDepth", false)) drawEdges(im1_, result);
+ break;
+ case Channel::Right:
+ result = im2_;
+ }
+ return result;
+}
+
bool ftl::gui::Camera::thumbnail(cv::Mat &thumb) {
UNIQUE_LOCK(mutex_, lk);
src_->grab(1,9);
@@ -474,12 +494,12 @@ const GLTexture &ftl::gui::Camera::captureFrame() {
texture2_.update(tmp);*/
break;
- //case Channel::Flow:
- case Channel::ColourNormals:
- case Channel::Right:
- if (im2_.rows == 0 || im2_.type() != CV_8UC3) { break; }
- texture2_.update(im2_);
- break;
+ //case Channel::Flow:
+ case Channel::ColourNormals:
+ case Channel::Right:
+ if (im2_.rows == 0 || im2_.type() != CV_8UC3) { break; }
+ texture2_.update(im2_);
+ break;
default:
break;
@@ -509,30 +529,33 @@ void ftl::gui::Camera::snapshot() {
char timestamp[18];
std::time_t t = std::time(NULL);
std::strftime(timestamp, sizeof(timestamp), "%F-%H%M%S", std::localtime(&t));
- cv::Mat image;
- cv::flip(im1_, image, 0);
- cv::imwrite(std::string(timestamp) + ".png", image);
+ cv::Mat blended;
+ cv::Mat visualized = visualizeActiveChannel();
+ if (!visualized.empty()) {
+ double alpha = screen_->root()->value("blending", 0.5);
+ cv::addWeighted(im1_, alpha, visualized, 1.0-alpha, 0, blended);
+ } else {
+ blended = im1_;
+ }
+ cv::Mat flipped;
+ cv::flip(blended, flipped, 0);
+ cv::imwrite(std::string(timestamp) + ".png", flipped);
}
-void ftl::gui::Camera::toggleVideoRecording() {
- if (recording_) {
- src_->removeRawCallback(recorder_);
- writer_->end();
- fileout_->close();
- recording_ = false;
- } else {
- char timestamp[18];
- std::time_t t=std::time(NULL);
- std::strftime(timestamp, sizeof(timestamp), "%F-%H%M%S", std::localtime(&t));
- fileout_->open(std::string(timestamp) + ".ftl");
+void ftl::gui::Camera::startVideoRecording(const std::string &filename) {
+ fileout_->open(filename);
- writer_->begin();
- src_->addRawCallback(recorder_);
+ writer_->begin();
+ src_->addRawCallback(recorder_);
- src_->inject(Channel::Calibration, src_->parameters(), Channel::Left, src_->getCapabilities());
- src_->inject(src_->getPose());
- recording_ = true;
- }
+ src_->inject(Channel::Calibration, src_->parameters(), Channel::Left, src_->getCapabilities());
+ src_->inject(src_->getPose());
+}
+
+void ftl::gui::Camera::stopVideoRecording() {
+ src_->removeRawCallback(recorder_);
+ writer_->end();
+ fileout_->close();
}
nlohmann::json ftl::gui::Camera::getMetaData() {
diff --git a/applications/gui/src/camera.hpp b/applications/gui/src/camera.hpp
index 43cf9c783..fe477f624 100644
--- a/applications/gui/src/camera.hpp
+++ b/applications/gui/src/camera.hpp
@@ -54,7 +54,9 @@ class Camera {
void snapshot();
- void toggleVideoRecording();
+ void startVideoRecording(const std::string &filename);
+
+ void stopVideoRecording();
nlohmann::json getMetaData();
@@ -69,6 +71,8 @@ class Camera {
#endif
private:
+ cv::Mat visualizeActiveChannel();
+
Screen *screen_;
ftl::rgbd::Source *src_;
GLTexture thumb_;
diff --git a/applications/gui/src/media_panel.cpp b/applications/gui/src/media_panel.cpp
index 142eb9c93..c576f28f6 100644
--- a/applications/gui/src/media_panel.cpp
+++ b/applications/gui/src/media_panel.cpp
@@ -1,5 +1,6 @@
#include "media_panel.hpp"
#include "screen.hpp"
+#include "record_window.hpp"
#include <nanogui/layout.h>
#include <nanogui/button.h>
@@ -13,7 +14,7 @@
using ftl::gui::MediaPanel;
using ftl::codecs::Channel;
-MediaPanel::MediaPanel(ftl::gui::Screen *screen) : nanogui::Window(screen, ""), screen_(screen) {
+MediaPanel::MediaPanel(ftl::gui::Screen *screen, ftl::gui::SourceWindow *sourceWindow) : nanogui::Window(screen, ""), screen_(screen) {
using namespace nanogui;
paused_ = false;
@@ -49,13 +50,31 @@ MediaPanel::MediaPanel(ftl::gui::Screen *screen) : nanogui::Window(screen, ""),
screen_->activeCamera()->snapshot();
recordbutton->setPushed(false);
});
+ itembutton = new Button(recordpopup, "3D snapshot (.ftl)");
+ itembutton->setCallback([this,recordbutton]() {
+ auto tag = screen_->activeCamera()->source()->get<std::string>("uri");
+ if (tag) {
+ auto tagvalue = tag.value();
+ auto configurables = ftl::config::findByTag(tagvalue);
+ if (configurables.size() > 0) {
+ ftl::Configurable *configurable = configurables[0];
+ char timestamp[18];
+ std::time_t t=std::time(NULL);
+ std::strftime(timestamp, sizeof(timestamp), "%F-%H%M%S", std::localtime(&t));
+ configurable->set("3D-snapshot", std::string(timestamp) + ".ftl");
+ }
+ }
+ recordbutton->setPushed(false);
+ });
itembutton = new Button(recordpopup, "Virtual camera recording (.ftl)");
itembutton->setCallback([this,recordbutton]() {
- auto activeCamera = screen_->activeCamera();
- activeCamera->toggleVideoRecording();
+ char timestamp[18];
+ std::time_t t=std::time(NULL);
+ std::strftime(timestamp, sizeof(timestamp), "%F-%H%M%S", std::localtime(&t));
+ auto filename = std::string(timestamp) + ".ftl";
+ toggleVirtualCameraRecording(screen_->activeCamera(), filename);
recordbutton->setTextColor(nanogui::Color(1.0f,0.1f,0.1f,1.0f));
recordbutton->setPushed(false);
- virtualCameraRecording_ = std::optional<ftl::gui::Camera*>(activeCamera);
});
itembutton = new Button(recordpopup, "3D scene recording (.ftl)");
itembutton->setCallback([this,recordbutton]() {
@@ -73,10 +92,15 @@ MediaPanel::MediaPanel(ftl::gui::Screen *screen) : nanogui::Window(screen, ""),
recordbutton->setPushed(false);
});
itembutton = new Button(recordpopup, "Detailed recording options");
+ itembutton->setCallback([this,sourceWindow,recordbutton] {
+ auto record_window = new RecordWindow(screen_, sourceWindow->getCameras(), this);
+ record_window->setTheme(screen_->windowtheme);
+ recordbutton->setPushed(false);
+ });
recordbutton->setCallback([this,recordbutton](){
if (virtualCameraRecording_) {
- virtualCameraRecording_.value()->toggleVideoRecording();
+ virtualCameraRecording_.value()->stopVideoRecording();
recordbutton->setTextColor(nanogui::Color(1.0f,1.0f,1.0f,1.0f));
// Prevents the popup from being opened, though it is shown while the button
@@ -280,3 +304,8 @@ void MediaPanel::cameraChanged() {
}
}
}
+
+void MediaPanel::toggleVirtualCameraRecording(ftl::gui::Camera *camera, const std::string &filename) {
+ camera->startVideoRecording(filename);
+ virtualCameraRecording_ = std::optional<ftl::gui::Camera*>(camera);
+}
\ No newline at end of file
diff --git a/applications/gui/src/media_panel.hpp b/applications/gui/src/media_panel.hpp
index df0b08022..b7e5957cc 100644
--- a/applications/gui/src/media_panel.hpp
+++ b/applications/gui/src/media_panel.hpp
@@ -5,6 +5,8 @@
#include <nanogui/window.h>
+#include "src_window.hpp"
+
namespace ftl {
namespace rgbd {
@@ -17,11 +19,13 @@ class Screen;
class MediaPanel : public nanogui::Window {
public:
- explicit MediaPanel(ftl::gui::Screen *);
+ explicit MediaPanel(ftl::gui::Screen *, ftl::gui::SourceWindow *);
~MediaPanel();
void cameraChanged();
+ void toggleVirtualCameraRecording(ftl::gui::Camera *camera, const std::string &filename);
+
private:
ftl::gui::Screen *screen_;
diff --git a/applications/gui/src/record_window.cpp b/applications/gui/src/record_window.cpp
new file mode 100644
index 000000000..82b333bc9
--- /dev/null
+++ b/applications/gui/src/record_window.cpp
@@ -0,0 +1,83 @@
+#include "record_window.hpp"
+
+#include <ftl/codecs/channels.hpp>
+
+#include <nanogui/layout.h>
+#include <nanogui/button.h>
+#include <nanogui/combobox.h>
+#include <nanogui/label.h>
+#include <nanogui/textbox.h>
+#include <nanogui/tabwidget.h>
+
+using ftl::gui::RecordWindow;
+
+RecordWindow::RecordWindow(nanogui::Widget *parent, const std::vector<ftl::gui::Camera *> &streams, ftl::gui::MediaPanel *media_panel)
+ : nanogui::Window(parent, "Recording options") {
+ using namespace nanogui;
+
+ setLayout(new GroupLayout());
+
+ new Label(this, "File name", "sans-bold");
+ char timestamp[18];
+ std::time_t t = std::time(NULL);
+ std::strftime(timestamp, sizeof(timestamp), "%F-%H%M%S", std::localtime(&t));
+ // TODO: Change the file type specifier to be set based on the type of the recording.
+ auto fileName = new TextBox(this, std::string(timestamp) + ".ftl");
+ fileName->setEditable(true);
+ new Label(this, "Select stream", "sans-bold");
+ auto streamNames = std::vector<std::string>();
+ streamNames.reserve(streams.size());
+ for (const auto s : streams) {
+ streamNames.push_back(s->source()->getURI());
+ }
+ auto streamSelect = new ComboBox(this, streamNames);
+ // TODO: Set the default stream to be the active stream if there is one.
+ // TODO: The function availableChannels() only finds those channels that
+ // have been set in camera.cpp. The only channels that are set in
+ // camera.cpp currently are Colour and Depth. This means that currently,
+ // the list of channels returned by availableChannels() is not accurate
+ // and should be fixed.
+ TabWidget *tabWidget = add<TabWidget>();
+ tabWidget->setFixedWidth(400);
+ auto snapshot2D = tabWidget->createTab("2D snapshot");
+ auto recording2D = tabWidget->createTab("2D recording");
+ auto snapshot3D = tabWidget->createTab("3D snapshot");
+ auto recording3D = tabWidget->createTab("3D recording");
+
+ snapshot2D->setLayout(new GroupLayout());
+ recording2D->setLayout(new GroupLayout());
+ snapshot3D->setLayout(new GroupLayout());
+ recording3D->setLayout(new GroupLayout());
+
+ new Label(snapshot2D, "Select channel (in addition to Left)", "sans-bold");
+ auto snapshotChannel = snapshot2D->add<ComboBox>();
+ new Label(recording2D, "Select channel (in addition to Left)", "sans-bold");
+ auto recordingChannel = recording2D->add<ComboBox>();
+ streamSelect->setCallback([this,streams,snapshotChannel,recordingChannel](int ix) {
+ channels_ = std::vector<std::string>();
+ ftl::codecs::Channels availableChannels = streams[ix]->availableChannels();
+ for (auto c : availableChannels) {
+ channels_.push_back(ftl::codecs::name(c));
+ }
+ snapshotChannel->setItems(channels_);
+ recordingChannel->setItems(channels_);
+ });
+
+ Widget *actionButtons = new Widget(this);
+ actionButtons->setLayout(new BoxLayout(Orientation::Horizontal));
+ auto button = new Button(actionButtons, "Start");
+ button->setCallback([streams,streamSelect,media_panel,fileName]() {
+ // Check the chosen stream type and channels, then record them.
+ auto stream = streams[streamSelect->selectedIndex()];
+ // TODO: If the camera isn't active, no frames are currently received.
+ media_panel->toggleVirtualCameraRecording(stream, fileName->value());
+ });
+ button = new Button(actionButtons, "Cancel");
+ button->setCallback([this]() {
+ dispose();
+ });
+}
+
+RecordWindow::~RecordWindow() {
+
+}
diff --git a/applications/gui/src/record_window.hpp b/applications/gui/src/record_window.hpp
new file mode 100644
index 000000000..4396c0456
--- /dev/null
+++ b/applications/gui/src/record_window.hpp
@@ -0,0 +1,19 @@
+#include <nanogui/window.h>
+
+#include "camera.hpp"
+#include "media_panel.hpp"
+
+namespace ftl {
+namespace gui {
+
+class RecordWindow : public nanogui::Window {
+ public:
+ explicit RecordWindow(nanogui::Widget *parent, const std::vector<ftl::gui::Camera *> &streams, ftl::gui::MediaPanel *media_panel);
+ ~RecordWindow();
+
+ private:
+ std::vector<std::string> channels_;
+};
+
+}
+}
\ No newline at end of file
diff --git a/applications/gui/src/screen.cpp b/applications/gui/src/screen.cpp
index 811d96cf8..3f432ebc3 100644
--- a/applications/gui/src/screen.cpp
+++ b/applications/gui/src/screen.cpp
@@ -266,7 +266,7 @@ ftl::gui::Screen::Screen(ftl::Configurable *proot, ftl::net::Universe *pnet, ftl
//configwindow_ = new ConfigWindow(parent, ctrl_);
cwindow_ = new ftl::gui::ControlWindow(this, controller);
swindow_ = new ftl::gui::SourceWindow(this);
- mwindow_ = new ftl::gui::MediaPanel(this);
+ mwindow_ = new ftl::gui::MediaPanel(this, swindow_);
mwindow_->setVisible(false);
mwindow_->setTheme(mediatheme);
diff --git a/applications/gui/src/src_window.cpp b/applications/gui/src/src_window.cpp
index 0126810a5..6b930c110 100644
--- a/applications/gui/src/src_window.cpp
+++ b/applications/gui/src/src_window.cpp
@@ -75,7 +75,8 @@ SourceWindow::SourceWindow(ftl::gui::Screen *screen)
}
std::vector<ftl::gui::Camera*> SourceWindow::getCameras() {
- auto cameras = std::vector<ftl::gui::Camera*>(cameras_.size());
+ auto cameras = std::vector<ftl::gui::Camera*>();
+ cameras.reserve(cameras_.size());
for (const auto &kv : cameras_) {
cameras.push_back(kv.second);
}
diff --git a/applications/reconstruct/src/main.cpp b/applications/reconstruct/src/main.cpp
index 551a2dfe3..66d6e3189 100644
--- a/applications/reconstruct/src/main.cpp
+++ b/applications/reconstruct/src/main.cpp
@@ -304,7 +304,7 @@ static void run(ftl::Configurable *root) {
for (size_t i=0; i<sources.size(); ++i) {
//writeSourceProperties(writer, i, sources[i]);
sources[i]->inject(Channel::Calibration, sources[i]->parameters(), Channel::Left, sources[i]->getCapabilities());
- sources[i]->inject(sources[i]->getPose());
+ sources[i]->inject(sources[i]->getPose());
}
} else {
group->removeRawCallback(recorder);
@@ -313,6 +313,51 @@ static void run(ftl::Configurable *root) {
}
});
+ std::ofstream snapshotout;
+ ftl::codecs::Writer snapshotwriter(snapshotout);
+
+ root->on("3D-snapshot", [&group,&snapshotout,&snapshotwriter,&scene_A](const ftl::config::Event &e) {
+ char timestamp[18];
+ std::time_t t=std::time(NULL);
+ std::strftime(timestamp, sizeof(timestamp), "%F-%H%M%S", std::localtime(&t));
+ snapshotout.open(e.entity->value<std::string>("3D-snapshot", std::string(timestamp) + ".ftl"));
+
+ snapshotwriter.begin();
+
+ auto sources = group->sources();
+ for (size_t i=0; i<sources.size(); ++i) {
+ auto packet_pair = sources[i]->make_packet(Channel::Calibration, sources[i]->parameters(), Channel::Left, sources[i]->getCapabilities());
+ packet_pair.first.streamID = group->streamID(sources[i]);
+ snapshotwriter.write(packet_pair.first, packet_pair.second);
+
+ packet_pair = sources[i]->make_packet(sources[i]->getPose());
+ packet_pair.first.streamID = group->streamID(sources[i]);
+ snapshotwriter.write(packet_pair.first, packet_pair.second);
+ }
+
+ for (auto &frame : scene_A.frames) {
+ // Write frames of each source as std::vector<uint8_t>.
+ frame.download(Channel::Left);
+ cv::Mat image = frame.get<cv::Mat>(Channel::Left);
+ cv::Mat flat = image.reshape(1, image.total()*image.channels());
+ std::vector<uint8_t> vec = image.isContinuous()? flat : flat.clone();
+ auto packet_pair = frame.source()->make_packet(Channel::Left, vec);
+ packet_pair.first.streamID = group->streamID(frame.source());
+ snapshotwriter.write(packet_pair.first, packet_pair.second);
+
+ frame.download(Channel::Depth);
+ image = frame.get<cv::Mat>(Channel::Depth);
+ flat = image.reshape(1, image.total()*image.channels());
+ vec = image.isContinuous()? flat : flat.clone();
+ packet_pair = frame.source()->make_packet(Channel::Depth, vec);
+ packet_pair.first.streamID = group->streamID(frame.source());
+ snapshotwriter.write(packet_pair.first, packet_pair.second);
+ }
+
+ snapshotwriter.end();
+ snapshotout.close();
+ });
+
// -------------------------------------------------------------------------
stream->setLatency(6); // FIXME: This depends on source!?
diff --git a/components/rgbd-sources/include/ftl/rgbd/source.hpp b/components/rgbd-sources/include/ftl/rgbd/source.hpp
index 895817359..f77f2456b 100644
--- a/components/rgbd-sources/include/ftl/rgbd/source.hpp
+++ b/components/rgbd-sources/include/ftl/rgbd/source.hpp
@@ -220,6 +220,11 @@ class Source : public ftl::Configurable {
void inject(const Eigen::Matrix4d &pose);
+ template <typename... ARGS>
+ std::pair<ftl::codecs::StreamPacket, ftl::codecs::Packet> make_packet(ftl::codecs::Channel c, ARGS... args);
+
+ std::pair<ftl::codecs::StreamPacket, ftl::codecs::Packet> make_packet(const Eigen::Matrix4d &pose);
+
protected:
detail::Source *impl_;
Eigen::Matrix4d pose_;
@@ -255,6 +260,30 @@ class VectorBuffer {
std::vector<unsigned char> &vector_;
};
+template <typename... ARGS>
+std::pair<ftl::codecs::StreamPacket, ftl::codecs::Packet> ftl::rgbd::Source::make_packet(ftl::codecs::Channel c, ARGS... args) {
+ auto data = std::make_tuple(args...);
+
+ ftl::codecs::StreamPacket spkt;
+ ftl::codecs::Packet pkt;
+
+ spkt.timestamp = impl_->timestamp_;
+ std::cout << "Timestamp set." << '\n';
+ spkt.channel = c;
+ spkt.channel_count = 0;
+ spkt.streamID = 0;
+ pkt.codec = ftl::codecs::codec_t::MSGPACK;
+ pkt.block_number = 0;
+ pkt.block_total = 1;
+ pkt.definition = ftl::codecs::definition_t::Any;
+ pkt.flags = 0;
+
+ VectorBuffer buf(pkt.data);
+ msgpack::pack(buf, data);
+
+ return std::make_pair(spkt, pkt);
+}
+
template <typename... ARGS>
void ftl::rgbd::Source::inject(ftl::codecs::Channel c, ARGS... args) {
if (!impl_) return;
diff --git a/components/rgbd-sources/src/source.cpp b/components/rgbd-sources/src/source.cpp
index 13cdd5487..5bf3a92f2 100644
--- a/components/rgbd-sources/src/source.cpp
+++ b/components/rgbd-sources/src/source.cpp
@@ -327,6 +327,24 @@ void Source::notify(int64_t ts, cv::cuda::GpuMat &c1, cv::cuda::GpuMat &c2) {
if (callback_) callback_(ts, c1, c2);
}
+std::pair<ftl::codecs::StreamPacket, ftl::codecs::Packet> Source::make_packet(const Eigen::Matrix4d &pose) {
+ ftl::codecs::StreamPacket spkt;
+ ftl::codecs::Packet pkt;
+
+ spkt.timestamp = impl_->timestamp_;
+ spkt.channel_count = 0;
+ spkt.channel = Channel::Pose;
+ spkt.streamID = 0;
+ pkt.codec = ftl::codecs::codec_t::POSE;
+ pkt.definition = ftl::codecs::definition_t::Any;
+ pkt.block_number = 0;
+ pkt.block_total = 1;
+ pkt.flags = 0;
+ pkt.data = std::move(std::vector<uint8_t>((uint8_t*)pose.data(), (uint8_t*)pose.data() + 4*4*sizeof(double)));
+
+ return std::make_pair(spkt, pkt);
+}
+
void Source::inject(const Eigen::Matrix4d &pose) {
ftl::codecs::StreamPacket spkt;
ftl::codecs::Packet pkt;
--
GitLab