From b73edd3cb3562fb34bfd9b9d555cbb695039d969 Mon Sep 17 00:00:00 2001
From: Iiro Rastas <iiro.t.rastas@utu.fi>
Date: Wed, 11 Dec 2019 15:54:45 +0200
Subject: [PATCH] Add GUI recording features
2D snapshots now have alpha blending between the channels.
3D snapshots have been added, though the feature is not yet functional.
Recording window for detailed options has been added, though it is still
missing most functionality.
---
applications/gui/CMakeLists.txt | 1 +
applications/gui/src/camera.cpp | 82 +++++++-----
applications/gui/src/camera.hpp | 8 +-
applications/gui/src/media_panel.cpp | 123 ++++++++++++-----
applications/gui/src/media_panel.hpp | 13 +-
applications/gui/src/record_window.cpp | 124 ++++++++++++++++++
applications/gui/src/record_window.hpp | 20 +++
applications/gui/src/screen.cpp | 2 +-
applications/gui/src/src_window.cpp | 3 +-
applications/reconstruct/src/main.cpp | 84 ++++++++++--
.../codecs/include/ftl/codecs/writer.hpp | 1 +
components/codecs/src/writer.cpp | 4 +
12 files changed, 387 insertions(+), 78 deletions(-)
create mode 100644 applications/gui/src/record_window.cpp
create mode 100644 applications/gui/src/record_window.hpp
diff --git a/applications/gui/CMakeLists.txt b/applications/gui/CMakeLists.txt
index fe553becc..9764984a7 100644
--- a/applications/gui/CMakeLists.txt
+++ b/applications/gui/CMakeLists.txt
@@ -13,6 +13,7 @@ set(GUISRC
src/camera.cpp
src/media_panel.cpp
src/thumbview.cpp
+ src/record_window.cpp
)
if (HAVE_OPENVR)
diff --git a/applications/gui/src/camera.cpp b/applications/gui/src/camera.cpp
index 64a100a59..fb5777740 100644
--- a/applications/gui/src/camera.cpp
+++ b/applications/gui/src/camera.cpp
@@ -139,7 +139,6 @@ ftl::gui::Camera::Camera(ftl::gui::Screen *screen, ftl::rgbd::Source *src) : scr
sdepth_ = false;
ftime_ = (float)glfwGetTime();
pause_ = false;
- recording_ = false;
fileout_ = new std::ofstream();
writer_ = new ftl::codecs::Writer(*fileout_);
recorder_ = std::function([this](ftl::rgbd::Source *src, const ftl::codecs::StreamPacket &spkt, const ftl::codecs::Packet &pkt) {
@@ -358,6 +357,27 @@ static void drawEdges( const cv::Mat &in, cv::Mat &out,
cv::addWeighted(edges, weight, out, 1.0, 0.0, out, CV_8UC3);
}
+cv::Mat ftl::gui::Camera::visualizeActiveChannel() {
+ cv::Mat result;
+ switch(channel_) {
+ case Channel::Smoothing:
+ case Channel::Confidence:
+ visualizeEnergy(im2_, result, 1.0);
+ break;
+ case Channel::Density:
+ case Channel::Energy:
+ visualizeEnergy(im2_, result, 10.0);
+ break;
+ case Channel::Depth:
+ visualizeDepthMap(im2_, result, 7.0);
+ if (screen_->root()->value("showEdgesInDepth", false)) drawEdges(im1_, result);
+ break;
+ case Channel::Right:
+ result = im2_;
+ }
+ return result;
+}
+
bool ftl::gui::Camera::thumbnail(cv::Mat &thumb) {
UNIQUE_LOCK(mutex_, lk);
src_->grab(1,9);
@@ -474,12 +494,12 @@ const GLTexture &ftl::gui::Camera::captureFrame() {
texture2_.update(tmp);*/
break;
- //case Channel::Flow:
- case Channel::ColourNormals:
- case Channel::Right:
- if (im2_.rows == 0 || im2_.type() != CV_8UC3) { break; }
- texture2_.update(im2_);
- break;
+ //case Channel::Flow:
+ case Channel::ColourNormals:
+ case Channel::Right:
+ if (im2_.rows == 0 || im2_.type() != CV_8UC3) { break; }
+ texture2_.update(im2_);
+ break;
default:
break;
@@ -504,35 +524,35 @@ const GLTexture &ftl::gui::Camera::captureFrame() {
return texture1_;
}
-void ftl::gui::Camera::snapshot() {
+void ftl::gui::Camera::snapshot(const std::string &filename) {
UNIQUE_LOCK(mutex_, lk);
- char timestamp[18];
- std::time_t t = std::time(NULL);
- std::strftime(timestamp, sizeof(timestamp), "%F-%H%M%S", std::localtime(&t));
- cv::Mat image;
- cv::flip(im1_, image, 0);
- cv::imwrite(std::string(timestamp) + ".png", image);
+ cv::Mat blended;
+ cv::Mat visualized = visualizeActiveChannel();
+ if (!visualized.empty()) {
+ double alpha = screen_->root()->value("blending", 0.5);
+ cv::addWeighted(im1_, alpha, visualized, 1.0-alpha, 0, blended);
+ } else {
+ blended = im1_;
+ }
+ cv::Mat flipped;
+ cv::flip(blended, flipped, 0);
+ cv::imwrite(filename, flipped);
}
-void ftl::gui::Camera::toggleVideoRecording() {
- if (recording_) {
- src_->removeRawCallback(recorder_);
- writer_->end();
- fileout_->close();
- recording_ = false;
- } else {
- char timestamp[18];
- std::time_t t=std::time(NULL);
- std::strftime(timestamp, sizeof(timestamp), "%F-%H%M%S", std::localtime(&t));
- fileout_->open(std::string(timestamp) + ".ftl");
+void ftl::gui::Camera::startVideoRecording(const std::string &filename) {
+ fileout_->open(filename);
- writer_->begin();
- src_->addRawCallback(recorder_);
+ writer_->begin();
+ src_->addRawCallback(recorder_);
- src_->inject(Channel::Calibration, src_->parameters(), Channel::Left, src_->getCapabilities());
- src_->inject(src_->getPose());
- recording_ = true;
- }
+ src_->inject(Channel::Calibration, src_->parameters(), Channel::Left, src_->getCapabilities());
+ src_->inject(src_->getPose());
+}
+
+void ftl::gui::Camera::stopVideoRecording() {
+ src_->removeRawCallback(recorder_);
+ writer_->end();
+ fileout_->close();
}
nlohmann::json ftl::gui::Camera::getMetaData() {
diff --git a/applications/gui/src/camera.hpp b/applications/gui/src/camera.hpp
index 43cf9c783..90914cf59 100644
--- a/applications/gui/src/camera.hpp
+++ b/applications/gui/src/camera.hpp
@@ -52,9 +52,11 @@ class Camera {
bool thumbnail(cv::Mat &thumb);
- void snapshot();
+ void snapshot(const std::string &filename);
- void toggleVideoRecording();
+ void startVideoRecording(const std::string &filename);
+
+ void stopVideoRecording();
nlohmann::json getMetaData();
@@ -69,6 +71,8 @@ class Camera {
#endif
private:
+ cv::Mat visualizeActiveChannel();
+
Screen *screen_;
ftl::rgbd::Source *src_;
GLTexture thumb_;
diff --git a/applications/gui/src/media_panel.cpp b/applications/gui/src/media_panel.cpp
index 693024bd3..c2fa4285e 100644
--- a/applications/gui/src/media_panel.cpp
+++ b/applications/gui/src/media_panel.cpp
@@ -1,5 +1,6 @@
#include "media_panel.hpp"
#include "screen.hpp"
+#include "record_window.hpp"
#include <nanogui/layout.h>
#include <nanogui/button.h>
@@ -13,7 +14,7 @@
using ftl::gui::MediaPanel;
using ftl::codecs::Channel;
-MediaPanel::MediaPanel(ftl::gui::Screen *screen) : nanogui::Window(screen, ""), screen_(screen) {
+MediaPanel::MediaPanel(ftl::gui::Screen *screen, ftl::gui::SourceWindow *sourceWindow) : nanogui::Window(screen, ""), screen_(screen) {
using namespace nanogui;
paused_ = false;
@@ -36,57 +37,70 @@ MediaPanel::MediaPanel(ftl::gui::Screen *screen) : nanogui::Window(screen, ""),
virtualCameraRecording_ = std::optional<ftl::gui::Camera*>();
sceneRecording_ = std::optional<ftl::Configurable*>();
- auto recordbutton = new PopupButton(this, "", ENTYPO_ICON_CONTROLLER_RECORD);
- recordbutton->setTooltip("Record");
- recordbutton->setSide(Popup::Side::Right);
- recordbutton->setChevronIcon(0);
- auto recordpopup = recordbutton->popup();
+ recordbutton_ = new PopupButton(this, "", ENTYPO_ICON_CONTROLLER_RECORD);
+ recordbutton_->setTooltip("Record");
+ recordbutton_->setSide(Popup::Side::Right);
+ recordbutton_->setChevronIcon(0);
+ auto recordpopup = recordbutton_->popup();
recordpopup->setLayout(new GroupLayout());
recordpopup->setTheme(screen->toolbuttheme);
recordpopup->setAnchorHeight(150);
auto itembutton = new Button(recordpopup, "2D snapshot (.png)");
- itembutton->setCallback([this,recordbutton]() {
- screen_->activeCamera()->snapshot();
- recordbutton->setPushed(false);
+ itembutton->setCallback([this]() {
+ char timestamp[18];
+ std::time_t t=std::time(NULL);
+ std::strftime(timestamp, sizeof(timestamp), "%F-%H%M%S", std::localtime(&t));
+ screen_->activeCamera()->snapshot(std::string(timestamp) + ".png");
+ recordbutton_->setPushed(false);
});
itembutton = new Button(recordpopup, "Virtual camera recording (.ftl)");
- itembutton->setCallback([this,recordbutton]() {
- auto activeCamera = screen_->activeCamera();
- activeCamera->toggleVideoRecording();
- recordbutton->setTextColor(nanogui::Color(1.0f,0.1f,0.1f,1.0f));
- recordbutton->setPushed(false);
- virtualCameraRecording_ = std::optional<ftl::gui::Camera*>(activeCamera);
+ itembutton->setCallback([this]() {
+ char timestamp[18];
+ std::time_t t=std::time(NULL);
+ std::strftime(timestamp, sizeof(timestamp), "%F-%H%M%S", std::localtime(&t));
+ auto filename = std::string(timestamp) + ".ftl";
+ startRecording2D(screen_->activeCamera(), filename);
+ recordbutton_->setTextColor(nanogui::Color(1.0f,0.1f,0.1f,1.0f));
+ recordbutton_->setPushed(false);
+ });
+ itembutton = new Button(recordpopup, "3D scene snapshot (.ftl)");
+ itembutton->setCallback([this]() {
+ char timestamp[18];
+ std::time_t t=std::time(NULL);
+ std::strftime(timestamp, sizeof(timestamp), "%F-%H%M%S", std::localtime(&t));
+ snapshot3D(screen_->activeCamera(), std::string(timestamp) + ".ftl");
+ recordbutton_->setPushed(false);
});
itembutton = new Button(recordpopup, "3D scene recording (.ftl)");
- itembutton->setCallback([this,recordbutton]() {
- auto tag = screen_->activeCamera()->source()->get<std::string>("uri");
- if (tag) {
- auto tagvalue = tag.value();
- auto configurables = ftl::config::findByTag(tagvalue);
- if (configurables.size() > 0) {
- ftl::Configurable *configurable = configurables[0];
- configurable->set("record", true);
- recordbutton->setTextColor(nanogui::Color(1.0f,0.1f,0.1f,1.0f));
- sceneRecording_ = std::optional<ftl::Configurable*>(configurable);
- }
- }
- recordbutton->setPushed(false);
+ itembutton->setCallback([this]() {
+ char timestamp[18];
+ std::time_t t=std::time(NULL);
+ std::strftime(timestamp, sizeof(timestamp), "%F-%H%M%S", std::localtime(&t));
+ startRecording3D(screen_->activeCamera(), std::string(timestamp) + ".ftl");
+ recordbutton_->setTextColor(nanogui::Color(1.0f,0.1f,0.1f,1.0f));
+ recordbutton_->setPushed(false);
});
itembutton = new Button(recordpopup, "Detailed recording options");
+ itembutton->setCallback([this,sourceWindow] {
+ auto record_window = new RecordWindow(screen_, screen_, sourceWindow->getCameras(), this);
+ record_window->setTheme(screen_->windowtheme);
+ recordbutton_->setPushed(false);
+ recordbutton_->setEnabled(false);
+ });
- recordbutton->setCallback([this,recordbutton](){
+ recordbutton_->setCallback([this](){
if (virtualCameraRecording_) {
- virtualCameraRecording_.value()->toggleVideoRecording();
- recordbutton->setTextColor(nanogui::Color(1.0f,1.0f,1.0f,1.0f));
+ virtualCameraRecording_.value()->stopVideoRecording();
+ recordbutton_->setTextColor(nanogui::Color(1.0f,1.0f,1.0f,1.0f));
// Prevents the popup from being opened, though it is shown while the button
// is being pressed.
- recordbutton->setPushed(false);
+ recordbutton_->setPushed(false);
virtualCameraRecording_ = std::nullopt;
} else if (sceneRecording_) {
sceneRecording_.value()->set("record", false);
- recordbutton->setTextColor(nanogui::Color(1.0f,1.0f,1.0f,1.0f));
- recordbutton->setPushed(false);
+ recordbutton_->setTextColor(nanogui::Color(1.0f,1.0f,1.0f,1.0f));
+ recordbutton_->setPushed(false);
sceneRecording_ = std::nullopt;
}
});
@@ -281,3 +295,44 @@ void MediaPanel::cameraChanged() {
}
}
}
+
+void MediaPanel::startRecording2D(ftl::gui::Camera *camera, const std::string &filename) {
+ camera->startVideoRecording(filename);
+ virtualCameraRecording_ = std::optional<ftl::gui::Camera*>(camera);
+ recordbutton_->setTextColor(nanogui::Color(1.0f,0.1f,0.1f,1.0f));
+}
+
+void MediaPanel::snapshot3D(ftl::gui::Camera *camera, const std::string &filename) {
+ auto tag = camera->source()->get<std::string>("uri");
+ if (tag) {
+ auto tagvalue = tag.value();
+ auto configurables = ftl::config::findByTag(tagvalue);
+ if (configurables.size() > 0) {
+ ftl::Configurable *configurable = ftl::config::find(configurables[0]->getID() + "/controls");
+ if (configurable) {
+ configurable->set("3D-snapshot", filename);
+ }
+ }
+ }
+}
+
+void MediaPanel::startRecording3D(ftl::gui::Camera *camera, const std::string &filename) {
+ auto tag = camera->source()->get<std::string>("uri");
+ if (tag) {
+ auto tagvalue = tag.value();
+ auto configurables = ftl::config::findByTag(tagvalue);
+ if (configurables.size() > 0) {
+ ftl::Configurable *configurable = ftl::config::find(configurables[0]->getID() + "/controls");
+ if (configurable) {
+ configurable->set("record-name", filename);
+ configurable->set("record", true);
+ sceneRecording_ = std::optional<ftl::Configurable*>(configurable);
+ recordbutton_->setTextColor(nanogui::Color(1.0f,0.1f,0.1f,1.0f));
+ }
+ }
+ }
+}
+
+void MediaPanel::recordWindowClosed() {
+ recordbutton_->setEnabled(true);
+}
\ No newline at end of file
diff --git a/applications/gui/src/media_panel.hpp b/applications/gui/src/media_panel.hpp
index df0b08022..211b84c6e 100644
--- a/applications/gui/src/media_panel.hpp
+++ b/applications/gui/src/media_panel.hpp
@@ -5,6 +5,8 @@
#include <nanogui/window.h>
+#include "src_window.hpp"
+
namespace ftl {
namespace rgbd {
@@ -17,11 +19,19 @@ class Screen;
class MediaPanel : public nanogui::Window {
public:
- explicit MediaPanel(ftl::gui::Screen *);
+ explicit MediaPanel(ftl::gui::Screen *, ftl::gui::SourceWindow *);
~MediaPanel();
void cameraChanged();
+ void startRecording2D(ftl::gui::Camera *camera, const std::string &filename);
+
+ void snapshot3D(ftl::gui::Camera *camera, const std::string &filename);
+
+ void startRecording3D(ftl::gui::Camera *camera, const std::string &filename);
+
+ void recordWindowClosed();
+
private:
ftl::gui::Screen *screen_;
@@ -32,6 +42,7 @@ class MediaPanel : public nanogui::Window {
nanogui::PopupButton *button_channels_;
nanogui::Button *right_button_;
nanogui::Button *depth_button_;
+ nanogui::PopupButton *recordbutton_;
/**
* These members indicate which type of recording is active, if any.
diff --git a/applications/gui/src/record_window.cpp b/applications/gui/src/record_window.cpp
new file mode 100644
index 000000000..87fbb9dc0
--- /dev/null
+++ b/applications/gui/src/record_window.cpp
@@ -0,0 +1,124 @@
+#include "record_window.hpp"
+
+#include "screen.hpp"
+
+#include <ftl/codecs/channels.hpp>
+
+#include <nanogui/layout.h>
+#include <nanogui/button.h>
+#include <nanogui/combobox.h>
+#include <nanogui/label.h>
+#include <nanogui/textbox.h>
+#include <nanogui/tabwidget.h>
+
+using ftl::gui::RecordWindow;
+
+RecordWindow::RecordWindow(nanogui::Widget *parent, ftl::gui::Screen *screen, const std::vector<ftl::gui::Camera *> &streams, ftl::gui::MediaPanel *media_panel)
+ : nanogui::Window(parent, "Recording options") {
+ using namespace nanogui;
+
+ setLayout(new GroupLayout());
+
+ new Label(this, "File name", "sans-bold");
+ char timestamp[18];
+ std::time_t t = std::time(NULL);
+ std::strftime(timestamp, sizeof(timestamp), "%F-%H%M%S", std::localtime(&t));
+ Widget *fileNameBox = new Widget(this);
+ fileNameBox->setLayout(new BoxLayout(Orientation::Horizontal, Alignment::Middle, 0, 6));
+ auto fileName = new TextBox(fileNameBox, std::string(timestamp));
+ fileName->setFixedWidth(350);
+ fileName->setEditable(true);
+ auto extension = new Label(fileNameBox, ".png", "sans-bold");
+ new Label(this, "Select stream", "sans-bold");
+ auto streamNames = std::vector<std::string>();
+ streamNames.reserve(streams.size());
+ std::optional<int> ix;
+ for (const auto s : streams) {
+ if (s == screen->activeCamera()) {
+ ix = std::optional<int>(streamNames.size());
+ }
+ streamNames.push_back(s->source()->getURI());
+ }
+ auto streamSelect = new ComboBox(this, streamNames);
+ // TODO: The function availableChannels() only finds those channels that
+ // have been set in camera.cpp. The only channels that are set in
+ // camera.cpp currently are Colour and Depth. This means that currently,
+ // the list of channels returned by availableChannels() is not accurate
+ // and should be fixed.
+ TabWidget *tabWidget = add<TabWidget>();
+ tabWidget->setFixedWidth(400);
+ auto snapshot2D = tabWidget->createTab("2D snapshot");
+ auto recording2D = tabWidget->createTab("2D recording");
+ auto snapshot3D = tabWidget->createTab("3D snapshot");
+ auto recording3D = tabWidget->createTab("3D recording");
+
+ snapshot2D->setLayout(new GroupLayout());
+ recording2D->setLayout(new GroupLayout());
+ snapshot3D->setLayout(new GroupLayout());
+ recording3D->setLayout(new GroupLayout());
+
+ // Set the file name extension based on the type of recording chosen.
+ tabWidget->setCallback([tabWidget,snapshot2D,extension](int ix) {
+ if (tabWidget->tab(ix) == snapshot2D) {
+ extension->setCaption(".png");
+ } else {
+ extension->setCaption(".ftl");
+ }
+ });
+
+ tabWidget->setActiveTab(0);
+
+ new Label(recording2D, "Select channel (in addition to Left)", "sans-bold");
+ auto recordingChannel = recording2D->add<ComboBox>();
+ auto streamCallback = [this,streams,recordingChannel](int ix) {
+ channels_ = std::vector<ftl::codecs::Channel>();
+ channel_names_ = std::vector<std::string>();
+ ftl::codecs::Channels availableChannels = streams[ix]->availableChannels();
+ for (auto c : availableChannels) {
+ channels_.push_back(c);
+ channel_names_.push_back(ftl::codecs::name(c));
+ }
+ recordingChannel->setItems(channel_names_);
+ };
+ streamSelect->setCallback(streamCallback);
+
+ // Set the selection to the active stream and set the channel list
+ // to be the channels available in that stream. The callback must
+ // be called explicitly, since setSelectedIndex() does not trigger it.
+ if (ix) {
+ streamSelect->setSelectedIndex(ix.value());
+ streamCallback(ix.value());
+ }
+
+ Widget *actionButtons = new Widget(this);
+ actionButtons->setLayout(new BoxLayout(Orientation::Horizontal));
+ auto button = new Button(actionButtons, "Start");
+ button->setCallback([this,streams,streamSelect,screen,media_panel,fileName,extension,tabWidget,snapshot2D,recording2D,snapshot3D,recording3D,recordingChannel]() {
+ // Check the chosen stream type and channels, then record them.
+ std::string name = fileName->value() + extension->caption();
+ auto stream = streams[streamSelect->selectedIndex()];
+ auto tab = tabWidget->tab(tabWidget->activeTab());
+ if (tab == snapshot2D) {
+ stream->snapshot(name);
+ } else if (tab == recording2D) {
+ stream->setChannel(channels_[recordingChannel->selectedIndex()]);
+ screen->setActiveCamera(stream);
+ media_panel->startRecording2D(stream, name);
+ } else if (tab == snapshot3D) {
+ media_panel->snapshot3D(stream, name);
+ } else if (tab == recording3D) {
+ media_panel->startRecording3D(stream, name);
+ }
+ dispose();
+ media_panel->recordWindowClosed();
+ });
+ button = new Button(actionButtons, "Cancel");
+ button->setCallback([this,media_panel]() {
+ dispose();
+ media_panel->recordWindowClosed();
+ });
+}
+
+RecordWindow::~RecordWindow() {
+
+}
diff --git a/applications/gui/src/record_window.hpp b/applications/gui/src/record_window.hpp
new file mode 100644
index 000000000..5a9b28fef
--- /dev/null
+++ b/applications/gui/src/record_window.hpp
@@ -0,0 +1,20 @@
+#include <nanogui/window.h>
+
+#include "camera.hpp"
+#include "media_panel.hpp"
+
+namespace ftl {
+namespace gui {
+
+class RecordWindow : public nanogui::Window {
+ public:
+ explicit RecordWindow(nanogui::Widget *parent, ftl::gui::Screen *screen, const std::vector<ftl::gui::Camera *> &streams, ftl::gui::MediaPanel *media_panel);
+ ~RecordWindow();
+
+ private:
+ std::vector<ftl::codecs::Channel> channels_;
+ std::vector<std::string> channel_names_;
+};
+
+}
+}
\ No newline at end of file
diff --git a/applications/gui/src/screen.cpp b/applications/gui/src/screen.cpp
index 76665281f..d0d994421 100644
--- a/applications/gui/src/screen.cpp
+++ b/applications/gui/src/screen.cpp
@@ -268,7 +268,7 @@ ftl::gui::Screen::Screen(ftl::Configurable *proot, ftl::net::Universe *pnet, ftl
//configwindow_ = new ConfigWindow(parent, ctrl_);
cwindow_ = new ftl::gui::ControlWindow(this, controller);
swindow_ = new ftl::gui::SourceWindow(this);
- mwindow_ = new ftl::gui::MediaPanel(this);
+ mwindow_ = new ftl::gui::MediaPanel(this, swindow_);
mwindow_->setVisible(false);
mwindow_->setTheme(mediatheme);
diff --git a/applications/gui/src/src_window.cpp b/applications/gui/src/src_window.cpp
index 0126810a5..6b930c110 100644
--- a/applications/gui/src/src_window.cpp
+++ b/applications/gui/src/src_window.cpp
@@ -75,7 +75,8 @@ SourceWindow::SourceWindow(ftl::gui::Screen *screen)
}
std::vector<ftl::gui::Camera*> SourceWindow::getCameras() {
- auto cameras = std::vector<ftl::gui::Camera*>(cameras_.size());
+ auto cameras = std::vector<ftl::gui::Camera*>();
+ cameras.reserve(cameras_.size());
for (const auto &kv : cameras_) {
cameras.push_back(kv.second);
}
diff --git a/applications/reconstruct/src/main.cpp b/applications/reconstruct/src/main.cpp
index 4b802d49f..33db96ae0 100644
--- a/applications/reconstruct/src/main.cpp
+++ b/applications/reconstruct/src/main.cpp
@@ -45,6 +45,9 @@
#include <ftl/cuda/normals.hpp>
#include <ftl/registration.hpp>
+#include <ftl/codecs/h264.hpp>
+#include <ftl/codecs/hevc.hpp>
+
#include <cuda_profiler_api.h>
#ifdef WIN32
@@ -134,10 +137,10 @@ static void run(ftl::Configurable *root) {
// Controls
auto *controls = ftl::create<ftl::Configurable>(root, "controls");
-
+
net->start();
net->waitConnections();
-
+
std::vector<int> sourcecounts;
// Add sources from the configuration file as a single group.
@@ -233,7 +236,9 @@ static void run(ftl::Configurable *root) {
//ftl::voxhash::SceneRep *scene = ftl::create<ftl::voxhash::SceneRep>(root, "voxelhash");
ftl::rgbd::Streamer *stream = ftl::create<ftl::rgbd::Streamer>(root, "stream", net);
ftl::rgbd::VirtualSource *vs = ftl::create<ftl::rgbd::VirtualSource>(root, "virtual");
- //root->set("tags", nlohmann::json::array({ root->getID()+"/virtual" }));
+ auto tags = root->value<std::vector<std::string>>("tags", nlohmann::json::array({}));
+ tags.push_back(root->getID()+"/virtual");
+ root->set("tags", tags);
int o = root->value("origin_pose", 0) % sources.size();
vs->setPose(sources[o]->getPose());
@@ -267,26 +272,70 @@ static void run(ftl::Configurable *root) {
std::ofstream fileout;
ftl::codecs::Writer writer(fileout);
- root->set("record", false);
+ std::ofstream snapshotout;
+ ftl::codecs::Writer snapshotwriter(snapshotout);
+
+ controls->set("record", false);
+ int64_t timestamp = -1;
+ bool writingSnapshot = false;
+ std::unordered_set<int64_t> precedingFrames, followingFrames;
// Add a recording callback to all reconstruction scenes
for (size_t i=0; i<sources.size(); ++i) {
- sources[i]->addRawCallback([&writer,&groups,i](ftl::rgbd::Source *src, const ftl::codecs::StreamPacket &spkt, const ftl::codecs::Packet &pkt) {
+ sources[i]->addRawCallback([&writer,&groups,&snapshotout,&snapshotwriter,×tamp,&writingSnapshot,&precedingFrames,&followingFrames,i](ftl::rgbd::Source *src, const ftl::codecs::StreamPacket &spkt, const ftl::codecs::Packet &pkt) {
ftl::codecs::StreamPacket s = spkt;
// Patch stream ID to match order in group
s.streamID = i;
writer.write(s, pkt);
+
+ if (snapshotwriter.active()) {
+ // The frame that is captured is the next IFrame, unless that
+ // IFrame is one of the first two frames seen. In this case a
+ // part of the frame might already have been missed, so the
+ // IFrame after that one is captured instead.
+
+ // Write all pose and calibration packets.
+ if ((int)spkt.channel >= 64) {
+ snapshotwriter.write(s, pkt);
+ } else if (precedingFrames.size() >= 2) {
+ bool isIFrame = true;
+ switch (pkt.codec) {
+ case ftl::codecs::codec_t::H264:
+ isIFrame = ftl::codecs::h264::isIFrame(pkt.data);
+ break;
+ case ftl::codecs::codec_t::HEVC:
+ isIFrame = ftl::codecs::hevc::isIFrame(pkt.data);
+ }
+
+ if (isIFrame && precedingFrames.count(s.timestamp) == 0) {
+ timestamp = s.timestamp;
+ writingSnapshot = true;
+ snapshotwriter.write(s, pkt);
+ } else if (writingSnapshot && s.timestamp > timestamp) {
+ followingFrames.insert(s.timestamp);
+ }
+
+ // Keep looking for packets of the captured frame until
+ // packets from two following frames have been seen.
+ if (followingFrames.size() >= 2) {
+ snapshotwriter.end();
+ snapshotout.close();
+ }
+ } else {
+ precedingFrames.insert(s.timestamp);
+ }
+ }
});
}
// Allow stream recording
- root->on("record", [&groups,&fileout,&writer,&sources](const ftl::config::Event &e) {
+ controls->on("record", [&fileout,&writer,&sources](const ftl::config::Event &e) {
if (e.entity->value("record", false)) {
char timestamp[18];
std::time_t t=std::time(NULL);
std::strftime(timestamp, sizeof(timestamp), "%F-%H%M%S", std::localtime(&t));
- fileout.open(std::string(timestamp) + ".ftl");
+ fileout.open(e.entity->value<std::string>("record-name", std::string(timestamp) + ".ftl"));
writer.begin();
@@ -295,7 +344,7 @@ static void run(ftl::Configurable *root) {
for (size_t i=0; i<sources.size(); ++i) {
//writeSourceProperties(writer, i, sources[i]);
sources[i]->inject(Channel::Calibration, sources[i]->parameters(), Channel::Left, sources[i]->getCapabilities());
- sources[i]->inject(sources[i]->getPose());
+ sources[i]->inject(sources[i]->getPose());
}
} else {
writer.end();
@@ -303,6 +352,25 @@ static void run(ftl::Configurable *root) {
}
});
+ controls->on("3D-snapshot", [&snapshotout,&snapshotwriter,&writingSnapshot,&precedingFrames,&followingFrames,&sources](const ftl::config::Event &e) {
+ if (!snapshotwriter.active()) {
+ char timestamp[18];
+ std::time_t t=std::time(NULL);
+ std::strftime(timestamp, sizeof(timestamp), "%F-%H%M%S", std::localtime(&t));
+ snapshotout.open(e.entity->value<std::string>("3D-snapshot", std::string(timestamp) + ".ftl"));
+ writingSnapshot = false;
+ precedingFrames.clear();
+ followingFrames.clear();
+ snapshotwriter.begin();
+
+ for (size_t i=0; i<sources.size(); ++i) {
+ //writeSourceProperties(writer, i, sources[i]);
+ sources[i]->inject(Channel::Calibration, sources[i]->parameters(), Channel::Left, sources[i]->getCapabilities());
+ sources[i]->inject(sources[i]->getPose());
+ }
+ }
+ });
+
// -------------------------------------------------------------------------
stream->setLatency(6); // FIXME: This depends on source!?
diff --git a/components/codecs/include/ftl/codecs/writer.hpp b/components/codecs/include/ftl/codecs/writer.hpp
index 94d82f1ec..c1a0ba6c3 100644
--- a/components/codecs/include/ftl/codecs/writer.hpp
+++ b/components/codecs/include/ftl/codecs/writer.hpp
@@ -19,6 +19,7 @@ class Writer {
bool begin();
bool write(const ftl::codecs::StreamPacket &, const ftl::codecs::Packet &);
bool end();
+ bool active();
private:
std::ostream *stream_;
diff --git a/components/codecs/src/writer.cpp b/components/codecs/src/writer.cpp
index 1e3841a8e..dba5853ac 100644
--- a/components/codecs/src/writer.cpp
+++ b/components/codecs/src/writer.cpp
@@ -46,3 +46,7 @@ bool Writer::write(const ftl::codecs::StreamPacket &s, const ftl::codecs::Packet
(*stream_).write(buffer.data(), buffer.size());
return true;
}
+
+bool Writer::active() {
+ return active_;
+}
\ No newline at end of file
--
GitLab