diff --git a/applications/gui/CMakeLists.txt b/applications/gui/CMakeLists.txt index fe553becc5ef883873f0385383b06ede5f3cc6ab..9764984a7b430c4bc256918aac2b39e356a1ee16 100644 --- a/applications/gui/CMakeLists.txt +++ b/applications/gui/CMakeLists.txt @@ -13,6 +13,7 @@ set(GUISRC src/camera.cpp src/media_panel.cpp src/thumbview.cpp + src/record_window.cpp ) if (HAVE_OPENVR) diff --git a/applications/gui/src/camera.cpp b/applications/gui/src/camera.cpp index 0a9d04c044e70b615f24733380818ff1a46a1739..38f6c5f779ec2c23d334bd31ee3f5c5a65727777 100644 --- a/applications/gui/src/camera.cpp +++ b/applications/gui/src/camera.cpp @@ -139,7 +139,6 @@ ftl::gui::Camera::Camera(ftl::gui::Screen *screen, ftl::rgbd::Source *src) : scr sdepth_ = false; ftime_ = (float)glfwGetTime(); pause_ = false; - recording_ = false; fileout_ = new std::ofstream(); writer_ = new ftl::codecs::Writer(*fileout_); recorder_ = std::function([this](ftl::rgbd::Source *src, const ftl::codecs::StreamPacket &spkt, const ftl::codecs::Packet &pkt) { @@ -358,6 +357,27 @@ static void drawEdges( const cv::Mat &in, cv::Mat &out, cv::addWeighted(edges, weight, out, 1.0, 0.0, out, CV_8UC3); } +cv::Mat ftl::gui::Camera::visualizeActiveChannel() { + cv::Mat result; + switch(channel_) { + case Channel::Smoothing: + case Channel::Confidence: + visualizeEnergy(im2_, result, 1.0); + break; + case Channel::Density: + case Channel::Energy: + visualizeEnergy(im2_, result, 10.0); + break; + case Channel::Depth: + visualizeDepthMap(im2_, result, 7.0); + if (screen_->root()->value("showEdgesInDepth", false)) drawEdges(im1_, result); + break; + case Channel::Right: + result = im2_; + } + return result; +} + bool ftl::gui::Camera::thumbnail(cv::Mat &thumb) { UNIQUE_LOCK(mutex_, lk); src_->grab(1,9); @@ -474,12 +494,12 @@ const GLTexture &ftl::gui::Camera::captureFrame() { texture2_.update(tmp);*/ break; - //case Channel::Flow: - case Channel::ColourNormals: - case Channel::Right: - if (im2_.rows == 0 || im2_.type() != CV_8UC3) { break; } - texture2_.update(im2_); - break; + //case Channel::Flow: + case Channel::ColourNormals: + case Channel::Right: + if (im2_.rows == 0 || im2_.type() != CV_8UC3) { break; } + texture2_.update(im2_); + break; default: break; @@ -509,30 +529,33 @@ void ftl::gui::Camera::snapshot() { char timestamp[18]; std::time_t t = std::time(NULL); std::strftime(timestamp, sizeof(timestamp), "%F-%H%M%S", std::localtime(&t)); - cv::Mat image; - cv::flip(im1_, image, 0); - cv::imwrite(std::string(timestamp) + ".png", image); + cv::Mat blended; + cv::Mat visualized = visualizeActiveChannel(); + if (!visualized.empty()) { + double alpha = screen_->root()->value("blending", 0.5); + cv::addWeighted(im1_, alpha, visualized, 1.0-alpha, 0, blended); + } else { + blended = im1_; + } + cv::Mat flipped; + cv::flip(blended, flipped, 0); + cv::imwrite(std::string(timestamp) + ".png", flipped); } -void ftl::gui::Camera::toggleVideoRecording() { - if (recording_) { - src_->removeRawCallback(recorder_); - writer_->end(); - fileout_->close(); - recording_ = false; - } else { - char timestamp[18]; - std::time_t t=std::time(NULL); - std::strftime(timestamp, sizeof(timestamp), "%F-%H%M%S", std::localtime(&t)); - fileout_->open(std::string(timestamp) + ".ftl"); +void ftl::gui::Camera::startVideoRecording(const std::string &filename) { + fileout_->open(filename); - writer_->begin(); - src_->addRawCallback(recorder_); + writer_->begin(); + src_->addRawCallback(recorder_); - src_->inject(Channel::Calibration, src_->parameters(), Channel::Left, src_->getCapabilities()); - src_->inject(src_->getPose()); - recording_ = true; - } + src_->inject(Channel::Calibration, src_->parameters(), Channel::Left, src_->getCapabilities()); + src_->inject(src_->getPose()); +} + +void ftl::gui::Camera::stopVideoRecording() { + src_->removeRawCallback(recorder_); + writer_->end(); + fileout_->close(); } nlohmann::json ftl::gui::Camera::getMetaData() { diff --git a/applications/gui/src/camera.hpp b/applications/gui/src/camera.hpp index 43cf9c783b1e4b5e3ae003c428b8d6126273ef40..fe477f624ed42c8fdca4535355b914a4676540e4 100644 --- a/applications/gui/src/camera.hpp +++ b/applications/gui/src/camera.hpp @@ -54,7 +54,9 @@ class Camera { void snapshot(); - void toggleVideoRecording(); + void startVideoRecording(const std::string &filename); + + void stopVideoRecording(); nlohmann::json getMetaData(); @@ -69,6 +71,8 @@ class Camera { #endif private: + cv::Mat visualizeActiveChannel(); + Screen *screen_; ftl::rgbd::Source *src_; GLTexture thumb_; diff --git a/applications/gui/src/media_panel.cpp b/applications/gui/src/media_panel.cpp index 142eb9c93a42c4b9403596c78fd31eedbd289bfe..c576f28f63cd4edc35a56768f50f27c2818631a2 100644 --- a/applications/gui/src/media_panel.cpp +++ b/applications/gui/src/media_panel.cpp @@ -1,5 +1,6 @@ #include "media_panel.hpp" #include "screen.hpp" +#include "record_window.hpp" #include <nanogui/layout.h> #include <nanogui/button.h> @@ -13,7 +14,7 @@ using ftl::gui::MediaPanel; using ftl::codecs::Channel; -MediaPanel::MediaPanel(ftl::gui::Screen *screen) : nanogui::Window(screen, ""), screen_(screen) { +MediaPanel::MediaPanel(ftl::gui::Screen *screen, ftl::gui::SourceWindow *sourceWindow) : nanogui::Window(screen, ""), screen_(screen) { using namespace nanogui; paused_ = false; @@ -49,13 +50,31 @@ MediaPanel::MediaPanel(ftl::gui::Screen *screen) : nanogui::Window(screen, ""), screen_->activeCamera()->snapshot(); recordbutton->setPushed(false); }); + itembutton = new Button(recordpopup, "3D snapshot (.ftl)"); + itembutton->setCallback([this,recordbutton]() { + auto tag = screen_->activeCamera()->source()->get<std::string>("uri"); + if (tag) { + auto tagvalue = tag.value(); + auto configurables = ftl::config::findByTag(tagvalue); + if (configurables.size() > 0) { + ftl::Configurable *configurable = configurables[0]; + char timestamp[18]; + std::time_t t=std::time(NULL); + std::strftime(timestamp, sizeof(timestamp), "%F-%H%M%S", std::localtime(&t)); + configurable->set("3D-snapshot", std::string(timestamp) + ".ftl"); + } + } + recordbutton->setPushed(false); + }); itembutton = new Button(recordpopup, "Virtual camera recording (.ftl)"); itembutton->setCallback([this,recordbutton]() { - auto activeCamera = screen_->activeCamera(); - activeCamera->toggleVideoRecording(); + char timestamp[18]; + std::time_t t=std::time(NULL); + std::strftime(timestamp, sizeof(timestamp), "%F-%H%M%S", std::localtime(&t)); + auto filename = std::string(timestamp) + ".ftl"; + toggleVirtualCameraRecording(screen_->activeCamera(), filename); recordbutton->setTextColor(nanogui::Color(1.0f,0.1f,0.1f,1.0f)); recordbutton->setPushed(false); - virtualCameraRecording_ = std::optional<ftl::gui::Camera*>(activeCamera); }); itembutton = new Button(recordpopup, "3D scene recording (.ftl)"); itembutton->setCallback([this,recordbutton]() { @@ -73,10 +92,15 @@ MediaPanel::MediaPanel(ftl::gui::Screen *screen) : nanogui::Window(screen, ""), recordbutton->setPushed(false); }); itembutton = new Button(recordpopup, "Detailed recording options"); + itembutton->setCallback([this,sourceWindow,recordbutton] { + auto record_window = new RecordWindow(screen_, sourceWindow->getCameras(), this); + record_window->setTheme(screen_->windowtheme); + recordbutton->setPushed(false); + }); recordbutton->setCallback([this,recordbutton](){ if (virtualCameraRecording_) { - virtualCameraRecording_.value()->toggleVideoRecording(); + virtualCameraRecording_.value()->stopVideoRecording(); recordbutton->setTextColor(nanogui::Color(1.0f,1.0f,1.0f,1.0f)); // Prevents the popup from being opened, though it is shown while the button @@ -280,3 +304,8 @@ void MediaPanel::cameraChanged() { } } } + +void MediaPanel::toggleVirtualCameraRecording(ftl::gui::Camera *camera, const std::string &filename) { + camera->startVideoRecording(filename); + virtualCameraRecording_ = std::optional<ftl::gui::Camera*>(camera); +} \ No newline at end of file diff --git a/applications/gui/src/media_panel.hpp b/applications/gui/src/media_panel.hpp index df0b0802294cbe64800c850faa60fadf4f3de55b..b7e5957cca4e6c171972013448229df8bd364f4c 100644 --- a/applications/gui/src/media_panel.hpp +++ b/applications/gui/src/media_panel.hpp @@ -5,6 +5,8 @@ #include <nanogui/window.h> +#include "src_window.hpp" + namespace ftl { namespace rgbd { @@ -17,11 +19,13 @@ class Screen; class MediaPanel : public nanogui::Window { public: - explicit MediaPanel(ftl::gui::Screen *); + explicit MediaPanel(ftl::gui::Screen *, ftl::gui::SourceWindow *); ~MediaPanel(); void cameraChanged(); + void toggleVirtualCameraRecording(ftl::gui::Camera *camera, const std::string &filename); + private: ftl::gui::Screen *screen_; diff --git a/applications/gui/src/record_window.cpp b/applications/gui/src/record_window.cpp new file mode 100644 index 0000000000000000000000000000000000000000..82b333bc9cd0b568697725b8b3e564a9df73c82f --- /dev/null +++ b/applications/gui/src/record_window.cpp @@ -0,0 +1,83 @@ +#include "record_window.hpp" + +#include <ftl/codecs/channels.hpp> + +#include <nanogui/layout.h> +#include <nanogui/button.h> +#include <nanogui/combobox.h> +#include <nanogui/label.h> +#include <nanogui/textbox.h> +#include <nanogui/tabwidget.h> + +using ftl::gui::RecordWindow; + +RecordWindow::RecordWindow(nanogui::Widget *parent, const std::vector<ftl::gui::Camera *> &streams, ftl::gui::MediaPanel *media_panel) + : nanogui::Window(parent, "Recording options") { + using namespace nanogui; + + setLayout(new GroupLayout()); + + new Label(this, "File name", "sans-bold"); + char timestamp[18]; + std::time_t t = std::time(NULL); + std::strftime(timestamp, sizeof(timestamp), "%F-%H%M%S", std::localtime(&t)); + // TODO: Change the file type specifier to be set based on the type of the recording. + auto fileName = new TextBox(this, std::string(timestamp) + ".ftl"); + fileName->setEditable(true); + new Label(this, "Select stream", "sans-bold"); + auto streamNames = std::vector<std::string>(); + streamNames.reserve(streams.size()); + for (const auto s : streams) { + streamNames.push_back(s->source()->getURI()); + } + auto streamSelect = new ComboBox(this, streamNames); + // TODO: Set the default stream to be the active stream if there is one. + // TODO: The function availableChannels() only finds those channels that + // have been set in camera.cpp. The only channels that are set in + // camera.cpp currently are Colour and Depth. This means that currently, + // the list of channels returned by availableChannels() is not accurate + // and should be fixed. + TabWidget *tabWidget = add<TabWidget>(); + tabWidget->setFixedWidth(400); + auto snapshot2D = tabWidget->createTab("2D snapshot"); + auto recording2D = tabWidget->createTab("2D recording"); + auto snapshot3D = tabWidget->createTab("3D snapshot"); + auto recording3D = tabWidget->createTab("3D recording"); + + snapshot2D->setLayout(new GroupLayout()); + recording2D->setLayout(new GroupLayout()); + snapshot3D->setLayout(new GroupLayout()); + recording3D->setLayout(new GroupLayout()); + + new Label(snapshot2D, "Select channel (in addition to Left)", "sans-bold"); + auto snapshotChannel = snapshot2D->add<ComboBox>(); + new Label(recording2D, "Select channel (in addition to Left)", "sans-bold"); + auto recordingChannel = recording2D->add<ComboBox>(); + streamSelect->setCallback([this,streams,snapshotChannel,recordingChannel](int ix) { + channels_ = std::vector<std::string>(); + ftl::codecs::Channels availableChannels = streams[ix]->availableChannels(); + for (auto c : availableChannels) { + channels_.push_back(ftl::codecs::name(c)); + } + snapshotChannel->setItems(channels_); + recordingChannel->setItems(channels_); + }); + + Widget *actionButtons = new Widget(this); + actionButtons->setLayout(new BoxLayout(Orientation::Horizontal)); + auto button = new Button(actionButtons, "Start"); + button->setCallback([streams,streamSelect,media_panel,fileName]() { + // Check the chosen stream type and channels, then record them. + auto stream = streams[streamSelect->selectedIndex()]; + // TODO: If the camera isn't active, no frames are currently received. + media_panel->toggleVirtualCameraRecording(stream, fileName->value()); + }); + button = new Button(actionButtons, "Cancel"); + button->setCallback([this]() { + dispose(); + }); +} + +RecordWindow::~RecordWindow() { + +} diff --git a/applications/gui/src/record_window.hpp b/applications/gui/src/record_window.hpp new file mode 100644 index 0000000000000000000000000000000000000000..4396c04569e908ee58bc64fdc9c0cca94a488c97 --- /dev/null +++ b/applications/gui/src/record_window.hpp @@ -0,0 +1,19 @@ +#include <nanogui/window.h> + +#include "camera.hpp" +#include "media_panel.hpp" + +namespace ftl { +namespace gui { + +class RecordWindow : public nanogui::Window { + public: + explicit RecordWindow(nanogui::Widget *parent, const std::vector<ftl::gui::Camera *> &streams, ftl::gui::MediaPanel *media_panel); + ~RecordWindow(); + + private: + std::vector<std::string> channels_; +}; + +} +} \ No newline at end of file diff --git a/applications/gui/src/screen.cpp b/applications/gui/src/screen.cpp index 811d96cf851781add465e4ec58ee97e93e5c98f2..3f432ebc3a772f145622cb8ab82c23467a497458 100644 --- a/applications/gui/src/screen.cpp +++ b/applications/gui/src/screen.cpp @@ -266,7 +266,7 @@ ftl::gui::Screen::Screen(ftl::Configurable *proot, ftl::net::Universe *pnet, ftl //configwindow_ = new ConfigWindow(parent, ctrl_); cwindow_ = new ftl::gui::ControlWindow(this, controller); swindow_ = new ftl::gui::SourceWindow(this); - mwindow_ = new ftl::gui::MediaPanel(this); + mwindow_ = new ftl::gui::MediaPanel(this, swindow_); mwindow_->setVisible(false); mwindow_->setTheme(mediatheme); diff --git a/applications/gui/src/src_window.cpp b/applications/gui/src/src_window.cpp index 0126810a511b85a1168fe06c638651e533f11a0d..6b930c110bd21280eb1df1395cb11149d3980fe8 100644 --- a/applications/gui/src/src_window.cpp +++ b/applications/gui/src/src_window.cpp @@ -75,7 +75,8 @@ SourceWindow::SourceWindow(ftl::gui::Screen *screen) } std::vector<ftl::gui::Camera*> SourceWindow::getCameras() { - auto cameras = std::vector<ftl::gui::Camera*>(cameras_.size()); + auto cameras = std::vector<ftl::gui::Camera*>(); + cameras.reserve(cameras_.size()); for (const auto &kv : cameras_) { cameras.push_back(kv.second); } diff --git a/applications/reconstruct/src/main.cpp b/applications/reconstruct/src/main.cpp index 551a2dfe309bf8a5a29c17cc4856acb48f70f09c..66d6e3189eb3b2ed13dbe86b8f7f47a2cf4905cb 100644 --- a/applications/reconstruct/src/main.cpp +++ b/applications/reconstruct/src/main.cpp @@ -304,7 +304,7 @@ static void run(ftl::Configurable *root) { for (size_t i=0; i<sources.size(); ++i) { //writeSourceProperties(writer, i, sources[i]); sources[i]->inject(Channel::Calibration, sources[i]->parameters(), Channel::Left, sources[i]->getCapabilities()); - sources[i]->inject(sources[i]->getPose()); + sources[i]->inject(sources[i]->getPose()); } } else { group->removeRawCallback(recorder); @@ -313,6 +313,51 @@ static void run(ftl::Configurable *root) { } }); + std::ofstream snapshotout; + ftl::codecs::Writer snapshotwriter(snapshotout); + + root->on("3D-snapshot", [&group,&snapshotout,&snapshotwriter,&scene_A](const ftl::config::Event &e) { + char timestamp[18]; + std::time_t t=std::time(NULL); + std::strftime(timestamp, sizeof(timestamp), "%F-%H%M%S", std::localtime(&t)); + snapshotout.open(e.entity->value<std::string>("3D-snapshot", std::string(timestamp) + ".ftl")); + + snapshotwriter.begin(); + + auto sources = group->sources(); + for (size_t i=0; i<sources.size(); ++i) { + auto packet_pair = sources[i]->make_packet(Channel::Calibration, sources[i]->parameters(), Channel::Left, sources[i]->getCapabilities()); + packet_pair.first.streamID = group->streamID(sources[i]); + snapshotwriter.write(packet_pair.first, packet_pair.second); + + packet_pair = sources[i]->make_packet(sources[i]->getPose()); + packet_pair.first.streamID = group->streamID(sources[i]); + snapshotwriter.write(packet_pair.first, packet_pair.second); + } + + for (auto &frame : scene_A.frames) { + // Write frames of each source as std::vector<uint8_t>. + frame.download(Channel::Left); + cv::Mat image = frame.get<cv::Mat>(Channel::Left); + cv::Mat flat = image.reshape(1, image.total()*image.channels()); + std::vector<uint8_t> vec = image.isContinuous()? flat : flat.clone(); + auto packet_pair = frame.source()->make_packet(Channel::Left, vec); + packet_pair.first.streamID = group->streamID(frame.source()); + snapshotwriter.write(packet_pair.first, packet_pair.second); + + frame.download(Channel::Depth); + image = frame.get<cv::Mat>(Channel::Depth); + flat = image.reshape(1, image.total()*image.channels()); + vec = image.isContinuous()? flat : flat.clone(); + packet_pair = frame.source()->make_packet(Channel::Depth, vec); + packet_pair.first.streamID = group->streamID(frame.source()); + snapshotwriter.write(packet_pair.first, packet_pair.second); + } + + snapshotwriter.end(); + snapshotout.close(); + }); + // ------------------------------------------------------------------------- stream->setLatency(6); // FIXME: This depends on source!? diff --git a/components/rgbd-sources/include/ftl/rgbd/source.hpp b/components/rgbd-sources/include/ftl/rgbd/source.hpp index 8958173597a89c00748712a3e1fae69fbc630af0..f77f2456b430141b58b3949e9370733c89f62ab5 100644 --- a/components/rgbd-sources/include/ftl/rgbd/source.hpp +++ b/components/rgbd-sources/include/ftl/rgbd/source.hpp @@ -220,6 +220,11 @@ class Source : public ftl::Configurable { void inject(const Eigen::Matrix4d &pose); + template <typename... ARGS> + std::pair<ftl::codecs::StreamPacket, ftl::codecs::Packet> make_packet(ftl::codecs::Channel c, ARGS... args); + + std::pair<ftl::codecs::StreamPacket, ftl::codecs::Packet> make_packet(const Eigen::Matrix4d &pose); + protected: detail::Source *impl_; Eigen::Matrix4d pose_; @@ -255,6 +260,30 @@ class VectorBuffer { std::vector<unsigned char> &vector_; }; +template <typename... ARGS> +std::pair<ftl::codecs::StreamPacket, ftl::codecs::Packet> ftl::rgbd::Source::make_packet(ftl::codecs::Channel c, ARGS... args) { + auto data = std::make_tuple(args...); + + ftl::codecs::StreamPacket spkt; + ftl::codecs::Packet pkt; + + spkt.timestamp = impl_->timestamp_; + std::cout << "Timestamp set." << '\n'; + spkt.channel = c; + spkt.channel_count = 0; + spkt.streamID = 0; + pkt.codec = ftl::codecs::codec_t::MSGPACK; + pkt.block_number = 0; + pkt.block_total = 1; + pkt.definition = ftl::codecs::definition_t::Any; + pkt.flags = 0; + + VectorBuffer buf(pkt.data); + msgpack::pack(buf, data); + + return std::make_pair(spkt, pkt); +} + template <typename... ARGS> void ftl::rgbd::Source::inject(ftl::codecs::Channel c, ARGS... args) { if (!impl_) return; diff --git a/components/rgbd-sources/src/source.cpp b/components/rgbd-sources/src/source.cpp index 13cdd5487edf0b7cbc99f7cd9dd7032b43d31185..5bf3a92f284f29d788617eb67fc25ab5eafcd400 100644 --- a/components/rgbd-sources/src/source.cpp +++ b/components/rgbd-sources/src/source.cpp @@ -327,6 +327,24 @@ void Source::notify(int64_t ts, cv::cuda::GpuMat &c1, cv::cuda::GpuMat &c2) { if (callback_) callback_(ts, c1, c2); } +std::pair<ftl::codecs::StreamPacket, ftl::codecs::Packet> Source::make_packet(const Eigen::Matrix4d &pose) { + ftl::codecs::StreamPacket spkt; + ftl::codecs::Packet pkt; + + spkt.timestamp = impl_->timestamp_; + spkt.channel_count = 0; + spkt.channel = Channel::Pose; + spkt.streamID = 0; + pkt.codec = ftl::codecs::codec_t::POSE; + pkt.definition = ftl::codecs::definition_t::Any; + pkt.block_number = 0; + pkt.block_total = 1; + pkt.flags = 0; + pkt.data = std::move(std::vector<uint8_t>((uint8_t*)pose.data(), (uint8_t*)pose.data() + 4*4*sizeof(double))); + + return std::make_pair(spkt, pkt); +} + void Source::inject(const Eigen::Matrix4d &pose) { ftl::codecs::StreamPacket spkt; ftl::codecs::Packet pkt;