diff --git a/applications/gui/CMakeLists.txt b/applications/gui/CMakeLists.txt index fe553becc5ef883873f0385383b06ede5f3cc6ab..9764984a7b430c4bc256918aac2b39e356a1ee16 100644 --- a/applications/gui/CMakeLists.txt +++ b/applications/gui/CMakeLists.txt @@ -13,6 +13,7 @@ set(GUISRC src/camera.cpp src/media_panel.cpp src/thumbview.cpp + src/record_window.cpp ) if (HAVE_OPENVR) diff --git a/applications/gui/src/camera.cpp b/applications/gui/src/camera.cpp index 0a9d04c044e70b615f24733380818ff1a46a1739..fb5777740397e855f291258826655125eb620ca3 100644 --- a/applications/gui/src/camera.cpp +++ b/applications/gui/src/camera.cpp @@ -139,7 +139,6 @@ ftl::gui::Camera::Camera(ftl::gui::Screen *screen, ftl::rgbd::Source *src) : scr sdepth_ = false; ftime_ = (float)glfwGetTime(); pause_ = false; - recording_ = false; fileout_ = new std::ofstream(); writer_ = new ftl::codecs::Writer(*fileout_); recorder_ = std::function([this](ftl::rgbd::Source *src, const ftl::codecs::StreamPacket &spkt, const ftl::codecs::Packet &pkt) { @@ -340,10 +339,10 @@ static void visualizeEnergy( const cv::Mat &depth, cv::Mat &out, depth.convertTo(out, CV_8U, 255.0f / max_depth); //out = 255 - out; - cv::Mat mask = (depth >= 39.0f); // TODO (mask for invalid pixels) + //cv::Mat mask = (depth >= 39.0f); // TODO (mask for invalid pixels) applyColorMap(out, out, cv::COLORMAP_JET); - out.setTo(cv::Scalar(255, 255, 255), mask); + //out.setTo(cv::Scalar(255, 255, 255), mask); } static void drawEdges( const cv::Mat &in, cv::Mat &out, @@ -358,6 +357,27 @@ static void drawEdges( const cv::Mat &in, cv::Mat &out, cv::addWeighted(edges, weight, out, 1.0, 0.0, out, CV_8UC3); } +cv::Mat ftl::gui::Camera::visualizeActiveChannel() { + cv::Mat result; + switch(channel_) { + case Channel::Smoothing: + case Channel::Confidence: + visualizeEnergy(im2_, result, 1.0); + break; + case Channel::Density: + case Channel::Energy: + visualizeEnergy(im2_, result, 10.0); + break; + case Channel::Depth: + visualizeDepthMap(im2_, result, 7.0); + if (screen_->root()->value("showEdgesInDepth", false)) drawEdges(im1_, result); + break; + case Channel::Right: + result = im2_; + } + return result; +} + bool ftl::gui::Camera::thumbnail(cv::Mat &thumb) { UNIQUE_LOCK(mutex_, lk); src_->grab(1,9); @@ -447,7 +467,7 @@ const GLTexture &ftl::gui::Camera::captureFrame() { case Channel::Smoothing: case Channel::Confidence: if (im2_.rows == 0) { break; } - visualizeEnergy(im2_, tmp, 1.0); + visualizeEnergy(im2_, tmp, screen_->root()->value("float_image_max", 1.0f)); texture2_.update(tmp); break; @@ -474,12 +494,12 @@ const GLTexture &ftl::gui::Camera::captureFrame() { texture2_.update(tmp);*/ break; - //case Channel::Flow: - case Channel::ColourNormals: - case Channel::Right: - if (im2_.rows == 0 || im2_.type() != CV_8UC3) { break; } - texture2_.update(im2_); - break; + //case Channel::Flow: + case Channel::ColourNormals: + case Channel::Right: + if (im2_.rows == 0 || im2_.type() != CV_8UC3) { break; } + texture2_.update(im2_); + break; default: break; @@ -504,35 +524,35 @@ const GLTexture &ftl::gui::Camera::captureFrame() { return texture1_; } -void ftl::gui::Camera::snapshot() { +void ftl::gui::Camera::snapshot(const std::string &filename) { UNIQUE_LOCK(mutex_, lk); - char timestamp[18]; - std::time_t t = std::time(NULL); - std::strftime(timestamp, sizeof(timestamp), "%F-%H%M%S", std::localtime(&t)); - cv::Mat image; - cv::flip(im1_, image, 0); - cv::imwrite(std::string(timestamp) + ".png", image); + cv::Mat blended; + cv::Mat visualized = visualizeActiveChannel(); + if (!visualized.empty()) { + double alpha = screen_->root()->value("blending", 0.5); + cv::addWeighted(im1_, alpha, visualized, 1.0-alpha, 0, blended); + } else { + blended = im1_; + } + cv::Mat flipped; + cv::flip(blended, flipped, 0); + cv::imwrite(filename, flipped); } -void ftl::gui::Camera::toggleVideoRecording() { - if (recording_) { - src_->removeRawCallback(recorder_); - writer_->end(); - fileout_->close(); - recording_ = false; - } else { - char timestamp[18]; - std::time_t t=std::time(NULL); - std::strftime(timestamp, sizeof(timestamp), "%F-%H%M%S", std::localtime(&t)); - fileout_->open(std::string(timestamp) + ".ftl"); +void ftl::gui::Camera::startVideoRecording(const std::string &filename) { + fileout_->open(filename); - writer_->begin(); - src_->addRawCallback(recorder_); + writer_->begin(); + src_->addRawCallback(recorder_); - src_->inject(Channel::Calibration, src_->parameters(), Channel::Left, src_->getCapabilities()); - src_->inject(src_->getPose()); - recording_ = true; - } + src_->inject(Channel::Calibration, src_->parameters(), Channel::Left, src_->getCapabilities()); + src_->inject(src_->getPose()); +} + +void ftl::gui::Camera::stopVideoRecording() { + src_->removeRawCallback(recorder_); + writer_->end(); + fileout_->close(); } nlohmann::json ftl::gui::Camera::getMetaData() { diff --git a/applications/gui/src/camera.hpp b/applications/gui/src/camera.hpp index 43cf9c783b1e4b5e3ae003c428b8d6126273ef40..90914cf599942107f8d7bbba8b8f5a9505f815e9 100644 --- a/applications/gui/src/camera.hpp +++ b/applications/gui/src/camera.hpp @@ -52,9 +52,11 @@ class Camera { bool thumbnail(cv::Mat &thumb); - void snapshot(); + void snapshot(const std::string &filename); - void toggleVideoRecording(); + void startVideoRecording(const std::string &filename); + + void stopVideoRecording(); nlohmann::json getMetaData(); @@ -69,6 +71,8 @@ class Camera { #endif private: + cv::Mat visualizeActiveChannel(); + Screen *screen_; ftl::rgbd::Source *src_; GLTexture thumb_; diff --git a/applications/gui/src/media_panel.cpp b/applications/gui/src/media_panel.cpp index 693024bd3c6a89ccb0ac6e893b01641dbd77c574..c2fa4285ebd433a8ac67a09eb0475e80f860100b 100644 --- a/applications/gui/src/media_panel.cpp +++ b/applications/gui/src/media_panel.cpp @@ -1,5 +1,6 @@ #include "media_panel.hpp" #include "screen.hpp" +#include "record_window.hpp" #include <nanogui/layout.h> #include <nanogui/button.h> @@ -13,7 +14,7 @@ using ftl::gui::MediaPanel; using ftl::codecs::Channel; -MediaPanel::MediaPanel(ftl::gui::Screen *screen) : nanogui::Window(screen, ""), screen_(screen) { +MediaPanel::MediaPanel(ftl::gui::Screen *screen, ftl::gui::SourceWindow *sourceWindow) : nanogui::Window(screen, ""), screen_(screen) { using namespace nanogui; paused_ = false; @@ -36,57 +37,70 @@ MediaPanel::MediaPanel(ftl::gui::Screen *screen) : nanogui::Window(screen, ""), virtualCameraRecording_ = std::optional<ftl::gui::Camera*>(); sceneRecording_ = std::optional<ftl::Configurable*>(); - auto recordbutton = new PopupButton(this, "", ENTYPO_ICON_CONTROLLER_RECORD); - recordbutton->setTooltip("Record"); - recordbutton->setSide(Popup::Side::Right); - recordbutton->setChevronIcon(0); - auto recordpopup = recordbutton->popup(); + recordbutton_ = new PopupButton(this, "", ENTYPO_ICON_CONTROLLER_RECORD); + recordbutton_->setTooltip("Record"); + recordbutton_->setSide(Popup::Side::Right); + recordbutton_->setChevronIcon(0); + auto recordpopup = recordbutton_->popup(); recordpopup->setLayout(new GroupLayout()); recordpopup->setTheme(screen->toolbuttheme); recordpopup->setAnchorHeight(150); auto itembutton = new Button(recordpopup, "2D snapshot (.png)"); - itembutton->setCallback([this,recordbutton]() { - screen_->activeCamera()->snapshot(); - recordbutton->setPushed(false); + itembutton->setCallback([this]() { + char timestamp[18]; + std::time_t t=std::time(NULL); + std::strftime(timestamp, sizeof(timestamp), "%F-%H%M%S", std::localtime(&t)); + screen_->activeCamera()->snapshot(std::string(timestamp) + ".png"); + recordbutton_->setPushed(false); }); itembutton = new Button(recordpopup, "Virtual camera recording (.ftl)"); - itembutton->setCallback([this,recordbutton]() { - auto activeCamera = screen_->activeCamera(); - activeCamera->toggleVideoRecording(); - recordbutton->setTextColor(nanogui::Color(1.0f,0.1f,0.1f,1.0f)); - recordbutton->setPushed(false); - virtualCameraRecording_ = std::optional<ftl::gui::Camera*>(activeCamera); + itembutton->setCallback([this]() { + char timestamp[18]; + std::time_t t=std::time(NULL); + std::strftime(timestamp, sizeof(timestamp), "%F-%H%M%S", std::localtime(&t)); + auto filename = std::string(timestamp) + ".ftl"; + startRecording2D(screen_->activeCamera(), filename); + recordbutton_->setTextColor(nanogui::Color(1.0f,0.1f,0.1f,1.0f)); + recordbutton_->setPushed(false); + }); + itembutton = new Button(recordpopup, "3D scene snapshot (.ftl)"); + itembutton->setCallback([this]() { + char timestamp[18]; + std::time_t t=std::time(NULL); + std::strftime(timestamp, sizeof(timestamp), "%F-%H%M%S", std::localtime(&t)); + snapshot3D(screen_->activeCamera(), std::string(timestamp) + ".ftl"); + recordbutton_->setPushed(false); }); itembutton = new Button(recordpopup, "3D scene recording (.ftl)"); - itembutton->setCallback([this,recordbutton]() { - auto tag = screen_->activeCamera()->source()->get<std::string>("uri"); - if (tag) { - auto tagvalue = tag.value(); - auto configurables = ftl::config::findByTag(tagvalue); - if (configurables.size() > 0) { - ftl::Configurable *configurable = configurables[0]; - configurable->set("record", true); - recordbutton->setTextColor(nanogui::Color(1.0f,0.1f,0.1f,1.0f)); - sceneRecording_ = std::optional<ftl::Configurable*>(configurable); - } - } - recordbutton->setPushed(false); + itembutton->setCallback([this]() { + char timestamp[18]; + std::time_t t=std::time(NULL); + std::strftime(timestamp, sizeof(timestamp), "%F-%H%M%S", std::localtime(&t)); + startRecording3D(screen_->activeCamera(), std::string(timestamp) + ".ftl"); + recordbutton_->setTextColor(nanogui::Color(1.0f,0.1f,0.1f,1.0f)); + recordbutton_->setPushed(false); }); itembutton = new Button(recordpopup, "Detailed recording options"); + itembutton->setCallback([this,sourceWindow] { + auto record_window = new RecordWindow(screen_, screen_, sourceWindow->getCameras(), this); + record_window->setTheme(screen_->windowtheme); + recordbutton_->setPushed(false); + recordbutton_->setEnabled(false); + }); - recordbutton->setCallback([this,recordbutton](){ + recordbutton_->setCallback([this](){ if (virtualCameraRecording_) { - virtualCameraRecording_.value()->toggleVideoRecording(); - recordbutton->setTextColor(nanogui::Color(1.0f,1.0f,1.0f,1.0f)); + virtualCameraRecording_.value()->stopVideoRecording(); + recordbutton_->setTextColor(nanogui::Color(1.0f,1.0f,1.0f,1.0f)); // Prevents the popup from being opened, though it is shown while the button // is being pressed. - recordbutton->setPushed(false); + recordbutton_->setPushed(false); virtualCameraRecording_ = std::nullopt; } else if (sceneRecording_) { sceneRecording_.value()->set("record", false); - recordbutton->setTextColor(nanogui::Color(1.0f,1.0f,1.0f,1.0f)); - recordbutton->setPushed(false); + recordbutton_->setTextColor(nanogui::Color(1.0f,1.0f,1.0f,1.0f)); + recordbutton_->setPushed(false); sceneRecording_ = std::nullopt; } }); @@ -281,3 +295,44 @@ void MediaPanel::cameraChanged() { } } } + +void MediaPanel::startRecording2D(ftl::gui::Camera *camera, const std::string &filename) { + camera->startVideoRecording(filename); + virtualCameraRecording_ = std::optional<ftl::gui::Camera*>(camera); + recordbutton_->setTextColor(nanogui::Color(1.0f,0.1f,0.1f,1.0f)); +} + +void MediaPanel::snapshot3D(ftl::gui::Camera *camera, const std::string &filename) { + auto tag = camera->source()->get<std::string>("uri"); + if (tag) { + auto tagvalue = tag.value(); + auto configurables = ftl::config::findByTag(tagvalue); + if (configurables.size() > 0) { + ftl::Configurable *configurable = ftl::config::find(configurables[0]->getID() + "/controls"); + if (configurable) { + configurable->set("3D-snapshot", filename); + } + } + } +} + +void MediaPanel::startRecording3D(ftl::gui::Camera *camera, const std::string &filename) { + auto tag = camera->source()->get<std::string>("uri"); + if (tag) { + auto tagvalue = tag.value(); + auto configurables = ftl::config::findByTag(tagvalue); + if (configurables.size() > 0) { + ftl::Configurable *configurable = ftl::config::find(configurables[0]->getID() + "/controls"); + if (configurable) { + configurable->set("record-name", filename); + configurable->set("record", true); + sceneRecording_ = std::optional<ftl::Configurable*>(configurable); + recordbutton_->setTextColor(nanogui::Color(1.0f,0.1f,0.1f,1.0f)); + } + } + } +} + +void MediaPanel::recordWindowClosed() { + recordbutton_->setEnabled(true); +} \ No newline at end of file diff --git a/applications/gui/src/media_panel.hpp b/applications/gui/src/media_panel.hpp index df0b0802294cbe64800c850faa60fadf4f3de55b..211b84c6e0beecdcfd5fd51cb3f413e628d31d87 100644 --- a/applications/gui/src/media_panel.hpp +++ b/applications/gui/src/media_panel.hpp @@ -5,6 +5,8 @@ #include <nanogui/window.h> +#include "src_window.hpp" + namespace ftl { namespace rgbd { @@ -17,11 +19,19 @@ class Screen; class MediaPanel : public nanogui::Window { public: - explicit MediaPanel(ftl::gui::Screen *); + explicit MediaPanel(ftl::gui::Screen *, ftl::gui::SourceWindow *); ~MediaPanel(); void cameraChanged(); + void startRecording2D(ftl::gui::Camera *camera, const std::string &filename); + + void snapshot3D(ftl::gui::Camera *camera, const std::string &filename); + + void startRecording3D(ftl::gui::Camera *camera, const std::string &filename); + + void recordWindowClosed(); + private: ftl::gui::Screen *screen_; @@ -32,6 +42,7 @@ class MediaPanel : public nanogui::Window { nanogui::PopupButton *button_channels_; nanogui::Button *right_button_; nanogui::Button *depth_button_; + nanogui::PopupButton *recordbutton_; /** * These members indicate which type of recording is active, if any. diff --git a/applications/gui/src/record_window.cpp b/applications/gui/src/record_window.cpp new file mode 100644 index 0000000000000000000000000000000000000000..87fbb9dc0e50562abe10b932b05f728f4e24208e --- /dev/null +++ b/applications/gui/src/record_window.cpp @@ -0,0 +1,124 @@ +#include "record_window.hpp" + +#include "screen.hpp" + +#include <ftl/codecs/channels.hpp> + +#include <nanogui/layout.h> +#include <nanogui/button.h> +#include <nanogui/combobox.h> +#include <nanogui/label.h> +#include <nanogui/textbox.h> +#include <nanogui/tabwidget.h> + +using ftl::gui::RecordWindow; + +RecordWindow::RecordWindow(nanogui::Widget *parent, ftl::gui::Screen *screen, const std::vector<ftl::gui::Camera *> &streams, ftl::gui::MediaPanel *media_panel) + : nanogui::Window(parent, "Recording options") { + using namespace nanogui; + + setLayout(new GroupLayout()); + + new Label(this, "File name", "sans-bold"); + char timestamp[18]; + std::time_t t = std::time(NULL); + std::strftime(timestamp, sizeof(timestamp), "%F-%H%M%S", std::localtime(&t)); + Widget *fileNameBox = new Widget(this); + fileNameBox->setLayout(new BoxLayout(Orientation::Horizontal, Alignment::Middle, 0, 6)); + auto fileName = new TextBox(fileNameBox, std::string(timestamp)); + fileName->setFixedWidth(350); + fileName->setEditable(true); + auto extension = new Label(fileNameBox, ".png", "sans-bold"); + new Label(this, "Select stream", "sans-bold"); + auto streamNames = std::vector<std::string>(); + streamNames.reserve(streams.size()); + std::optional<int> ix; + for (const auto s : streams) { + if (s == screen->activeCamera()) { + ix = std::optional<int>(streamNames.size()); + } + streamNames.push_back(s->source()->getURI()); + } + auto streamSelect = new ComboBox(this, streamNames); + // TODO: The function availableChannels() only finds those channels that + // have been set in camera.cpp. The only channels that are set in + // camera.cpp currently are Colour and Depth. This means that currently, + // the list of channels returned by availableChannels() is not accurate + // and should be fixed. + TabWidget *tabWidget = add<TabWidget>(); + tabWidget->setFixedWidth(400); + auto snapshot2D = tabWidget->createTab("2D snapshot"); + auto recording2D = tabWidget->createTab("2D recording"); + auto snapshot3D = tabWidget->createTab("3D snapshot"); + auto recording3D = tabWidget->createTab("3D recording"); + + snapshot2D->setLayout(new GroupLayout()); + recording2D->setLayout(new GroupLayout()); + snapshot3D->setLayout(new GroupLayout()); + recording3D->setLayout(new GroupLayout()); + + // Set the file name extension based on the type of recording chosen. + tabWidget->setCallback([tabWidget,snapshot2D,extension](int ix) { + if (tabWidget->tab(ix) == snapshot2D) { + extension->setCaption(".png"); + } else { + extension->setCaption(".ftl"); + } + }); + + tabWidget->setActiveTab(0); + + new Label(recording2D, "Select channel (in addition to Left)", "sans-bold"); + auto recordingChannel = recording2D->add<ComboBox>(); + auto streamCallback = [this,streams,recordingChannel](int ix) { + channels_ = std::vector<ftl::codecs::Channel>(); + channel_names_ = std::vector<std::string>(); + ftl::codecs::Channels availableChannels = streams[ix]->availableChannels(); + for (auto c : availableChannels) { + channels_.push_back(c); + channel_names_.push_back(ftl::codecs::name(c)); + } + recordingChannel->setItems(channel_names_); + }; + streamSelect->setCallback(streamCallback); + + // Set the selection to the active stream and set the channel list + // to be the channels available in that stream. The callback must + // be called explicitly, since setSelectedIndex() does not trigger it. + if (ix) { + streamSelect->setSelectedIndex(ix.value()); + streamCallback(ix.value()); + } + + Widget *actionButtons = new Widget(this); + actionButtons->setLayout(new BoxLayout(Orientation::Horizontal)); + auto button = new Button(actionButtons, "Start"); + button->setCallback([this,streams,streamSelect,screen,media_panel,fileName,extension,tabWidget,snapshot2D,recording2D,snapshot3D,recording3D,recordingChannel]() { + // Check the chosen stream type and channels, then record them. + std::string name = fileName->value() + extension->caption(); + auto stream = streams[streamSelect->selectedIndex()]; + auto tab = tabWidget->tab(tabWidget->activeTab()); + if (tab == snapshot2D) { + stream->snapshot(name); + } else if (tab == recording2D) { + stream->setChannel(channels_[recordingChannel->selectedIndex()]); + screen->setActiveCamera(stream); + media_panel->startRecording2D(stream, name); + } else if (tab == snapshot3D) { + media_panel->snapshot3D(stream, name); + } else if (tab == recording3D) { + media_panel->startRecording3D(stream, name); + } + dispose(); + media_panel->recordWindowClosed(); + }); + button = new Button(actionButtons, "Cancel"); + button->setCallback([this,media_panel]() { + dispose(); + media_panel->recordWindowClosed(); + }); +} + +RecordWindow::~RecordWindow() { + +} diff --git a/applications/gui/src/record_window.hpp b/applications/gui/src/record_window.hpp new file mode 100644 index 0000000000000000000000000000000000000000..5a9b28fef85b5ef2832b7a43b2db1be2b09aa202 --- /dev/null +++ b/applications/gui/src/record_window.hpp @@ -0,0 +1,20 @@ +#include <nanogui/window.h> + +#include "camera.hpp" +#include "media_panel.hpp" + +namespace ftl { +namespace gui { + +class RecordWindow : public nanogui::Window { + public: + explicit RecordWindow(nanogui::Widget *parent, ftl::gui::Screen *screen, const std::vector<ftl::gui::Camera *> &streams, ftl::gui::MediaPanel *media_panel); + ~RecordWindow(); + + private: + std::vector<ftl::codecs::Channel> channels_; + std::vector<std::string> channel_names_; +}; + +} +} \ No newline at end of file diff --git a/applications/gui/src/screen.cpp b/applications/gui/src/screen.cpp index 76665281fc1b9fcbb1b0b45609c509d02f88a69d..d0d9944214343699bd04230b07da47799f1f415f 100644 --- a/applications/gui/src/screen.cpp +++ b/applications/gui/src/screen.cpp @@ -268,7 +268,7 @@ ftl::gui::Screen::Screen(ftl::Configurable *proot, ftl::net::Universe *pnet, ftl //configwindow_ = new ConfigWindow(parent, ctrl_); cwindow_ = new ftl::gui::ControlWindow(this, controller); swindow_ = new ftl::gui::SourceWindow(this); - mwindow_ = new ftl::gui::MediaPanel(this); + mwindow_ = new ftl::gui::MediaPanel(this, swindow_); mwindow_->setVisible(false); mwindow_->setTheme(mediatheme); diff --git a/applications/gui/src/src_window.cpp b/applications/gui/src/src_window.cpp index 0126810a511b85a1168fe06c638651e533f11a0d..6b930c110bd21280eb1df1395cb11149d3980fe8 100644 --- a/applications/gui/src/src_window.cpp +++ b/applications/gui/src/src_window.cpp @@ -75,7 +75,8 @@ SourceWindow::SourceWindow(ftl::gui::Screen *screen) } std::vector<ftl::gui::Camera*> SourceWindow::getCameras() { - auto cameras = std::vector<ftl::gui::Camera*>(cameras_.size()); + auto cameras = std::vector<ftl::gui::Camera*>(); + cameras.reserve(cameras_.size()); for (const auto &kv : cameras_) { cameras.push_back(kv.second); } diff --git a/applications/player/src/main.cpp b/applications/player/src/main.cpp index 7eeb6e6bf5f96fd8f32360949c566c93860518db..33e146880668fb29224aaffc8942cb4255ebf0c1 100644 --- a/applications/player/src/main.cpp +++ b/applications/player/src/main.cpp @@ -7,6 +7,7 @@ #include <ftl/timer.hpp> #include <fstream> +#include <bitset> #include <Eigen/Eigen> @@ -57,8 +58,23 @@ int main(int argc, char **argv) { int current_stream = 0; int current_channel = 0; - ftl::timer::add(ftl::timer::kTimerMain, [¤t_stream,¤t_channel,&r](int64_t ts) { - bool res = r.read(ts, [¤t_stream,¤t_channel,&r](const ftl::codecs::StreamPacket &spkt, const ftl::codecs::Packet &pkt) { + int stream_mask = 0; + std::vector<std::bitset<128>> channel_mask; + + ftl::timer::add(ftl::timer::kTimerMain, [¤t_stream,¤t_channel,&r,&stream_mask,&channel_mask](int64_t ts) { + bool res = r.read(ts, [¤t_stream,¤t_channel,&r,&stream_mask,&channel_mask](const ftl::codecs::StreamPacket &spkt, const ftl::codecs::Packet &pkt) { + if (!(stream_mask & (1 << spkt.streamID))) { + stream_mask |= 1 << spkt.streamID; + LOG(INFO) << " - Stream found (" << (int)spkt.streamID << ")"; + + channel_mask.push_back(0); + } + + if (!(channel_mask[spkt.streamID][(int)spkt.channel])) { + channel_mask[spkt.streamID].set((int)spkt.channel); + LOG(INFO) << " - Channel " << (int)spkt.channel << " found (" << (int)spkt.streamID << ")"; + } + if (spkt.streamID == current_stream) { if (pkt.codec == codec_t::POSE) { diff --git a/applications/reconstruct/src/main.cpp b/applications/reconstruct/src/main.cpp index 4b802d49f85ebe618fd27ecbfe48b46b2f40b5b1..33db96ae046a1ee5c29898a503d0c483446e7f91 100644 --- a/applications/reconstruct/src/main.cpp +++ b/applications/reconstruct/src/main.cpp @@ -45,6 +45,9 @@ #include <ftl/cuda/normals.hpp> #include <ftl/registration.hpp> +#include <ftl/codecs/h264.hpp> +#include <ftl/codecs/hevc.hpp> + #include <cuda_profiler_api.h> #ifdef WIN32 @@ -134,10 +137,10 @@ static void run(ftl::Configurable *root) { // Controls auto *controls = ftl::create<ftl::Configurable>(root, "controls"); - + net->start(); net->waitConnections(); - + std::vector<int> sourcecounts; // Add sources from the configuration file as a single group. @@ -233,7 +236,9 @@ static void run(ftl::Configurable *root) { //ftl::voxhash::SceneRep *scene = ftl::create<ftl::voxhash::SceneRep>(root, "voxelhash"); ftl::rgbd::Streamer *stream = ftl::create<ftl::rgbd::Streamer>(root, "stream", net); ftl::rgbd::VirtualSource *vs = ftl::create<ftl::rgbd::VirtualSource>(root, "virtual"); - //root->set("tags", nlohmann::json::array({ root->getID()+"/virtual" })); + auto tags = root->value<std::vector<std::string>>("tags", nlohmann::json::array({})); + tags.push_back(root->getID()+"/virtual"); + root->set("tags", tags); int o = root->value("origin_pose", 0) % sources.size(); vs->setPose(sources[o]->getPose()); @@ -267,26 +272,70 @@ static void run(ftl::Configurable *root) { std::ofstream fileout; ftl::codecs::Writer writer(fileout); - root->set("record", false); + std::ofstream snapshotout; + ftl::codecs::Writer snapshotwriter(snapshotout); + + controls->set("record", false); + int64_t timestamp = -1; + bool writingSnapshot = false; + std::unordered_set<int64_t> precedingFrames, followingFrames; // Add a recording callback to all reconstruction scenes for (size_t i=0; i<sources.size(); ++i) { - sources[i]->addRawCallback([&writer,&groups,i](ftl::rgbd::Source *src, const ftl::codecs::StreamPacket &spkt, const ftl::codecs::Packet &pkt) { + sources[i]->addRawCallback([&writer,&groups,&snapshotout,&snapshotwriter,×tamp,&writingSnapshot,&precedingFrames,&followingFrames,i](ftl::rgbd::Source *src, const ftl::codecs::StreamPacket &spkt, const ftl::codecs::Packet &pkt) { ftl::codecs::StreamPacket s = spkt; // Patch stream ID to match order in group s.streamID = i; writer.write(s, pkt); + + if (snapshotwriter.active()) { + // The frame that is captured is the next IFrame, unless that + // IFrame is one of the first two frames seen. In this case a + // part of the frame might already have been missed, so the + // IFrame after that one is captured instead. + + // Write all pose and calibration packets. + if ((int)spkt.channel >= 64) { + snapshotwriter.write(s, pkt); + } else if (precedingFrames.size() >= 2) { + bool isIFrame = true; + switch (pkt.codec) { + case ftl::codecs::codec_t::H264: + isIFrame = ftl::codecs::h264::isIFrame(pkt.data); + break; + case ftl::codecs::codec_t::HEVC: + isIFrame = ftl::codecs::hevc::isIFrame(pkt.data); + } + + if (isIFrame && precedingFrames.count(s.timestamp) == 0) { + timestamp = s.timestamp; + writingSnapshot = true; + snapshotwriter.write(s, pkt); + } else if (writingSnapshot && s.timestamp > timestamp) { + followingFrames.insert(s.timestamp); + } + + // Keep looking for packets of the captured frame until + // packets from two following frames have been seen. + if (followingFrames.size() >= 2) { + snapshotwriter.end(); + snapshotout.close(); + } + } else { + precedingFrames.insert(s.timestamp); + } + } }); } // Allow stream recording - root->on("record", [&groups,&fileout,&writer,&sources](const ftl::config::Event &e) { + controls->on("record", [&fileout,&writer,&sources](const ftl::config::Event &e) { if (e.entity->value("record", false)) { char timestamp[18]; std::time_t t=std::time(NULL); std::strftime(timestamp, sizeof(timestamp), "%F-%H%M%S", std::localtime(&t)); - fileout.open(std::string(timestamp) + ".ftl"); + fileout.open(e.entity->value<std::string>("record-name", std::string(timestamp) + ".ftl")); writer.begin(); @@ -295,7 +344,7 @@ static void run(ftl::Configurable *root) { for (size_t i=0; i<sources.size(); ++i) { //writeSourceProperties(writer, i, sources[i]); sources[i]->inject(Channel::Calibration, sources[i]->parameters(), Channel::Left, sources[i]->getCapabilities()); - sources[i]->inject(sources[i]->getPose()); + sources[i]->inject(sources[i]->getPose()); } } else { writer.end(); @@ -303,6 +352,25 @@ static void run(ftl::Configurable *root) { } }); + controls->on("3D-snapshot", [&snapshotout,&snapshotwriter,&writingSnapshot,&precedingFrames,&followingFrames,&sources](const ftl::config::Event &e) { + if (!snapshotwriter.active()) { + char timestamp[18]; + std::time_t t=std::time(NULL); + std::strftime(timestamp, sizeof(timestamp), "%F-%H%M%S", std::localtime(&t)); + snapshotout.open(e.entity->value<std::string>("3D-snapshot", std::string(timestamp) + ".ftl")); + writingSnapshot = false; + precedingFrames.clear(); + followingFrames.clear(); + snapshotwriter.begin(); + + for (size_t i=0; i<sources.size(); ++i) { + //writeSourceProperties(writer, i, sources[i]); + sources[i]->inject(Channel::Calibration, sources[i]->parameters(), Channel::Left, sources[i]->getCapabilities()); + sources[i]->inject(sources[i]->getPose()); + } + } + }); + // ------------------------------------------------------------------------- stream->setLatency(6); // FIXME: This depends on source!? diff --git a/components/codecs/include/ftl/codecs/writer.hpp b/components/codecs/include/ftl/codecs/writer.hpp index 94d82f1ecbfb697bdbad35acc5a4cf29dad22410..c1a0ba6c37c03cf73040fb13adc494420d0c4ca2 100644 --- a/components/codecs/include/ftl/codecs/writer.hpp +++ b/components/codecs/include/ftl/codecs/writer.hpp @@ -19,6 +19,7 @@ class Writer { bool begin(); bool write(const ftl::codecs::StreamPacket &, const ftl::codecs::Packet &); bool end(); + bool active(); private: std::ostream *stream_; diff --git a/components/codecs/src/writer.cpp b/components/codecs/src/writer.cpp index 1e3841a8e05f840f808aaddf127833392212af05..dba5853ac985fe71028db6152d4c74028e2d91b6 100644 --- a/components/codecs/src/writer.cpp +++ b/components/codecs/src/writer.cpp @@ -46,3 +46,7 @@ bool Writer::write(const ftl::codecs::StreamPacket &s, const ftl::codecs::Packet (*stream_).write(buffer.data(), buffer.size()); return true; } + +bool Writer::active() { + return active_; +} \ No newline at end of file diff --git a/components/operators/src/correspondence.cu b/components/operators/src/correspondence.cu index 36e91d9f483d62164b284f43b82443641eadd664..05d60106059df6855b65003af076bb751176e56f 100644 --- a/components/operators/src/correspondence.cu +++ b/components/operators/src/correspondence.cu @@ -1,6 +1,7 @@ #include "mvmls_cuda.hpp" #include <ftl/cuda/weighting.hpp> #include <ftl/cuda/mask.hpp> +#include <ftl/cuda/warp.hpp> using ftl::cuda::TextureObject; using ftl::rgbd::Camera; @@ -8,6 +9,8 @@ using ftl::cuda::Mask; using ftl::cuda::MvMLSParams; #define T_PER_BLOCK 8 +#define WARP_SIZE 32 +#define INTERVAL 1 template<int FUNCTION> __device__ float weightFunction(const ftl::cuda::MvMLSParams ¶ms, float dweight, float cweight); @@ -42,6 +45,54 @@ __device__ inline float weightFunction<5>(const ftl::cuda::MvMLSParams ¶ms, return (cweight > 0.0f) ? dweight : 0.0f; } +#ifndef PINF +#define PINF __int_as_float(0x7f800000) +#endif + +__device__ inline float distance(float4 p1, float4 p2) { + return min(1.0f, max(max(fabsf(p1.x - p2.x),fabsf(p1.y - p2.y)), fabsf(p1.z - p2.z))/ 10.0f); + //return min(1.0f, ftl::cuda::colourDistance(p1, p2) / 10.0f); +} + +__device__ inline int halfWarpCensus(float e) { + float e0 = __shfl_sync(FULL_MASK, e, (threadIdx.x >= 16) ? 16 : 0, WARP_SIZE); + int c = (e > e0) ? 1 << (threadIdx.x % 16) : 0; + for (int i = WARP_SIZE/4; i > 0; i /= 2) { + const int other = __shfl_xor_sync(FULL_MASK, c, i, WARP_SIZE); + c |= other; + } + return c; +} + +__device__ inline float halfWarpBest(float e, float c) { + for (int i = WARP_SIZE/4; i > 0; i /= 2) { + const float o1 = __shfl_xor_sync(FULL_MASK, e, i, WARP_SIZE); + const float o2 = __shfl_xor_sync(FULL_MASK, c, i, WARP_SIZE); + e = (o2 > c) ? o1 : e; + } + return e; +} + +__device__ inline float4 relativeDelta(const float4 &e) { + const float e0x = __shfl_sync(FULL_MASK, e.x, 0, WARP_SIZE/2); + const float e0y = __shfl_sync(FULL_MASK, e.y, 0, WARP_SIZE/2); + const float e0z = __shfl_sync(FULL_MASK, e.z, 0, WARP_SIZE/2); + return make_float4(e.x-e0x, e.y-e0y, e.z-e0z, 0.0f); +} + +/** + * See: Birchfield S. et al. (1998). A pixel dissimilarity measure that is + * insensitive to image sampling. IEEE Transactions on Pattern Analysis and + * Machine Intelligence. + */ +__device__ float dissimilarity(const float4 &l, const float4 &rp, const float4 &rc, const float4 &rn) { + const float rpd = distance((rc - rp) * 0.5f + rp, l); + const float rnd = distance((rc - rn) * 0.5f + rn, l); + const float rcd = distance(rc, l); + return min(min(rpd, rnd), rcd); +} + + template<int COR_STEPS, int FUNCTION> __global__ void corresponding_point_kernel( TextureObject<float> d1, @@ -51,16 +102,14 @@ __global__ void corresponding_point_kernel( TextureObject<short2> screenOut, TextureObject<float> conf, TextureObject<int> mask, - float4x4 pose1, - float4x4 pose1_inv, - float4x4 pose2, // Inverse + float4x4 pose, Camera cam1, Camera cam2, ftl::cuda::MvMLSParams params) { + + //const int tid = (threadIdx.x + threadIdx.y * blockDim.x); + const int x = (blockIdx.x*8 + (threadIdx.x%4) + 4*(threadIdx.x / 16)); // / WARP_SIZE; + const int y = blockIdx.y*8 + threadIdx.x/4 + 4*threadIdx.y; - // Each warp picks point in p1 - //const int tid = (threadIdx.x + threadIdx.y * blockDim.x); - const int x = (blockIdx.x*blockDim.x + threadIdx.x); // / WARP_SIZE; - const int y = blockIdx.y*blockDim.y + threadIdx.y; if (x >= 0 && y >=0 && x < screenOut.width() && y < screenOut.height()) { screenOut(x,y) = make_short2(-1,-1); @@ -73,86 +122,112 @@ __global__ void corresponding_point_kernel( //const float4 temp = vout.tex2D(x,y); //vout(x,y) = make_float4(depth1, 0.0f, temp.z, temp.w); - const float3 world1 = pose1 * cam1.screenToCam(x,y,depth1); + //const float3 world1 = pose1 * cam1.screenToCam(x,y,depth1); const auto colour1 = c1.tex2D((float)x+0.5f, (float)y+0.5f); //float bestdepth = 0.0f; short2 bestScreen = make_short2(-1,-1); - float bestdepth = 0.0f; - float bestdepth2 = 0.0f; + //float bestdepth = 0.0f; + //float bestdepth2 = 0.0f; float bestweight = 0.0f; float bestcolour = 0.0f; - float bestdweight = 0.0f; + //float bestdweight = 0.0f; float totalcolour = 0.0f; - int count = 0; - float contrib = 0.0f; + //int count = 0; + //float contrib = 0.0f; + + const float3 camPosOrigin = pose * cam1.screenToCam(x,y,depth1); + const float2 lineOrigin = cam2.camToScreen<float2>(camPosOrigin); + const float3 camPosDistant = pose * cam1.screenToCam(x,y,depth1 + 10.0f); + const float2 lineDistant = cam2.camToScreen<float2>(camPosDistant); + const float lineM = (lineDistant.y - lineOrigin.y) / (lineDistant.x - lineOrigin.x); + const float depthM = 10.0f / (lineDistant.x - lineOrigin.x); + const float depthM2 = (camPosDistant.z - camPosOrigin.z) / (lineDistant.x - lineOrigin.x); + float2 linePos; + linePos.x = lineOrigin.x - ((COR_STEPS/2)); + linePos.y = lineOrigin.y - (((COR_STEPS/2)) * lineM); + //float depthPos = depth1 - (float((COR_STEPS/2)) * depthM); + float depthPos2 = camPosOrigin.z - (float((COR_STEPS/2)) * depthM2); - const float step_interval = params.range / (COR_STEPS / 2); - - const float3 rayStep_world = pose1.getFloat3x3() * cam1.screenToCam(x,y,step_interval); - const float3 rayStart_2 = pose2 * world1; - const float3 rayStep_2 = pose2.getFloat3x3() * rayStep_world; + uint badMask = 0; + int bestStep = COR_STEPS/2; + // Project to p2 using cam2 // Each thread takes a possible correspondence and calculates a weighting //const int lane = tid % WARP_SIZE; - for (int i=0; i<COR_STEPS; ++i) { - const int j = i - (COR_STEPS/2); - const float depth_adjust = (float)j * step_interval; - - // Calculate adjusted depth 3D point in camera 2 space - const float3 worldPos = world1 + j * rayStep_world; //(pose1 * cam1.screenToCam(x, y, depth_adjust)); - const float3 camPos = rayStart_2 + j * rayStep_2; //pose2 * worldPos; - const float2 screen = cam2.camToScreen<float2>(camPos); - - float weight = (screen.x >= cam2.width || screen.y >= cam2.height) ? 0.0f : 1.0f; + for (int i=0; i<COR_STEPS; ++i) { + //float weight = 1.0f; //(linePos.x >= cam2.width || linePos.y >= cam2.height) ? 0.0f : 1.0f; // Generate a colour correspondence value - const auto colour2 = c2.tex2D(screen.x, screen.y); + const auto colour2 = c2.tex2D(linePos.x, linePos.y); + + // TODO: Check if other colour dissimilarities are better... const float cweight = ftl::cuda::colourWeighting(colour1, colour2, params.colour_smooth); // Generate a depth correspondence value - const float depth2 = d2.tex2D(int(screen.x+0.5f), int(screen.y+0.5f)); + const float depth2 = d2.tex2D(int(linePos.x+0.5f), int(linePos.y+0.5f)); + + // Record which correspondences are invalid + badMask |= ( + depth2 <= cam2.minDepth || + depth2 >= cam2.maxDepth || + linePos.x < 0.5f || + linePos.y < 0.5f || + linePos.x >= d2.width()-0.5f || + linePos.y >= d2.height()-0.5f + ) ? 1 << i : 0; - if (FUNCTION == 1) { - weight *= ftl::cuda::weighting(fabs(depth2 - camPos.z), cweight*params.spatial_smooth); - } else { - const float dweight = ftl::cuda::weighting(fabs(depth2 - camPos.z), params.spatial_smooth); - weight *= weightFunction<FUNCTION>(params, dweight, cweight); - } + //if (FUNCTION == 1) { + float weight = ftl::cuda::weighting(fabs(depth2 - depthPos2), cweight*params.spatial_smooth); + //weight = ftl::cuda::halfWarpSum(weight); + //} else { + // const float dweight = ftl::cuda::weighting(fabs(depth2 - depthPos2), params.spatial_smooth); + // weight *= weightFunction<FUNCTION>(params, dweight, cweight); + //} //const float dweight = ftl::cuda::weighting(fabs(depth_adjust), 10.0f*params.range); //weight *= weightFunction<FUNCTION>(params, dweight, cweight); - ++count; - contrib += weight; + //++count; + bestcolour = max(cweight, bestcolour); - //bestdweight = max(dweight, bestdweight); totalcolour += cweight; - bestdepth = (weight > bestweight) ? depth_adjust : bestdepth; - //bestdepth2 = (weight > bestweight) ? camPos.z : bestdepth2; - //bestScreen = (weight > bestweight) ? make_short2(screen.x+0.5f, screen.y+0.5f) : bestScreen; + + //bestdepth = (weight > bestweight) ? depthPos : bestdepth; + bestStep = (weight > bestweight) ? i : bestStep; + bestweight = max(bestweight, weight); - //bestweight = weight; - //bestdepth = depth_adjust; - //bestScreen = make_short2(screen.x+0.5f, screen.y+0.5f); - //} + + + //depthPos += depthM; + depthPos2 += depthM2; + linePos.x += 1.0f; + linePos.y += lineM; } - const float avgcolour = totalcolour/(float)count; - const float confidence = bestcolour / totalcolour; //bestcolour - avgcolour; - + //const float avgcolour = totalcolour/(float)count; + const float confidence = ((bestcolour / totalcolour) - (1.0f / 16.0f)) * (1.0f + (1.0f/16.0f)); + float bestadjust = float(bestStep-(COR_STEPS/2))*depthM; + + // Detect matches to boundaries, and discard those + uint stepMask = 1 << bestStep; + if ((stepMask & badMask) || (stepMask & (badMask << 1)) || (stepMask & (badMask >> 1))) bestweight = 0.0f; + + //bestadjust = halfWarpBest(bestadjust, (bestweight > 0.0f) ? confidence : 0.0f); + //Mask m(mask.tex2D(x,y)); //if (bestweight > 0.0f) { float old = conf.tex2D(x,y); - if (bestweight * confidence > old) { - d1(x,y) = 0.4f*bestdepth + depth1; + if (bestweight > 0.0f) { + d1(x,y) = (0.4f*bestadjust) + depth1; //d2(bestScreen.x, bestScreen.y) = bestdepth2; //screenOut(x,y) = bestScreen; - conf(x,y) = bestweight * confidence; + conf(x,y) = max(old,confidence); //bestweight * confidence; + //conf(x,y) = max(old,fabs(bestadjust)); } //} @@ -169,26 +244,35 @@ void ftl::cuda::correspondence( TextureObject<short2> &screen, TextureObject<float> &conf, TextureObject<int> &mask, - float4x4 &pose1, - float4x4 &pose1_inv, float4x4 &pose2, const Camera &cam1, const Camera &cam2, const MvMLSParams ¶ms, int func, cudaStream_t stream) { - const dim3 gridSize((d1.width() + T_PER_BLOCK - 1)/T_PER_BLOCK, (d1.height() + T_PER_BLOCK - 1)/T_PER_BLOCK); - const dim3 blockSize(T_PER_BLOCK, T_PER_BLOCK); + //const dim3 gridSize((d1.width() + T_PER_BLOCK - 1)/T_PER_BLOCK, (d1.height() + T_PER_BLOCK - 1)/T_PER_BLOCK); + //const dim3 blockSize(T_PER_BLOCK, T_PER_BLOCK); + + const dim3 gridSize((d1.width() + 1), (d1.height() + T_PER_BLOCK - 1)/T_PER_BLOCK); + const dim3 blockSize(WARP_SIZE, 2); //printf("COR SIZE %d,%d\n", p1.width(), p1.height()); - switch (func) { - case 0: corresponding_point_kernel<16,0><<<gridSize, blockSize, 0, stream>>>(d1, d2, c1, c2, screen, conf, mask, pose1, pose1_inv, pose2, cam1, cam2, params); break; - case 1: corresponding_point_kernel<16,1><<<gridSize, blockSize, 0, stream>>>(d1, d2, c1, c2, screen, conf, mask, pose1, pose1_inv, pose2, cam1, cam2, params); break; - case 2: corresponding_point_kernel<16,2><<<gridSize, blockSize, 0, stream>>>(d1, d2, c1, c2, screen, conf, mask, pose1, pose1_inv, pose2, cam1, cam2, params); break; - case 3: corresponding_point_kernel<16,3><<<gridSize, blockSize, 0, stream>>>(d1, d2, c1, c2, screen, conf, mask, pose1, pose1_inv, pose2, cam1, cam2, params); break; - case 4: corresponding_point_kernel<16,4><<<gridSize, blockSize, 0, stream>>>(d1, d2, c1, c2, screen, conf, mask, pose1, pose1_inv, pose2, cam1, cam2, params); break; - case 5: corresponding_point_kernel<16,5><<<gridSize, blockSize, 0, stream>>>(d1, d2, c1, c2, screen, conf, mask, pose1, pose1_inv, pose2, cam1, cam2, params); break; - } + /*switch (func) { + case 0: corresponding_point_kernel<16,0><<<gridSize, blockSize, 0, stream>>>(d1, d2, c1, c2, screen, conf, mask, pose2, cam1, cam2, params); break; + case 1: corresponding_point_kernel<16,1><<<gridSize, blockSize, 0, stream>>>(d1, d2, c1, c2, screen, conf, mask, pose2, cam1, cam2, params); break; + case 2: corresponding_point_kernel<16,2><<<gridSize, blockSize, 0, stream>>>(d1, d2, c1, c2, screen, conf, mask, pose2, cam1, cam2, params); break; + case 3: corresponding_point_kernel<16,3><<<gridSize, blockSize, 0, stream>>>(d1, d2, c1, c2, screen, conf, mask, pose2, cam1, cam2, params); break; + case 4: corresponding_point_kernel<16,4><<<gridSize, blockSize, 0, stream>>>(d1, d2, c1, c2, screen, conf, mask, pose2, cam1, cam2, params); break; + case 5: corresponding_point_kernel<16,5><<<gridSize, blockSize, 0, stream>>>(d1, d2, c1, c2, screen, conf, mask, pose2, cam1, cam2, params); break; + }*/ + + switch (func) { + case 32: corresponding_point_kernel<32,1><<<gridSize, blockSize, 0, stream>>>(d1, d2, c1, c2, screen, conf, mask, pose2, cam1, cam2, params); break; + case 16: corresponding_point_kernel<16,1><<<gridSize, blockSize, 0, stream>>>(d1, d2, c1, c2, screen, conf, mask, pose2, cam1, cam2, params); break; + case 8: corresponding_point_kernel<8,1><<<gridSize, blockSize, 0, stream>>>(d1, d2, c1, c2, screen, conf, mask, pose2, cam1, cam2, params); break; + case 4: corresponding_point_kernel<4,1><<<gridSize, blockSize, 0, stream>>>(d1, d2, c1, c2, screen, conf, mask, pose2, cam1, cam2, params); break; + case 2: corresponding_point_kernel<2,1><<<gridSize, blockSize, 0, stream>>>(d1, d2, c1, c2, screen, conf, mask, pose2, cam1, cam2, params); break; + } cudaSafeCall( cudaGetLastError() ); } diff --git a/components/operators/src/mvmls.cpp b/components/operators/src/mvmls.cpp index e85f8271149537c920b838e9885c3a406bbb5b5b..8ff3c89dd8ce9f10882e4c5386b4de631f7836b9 100644 --- a/components/operators/src/mvmls.cpp +++ b/components/operators/src/mvmls.cpp @@ -24,7 +24,8 @@ bool MultiViewMLS::apply(ftl::rgbd::FrameSet &in, ftl::rgbd::FrameSet &out, cuda int iters = config()->value("mls_iterations", 3); int radius = config()->value("mls_radius",5); //bool aggre = config()->value("aggregation", true); - int win = config()->value("cost_function",1); + //int win = config()->value("cost_function",1); + int win = config()->value("window_size",16); bool do_corr = config()->value("merge_corresponding", true); bool do_aggr = config()->value("merge_mls", false); bool cull_zero = config()->value("cull_no_confidence", false); @@ -71,6 +72,8 @@ bool MultiViewMLS::apply(ftl::rgbd::FrameSet &in, ftl::rgbd::FrameSet &out, cuda f.createTexture<float>(Channel::Confidence); f.create<GpuMat>(Channel::Screen, Format<short2>(size)); f.createTexture<short2>(Channel::Screen); + + f.get<GpuMat>(Channel::Confidence).setTo(cv::Scalar(0.0f), cvstream); } for (int iter=0; iter<iters; ++iter) { @@ -83,7 +86,7 @@ bool MultiViewMLS::apply(ftl::rgbd::FrameSet &in, ftl::rgbd::FrameSet &out, cuda for (size_t i=0; i<in.frames.size(); ++i) { auto &f1 = in.frames[i]; //f1.get<GpuMat>(Channel::Depth2).setTo(cv::Scalar(0.0f), cvstream); - f1.get<GpuMat>(Channel::Confidence).setTo(cv::Scalar(0.0f), cvstream); + //f1.get<GpuMat>(Channel::Confidence).setTo(cv::Scalar(0.0f), cvstream); Eigen::Vector4d d1(0.0, 0.0, 1.0, 0.0); d1 = in.sources[i]->getPose() * d1; @@ -103,12 +106,9 @@ bool MultiViewMLS::apply(ftl::rgbd::FrameSet &in, ftl::rgbd::FrameSet &out, cuda // No, so skip this combination if (d1.dot(d2) <= 0.0) continue; - auto pose1 = MatrixConversion::toCUDA(s1->getPose().cast<float>()); - auto pose1_inv = MatrixConversion::toCUDA(s1->getPose().cast<float>().inverse()); - auto pose2 = MatrixConversion::toCUDA(s2->getPose().cast<float>().inverse()); - auto pose2_inv = MatrixConversion::toCUDA(s2->getPose().cast<float>()); + auto pose2 = MatrixConversion::toCUDA(s2->getPose().cast<float>().inverse() * s1->getPose().cast<float>()); - auto transform = pose2 * pose1; + //auto transform = pose2 * pose1; //Calculate screen positions of estimated corresponding points ftl::cuda::correspondence( @@ -120,8 +120,6 @@ bool MultiViewMLS::apply(ftl::rgbd::FrameSet &in, ftl::rgbd::FrameSet &out, cuda f1.getTexture<short2>(Channel::Screen), f1.getTexture<float>(Channel::Confidence), f1.getTexture<int>(Channel::Mask), - pose1, - pose1_inv, pose2, s1->parameters(), s2->parameters(), @@ -144,6 +142,9 @@ bool MultiViewMLS::apply(ftl::rgbd::FrameSet &in, ftl::rgbd::FrameSet &out, cuda );*/ } } + + // Reduce window size for next iteration + win = max(win>>1, 4); } // Find best source for every pixel @@ -250,7 +251,7 @@ bool MultiViewMLS::apply(ftl::rgbd::FrameSet &in, ftl::rgbd::FrameSet &out, cuda for (size_t i=0; i<in.frames.size(); ++i) { auto &f1 = in.frames[i]; //f1.get<GpuMat>(Channel::Depth2).setTo(cv::Scalar(0.0f), cvstream); - f1.get<GpuMat>(Channel::Confidence).setTo(cv::Scalar(0.0f), cvstream); + //f1.get<GpuMat>(Channel::Confidence).setTo(cv::Scalar(0.0f), cvstream); Eigen::Vector4d d1(0.0, 0.0, 1.0, 0.0); d1 = in.sources[i]->getPose() * d1; diff --git a/components/operators/src/mvmls_cuda.hpp b/components/operators/src/mvmls_cuda.hpp index 93b1e8d882848490aec96a291d58805c2d2dbf03..5faeb47533a16e7c3e13bfc5e23e826aa8b836e3 100644 --- a/components/operators/src/mvmls_cuda.hpp +++ b/components/operators/src/mvmls_cuda.hpp @@ -28,9 +28,7 @@ void correspondence( ftl::cuda::TextureObject<short2> &screen, ftl::cuda::TextureObject<float> &conf, ftl::cuda::TextureObject<int> &mask, - float4x4 &pose1, - float4x4 &pose1_inv, - float4x4 &pose2, + float4x4 &pose, const ftl::rgbd::Camera &cam1, const ftl::rgbd::Camera &cam2, const ftl::cuda::MvMLSParams ¶ms, int func, cudaStream_t stream); diff --git a/components/renderers/cpp/include/ftl/cuda/warp.hpp b/components/renderers/cpp/include/ftl/cuda/warp.hpp index 9164b0eeeb8b3ef606aef4930f55b38a1afacdc4..10fc460f3f33a30fadce45912caf72d4a7a86ac3 100644 --- a/components/renderers/cpp/include/ftl/cuda/warp.hpp +++ b/components/renderers/cpp/include/ftl/cuda/warp.hpp @@ -42,6 +42,49 @@ __device__ inline int warpSum(int e) { return e; } +// Half warp + +__device__ inline float halfWarpMin(float e) { + for (int i = WARP_SIZE/4; i > 0; i /= 2) { + const float other = __shfl_xor_sync(FULL_MASK, e, i, WARP_SIZE); + e = min(e, other); + } + return e; +} + +__device__ inline float halfWarpMax(float e) { + for (int i = WARP_SIZE/4; i > 0; i /= 2) { + const float other = __shfl_xor_sync(FULL_MASK, e, i, WARP_SIZE); + e = max(e, other); + } + return e; +} + +__device__ inline float halfWarpSum(float e) { + for (int i = WARP_SIZE/4; i > 0; i /= 2) { + const float other = __shfl_xor_sync(FULL_MASK, e, i, WARP_SIZE); + e += other; + } + return e; +} + +__device__ inline int halfWarpSum(int e) { + for (int i = WARP_SIZE/4; i > 0; i /= 2) { + const float other = __shfl_xor_sync(FULL_MASK, e, i, WARP_SIZE); + e += other; + } + return e; +} + +__device__ inline float halfWarpMul(float e) { + for (int i = WARP_SIZE/4; i > 0; i /= 2) { + const float other = __shfl_xor_sync(FULL_MASK, e, i, WARP_SIZE); + e *= other; + } + return e; +} + + } } diff --git a/components/renderers/cpp/src/reprojection.cu b/components/renderers/cpp/src/reprojection.cu index e6490c47eaf9033c2077cdd06457b3ff4f59103e..9c414f8927572f93795ea4bfb2e17c491f2deb44 100644 --- a/components/renderers/cpp/src/reprojection.cu +++ b/components/renderers/cpp/src/reprojection.cu @@ -89,13 +89,16 @@ __global__ void reprojection_kernel( const float3 n = transformR * make_float3(normals.tex2D((int)x, (int)y)); float3 ray = camera.screenToCam(screenPos.x, screenPos.y, 1.0f); ray = ray / length(ray); - const float dotproduct = max(dot(ray,n),0.0f); + + // Allow slightly beyond 90 degrees due to normal estimation errors + const float dotproduct = (max(dot(ray,n),-0.1f)+0.1) / 1.1f; const float d2 = depth_src.tex2D(int(screenPos.x+0.5f), int(screenPos.y+0.5f)); const auto input = in.tex2D(screenPos.x, screenPos.y); //generateInput(in.tex2D((int)screenPos.x, (int)screenPos.y), params, worldPos); // TODO: Z checks need to interpolate between neighbors if large triangles are used - float weight = ftl::cuda::weighting(fabs(camPos.z - d2), 0.02f); + //float weight = ftl::cuda::weighting(fabs(camPos.z - d2), params.depthThreshold); + float weight = (fabs(camPos.z - d2) <= params.depthThreshold) ? 1.0f : 0.0f; /* Buehler C. et al. 2001. Unstructured Lumigraph Rendering. */ /* Orts-Escolano S. et al. 2016. Holoportation: Virtual 3D teleportation in real-time. */ diff --git a/components/renderers/cpp/src/splatter.cu b/components/renderers/cpp/src/splatter.cu index 2986234bb3bbc24f762ff5ba0103ba173f4a0093..55706b0856750738134a3417dfdb017205ab2bdf 100644 --- a/components/renderers/cpp/src/splatter.cu +++ b/components/renderers/cpp/src/splatter.cu @@ -131,7 +131,7 @@ using ftl::cuda::warpSum; const uint2 screenPos = params.camera.camToScreen<uint2>(camPos); const unsigned int cx = screenPos.x; const unsigned int cy = screenPos.y; - if (d > params.camera.minDepth && d < params.camera.maxDepth && cx < depth.width() && cy < depth.height()) { + if (d > params.camera.minDepth && d < params.camera.maxDepth && cx < depth_out.width() && cy < depth_out.height()) { // Transform estimated point to virtual cam space and output z atomicMin(&depth_out(cx,cy), d * 100000.0f); } @@ -155,7 +155,7 @@ void ftl::cuda::dibr_merge(TextureObject<float4> &points, TextureObject<int> &de } void ftl::cuda::dibr_merge(TextureObject<float> &depth, TextureObject<int> &depth_out, const float4x4 &transform, const ftl::rgbd::Camera &cam, SplatParams params, cudaStream_t stream) { - const dim3 gridSize((depth.width() + T_PER_BLOCK - 1)/T_PER_BLOCK, (depth.height() + T_PER_BLOCK - 1)/T_PER_BLOCK); + const dim3 gridSize((depth_out.width() + T_PER_BLOCK - 1)/T_PER_BLOCK, (depth_out.height() + T_PER_BLOCK - 1)/T_PER_BLOCK); const dim3 blockSize(T_PER_BLOCK, T_PER_BLOCK); dibr_merge_kernel<<<gridSize, blockSize, 0, stream>>>(depth, depth_out, transform, cam, params); diff --git a/components/renderers/cpp/src/tri_render.cpp b/components/renderers/cpp/src/tri_render.cpp index 567a9d139e45678c4d64534e481a8d5c107e3ad5..06d4fe2626f01e989645e19e795ef4df925c96c2 100644 --- a/components/renderers/cpp/src/tri_render.cpp +++ b/components/renderers/cpp/src/tri_render.cpp @@ -621,7 +621,7 @@ bool Triangular::render(ftl::rgbd::VirtualSource *src, ftl::rgbd::Frame &out, co ); } - if (value("show_bad_colour", false)) { + if (value("show_bad_colour", true)) { ftl::cuda::show_missing_colour( out.getTexture<float>(Channel::Depth), out.getTexture<uchar4>(Channel::Colour), diff --git a/components/renderers/cpp/src/triangle_render.cu b/components/renderers/cpp/src/triangle_render.cu index 2e1966c4252fcf751639cc70a94dbfbdccd43b3b..61f44f320e0b889649d1f82c5f2c090d326e1221 100644 --- a/components/renderers/cpp/src/triangle_render.cu +++ b/components/renderers/cpp/src/triangle_render.cu @@ -77,7 +77,7 @@ __device__ static float3 calculateBarycentricCoordinate(const short2 (&tri)[3], const short2 &point) { float beta = calculateBarycentricCoordinateValue(tri[0], point, tri[2], tri); float gamma = calculateBarycentricCoordinateValue(tri[0], tri[1], point, tri); - float alpha = 1.0 - beta - gamma; + float alpha = 1.0f - beta - gamma; return make_float3(alpha, beta, gamma); } @@ -86,9 +86,9 @@ __device__ static */ __host__ __device__ static bool isBarycentricCoordInBounds(const float3 &barycentricCoord) { - return barycentricCoord.x >= 0.0 && barycentricCoord.x <= 1.0 && - barycentricCoord.y >= 0.0 && barycentricCoord.y <= 1.0 && - barycentricCoord.z >= 0.0 && barycentricCoord.z <= 1.0; + return barycentricCoord.x >= -0.0001f && //barycentricCoord.x <= 1.0f && + barycentricCoord.y >= -0.0001f && //barycentricCoord.y <= 1.0f && + barycentricCoord.z >= -0.0001f; // &&barycentricCoord.z <= 1.0f; } /** @@ -137,6 +137,11 @@ float getZAtCoordinate(const float3 &barycentricCoord, const float (&tri)[3]) { const int maxX = max(v[0].x, max(v[1].x, v[2].x)); const int maxY = max(v[0].y, max(v[1].y, v[2].y)); + // Ensure the points themselves are drawn + //atomicMin(&depth_out(v[0].x,v[0].y), int(d[0]*100000.0f)); + //atomicMin(&depth_out(v[1].x,v[1].y), int(d[1]*100000.0f)); + //atomicMin(&depth_out(v[2].x,v[2].y), int(d[2]*100000.0f)); + // Remove really large triangles if ((maxX - minX) * (maxY - minY) > params.triangle_limit) return;