diff --git a/CMakeLists.txt b/CMakeLists.txt
index fb73e5cf9d35ad7de7e1a2cbf66826d802c0b749..6640f5d7ee46c3511254604ae0840e49cb9e91d6 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -230,6 +230,7 @@ add_subdirectory(applications/calibration)
add_subdirectory(applications/groupview)
add_subdirectory(applications/player)
add_subdirectory(applications/recorder)
+add_subdirectory(applications/merger)
if (HAVE_AVFORMAT)
add_subdirectory(applications/ftl2mkv)
diff --git a/applications/ftl2mkv/src/main.cpp b/applications/ftl2mkv/src/main.cpp
index b555dcbf7f3494dbd66f87415f51cbb0cec3c49e..598e372ca524d33e358c1f312fdeba5429fa0ea8 100644
--- a/applications/ftl2mkv/src/main.cpp
+++ b/applications/ftl2mkv/src/main.cpp
@@ -81,7 +81,7 @@ int main(int argc, char **argv) {
LOG(ERROR) << "Missing input ftl file.";
return -1;
} else {
- filename = paths[0];
+ filename = paths[paths.size()-1];
}
std::ifstream f;
@@ -99,7 +99,7 @@ int main(int argc, char **argv) {
AVOutputFormat *fmt;
AVFormatContext *oc;
- AVStream *video_st[10] = {nullptr};
+ AVStream *video_st[10][2] = {nullptr};
av_register_all();
@@ -130,28 +130,24 @@ int main(int argc, char **argv) {
LOG(INFO) << "Converting...";
int current_stream = root->value("stream", 0);
- int current_channel = 0;
+ int current_channel = root->value("channel", -1);
//bool stream_added[10] = {false};
// TODO: In future, find a better way to discover number of streams...
// Read entire file to find all streams before reading again to write data
bool res = r.read(90000000000000, [¤t_stream,¤t_channel,&r,&video_st,oc](const ftl::codecs::StreamPacket &spkt, const ftl::codecs::Packet &pkt) {
- if (spkt.channel != static_cast<ftl::codecs::Channel>(current_channel)) return;
+ if (spkt.channel != static_cast<ftl::codecs::Channel>(current_channel) && current_channel != -1) return;
if (spkt.streamID == current_stream || current_stream == 255) {
- if (pkt.codec == codec_t::POSE) {
- return;
- }
-
- if (pkt.codec == codec_t::CALIBRATION) {
+ if (pkt.codec != codec_t::HEVC) {
return;
}
if (spkt.streamID >= 10) return; // TODO: Allow for more than 10
- if (video_st[spkt.streamID] == nullptr) {
- video_st[spkt.streamID] = add_video_stream(oc, pkt);
+ if (video_st[spkt.streamID][(spkt.channel == Channel::Left) ? 0 : 1] == nullptr) {
+ video_st[spkt.streamID][(spkt.channel == Channel::Left) ? 0 : 1] = add_video_stream(oc, pkt);
}
}
});
@@ -172,14 +168,10 @@ int main(int argc, char **argv) {
bool seen_key[10] = {false};
res = r.read(90000000000000, [¤t_stream,¤t_channel,&r,&video_st,oc,&seen_key](const ftl::codecs::StreamPacket &spkt, const ftl::codecs::Packet &pkt) {
- if (spkt.channel != static_cast<ftl::codecs::Channel>(current_channel)) return;
+ if (spkt.channel != static_cast<ftl::codecs::Channel>(current_channel) && current_channel != -1) return;
if (spkt.streamID == current_stream || current_stream == 255) {
- if (pkt.codec == codec_t::POSE) {
- return;
- }
-
- if (pkt.codec == codec_t::CALIBRATION) {
+ if (pkt.codec != codec_t::HEVC) {
return;
}
@@ -201,7 +193,7 @@ int main(int argc, char **argv) {
if (keyframe) avpkt.flags |= AV_PKT_FLAG_KEY;
avpkt.pts = spkt.timestamp - r.getStartTime();
avpkt.dts = avpkt.pts;
- avpkt.stream_index= video_st[spkt.streamID]->index;
+ avpkt.stream_index= video_st[spkt.streamID][(spkt.channel == Channel::Left) ? 0 : 1]->index;
avpkt.data= const_cast<uint8_t*>(pkt.data.data());
avpkt.size= pkt.data.size();
avpkt.duration = 1;
@@ -220,7 +212,8 @@ int main(int argc, char **argv) {
//avcodec_close(video_st->codec);
for (int i=0; i<10; ++i) {
- if (video_st[i]) av_free(video_st[i]);
+ if (video_st[i][0]) av_free(video_st[i][0]);
+ if (video_st[i][1]) av_free(video_st[i][1]);
}
if (!(fmt->flags & AVFMT_NOFILE)) {
diff --git a/applications/gui/src/camera.cpp b/applications/gui/src/camera.cpp
index ae0d5627a94a84afb819a9b1daf62b6bddacccae..0a9d04c044e70b615f24733380818ff1a46a1739 100644
--- a/applications/gui/src/camera.cpp
+++ b/applications/gui/src/camera.cpp
@@ -3,6 +3,8 @@
#include "screen.hpp"
#include <nanogui/glutil.h>
+#include <fstream>
+
#ifdef HAVE_OPENVR
#include "vr.hpp"
#endif
@@ -137,6 +139,13 @@ ftl::gui::Camera::Camera(ftl::gui::Screen *screen, ftl::rgbd::Source *src) : scr
sdepth_ = false;
ftime_ = (float)glfwGetTime();
pause_ = false;
+ recording_ = false;
+ fileout_ = new std::ofstream();
+ writer_ = new ftl::codecs::Writer(*fileout_);
+ recorder_ = std::function([this](ftl::rgbd::Source *src, const ftl::codecs::StreamPacket &spkt, const ftl::codecs::Packet &pkt) {
+ ftl::codecs::StreamPacket s = spkt;
+ writer_->write(s, pkt);
+ });
channel_ = Channel::Left;
@@ -166,7 +175,8 @@ ftl::gui::Camera::Camera(ftl::gui::Screen *screen, ftl::rgbd::Source *src) : scr
}
ftl::gui::Camera::~Camera() {
-
+ delete writer_;
+ delete fileout_;
}
ftl::rgbd::Source *ftl::gui::Camera::source() {
@@ -258,7 +268,6 @@ bool ftl::gui::Camera::setVR(bool on) {
src_->set("focal", intrinsic(0, 0));
src_->set("centre_x", intrinsic(0, 2));
src_->set("centre_y", intrinsic(1, 2));
- LOG(INFO) << intrinsic;
intrinsic = getCameraMatrix(screen_->getVR(), vr::Eye_Right);
CHECK(intrinsic(0, 2) < 0 && intrinsic(1, 2) < 0);
@@ -366,8 +375,9 @@ const GLTexture &ftl::gui::Camera::captureFrame() {
if (src_ && src_->isReady()) {
UNIQUE_LOCK(mutex_, lk);
- if (isVR()) {
+ if (screen_->isVR()) {
#ifdef HAVE_OPENVR
+
vr::VRCompositor()->WaitGetPoses(rTrackedDevicePose_, vr::k_unMaxTrackedDeviceCount, NULL, 0 );
if ((channel_ == Channel::Right) && rTrackedDevicePose_[vr::k_unTrackedDeviceIndex_Hmd].bPoseIsValid )
@@ -494,6 +504,37 @@ const GLTexture &ftl::gui::Camera::captureFrame() {
return texture1_;
}
+void ftl::gui::Camera::snapshot() {
+ UNIQUE_LOCK(mutex_, lk);
+ char timestamp[18];
+ std::time_t t = std::time(NULL);
+ std::strftime(timestamp, sizeof(timestamp), "%F-%H%M%S", std::localtime(&t));
+ cv::Mat image;
+ cv::flip(im1_, image, 0);
+ cv::imwrite(std::string(timestamp) + ".png", image);
+}
+
+void ftl::gui::Camera::toggleVideoRecording() {
+ if (recording_) {
+ src_->removeRawCallback(recorder_);
+ writer_->end();
+ fileout_->close();
+ recording_ = false;
+ } else {
+ char timestamp[18];
+ std::time_t t=std::time(NULL);
+ std::strftime(timestamp, sizeof(timestamp), "%F-%H%M%S", std::localtime(&t));
+ fileout_->open(std::string(timestamp) + ".ftl");
+
+ writer_->begin();
+ src_->addRawCallback(recorder_);
+
+ src_->inject(Channel::Calibration, src_->parameters(), Channel::Left, src_->getCapabilities());
+ src_->inject(src_->getPose());
+ recording_ = true;
+ }
+}
+
nlohmann::json ftl::gui::Camera::getMetaData() {
return nlohmann::json();
}
diff --git a/applications/gui/src/camera.hpp b/applications/gui/src/camera.hpp
index 24dcbacfdc51d20166ffc49175f268140e4dcdf3..43cf9c783b1e4b5e3ae003c428b8d6126273ef40 100644
--- a/applications/gui/src/camera.hpp
+++ b/applications/gui/src/camera.hpp
@@ -2,6 +2,7 @@
#define _FTL_GUI_CAMERA_HPP_
#include <ftl/rgbd/source.hpp>
+#include <ftl/codecs/writer.hpp>
#include "gltexture.hpp"
#include <string>
@@ -43,7 +44,7 @@ class Camera {
void togglePause();
void isPaused();
- const ftl::codecs::Channels &availableChannels();
+ const ftl::codecs::Channels &availableChannels() { return channels_; }
const GLTexture &captureFrame();
const GLTexture &getLeft() const { return texture1_; }
@@ -51,6 +52,10 @@ class Camera {
bool thumbnail(cv::Mat &thumb);
+ void snapshot();
+
+ void toggleVideoRecording();
+
nlohmann::json getMetaData();
StatisticsImage *stats_ = nullptr;
@@ -85,6 +90,10 @@ class Camera {
ftl::codecs::Channels channels_;
cv::Mat im1_; // first channel (left)
cv::Mat im2_; // second channel ("right")
+ bool recording_;
+ std::ofstream *fileout_;
+ ftl::codecs::Writer *writer_;
+ ftl::rgbd::RawCallback recorder_;
MUTEX mutex_;
diff --git a/applications/gui/src/config_window.cpp b/applications/gui/src/config_window.cpp
index fcba8ff7ad2e033ab790790b03aed4a3af25af15..322f817ae264386ab3a54bfddc72c7d5f588fc36 100644
--- a/applications/gui/src/config_window.cpp
+++ b/applications/gui/src/config_window.cpp
@@ -2,10 +2,10 @@
#include <nanogui/layout.h>
#include <nanogui/label.h>
-#include <nanogui/combobox.h>
#include <nanogui/button.h>
#include <nanogui/entypo.h>
#include <nanogui/formhelper.h>
+#include <nanogui/vscrollpanel.h>
#include <vector>
#include <string>
@@ -15,28 +15,25 @@ using std::string;
using std::vector;
using ftl::config::json_t;
-ConfigWindow::ConfigWindow(nanogui::Widget *parent, ftl::ctrl::Master *ctrl, const ftl::UUID &peer) : ConfigWindow(parent, ctrl, std::optional<ftl::UUID>(peer)) {
-
-}
-
-ConfigWindow::ConfigWindow(nanogui::Widget *parent, ftl::ctrl::Master *ctrl, const std::optional<ftl::UUID> &peer)
- : nanogui::Window(parent, "Settings"), ctrl_(ctrl), peer_(peer) {
+ConfigWindow::ConfigWindow(nanogui::Widget *parent, ftl::ctrl::Master *ctrl)
+ : nanogui::Window(parent, "Settings"), ctrl_(ctrl) {
using namespace nanogui;
setLayout(new GroupLayout());
setPosition(Vector2i(parent->width()/2.0f - 100.0f, parent->height()/2.0f - 100.0f));
//setModal(true);
- if (peer) {
- configurables_ = ctrl->getConfigurables(peer.value());
- } else {
- configurables_ = ftl::config::list();
- }
+ configurables_ = ftl::config::list();
new Label(this, "Select Configurable","sans-bold");
+ auto vscroll = new VScrollPanel(this);
+ vscroll->setFixedHeight(300);
+ Widget *buttons = new Widget(vscroll);
+ buttons->setLayout(new BoxLayout(Orientation::Vertical, Alignment::Fill));
+
for (auto c : configurables_) {
- auto itembutton = new Button(this, c);
+ auto itembutton = new Button(buttons, c);
itembutton->setCallback([this,c]() {
LOG(INFO) << "Change configurable: " << c;
_buildForm(c);
@@ -52,67 +49,51 @@ ConfigWindow::~ConfigWindow() {
}
-class ConfigWindow::References {
- public:
- References(ftl::NetConfigurable* nc, ftl::config::json_t* config, const std::string* suri) : nc(nc), config(config), suri(suri) {
- }
-
- ~References() {
- delete nc;
- delete config;
- delete suri;
- }
-
- private:
- ftl::NetConfigurable* nc;
- ftl::config::json_t* config;
- const std::string* suri;
-};
-
-std::vector<ftl::gui::ConfigWindow::References *> ConfigWindow::_addElements(nanogui::FormHelper *form, ftl::Configurable &nc, const std::string &suri, std::function<ftl::Configurable*(const std::string*, std::vector<References *>&)> construct) {
+void ConfigWindow::_addElements(nanogui::FormHelper *form, const std::string &suri) {
using namespace nanogui;
- std::vector<References *> references;
-
- auto data = nc.getConfig();
+ Configurable *configurable = ftl::config::find(suri);
+ ftl::config::json_t data;
+ if (configurable) {
+ configurable->refresh();
+ data = configurable->getConfig();
+ }
for (auto i=data.begin(); i!=data.end(); ++i) {
if (i.key() == "$id") continue;
if (i.key() == "$ref" && i.value().is_string()) {
LOG(INFO) << "Follow $ref: " << i.value();
- const std::string* suri = new std::string(i.value().get<string>());
- ftl::Configurable* rc = construct(suri, references);
- auto new_references = _addElements(form, *rc, *suri, construct);
- references.insert(references.end(), new_references.begin(), new_references.end());
+ const std::string suri = std::string(i.value().get<string>());
+ _addElements(form, suri);
continue;
}
if (i.value().is_boolean()) {
string key = i.key();
- form->addVariable<bool>(i.key(), [this,data,key,&nc](const bool &b){
- nc.set(key, b);
+ form->addVariable<bool>(i.key(), [this,data,key,suri](const bool &b){
+ ftl::config::update(suri+"/"+key, b);
}, [data,key]() -> bool {
return data[key].get<bool>();
});
} else if (i.value().is_number_integer()) {
string key = i.key();
- form->addVariable<int>(i.key(), [this,data,key,&nc](const int &f){
- nc.set(key, f);
+ form->addVariable<int>(i.key(), [this,data,key,suri](const int &f){
+ ftl::config::update(suri+"/"+key, f);
}, [data,key]() -> int {
return data[key].get<int>();
});
} else if (i.value().is_number_float()) {
string key = i.key();
- form->addVariable<float>(i.key(), [this,data,key,&nc](const float &f){
- nc.set(key, f);
+ form->addVariable<float>(i.key(), [this,data,key,suri](const float &f){
+ ftl::config::update(suri+"/"+key, f);
}, [data,key]() -> float {
return data[key].get<float>();
});
} else if (i.value().is_string()) {
string key = i.key();
- form->addVariable<string>(i.key(), [this,data,key,&nc](const string &f){
- nc.set(key, f);
+ form->addVariable<string>(i.key(), [this,data,key,suri](const string &f){
+ ftl::config::update(suri+"/"+key, f);
}, [data,key]() -> string {
return data[key].get<string>();
});
@@ -131,8 +112,6 @@ std::vector<ftl::gui::ConfigWindow::References *> ConfigWindow::_addElements(nan
}
}
}
-
- return references;
}
void ConfigWindow::_buildForm(const std::string &suri) {
@@ -145,50 +124,15 @@ void ConfigWindow::_buildForm(const std::string &suri) {
form->addWindow(Vector2i(100,50), uri.getFragment());
form->window()->setTheme(theme());
- ftl::config::json_t* config;
- config = new ftl::config::json_t;
- const std::string* allocated_suri = new std::string(suri);
- std::vector<ftl::gui::ConfigWindow::References *> references;
-
- ftl::Configurable* nc;
-
- if (peer_) {
- *config = ctrl_->get(peer_.value(), suri);
- nc = new ftl::NetConfigurable(peer_.value(), *allocated_suri, *ctrl_, *config);
+ _addElements(form, suri);
- references = _addElements(form, *nc, *allocated_suri, [this](auto suri, auto &references) {
- ftl::config::json_t* config = new ftl::config::json_t;
- *config = ctrl_->get(peer_.value(), *suri);
- auto nc = new ftl::NetConfigurable(peer_.value(), *suri, *ctrl_, *config);
- auto r = new References(nc, config, suri);
- references.push_back(r);
- return nc;
- });
- } else {
- nc = ftl::config::find(suri);
- if (nc) {
- references = _addElements(form, *nc, *allocated_suri, [this](auto suri, auto &references) {
- return ftl::config::find(*suri);
- });
- }
- }
-
- auto closebutton = form->addButton("Close", [this,form,config,allocated_suri,nc,references]() {
+ auto closebutton = form->addButton("Close", [this,form]() {
form->window()->setVisible(false);
- for(auto r : references) {
- delete r;
- }
- if (peer_) {
- delete nc;
- }
- delete config;
- delete allocated_suri;
delete form;
});
closebutton->setIcon(ENTYPO_ICON_CROSS);
}
bool ConfigWindow::exists(const std::string &uri) {
- // If the Configurable is a NetConfigurable, the URI is not checked.
- return peer_ || ftl::config::find(uri);
-}
\ No newline at end of file
+ return ftl::config::find(uri) != nullptr;
+}
diff --git a/applications/gui/src/config_window.hpp b/applications/gui/src/config_window.hpp
index 23279d93cee1d601e53e0147b4edee8d5309a483..a0fe74155fe5320f983871d8c9237c7571625670 100644
--- a/applications/gui/src/config_window.hpp
+++ b/applications/gui/src/config_window.hpp
@@ -16,23 +16,15 @@ namespace gui {
*/
class ConfigWindow : public nanogui::Window {
public:
- ConfigWindow(nanogui::Widget *parent, ftl::ctrl::Master *ctrl, const ftl::UUID &peer);
- ConfigWindow(nanogui::Widget *parent, ftl::ctrl::Master *ctrl, const std::optional<ftl::UUID> &peer = std::nullopt);
+ ConfigWindow(nanogui::Widget *parent, ftl::ctrl::Master *ctrl);
~ConfigWindow();
private:
- /*
- References holds the pointers to a NetConfigurable and all its members so that
- they can all be returned from _addElements() and then simultaneously deleted
- as the form is closed.
- */
- class References;
ftl::ctrl::Master *ctrl_;
- std::optional<ftl::UUID> peer_;
std::vector<std::string> configurables_;
void _buildForm(const std::string &uri);
- std::vector<References *> _addElements(nanogui::FormHelper *form, ftl::Configurable &nc, const std::string &suri, std::function<ftl::Configurable*(const std::string*, std::vector<References *>&)> construct);
+ void _addElements(nanogui::FormHelper *form, const std::string &suri);
bool exists(const std::string &uri);
};
diff --git a/applications/gui/src/ctrl_window.cpp b/applications/gui/src/ctrl_window.cpp
index 67a3682bd2fcfcb55a68fb9c71b6dd55aad36491..fc82c45838a568efc8b4e8d456cdb3fbbef2c8b4 100644
--- a/applications/gui/src/ctrl_window.cpp
+++ b/applications/gui/src/ctrl_window.cpp
@@ -69,7 +69,7 @@ ControlWindow::ControlWindow(nanogui::Widget *parent, ftl::ctrl::Master *ctrl)
button = new Button(tools, "", ENTYPO_ICON_COG);
button->setCallback([this,parent] {
- auto cfgwin = new ConfigWindow(parent, ctrl_, _getActiveID());
+ auto cfgwin = new ConfigWindow(parent, ctrl_);
cfgwin->setTheme(theme());
});
button->setTooltip("Edit node configuration");
diff --git a/applications/gui/src/main.cpp b/applications/gui/src/main.cpp
index 50bc3638c2e26197fbcf5b74dadacd4b6a49d0ad..5ebc552fa0ee0664c7a0b7898a723b90bf87f29c 100644
--- a/applications/gui/src/main.cpp
+++ b/applications/gui/src/main.cpp
@@ -2,6 +2,7 @@
#include <ftl/net/universe.hpp>
#include <ftl/rgbd.hpp>
#include <ftl/master.hpp>
+#include <ftl/net_configurable.hpp>
#include <loguru.hpp>
@@ -12,9 +13,6 @@ int main(int argc, char **argv) {
auto root = ftl::configure(argc, argv, "gui_default");
ftl::net::Universe *net = ftl::create<ftl::net::Universe>(root, "net");
- net->start();
- net->waitConnections();
-
ftl::ctrl::Master *controller = new ftl::ctrl::Master(root, net);
controller->onLog([](const ftl::ctrl::LogEvent &e){
const int v = e.verbosity;
@@ -25,6 +23,35 @@ int main(int argc, char **argv) {
}
});
+ std::map<ftl::UUID, std::vector<ftl::NetConfigurable*>> peerConfigurables;
+
+ // FIXME: Move this elsewhere, it is not just for GUI
+ net->onConnect([&controller, &peerConfigurables](ftl::net::Peer *p) {
+ ftl::UUID peer = p->id();
+ auto cs = controller->getConfigurables(peer);
+ for (auto c : cs) {
+ //LOG(INFO) << "NET CONFIG: " << c;
+ ftl::config::json_t *configuration = new ftl::config::json_t;
+ *configuration = controller->get(peer, c);
+ if (!configuration->empty()) {
+ ftl::NetConfigurable *nc = new ftl::NetConfigurable(peer, c, *controller, *configuration);
+ peerConfigurables[peer].push_back(nc);
+ }
+ }
+ });
+
+ net->onDisconnect([&peerConfigurables](ftl::net::Peer *p) {
+ ftl::UUID peer = p->id();
+ for (ftl::NetConfigurable *nc : peerConfigurables[peer]) {
+ ftl::config::json_t *configuration = &(nc->getConfig());
+ delete nc;
+ delete configuration;
+ }
+ });
+
+ net->start();
+ net->waitConnections();
+
/*auto available = net.findAll<string>("list_streams");
for (auto &a : available) {
std::cout << " -- " << a << std::endl;
@@ -57,4 +84,4 @@ int main(int argc, char **argv) {
delete root;
return 0;
-}
\ No newline at end of file
+}
diff --git a/applications/gui/src/media_panel.cpp b/applications/gui/src/media_panel.cpp
index 800940199e2488ca1b11bd723498059efd63d374..693024bd3c6a89ccb0ac6e893b01641dbd77c574 100644
--- a/applications/gui/src/media_panel.cpp
+++ b/applications/gui/src/media_panel.cpp
@@ -1,6 +1,5 @@
#include "media_panel.hpp"
#include "screen.hpp"
-#include "camera.hpp"
#include <nanogui/layout.h>
#include <nanogui/button.h>
@@ -18,7 +17,6 @@ MediaPanel::MediaPanel(ftl::gui::Screen *screen) : nanogui::Window(screen, ""),
using namespace nanogui;
paused_ = false;
- writer_ = nullptr;
disable_switch_channels_ = false;
setLayout(new BoxLayout(Orientation::Horizontal,
@@ -35,29 +33,62 @@ MediaPanel::MediaPanel(ftl::gui::Screen *screen) : nanogui::Window(screen, ""),
if (cam) cam->showPoseWindow();
});
- button = new Button(this, "", ENTYPO_ICON_CONTROLLER_RECORD);
- button->setFlags(Button::ToggleButton);
- button->setChangeCallback([this,button](bool state) {
- if (state){
- auto *cam = screen_->activeCamera();
-
- button->setTextColor(nanogui::Color(1.0f,0.1f,0.1f,1.0f));
- char timestamp[18];
- std::time_t t=std::time(NULL);
- std::strftime(timestamp, sizeof(timestamp), "%F-%H%M%S", std::localtime(&t));
- writer_ = new ftl::rgbd::SnapshotStreamWriter(std::string(timestamp) + ".tar.gz", 1000 / 25);
- writer_->addSource(cam->source());
- writer_->start();
- } else {
- button->setTextColor(nanogui::Color(1.0f,1.0f,1.0f,1.0f));
- if (writer_) {
- writer_->stop();
- delete writer_;
- writer_ = nullptr;
+ virtualCameraRecording_ = std::optional<ftl::gui::Camera*>();
+ sceneRecording_ = std::optional<ftl::Configurable*>();
+
+ auto recordbutton = new PopupButton(this, "", ENTYPO_ICON_CONTROLLER_RECORD);
+ recordbutton->setTooltip("Record");
+ recordbutton->setSide(Popup::Side::Right);
+ recordbutton->setChevronIcon(0);
+ auto recordpopup = recordbutton->popup();
+ recordpopup->setLayout(new GroupLayout());
+ recordpopup->setTheme(screen->toolbuttheme);
+ recordpopup->setAnchorHeight(150);
+ auto itembutton = new Button(recordpopup, "2D snapshot (.png)");
+ itembutton->setCallback([this,recordbutton]() {
+ screen_->activeCamera()->snapshot();
+ recordbutton->setPushed(false);
+ });
+ itembutton = new Button(recordpopup, "Virtual camera recording (.ftl)");
+ itembutton->setCallback([this,recordbutton]() {
+ auto activeCamera = screen_->activeCamera();
+ activeCamera->toggleVideoRecording();
+ recordbutton->setTextColor(nanogui::Color(1.0f,0.1f,0.1f,1.0f));
+ recordbutton->setPushed(false);
+ virtualCameraRecording_ = std::optional<ftl::gui::Camera*>(activeCamera);
+ });
+ itembutton = new Button(recordpopup, "3D scene recording (.ftl)");
+ itembutton->setCallback([this,recordbutton]() {
+ auto tag = screen_->activeCamera()->source()->get<std::string>("uri");
+ if (tag) {
+ auto tagvalue = tag.value();
+ auto configurables = ftl::config::findByTag(tagvalue);
+ if (configurables.size() > 0) {
+ ftl::Configurable *configurable = configurables[0];
+ configurable->set("record", true);
+ recordbutton->setTextColor(nanogui::Color(1.0f,0.1f,0.1f,1.0f));
+ sceneRecording_ = std::optional<ftl::Configurable*>(configurable);
}
}
- //if (state) ... start
- //else ... stop
+ recordbutton->setPushed(false);
+ });
+ itembutton = new Button(recordpopup, "Detailed recording options");
+
+ recordbutton->setCallback([this,recordbutton](){
+ if (virtualCameraRecording_) {
+ virtualCameraRecording_.value()->toggleVideoRecording();
+ recordbutton->setTextColor(nanogui::Color(1.0f,1.0f,1.0f,1.0f));
+
+ // Prevents the popup from being opened, though it is shown while the button
+ // is being pressed.
+ recordbutton->setPushed(false);
+ virtualCameraRecording_ = std::nullopt;
+ } else if (sceneRecording_) {
+ sceneRecording_.value()->set("record", false);
+ recordbutton->setTextColor(nanogui::Color(1.0f,1.0f,1.0f,1.0f));
+ recordbutton->setPushed(false);
+ sceneRecording_ = std::nullopt;
+ }
});
button = new Button(this, "", ENTYPO_ICON_CONTROLLER_STOP);
@@ -67,8 +98,9 @@ MediaPanel::MediaPanel(ftl::gui::Screen *screen) : nanogui::Window(screen, ""),
button = new Button(this, "", ENTYPO_ICON_CONTROLLER_PAUS);
button->setCallback([this,button]() {
- paused_ = !paused_;
- screen_->control()->pause();
+ //paused_ = !paused_;
+ paused_ = !(bool)ftl::config::get("[reconstruction]/controls/paused");
+ ftl::config::update("[reconstruction]/controls/paused", paused_);
if (paused_) {
button->setIcon(ENTYPO_ICON_CONTROLLER_PLAY);
} else {
@@ -114,11 +146,11 @@ MediaPanel::MediaPanel(ftl::gui::Screen *screen) : nanogui::Window(screen, ""),
*/
#ifdef HAVE_OPENVR
- if (this->screen_->hasVR()) {
+ if (this->screen_->isHmdPresent()) {
auto button_vr = new Button(this, "VR");
button_vr->setFlags(Button::ToggleButton);
button_vr->setChangeCallback([this, button_vr](bool state) {
- if (!screen_->useVR()) {
+ if (!screen_->isVR()) {
if (screen_->switchVR(true) == true) {
button_vr->setTextColor(nanogui::Color(0.5f,0.5f,1.0f,1.0f));
this->button_channels_->setEnabled(false);
diff --git a/applications/gui/src/media_panel.hpp b/applications/gui/src/media_panel.hpp
index 0279cb3fadab41003ceefef538e8f42a20f00139..df0b0802294cbe64800c850faa60fadf4f3de55b 100644
--- a/applications/gui/src/media_panel.hpp
+++ b/applications/gui/src/media_panel.hpp
@@ -1,6 +1,8 @@
#ifndef _FTL_GUI_MEDIAPANEL_HPP_
#define _FTL_GUI_MEDIAPANEL_HPP_
+#include "camera.hpp"
+
#include <nanogui/window.h>
namespace ftl {
@@ -30,6 +32,15 @@ class MediaPanel : public nanogui::Window {
nanogui::PopupButton *button_channels_;
nanogui::Button *right_button_;
nanogui::Button *depth_button_;
+
+ /**
+ * These members indicate which type of recording is active, if any.
+ * They also include a pointer to an object which is used
+ * to end the recording. Only one of these members should have a value
+ * at any given time.
+ */
+ std::optional<ftl::gui::Camera*> virtualCameraRecording_;
+ std::optional<ftl::Configurable*> sceneRecording_;
};
}
diff --git a/applications/gui/src/screen.cpp b/applications/gui/src/screen.cpp
index c359e227d9be39ab98df2cd8e816323b6a0ecbef..76665281fc1b9fcbb1b0b45609c509d02f88a69d 100644
--- a/applications/gui/src/screen.cpp
+++ b/applications/gui/src/screen.cpp
@@ -6,7 +6,6 @@
#include <nanogui/window.h>
#include <nanogui/layout.h>
#include <nanogui/imageview.h>
-#include <nanogui/combobox.h>
#include <nanogui/label.h>
#include <nanogui/toolbutton.h>
#include <nanogui/popupbutton.h>
@@ -69,7 +68,6 @@ ftl::gui::Screen::Screen(ftl::Configurable *proot, ftl::net::Universe *pnet, ftl
#ifdef HAVE_OPENVR
HMD_ = nullptr;
- has_vr_ = vr::VR_IsHmdPresent();
#endif
zoom_ = root_->value("zoom", 1.0f);
@@ -86,6 +84,8 @@ ftl::gui::Screen::Screen(ftl::Configurable *proot, ftl::net::Universe *pnet, ftl
pos_y_ = root_->value("position_y", 0.0f);
});
+ shortcuts_ = ftl::create<ftl::Configurable>(root_, "shortcuts");
+
setSize(Vector2i(1280,720));
toolbuttheme = new Theme(*theme());
@@ -229,30 +229,29 @@ ftl::gui::Screen::Screen(ftl::Configurable *proot, ftl::net::Universe *pnet, ftl
popup->setVisible(false);
});
- popbutton = new PopupButton(innertool, "", ENTYPO_ICON_COG);
- popbutton->setIconExtraScale(1.5f);
- popbutton->setTheme(toolbuttheme);
- popbutton->setTooltip("Settings");
- popbutton->setFixedSize(Vector2i(40,40));
- popbutton->setSide(Popup::Side::Right);
- popbutton->setChevronIcon(0);
- // popbutton->setPosition(Vector2i(5,height()-50));
- popup = popbutton->popup();
- popup->setLayout(new GroupLayout());
- popup->setTheme(toolbuttheme);
+ itembutton = new Button(innertool, "", ENTYPO_ICON_COG);
+ itembutton->setIconExtraScale(1.5f);
+ itembutton->setTheme(toolbuttheme);
+ itembutton->setTooltip("Settings");
+ itembutton->setFixedSize(Vector2i(40,40));
+
+ itembutton->setCallback([this]() {
+ auto config_window = new ConfigWindow(this, ctrl_);
+ config_window->setTheme(windowtheme);
+ });
+ /*
//net_->onConnect([this,popup](ftl::net::Peer *p) {
{
LOG(INFO) << "NET CONNECT";
auto node_details = ctrl_->getSlaves();
- std::vector<std::string> node_titles;
for (auto &d : node_details) {
LOG(INFO) << "ADDING TITLE: " << d.dump();
auto peer = ftl::UUID(d["id"].get<std::string>());
auto itembutton = new Button(popup, d["title"].get<std::string>());
itembutton->setCallback([this,popup,peer]() {
- auto config_window = new ConfigWindow(this, ctrl_, peer);
+ auto config_window = new ConfigWindow(this, ctrl_);
config_window->setTheme(windowtheme);
});
}
@@ -264,6 +263,7 @@ ftl::gui::Screen::Screen(ftl::Configurable *proot, ftl::net::Universe *pnet, ftl
auto config_window = new ConfigWindow(this, ctrl_);
config_window->setTheme(windowtheme);
});
+ */
//configwindow_ = new ConfigWindow(parent, ctrl_);
cwindow_ = new ftl::gui::ControlWindow(this, controller);
@@ -320,14 +320,14 @@ bool ftl::gui::Screen::initVR() {
return true;
}
-bool ftl::gui::Screen::useVR() {
+bool ftl::gui::Screen::isVR() {
auto *cam = activeCamera();
if (HMD_ == nullptr || cam == nullptr) { return false; }
return cam->isVR();
}
bool ftl::gui::Screen::switchVR(bool on) {
- if (useVR() == on) { return on; }
+ if (isVR() == on) { return on; }
if (on && (HMD_ == nullptr) && !initVR()) {
return false;
@@ -339,8 +339,13 @@ bool ftl::gui::Screen::switchVR(bool on) {
activeCamera()->setVR(false);
}
- return useVR();
+ return isVR();
}
+
+bool ftl::gui::Screen::isHmdPresent() {
+ return vr::VR_IsHmdPresent();
+}
+
#endif
ftl::gui::Screen::~Screen() {
@@ -431,12 +436,31 @@ bool ftl::gui::Screen::mouseButtonEvent(const nanogui::Vector2i &p, int button,
}
}
+static std::string generateKeyComboStr(int key, int modifiers) {
+ std::string res = "";
+
+ switch(modifiers) {
+ case 1: res += "Shift+"; break;
+ case 2: res += "Ctrl+"; break;
+ case 3: res += "Ctrl+Shift+"; break;
+ case 4: res += "Alt+"; break;
+ default: break;
+ }
+
+ if (key < 127 && key >= 32) {
+ char buf[2] = { (char)key, 0 };
+ return res + std::string(buf);
+ } else {
+ return "";
+ }
+}
+
bool ftl::gui::Screen::keyboardEvent(int key, int scancode, int action, int modifiers) {
using namespace Eigen;
if (nanogui::Screen::keyboardEvent(key, scancode, action, modifiers)) {
return true;
} else {
- LOG(INFO) << "Key press " << key << " - " << action << " - " << modifiers;
+ //LOG(INFO) << "Key press " << key << " - " << action << " - " << modifiers;
if (key >= 262 && key <= 267) {
if (camera_) camera_->keyMovement(key, modifiers);
@@ -444,9 +468,42 @@ bool ftl::gui::Screen::keyboardEvent(int key, int scancode, int action, int modi
} else if (action == 1 && key == 'H') {
swindow_->setVisible(false);
cwindow_->setVisible(false);
- } else if (action == 1 && key == 32) {
- ctrl_->pause();
- return true;
+ } else if (action == 1) {
+ std::string combo = generateKeyComboStr(key, modifiers);
+
+ if (combo.size() > 0) {
+ LOG(INFO) << "Key combo = " << combo;
+
+ auto s = shortcuts_->get<nlohmann::json>(combo);
+ if (s) {
+ //LOG(INFO) << "FOUND KEYBOARD SHORTCUT";
+ std::string op = (*s).value("op",std::string("="));
+ std::string uri = (*s).value("uri",std::string(""));
+
+ if (op == "toggle") {
+ auto v = ftl::config::get(uri);
+ if (v.is_boolean()) {
+ ftl::config::update(uri, !v.get<bool>());
+ }
+ } else if (op == "+=") {
+ auto v = ftl::config::get(uri);
+ if (v.is_number_float()) {
+ ftl::config::update(uri, v.get<float>() + (*s).value("value",0.0f));
+ } else if (v.is_number_integer()) {
+ ftl::config::update(uri, v.get<int>() + (*s).value("value",0));
+ }
+ } else if (op == "-=") {
+ auto v = ftl::config::get(uri);
+ if (v.is_number_float()) {
+ ftl::config::update(uri, v.get<float>() - (*s).value("value",0.0f));
+ } else if (v.is_number_integer()) {
+ ftl::config::update(uri, v.get<int>() - (*s).value("value",0));
+ }
+ } else if (op == "=") {
+ ftl::config::update(uri, (*s)["value"]);
+ }
+ }
+ }
}
return false;
}
@@ -467,7 +524,7 @@ void ftl::gui::Screen::draw(NVGcontext *ctx) {
if (camera_->getChannel() != ftl::codecs::Channel::Left) { mImageID = rightEye_; }
#ifdef HAVE_OPENVR
- if (useVR() && imageSize[0] > 0 && camera_->getLeft().isValid() && camera_->getRight().isValid()) {
+ if (isVR() && imageSize[0] > 0 && camera_->getLeft().isValid() && camera_->getRight().isValid()) {
vr::Texture_t leftEyeTexture = {(void*)(uintptr_t)leftEye_, vr::TextureType_OpenGL, vr::ColorSpace_Gamma };
vr::VRCompositor()->Submit(vr::Eye_Left, &leftEyeTexture );
diff --git a/applications/gui/src/screen.hpp b/applications/gui/src/screen.hpp
index a195f4ccdd45500c534039709ba6762fdd640b22..90cd519bb3681126f4dc4f0d258e342953562433 100644
--- a/applications/gui/src/screen.hpp
+++ b/applications/gui/src/screen.hpp
@@ -48,19 +48,18 @@ class Screen : public nanogui::Screen {
// initialize OpenVR
bool initVR();
- // is VR available (HMD was found at initialization)
- bool hasVR() const { return has_vr_; }
-
// is VR mode on/off
- bool useVR();
+ bool isVR();
// toggle VR on/off
bool switchVR(bool mode);
+ bool isHmdPresent();
+
vr::IVRSystem* getVR() { return HMD_; }
#else
- bool hasVR() const { return false; }
+ bool isVR() { return false; }
#endif
nanogui::Theme *windowtheme;
@@ -102,8 +101,9 @@ class Screen : public nanogui::Screen {
bool show_two_images_ = false;
+ ftl::Configurable *shortcuts_;
+
#ifdef HAVE_OPENVR
- bool has_vr_;
vr::IVRSystem *HMD_;
#endif
};
diff --git a/applications/gui/src/src_window.cpp b/applications/gui/src/src_window.cpp
index 8e087d1dd42e25b00afd0efaf7b5dd22d5139a0d..0126810a511b85a1168fe06c638651e533f11a0d 100644
--- a/applications/gui/src/src_window.cpp
+++ b/applications/gui/src/src_window.cpp
@@ -63,7 +63,7 @@ SourceWindow::SourceWindow(ftl::gui::Screen *screen)
UNIQUE_LOCK(mutex_, lk);
_updateCameras(screen_->net()->findAll<string>("list_streams"));
});
-
+
UNIQUE_LOCK(mutex_, lk);
std::vector<ftl::rgbd::Source*> srcs = ftl::createArray<ftl::rgbd::Source>(screen_->root(), "sources", screen_->net());
@@ -74,6 +74,14 @@ SourceWindow::SourceWindow(ftl::gui::Screen *screen)
_updateCameras(screen_->control()->getNet()->findAll<string>("list_streams"));
}
+std::vector<ftl::gui::Camera*> SourceWindow::getCameras() {
+ auto cameras = std::vector<ftl::gui::Camera*>(cameras_.size());
+ for (const auto &kv : cameras_) {
+ cameras.push_back(kv.second);
+ }
+ return cameras;
+}
+
void SourceWindow::_updateCameras(const vector<string> &netcams) {
for (auto s : netcams) {
if (cameras_.find(s) == cameras_.end()) {
diff --git a/applications/gui/src/src_window.hpp b/applications/gui/src/src_window.hpp
index b2fe8a9e0957f3345a719a0c37bac46bf473089c..dab3f5d4301de73d0dda2bbd32b321c3f796d160 100644
--- a/applications/gui/src/src_window.hpp
+++ b/applications/gui/src/src_window.hpp
@@ -25,7 +25,7 @@ class SourceWindow : public nanogui::Window {
explicit SourceWindow(ftl::gui::Screen *screen);
~SourceWindow();
- const std::vector<ftl::gui::Camera*> &getCameras();
+ std::vector<ftl::gui::Camera*> getCameras();
virtual void draw(NVGcontext *ctx);
diff --git a/applications/merger/CMakeLists.txt b/applications/merger/CMakeLists.txt
new file mode 100644
index 0000000000000000000000000000000000000000..ccc625afb0241966628bf8bffa790cf8a041afa7
--- /dev/null
+++ b/applications/merger/CMakeLists.txt
@@ -0,0 +1,11 @@
+set(FTLMERGER
+ src/main.cpp
+)
+
+add_executable(ftl-merge ${FTLMERGER})
+
+target_include_directories(ftl-merge PRIVATE src)
+
+target_link_libraries(ftl-merge ftlcommon ftlcodecs ftlrgbd Threads::Threads ${OpenCV_LIBS})
+
+
diff --git a/applications/merger/src/main.cpp b/applications/merger/src/main.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..d7b3f2b92d5976082f1995420cc6486061b714b2
--- /dev/null
+++ b/applications/merger/src/main.cpp
@@ -0,0 +1,104 @@
+#include <loguru.hpp>
+#include <ftl/configuration.hpp>
+#include <ftl/codecs/reader.hpp>
+#include <ftl/codecs/writer.hpp>
+#include <ftl/codecs/packet.hpp>
+#include <ftl/rgbd/camera.hpp>
+#include <ftl/codecs/hevc.hpp>
+
+#include <fstream>
+
+int main(int argc, char **argv) {
+ auto root = ftl::configure(argc, argv, "merger_default");
+
+ std::string outputfile = root->value("out", std::string("output.ftl"));
+ std::vector<std::string> paths = *root->get<std::vector<std::string>>("paths");
+ int timeoff = int(root->value("offset", 0.0f) * 1000.0f);
+ int stream_mask1 = root->value("mask1",0xFF);
+ int stream_mask2 = root->value("mask2",0xFF);
+
+ if (paths.size() == 0) {
+ LOG(ERROR) << "Missing input ftl file(s).";
+ return -1;
+ }
+
+ // Generate the output writer...
+ std::ofstream of;
+ of.open(outputfile);
+ if (!of.is_open()) {
+ LOG(ERROR) << "Could not open output file: " << outputfile;
+ return -1;
+ }
+
+ ftl::codecs::Writer out(of);
+ out.begin();
+
+ std::vector<std::ifstream> fs;
+ std::vector<ftl::codecs::Reader*> rs;
+ fs.resize(paths.size());
+ rs.resize(paths.size());
+
+ for (size_t i=0; i<paths.size(); ++i) {
+ fs[i].open(paths[i]);
+ if (!fs[i].is_open()) {
+ LOG(ERROR) << "Could not open file: " << paths[i];
+ return -1;
+ }
+
+ LOG(INFO) << "Opening("<< i <<"): " << paths[i];
+
+ rs[i] = new ftl::codecs::Reader(fs[i]);
+ if (!rs[i]->begin()) {
+ LOG(ERROR) << "Bad ftl file format";
+ return -1;
+ }
+ }
+
+ std::map<int,int> idmap;
+ int lastid = 0;
+
+ bool res = rs[0]->read(90000000000000, [&rs,&out,&idmap,&lastid,stream_mask1,stream_mask2,timeoff](const ftl::codecs::StreamPacket &spkt, const ftl::codecs::Packet &pkt) {
+ if (((0x1 << spkt.streamID) & stream_mask1) == 0) return;
+
+ ftl::codecs::StreamPacket spkt2 = spkt;
+ if (idmap.find(spkt.streamID) == idmap.end()) {
+ idmap[spkt.streamID] = lastid++;
+ }
+ spkt2.streamID = idmap[spkt.streamID];
+
+ // Now read all other sources up to the same packet timestamp.
+ out.write(spkt2, pkt);
+
+ for (size_t j=1; j<rs.size(); ++j) {
+ ftl::codecs::Reader *r = rs[j];
+
+ // FIXME: Need to truncate other stream if the following returns
+ // no frames, meaning the timeshift causes this stream to run out
+ // before the main stream.
+ rs[j]->read(spkt.timestamp+timeoff+1, [&out,&idmap,&lastid,j,r,stream_mask2,timeoff](const ftl::codecs::StreamPacket &spkt, const ftl::codecs::Packet &pkt) {
+ if (((0x1 << spkt.streamID) & stream_mask2) == 0) return;
+ if (int(spkt.channel) < 32 && spkt.timestamp < r->getStartTime()+timeoff) return;
+
+ ftl::codecs::StreamPacket spkt2 = spkt;
+ if (idmap.find(spkt.streamID + (j << 16)) == idmap.end()) {
+ idmap[spkt.streamID+(j << 16)] = lastid++;
+ }
+ spkt2.streamID = idmap[spkt.streamID + (j << 16)];
+ spkt2.timestamp -= timeoff;
+
+ out.write(spkt2, pkt);
+ });
+ }
+ });
+
+ out.end();
+ of.close();
+
+ for (size_t i=0; i<rs.size(); ++i) {
+ rs[i]->end();
+ delete rs[i];
+ fs[i].close();
+ }
+
+ return 0;
+}
\ No newline at end of file
diff --git a/applications/reconstruct/CMakeLists.txt b/applications/reconstruct/CMakeLists.txt
index b089d4a1aeae9654b50865c7400ca0e381f874e6..5dc0ef5dbce517fb88cdf0fb0cc9fdc7b84a778a 100644
--- a/applications/reconstruct/CMakeLists.txt
+++ b/applications/reconstruct/CMakeLists.txt
@@ -20,6 +20,7 @@ set(REPSRC
src/ilw/fill.cu
src/ilw/discontinuity.cu
src/ilw/correspondence.cu
+ src/reconstruction.cpp
)
add_executable(ftl-reconstruct ${REPSRC})
diff --git a/applications/reconstruct/src/main.cpp b/applications/reconstruct/src/main.cpp
index 0603dad4028d69223b0ce5ddc31d36eafc900c14..4b802d49f85ebe618fd27ecbfe48b46b2f40b5b1 100644
--- a/applications/reconstruct/src/main.cpp
+++ b/applications/reconstruct/src/main.cpp
@@ -18,6 +18,8 @@
#include <ftl/codecs/writer.hpp>
#include <ftl/codecs/reader.hpp>
+#include "reconstruction.hpp"
+
#include "ilw/ilw.hpp"
#include <ftl/render/tri_render.hpp>
@@ -136,7 +138,16 @@ static void run(ftl::Configurable *root) {
net->start();
net->waitConnections();
- // Check paths for an FTL file to load...
+ std::vector<int> sourcecounts;
+
+ // Add sources from the configuration file as a single group.
+ auto configuration_sources = root->getConfig()["sources"];
+ size_t configuration_size = configuration_sources.size();
+ if (configuration_size > 0) {
+ sourcecounts.push_back(configuration_size);
+ }
+
+ // Check paths for FTL files to load.
auto paths = (*root->get<nlohmann::json>("paths"));
for (auto &x : paths.items()) {
std::string path = x.value().get<std::string>();
@@ -162,12 +173,14 @@ static void run(ftl::Configurable *root) {
int N = root->value("N", 100);
// For each stream found, add a source object
- for (int i=0; i<=min(max_stream,N-1); ++i) {
+ int count = min(max_stream+1, N);
+ for (int i=0; i<count; ++i) {
root->getConfig()["sources"].push_back(nlohmann::json{{"uri",std::string("file://") + path + std::string("#") + std::to_string(i)}});
}
+ sourcecounts.push_back(count);
}
}
-
+
// Create a vector of all input RGB-Depth sources
auto sources = ftl::createArray<Source>(root, "sources", net);
@@ -175,7 +188,7 @@ static void run(ftl::Configurable *root) {
LOG(ERROR) << "No sources configured!";
return;
}
-
+
ConfigProxy *configproxy = nullptr;
if (net->numberOfPeers() > 0) {
configproxy = new ConfigProxy(net); // TODO delete
@@ -187,26 +200,6 @@ static void run(ftl::Configurable *root) {
configproxy->add(disparity, "source/disparity/cross", "cross");
}
- // Create scene transform, intended for axis aligning the walls and floor
- Eigen::Matrix4d transform;
- if (root->getConfig()["transform"].is_object()) {
- auto &c = root->getConfig()["transform"];
- float rx = c.value("pitch", 0.0f);
- float ry = c.value("yaw", 0.0f);
- float rz = c.value("roll", 0.0f);
- float x = c.value("x", 0.0f);
- float y = c.value("y", 0.0f);
- float z = c.value("z", 0.0f);
-
- Eigen::Affine3d r = create_rotation_matrix(rx, ry, rz);
- Eigen::Translation3d trans(Eigen::Vector3d(x,y,z));
- Eigen::Affine3d t(trans);
- transform = t.matrix() * r.matrix();
- LOG(INFO) << "Set transform: " << transform;
- } else {
- transform.setIdentity();
- }
-
// Must find pose for each source...
if (sources.size() > 1) {
std::map<std::string, Eigen::Matrix4d> transformations;
@@ -222,73 +215,73 @@ static void run(ftl::Configurable *root) {
string uri = input->getURI();
auto T = transformations.find(uri);
if (T == transformations.end()) {
- LOG(ERROR) << "Camera pose for " + uri + " not found in transformations";
+ LOG(WARNING) << "Camera pose for " + uri + " not found in transformations";
//LOG(WARNING) << "Using only first configured source";
// TODO: use target source if configured and found
//sources = { sources[0] };
//sources[0]->setPose(Eigen::Matrix4d::Identity());
//break;
- input->setPose(transform * input->getPose());
+ input->setPose(input->getPose());
continue;
}
- input->setPose(transform * T->second);
+ input->setPose(T->second);
}
}
- ftl::rgbd::FrameSet scene_A; // Output of align process
- ftl::rgbd::FrameSet scene_B; // Input of render process
+ ftl::rgbd::FrameSet fs_out;
//ftl::voxhash::SceneRep *scene = ftl::create<ftl::voxhash::SceneRep>(root, "voxelhash");
ftl::rgbd::Streamer *stream = ftl::create<ftl::rgbd::Streamer>(root, "stream", net);
- ftl::rgbd::VirtualSource *virt = ftl::create<ftl::rgbd::VirtualSource>(root, "virtual");
- ftl::render::Triangular *splat = ftl::create<ftl::render::Triangular>(root, "renderer", &scene_B);
- ftl::rgbd::Group *group = new ftl::rgbd::Group;
- ftl::ILW *align = ftl::create<ftl::ILW>(root, "merge");
+ ftl::rgbd::VirtualSource *vs = ftl::create<ftl::rgbd::VirtualSource>(root, "virtual");
+ //root->set("tags", nlohmann::json::array({ root->getID()+"/virtual" }));
int o = root->value("origin_pose", 0) % sources.size();
- virt->setPose(sources[o]->getPose());
+ vs->setPose(sources[o]->getPose());
+
+ vector<ftl::Reconstruction*> groups;
+
+ size_t cumulative = 0;
+ for (auto c : sourcecounts) {
+ std::string id = std::to_string(cumulative);
+ auto reconstr = ftl::create<ftl::Reconstruction>(root, id, id);
+ for (size_t i=cumulative; i<cumulative+c; i++) {
+ reconstr->addSource(sources[i]);
+ }
+ groups.push_back(reconstr);
+ cumulative += c;
+ }
auto *renderpipe = ftl::config::create<ftl::operators::Graph>(root, "render_pipe");
renderpipe->append<ftl::operators::ColourChannels>("colour"); // Generate interpolation texture...
renderpipe->append<ftl::operators::FXAA>("antialiasing");
- // Generate virtual camera render when requested by streamer
- virt->onRender([splat,virt,&scene_B,align,renderpipe](ftl::rgbd::Frame &out) {
- //virt->setTimestamp(scene_B.timestamp);
- // Do we need to convert Lab to BGR?
- if (align->isLabColour()) {
- for (auto &f : scene_B.frames) {
- auto &col = f.get<cv::cuda::GpuMat>(Channel::Colour);
- cv::cuda::cvtColor(col,col, cv::COLOR_Lab2BGR); // TODO: Add stream
- }
+ vs->onRender([vs, &groups, &renderpipe](ftl::rgbd::Frame &out) {
+ for (auto &reconstr : groups) {
+ reconstr->render(vs, out);
}
- splat->render(virt, out);
- renderpipe->apply(out, out, virt, 0);
+ renderpipe->apply(out, out, vs, 0);
});
- stream->add(virt);
-
- for (size_t i=0; i<sources.size(); i++) {
- Source *in = sources[i];
- in->setChannel(Channel::Depth);
- group->addSource(in);
- }
+ stream->add(vs);
// ---- Recording code -----------------------------------------------------
-
std::ofstream fileout;
ftl::codecs::Writer writer(fileout);
- auto recorder = [&writer,&group](ftl::rgbd::Source *src, const ftl::codecs::StreamPacket &spkt, const ftl::codecs::Packet &pkt) {
- ftl::codecs::StreamPacket s = spkt;
-
- // Patch stream ID to match order in group
- s.streamID = group->streamID(src);
- writer.write(s, pkt);
- };
root->set("record", false);
+ // Add a recording callback to all reconstruction scenes
+ for (size_t i=0; i<sources.size(); ++i) {
+ sources[i]->addRawCallback([&writer,&groups,i](ftl::rgbd::Source *src, const ftl::codecs::StreamPacket &spkt, const ftl::codecs::Packet &pkt) {
+ ftl::codecs::StreamPacket s = spkt;
+
+ // Patch stream ID to match order in group
+ s.streamID = i;
+ writer.write(s, pkt);
+ });
+ }
+
// Allow stream recording
- root->on("record", [&group,&fileout,&writer,&recorder](const ftl::config::Event &e) {
+ root->on("record", [&groups,&fileout,&writer,&sources](const ftl::config::Event &e) {
if (e.entity->value("record", false)) {
char timestamp[18];
std::time_t t=std::time(NULL);
@@ -296,17 +289,15 @@ static void run(ftl::Configurable *root) {
fileout.open(std::string(timestamp) + ".ftl");
writer.begin();
- group->addRawCallback(std::function(recorder));
// TODO: Write pose+calibration+config packets
- auto sources = group->sources();
+
for (size_t i=0; i<sources.size(); ++i) {
//writeSourceProperties(writer, i, sources[i]);
sources[i]->inject(Channel::Calibration, sources[i]->parameters(), Channel::Left, sources[i]->getCapabilities());
sources[i]->inject(sources[i]->getPose());
}
} else {
- group->removeRawCallback(recorder);
writer.end();
fileout.close();
}
@@ -318,62 +309,6 @@ static void run(ftl::Configurable *root) {
//stream->add(group);
stream->run();
- bool busy = false;
-
- // Create the source depth map pipeline
- auto *pipeline1 = ftl::config::create<ftl::operators::Graph>(root, "pre_filters");
- pipeline1->append<ftl::operators::ClipScene>("clipping");
- pipeline1->append<ftl::operators::ColourChannels>("colour"); // Convert BGR to BGRA
- //pipeline1->append<ftl::operators::HFSmoother>("hfnoise"); // Remove high-frequency noise
- pipeline1->append<ftl::operators::Normals>("normals"); // Estimate surface normals
- //pipeline1->append<ftl::operators::SmoothChannel>("smoothing"); // Generate a smoothing channel
- //pipeline1->append<ftl::operators::ScanFieldFill>("filling"); // Generate a smoothing channel
- pipeline1->append<ftl::operators::CrossSupport>("cross");
- pipeline1->append<ftl::operators::DiscontinuityMask>("discontinuity");
- pipeline1->append<ftl::operators::CullDiscontinuity>("remove_discontinuity");
- //pipeline1->append<ftl::operators::AggreMLS>("mls"); // Perform MLS (using smoothing channel)
- pipeline1->append<ftl::operators::VisCrossSupport>("viscross")->set("enabled", false);
- pipeline1->append<ftl::operators::MultiViewMLS>("mvmls");
- // Alignment
-
-
- group->setLatency(4);
- group->setName("ReconGroup");
- group->sync([splat,virt,&busy,&slave,&scene_A,&scene_B,&align,controls,pipeline1](ftl::rgbd::FrameSet &fs) -> bool {
- //cudaSetDevice(scene->getCUDADevice());
-
- //if (slave.isPaused()) return true;
- if (controls->value("paused", false)) return true;
-
- if (busy) {
- LOG(INFO) << "Group frameset dropped: " << fs.timestamp;
- return true;
- }
- busy = true;
-
- // Swap the entire frameset to allow rapid return
- fs.swapTo(scene_A);
-
- ftl::pool.push([&scene_B,&scene_A,&busy,&slave,&align, pipeline1](int id) {
- //cudaSetDevice(scene->getCUDADevice());
- // TODO: Release frameset here...
- //cudaSafeCall(cudaStreamSynchronize(scene->getIntegrationStream()));
-
- UNIQUE_LOCK(scene_A.mtx, lk);
-
- pipeline1->apply(scene_A, scene_A, 0);
- align->process(scene_A);
-
-
- // TODO: To use second GPU, could do a download, swap, device change,
- // then upload to other device. Or some direct device-2-device copy.
- scene_A.swapTo(scene_B);
- LOG(INFO) << "Align complete... " << scene_A.timestamp;
- busy = false;
- });
- return true;
- });
-
LOG(INFO) << "Start timer";
ftl::timer::start(true);
@@ -387,12 +322,12 @@ static void run(ftl::Configurable *root) {
LOG(INFO) << "Deleting...";
- delete align;
- delete splat;
delete stream;
- delete virt;
+ delete vs;
delete net;
- delete group;
+ for (auto g : groups) {
+ delete g;
+ }
ftl::config::cleanup(); // Remove any last configurable objects.
LOG(INFO) << "Done.";
diff --git a/applications/reconstruct/src/reconstruction.cpp b/applications/reconstruct/src/reconstruction.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..be99d1581c3555b1d52aab156bb9ed316d559de6
--- /dev/null
+++ b/applications/reconstruct/src/reconstruction.cpp
@@ -0,0 +1,112 @@
+#include "reconstruction.hpp"
+
+#include "ftl/operators/smoothing.hpp"
+#include "ftl/operators/colours.hpp"
+#include "ftl/operators/normals.hpp"
+#include "ftl/operators/filling.hpp"
+#include "ftl/operators/segmentation.hpp"
+#include "ftl/operators/mask.hpp"
+#include "ftl/operators/antialiasing.hpp"
+#include "ftl/operators/mvmls.hpp"
+#include "ftl/operators/clipping.hpp"
+
+using ftl::Reconstruction;
+using ftl::codecs::Channel;
+
+static Eigen::Affine3d create_rotation_matrix(float ax, float ay, float az) {
+ Eigen::Affine3d rx =
+ Eigen::Affine3d(Eigen::AngleAxisd(ax, Eigen::Vector3d(1, 0, 0)));
+ Eigen::Affine3d ry =
+ Eigen::Affine3d(Eigen::AngleAxisd(ay, Eigen::Vector3d(0, 1, 0)));
+ Eigen::Affine3d rz =
+ Eigen::Affine3d(Eigen::AngleAxisd(az, Eigen::Vector3d(0, 0, 1)));
+ return rz * rx * ry;
+}
+
+Reconstruction::Reconstruction(nlohmann::json &config, const std::string name) :
+ ftl::Configurable(config), busy_(false), fs_render_(), fs_align_() {
+ group_ = new ftl::rgbd::Group;
+ group_->setName("ReconGroup-" + name);
+ group_->setLatency(4);
+
+ renderer_ = ftl::create<ftl::render::Triangular>(this, "renderer", &fs_render_);
+
+ pipeline_ = ftl::config::create<ftl::operators::Graph>(this, "pre_filters");
+ pipeline_->append<ftl::operators::ClipScene>("clipping")->set("enabled", false);
+ pipeline_->append<ftl::operators::ColourChannels>("colour"); // Convert BGR to BGRA
+ //pipeline_->append<ftl::operators::HFSmoother>("hfnoise"); // Remove high-frequency noise
+ pipeline_->append<ftl::operators::Normals>("normals"); // Estimate surface normals
+ //pipeline_->append<ftl::operators::SmoothChannel>("smoothing"); // Generate a smoothing channel
+ //pipeline_->append<ftl::operators::ScanFieldFill>("filling"); // Generate a smoothing channel
+ pipeline_->append<ftl::operators::CrossSupport>("cross");
+ pipeline_->append<ftl::operators::DiscontinuityMask>("discontinuity");
+ pipeline_->append<ftl::operators::CrossSupport>("cross2")->set("discon_support", true);
+ pipeline_->append<ftl::operators::CullDiscontinuity>("remove_discontinuity");
+ //pipeline_->append<ftl::operators::AggreMLS>("mls"); // Perform MLS (using smoothing channel)
+ pipeline_->append<ftl::operators::VisCrossSupport>("viscross")->set("enabled", false);
+ pipeline_->append<ftl::operators::MultiViewMLS>("mvmls");
+
+ group_->sync([this](ftl::rgbd::FrameSet &fs) -> bool {
+ // TODO: pause
+
+ if (busy_) {
+ LOG(INFO) << "Group frameset dropped: " << fs.timestamp;
+ return true;
+ }
+ busy_ = true;
+
+ // Swap the entire frameset to allow rapid return
+ fs.swapTo(fs_align_);
+
+ ftl::pool.push([this](int id) {
+ UNIQUE_LOCK(fs_align_.mtx, lk);
+ pipeline_->apply(fs_align_, fs_align_, 0);
+
+ // TODO: To use second GPU, could do a download, swap, device change,
+ // then upload to other device. Or some direct device-2-device copy.
+ fs_align_.swapTo(fs_render_);
+
+ LOG(INFO) << "Align complete... " << fs_align_.timestamp;
+ busy_ = false;
+ });
+ return true;
+ });
+}
+
+Reconstruction::~Reconstruction() {
+ // TODO delete
+}
+
+void Reconstruction::addSource(ftl::rgbd::Source *src) {
+ src->setChannel(Channel::Depth);
+ group_->addSource(src); // TODO: check if source is already in group?
+}
+
+void Reconstruction::addRawCallback(const std::function<void(ftl::rgbd::Source *src, const ftl::codecs::StreamPacket &spkt, const ftl::codecs::Packet &pkt)> &cb) {
+ group_->addRawCallback(cb);
+}
+
+void Reconstruction::render(ftl::rgbd::VirtualSource *vs, ftl::rgbd::Frame &out) {
+ // Create scene transform, intended for axis aligning the walls and floor
+ Eigen::Matrix4d transform;
+ //if (getConfig()["transform"].is_object()) {
+ //auto &c = getConfig()["transform"];
+ float rx = value("transform_pitch", 0.0f);
+ float ry = value("transform_yaw", 0.0f);
+ float rz = value("transform_roll", 0.0f);
+ float x = value("transform_x", 0.0f);
+ float y = value("transform_y", 0.0f);
+ float z = value("transform_z", 0.0f);
+
+ Eigen::Affine3d r = create_rotation_matrix(rx, ry, rz);
+ Eigen::Translation3d trans(Eigen::Vector3d(x,y,z));
+ Eigen::Affine3d t(trans);
+ transform = t.matrix() * r.matrix();
+ //LOG(INFO) << "Set transform: " << transform;
+ //} else {
+ // transform.setIdentity();
+ //}
+
+ Eigen::Affine3d sm = Eigen::Affine3d(Eigen::Scaling(double(value("scale", 1.0f))));
+ renderer_->render(vs, out, sm.matrix() * transform);
+}
\ No newline at end of file
diff --git a/applications/reconstruct/src/reconstruction.hpp b/applications/reconstruct/src/reconstruction.hpp
new file mode 100644
index 0000000000000000000000000000000000000000..6546f85c163142d4b1cbdf9f84081ca21d703907
--- /dev/null
+++ b/applications/reconstruct/src/reconstruction.hpp
@@ -0,0 +1,39 @@
+#ifndef _FTL_RECONSTRUCTION_HPP_
+#define _FTL_RECONSTRUCTION_HPP_
+
+#include "ftl/configurable.hpp"
+#include "ftl/rgbd/source.hpp"
+#include "ftl/rgbd/frame.hpp"
+#include "ftl/rgbd/group.hpp"
+#include "ftl/rgbd/frameset.hpp"
+#include "ftl/operators/operator.hpp"
+#include "ftl/render/tri_render.hpp"
+
+namespace ftl {
+
+class Reconstruction : public ftl::Configurable {
+ public:
+ Reconstruction(nlohmann::json &config, const std::string name);
+ ~Reconstruction();
+
+ void addSource(ftl::rgbd::Source *);
+
+ void addRawCallback(const std::function<void(ftl::rgbd::Source *src, const ftl::codecs::StreamPacket &spkt, const ftl::codecs::Packet &pkt)> &cb);
+
+ /**
+ * Do the render for a specified virtual camera.
+ */
+ void render(ftl::rgbd::VirtualSource *vs, ftl::rgbd::Frame &out);
+
+ private:
+ bool busy_;
+ ftl::rgbd::FrameSet fs_render_;
+ ftl::rgbd::FrameSet fs_align_;
+ ftl::rgbd::Group *group_;
+ ftl::operators::Graph *pipeline_;
+ ftl::render::Triangular *renderer_;
+};
+
+}
+
+#endif // _FTL_RECONSTRUCTION_HPP_
diff --git a/components/common/cpp/include/ftl/configurable.hpp b/components/common/cpp/include/ftl/configurable.hpp
index 219bd171edf4b7997cc35f97fa7787a0c4bb4559..61dca8ea3b883e97f2e26c2f1a5126c92f8aa520 100644
--- a/components/common/cpp/include/ftl/configurable.hpp
+++ b/components/common/cpp/include/ftl/configurable.hpp
@@ -101,6 +101,12 @@ class Configurable {
void patchPtr(nlohmann::json &newcfg) { config_ = &newcfg; }
+ /**
+ * Allow configurables to refresh their internal state, perhaps from a
+ * remote source.
+ */
+ virtual void refresh();
+
protected:
nlohmann::json *config_;
diff --git a/components/common/cpp/include/ftl/configuration.hpp b/components/common/cpp/include/ftl/configuration.hpp
index 94fcc7dfa6b57e9bfd59fbcf36c5be80e0b82638..18aaf89f9433911dd71ec3d3519ff2125317e78e 100644
--- a/components/common/cpp/include/ftl/configuration.hpp
+++ b/components/common/cpp/include/ftl/configuration.hpp
@@ -47,6 +47,8 @@ void removeConfigurable(Configurable *cfg);
*/
bool update(const std::string &puri, const json_t &value);
+json_t &get(const std::string &puri);
+
/**
* Resolve a JSON schema reference, but do not wait for a remote reference
* if it is not available. A null entity is returned if not resolved.
@@ -72,6 +74,13 @@ json_t &resolveWait(const std::string &);
*/
Configurable *find(const std::string &uri);
+/**
+ * Get all configurables that contain a specified tag. Tags are given under the
+ * "tags" property as an array of strings, but only during configurable
+ * construction.
+ */
+const std::vector<Configurable *> &findByTag(const std::string &tag);
+
std::vector<std::string> list();
/**
diff --git a/components/common/cpp/src/configurable.cpp b/components/common/cpp/src/configurable.cpp
index 5116292adab92672da3adb239ecb2ed3f4630011..8186713b11025f1799afbf26ca8f56532f1deb3f 100644
--- a/components/common/cpp/src/configurable.cpp
+++ b/components/common/cpp/src/configurable.cpp
@@ -59,4 +59,8 @@ void Configurable::on(const string &prop, function<void(const ftl::config::Event
} else {
(*ix).second.push_back(f);
}
-}
\ No newline at end of file
+}
+
+void Configurable::refresh() {
+ // Do nothing by default
+}
diff --git a/components/common/cpp/src/configuration.cpp b/components/common/cpp/src/configuration.cpp
index 89d539cbec5270b7d7a5bc57a0a113eb57456553..f8c982a811652252f14205b00e7f778d906ab8ed 100644
--- a/components/common/cpp/src/configuration.cpp
+++ b/components/common/cpp/src/configuration.cpp
@@ -165,6 +165,7 @@ static bool mergeConfig(const string path) {
static std::map<std::string, json_t*> config_index;
static std::map<std::string, ftl::Configurable*> config_instance;
+static std::map<std::string, std::vector<ftl::Configurable*>> tag_index;
/*
* Recursively URI index the JSON structure.
@@ -193,11 +194,16 @@ ftl::Configurable *ftl::config::find(const std::string &uri) {
actual_uri = rootCFG->getID() + uri;
}
}
+
auto ix = config_instance.find(actual_uri);
if (ix == config_instance.end()) return nullptr;
else return (*ix).second;
}
+const std::vector<Configurable*> &ftl::config::findByTag(const std::string &tag) {
+ return tag_index[tag];
+}
+
std::vector<std::string> ftl::config::list() {
vector<string> r;
for (auto i : config_instance) {
@@ -219,27 +225,60 @@ void ftl::config::registerConfigurable(ftl::Configurable *cfg) {
} else {
config_instance[*uri] = cfg;
LOG(INFO) << "Registering instance: " << *uri;
+
+ auto tags = cfg->get<vector<string>>("tags");
+ if (tags) {
+ for (auto &t : *tags) {
+ //LOG(INFO) << "REGISTER TAG: " << t;
+ tag_index[t].push_back(cfg);
+ }
+ }
}
}
json_t null_json;
+/* To allow for custom tag format */
+static std::string preprocessURI(const std::string &uri) {
+ if (uri[0] == '[') {
+ size_t closing = uri.find_last_of(']');
+ string tags = uri.substr(1, closing-1);
+
+ // TODO: Allow for multiple tags
+
+ const auto &cfgs = ftl::config::findByTag(tags);
+
+ // FIXME: Check for more than one tag result
+ if (cfgs.size() > 0) {
+ //LOG(INFO) << "PREPROC URI " << cfgs[0]->getID() + uri.substr(closing+1);
+ return cfgs[0]->getID() + uri.substr(closing+1);
+ } else {
+ return uri;
+ }
+ } else if (uri[0] == '/') {
+ return rootCFG->getID() + uri;
+ } else {
+ return uri;
+ }
+}
+
bool ftl::config::update(const std::string &puri, const json_t &value) {
// Remove last component of URI
string tail = "";
string head = "";
- size_t last_hash = puri.find_last_of('#');
+ string uri = preprocessURI(puri);
+ size_t last_hash = uri.find_last_of('#');
if (last_hash != string::npos) {
- size_t last = puri.find_last_of('/');
+ size_t last = uri.find_last_of('/');
if (last != string::npos && last > last_hash) {
- tail = puri.substr(last+1);
- head = puri.substr(0, last);
+ tail = uri.substr(last+1);
+ head = uri.substr(0, last);
} else {
- tail = puri.substr(last_hash+1);
- head = puri.substr(0, last_hash);
+ tail = uri.substr(last_hash+1);
+ head = uri.substr(0, last_hash);
}
} else {
- LOG(WARNING) << "Expected a # in an update URI: " << puri;
+ LOG(WARNING) << "Expected a # in an update URI: " << uri;
return false;
}
@@ -272,6 +311,35 @@ bool ftl::config::update(const std::string &puri, const json_t &value) {
}
}
+json_t &ftl::config::get(const std::string &puri) {
+ // Remove last component of URI
+ string tail = "";
+ string head = "";
+ string uri = preprocessURI(puri);
+ size_t last_hash = uri.find_last_of('#');
+ if (last_hash != string::npos) {
+ size_t last = uri.find_last_of('/');
+ if (last != string::npos && last > last_hash) {
+ tail = uri.substr(last+1);
+ head = uri.substr(0, last);
+ } else {
+ tail = uri.substr(last_hash+1);
+ head = uri.substr(0, last_hash);
+ }
+ } else {
+ LOG(WARNING) << "Expected a # in an update URI: " << uri;
+ return null_json;
+ }
+
+ Configurable *cfg = find(head);
+
+ if (cfg) {
+ return cfg->getConfig()[tail];
+ } else {
+ return null_json;
+ }
+}
+
json_t &ftl::config::resolve(const std::string &puri, bool eager) {
string uri_str = puri;
@@ -513,11 +581,12 @@ Configurable *ftl::config::configure(int argc, char **argv, const std::string &r
// Process Arguments
auto options = ftl::config::read_options(&argv, &argc);
- vector<string> paths;
+ vector<string> paths(argc);
while (argc-- > 0) {
paths.push_back(argv[0]);
+ argv++;
}
-
+
if (!findConfiguration(options["config"], paths)) {
LOG(FATAL) << "Could not find any configuration!";
}
diff --git a/components/net/cpp/include/ftl/net_configurable.hpp b/components/net/cpp/include/ftl/net_configurable.hpp
index bdd21c4700e7664cff996672585054c30b6e8e6a..2c6495410321c03b62e18a83f214d00b807bccdb 100644
--- a/components/net/cpp/include/ftl/net_configurable.hpp
+++ b/components/net/cpp/include/ftl/net_configurable.hpp
@@ -12,12 +12,14 @@ namespace ftl {
NetConfigurable(ftl::UUID peer, const std::string &suri, ftl::ctrl::Master &ctrl, ftl::config::json_t &config);
~NetConfigurable();
+ void refresh() override;
+
protected:
void inject(const std::string &name, nlohmann::json &value);
private:
ftl::UUID peer;
- const std::string &suri;
+ const std::string suri;
ftl::ctrl::Master &ctrl;
};
diff --git a/components/net/cpp/src/net_configurable.cpp b/components/net/cpp/src/net_configurable.cpp
index be98cf7edbb221c61e9732a9e59d8546f1185644..cf597c5c77eec05205b6bf0de3f360c7fac178b2 100644
--- a/components/net/cpp/src/net_configurable.cpp
+++ b/components/net/cpp/src/net_configurable.cpp
@@ -10,3 +10,7 @@ ftl::NetConfigurable::~NetConfigurable(){}
void ftl::NetConfigurable::inject(const std::string &name, nlohmann::json &value) {
ctrl.set(peer, suri + std::string("/") + name, value);
}
+
+void ftl::NetConfigurable::refresh() {
+ (*config_) = ctrl.get(peer, suri);
+}
diff --git a/components/operators/src/mvmls.cpp b/components/operators/src/mvmls.cpp
index 4c02a7607f1d6a506c1f60cbd1f25eabd2cb5732..e85f8271149537c920b838e9885c3a406bbb5b5b 100644
--- a/components/operators/src/mvmls.cpp
+++ b/components/operators/src/mvmls.cpp
@@ -61,7 +61,7 @@ bool MultiViewMLS::apply(ftl::rgbd::FrameSet &in, ftl::rgbd::FrameSet &out, cuda
LOG(ERROR) << "Required normals channel missing for MLS";
return false;
}
- if (!f.hasChannel(Channel::Support1)) {
+ if (!f.hasChannel(Channel::Support2)) {
LOG(ERROR) << "Required cross support channel missing for MLS";
return false;
}
@@ -214,7 +214,7 @@ bool MultiViewMLS::apply(ftl::rgbd::FrameSet &in, ftl::rgbd::FrameSet &out, cuda
}
ftl::cuda::mls_aggr_horiz(
- f.createTexture<uchar4>(Channel::Support1),
+ f.createTexture<uchar4>(Channel::Support2),
f.createTexture<float4>(Channel::Normals),
normals_horiz_[i],
f.createTexture<float>(Channel::Depth),
@@ -228,7 +228,7 @@ bool MultiViewMLS::apply(ftl::rgbd::FrameSet &in, ftl::rgbd::FrameSet &out, cuda
);
ftl::cuda::mls_aggr_vert(
- f.getTexture<uchar4>(Channel::Support1),
+ f.getTexture<uchar4>(Channel::Support2),
normals_horiz_[i],
f.getTexture<float4>(Channel::Normals),
centroid_horiz_[i],
diff --git a/components/operators/src/segmentation.cpp b/components/operators/src/segmentation.cpp
index fb015b6b16e1aa7e1dcac9735103c8c6af41567e..9bf7605e1ea9ef6b5dd73a4c3078ba70bb61abb9 100644
--- a/components/operators/src/segmentation.cpp
+++ b/components/operators/src/segmentation.cpp
@@ -14,18 +14,17 @@ CrossSupport::~CrossSupport() {
}
bool CrossSupport::apply(ftl::rgbd::Frame &in, ftl::rgbd::Frame &out, ftl::rgbd::Source *s, cudaStream_t stream) {
- bool use_depth = config()->value("depth_region", false);
+ bool use_mask = config()->value("discon_support", false);
- if (use_depth) {
+ if (use_mask) {
ftl::cuda::support_region(
- in.createTexture<float>(Channel::Depth),
+ in.createTexture<int>(Channel::Mask),
out.createTexture<uchar4>(Channel::Support2, ftl::rgbd::Format<uchar4>(in.get<cv::cuda::GpuMat>(Channel::Colour).size())),
- config()->value("depth_tau", 0.04f),
config()->value("v_max", 5),
config()->value("h_max", 5),
- config()->value("symmetric", true), stream
+ config()->value("symmetric", false), stream
);
- } //else {
+ } else {
ftl::cuda::support_region(
in.createTexture<uchar4>(Channel::Colour),
out.createTexture<uchar4>(Channel::Support1, ftl::rgbd::Format<uchar4>(in.get<cv::cuda::GpuMat>(Channel::Colour).size())),
@@ -34,7 +33,7 @@ bool CrossSupport::apply(ftl::rgbd::Frame &in, ftl::rgbd::Frame &out, ftl::rgbd:
config()->value("h_max", 5),
config()->value("symmetric", true), stream
);
- //}
+ }
return true;
}
diff --git a/components/operators/src/segmentation.cu b/components/operators/src/segmentation.cu
index c16e647931f7d4c4d92551f44f6242fc415bd91d..aaa81e2ad98b5171572b6768754b93f32abcd1eb 100644
--- a/components/operators/src/segmentation.cu
+++ b/components/operators/src/segmentation.cu
@@ -1,8 +1,10 @@
#include "segmentation_cuda.hpp"
+#include "mask_cuda.hpp"
#define T_PER_BLOCK 8
using ftl::cuda::TextureObject;
+using ftl::cuda::Mask;
template <typename T>
__device__ inline float cross(T p1, T p2);
@@ -89,6 +91,64 @@ __device__ uchar4 calculate_support_region(const TextureObject<T> &img, int x, i
return result;
}
+__device__ uchar4 calculate_support_region(const TextureObject<int> &img, int x, int y, int v_max, int h_max) {
+ int x_min = max(0, x - h_max);
+ int x_max = min(img.width()-1, x + h_max);
+ int y_min = max(0, y - v_max);
+ int y_max = min(img.height()-1, y + v_max);
+
+ uchar4 result = make_uchar4(0, 0, 0, 0);
+
+ Mask m1(img.tex2D(x,y));
+
+ int u;
+ for (u=x-1; u >= x_min; --u) {
+ Mask m2(img.tex2D(u,y));
+ if (m2.isDiscontinuity()) {
+ result.x = x - u - 1;
+ break;
+ }
+ }
+ if (u < x_min) result.x = x - x_min;
+
+ for (u=x+1; u <= x_max; ++u) {
+ Mask m2(img.tex2D(u,y));
+ if (m2.isDiscontinuity()) {
+ result.y = u - x - 1;
+ break;
+ }
+ }
+ if (u > x_max) result.y = x_max - x;
+
+ int v;
+ for (v=y-1; v >= y_min; --v) {
+ Mask m2(img.tex2D(x,v));
+ if (m2.isDiscontinuity()) {
+ result.z = y - v - 1;
+ break;
+ }
+ }
+ if (v < y_min) result.z = y - y_min;
+
+ for (v=y+1; v <= y_max; ++v) {
+ Mask m2(img.tex2D(x,v));
+ if (m2.isDiscontinuity()) {
+ result.w = v - y - 1;
+ break;
+ }
+ }
+ if (v > y_max) result.w = y_max - y;
+
+ // Make symetric left/right and up/down
+ if (false) {
+ result.x = min(result.x, result.y);
+ result.y = result.x;
+ result.z = min(result.z, result.w);
+ result.w = result.z;
+ }
+ return result;
+}
+
template <typename T, bool SYM>
__global__ void support_region_kernel(TextureObject<T> img, TextureObject<uchar4> region, float tau, int v_max, int h_max) {
const int x = blockIdx.x*blockDim.x + threadIdx.x;
@@ -99,6 +159,15 @@ __global__ void support_region_kernel(TextureObject<T> img, TextureObject<uchar4
region(x,y) = calculate_support_region<T,SYM>(img, x, y, tau, v_max, h_max);
}
+__global__ void support_region_kernel(TextureObject<int> img, TextureObject<uchar4> region, int v_max, int h_max) {
+ const int x = blockIdx.x*blockDim.x + threadIdx.x;
+ const int y = blockIdx.y*blockDim.y + threadIdx.y;
+
+ if (x < 0 || y < 0 || x >= img.width() || y >= img.height()) return;
+
+ region(x,y) = calculate_support_region(img, x, y, v_max, h_max);
+}
+
void ftl::cuda::support_region(
ftl::cuda::TextureObject<uchar4> &colour,
ftl::cuda::TextureObject<uchar4> ®ion,
@@ -142,6 +211,26 @@ void ftl::cuda::support_region(
#endif
}
+void ftl::cuda::support_region(
+ ftl::cuda::TextureObject<int> &mask,
+ ftl::cuda::TextureObject<uchar4> ®ion,
+ int v_max,
+ int h_max,
+ bool sym,
+ cudaStream_t stream) {
+
+ const dim3 gridSize((region.width() + T_PER_BLOCK - 1)/T_PER_BLOCK, (region.height() + T_PER_BLOCK - 1)/T_PER_BLOCK);
+ const dim3 blockSize(T_PER_BLOCK, T_PER_BLOCK);
+
+ support_region_kernel<<<gridSize, blockSize, 0, stream>>>(mask, region, v_max, h_max);
+ cudaSafeCall( cudaGetLastError() );
+
+
+ #ifdef _DEBUG
+ cudaSafeCall(cudaDeviceSynchronize());
+ #endif
+}
+
__global__ void vis_support_region_kernel(TextureObject<uchar4> colour, TextureObject<uchar4> region, uchar4 bcolour, uchar4 acolour,
int ox, int oy, int dx, int dy) {
const int x = blockIdx.x*blockDim.x + threadIdx.x;
diff --git a/components/operators/src/segmentation_cuda.hpp b/components/operators/src/segmentation_cuda.hpp
index c2cb390d9c0ee62a33127eec1720cf0d6fee8cae..1383489337dc968a33ec630facb597920518e2bf 100644
--- a/components/operators/src/segmentation_cuda.hpp
+++ b/components/operators/src/segmentation_cuda.hpp
@@ -18,6 +18,12 @@ void support_region(
float tau, int v_max, int h_max, bool sym,
cudaStream_t stream);
+void support_region(
+ ftl::cuda::TextureObject<int> &mask,
+ ftl::cuda::TextureObject<uchar4> ®ion,
+ int v_max, int h_max, bool sym,
+ cudaStream_t stream);
+
void vis_support_region(
ftl::cuda::TextureObject<uchar4> &colour,
ftl::cuda::TextureObject<uchar4> ®ion,
diff --git a/components/renderers/cpp/include/ftl/cuda/normals.hpp b/components/renderers/cpp/include/ftl/cuda/normals.hpp
index dc3d0265ce4c142861bb0ca0b2e841dc81887449..bbf690f4f66178297158dea35528ba01629445a1 100644
--- a/components/renderers/cpp/include/ftl/cuda/normals.hpp
+++ b/components/renderers/cpp/include/ftl/cuda/normals.hpp
@@ -42,6 +42,11 @@ void normal_visualise(ftl::cuda::TextureObject<float4> &norm,
const float3 &light, const uchar4 &diffuse, const uchar4 &ambient,
cudaStream_t stream);
+void cool_blue(ftl::cuda::TextureObject<float4> &norm,
+ ftl::cuda::TextureObject<uchar4> &output,
+ const uchar4 &colouring, const float3x3 &pose,
+ cudaStream_t stream);
+
void normal_filter(ftl::cuda::TextureObject<float4> &norm,
ftl::cuda::TextureObject<float4> &points,
const ftl::rgbd::Camera &camera, const float4x4 &pose,
diff --git a/components/renderers/cpp/include/ftl/render/renderer.hpp b/components/renderers/cpp/include/ftl/render/renderer.hpp
index 432be6839de24e94448afbaf407260ea44c5a508..605fa27d182fec5c6463faff397af83b25b43d85 100644
--- a/components/renderers/cpp/include/ftl/render/renderer.hpp
+++ b/components/renderers/cpp/include/ftl/render/renderer.hpp
@@ -26,7 +26,7 @@ class Renderer : public ftl::Configurable {
* the virtual camera object passed, and writes the result into the
* virtual camera.
*/
- virtual bool render(ftl::rgbd::VirtualSource *, ftl::rgbd::Frame &)=0;
+ virtual bool render(ftl::rgbd::VirtualSource *, ftl::rgbd::Frame &, const Eigen::Matrix4d &)=0;
};
}
diff --git a/components/renderers/cpp/include/ftl/render/splat_render.hpp b/components/renderers/cpp/include/ftl/render/splat_render.hpp
index 3b36e8ec98dd37aaf40b0e3a7e12cad6b3b5a14e..8e51aadf15b8e32d05b3253e72cb2adc7e69f98b 100644
--- a/components/renderers/cpp/include/ftl/render/splat_render.hpp
+++ b/components/renderers/cpp/include/ftl/render/splat_render.hpp
@@ -22,7 +22,7 @@ class Splatter : public ftl::render::Renderer {
explicit Splatter(nlohmann::json &config, ftl::rgbd::FrameSet *fs);
~Splatter();
- bool render(ftl::rgbd::VirtualSource *src, ftl::rgbd::Frame &out) override;
+ bool render(ftl::rgbd::VirtualSource *src, ftl::rgbd::Frame &out, const Eigen::Matrix4d &t) override;
//void setOutputDevice(int);
protected:
diff --git a/components/renderers/cpp/include/ftl/render/tri_render.hpp b/components/renderers/cpp/include/ftl/render/tri_render.hpp
index 3d9183643e9f2fe183499cf8bbcc97699328ef60..27b6db318c1a751cfcb99dec829828b69237b618 100644
--- a/components/renderers/cpp/include/ftl/render/tri_render.hpp
+++ b/components/renderers/cpp/include/ftl/render/tri_render.hpp
@@ -19,11 +19,11 @@ class Triangular : public ftl::render::Renderer {
explicit Triangular(nlohmann::json &config, ftl::rgbd::FrameSet *fs);
~Triangular();
- bool render(ftl::rgbd::VirtualSource *src, ftl::rgbd::Frame &out) override;
+ bool render(ftl::rgbd::VirtualSource *src, ftl::rgbd::Frame &out, const Eigen::Matrix4d &t) override;
//void setOutputDevice(int);
protected:
- void _renderChannel(ftl::rgbd::Frame &out, ftl::codecs::Channel channel_in, ftl::codecs::Channel channel_out, cudaStream_t stream);
+ void _renderChannel(ftl::rgbd::Frame &out, ftl::codecs::Channel channel_in, ftl::codecs::Channel channel_out, const Eigen::Matrix4d &t, cudaStream_t stream);
private:
int device_;
@@ -42,6 +42,8 @@ class Triangular : public ftl::render::Renderer {
ftl::render::SplatParams params_;
cudaStream_t stream_;
float3 light_pos_;
+ Eigen::Matrix4d transform_;
+ float scale_;
cv::cuda::GpuMat env_image_;
ftl::cuda::TextureObject<uchar4> env_tex_;
@@ -49,10 +51,10 @@ class Triangular : public ftl::render::Renderer {
//ftl::Filters *filters_;
template <typename T>
- void __reprojectChannel(ftl::rgbd::Frame &, ftl::codecs::Channel in, ftl::codecs::Channel out, cudaStream_t);
- void _reprojectChannel(ftl::rgbd::Frame &, ftl::codecs::Channel in, ftl::codecs::Channel out, cudaStream_t);
- void _dibr(ftl::rgbd::Frame &, cudaStream_t);
- void _mesh(ftl::rgbd::Frame &, ftl::rgbd::Source *, cudaStream_t);
+ void __reprojectChannel(ftl::rgbd::Frame &, ftl::codecs::Channel in, ftl::codecs::Channel out, const Eigen::Matrix4d &t, cudaStream_t);
+ void _reprojectChannel(ftl::rgbd::Frame &, ftl::codecs::Channel in, ftl::codecs::Channel out, const Eigen::Matrix4d &t, cudaStream_t);
+ void _dibr(ftl::rgbd::Frame &, const Eigen::Matrix4d &t, cudaStream_t);
+ void _mesh(ftl::rgbd::Frame &, ftl::rgbd::Source *, const Eigen::Matrix4d &t, cudaStream_t);
};
}
diff --git a/components/renderers/cpp/src/normals.cu b/components/renderers/cpp/src/normals.cu
index 97452555dfb61fd5015aa6a1745b39e2ddcca409..31034c0af6059f2b8014d7af780bd082bb34868a 100644
--- a/components/renderers/cpp/src/normals.cu
+++ b/components/renderers/cpp/src/normals.cu
@@ -399,6 +399,50 @@ void ftl::cuda::normal_visualise(ftl::cuda::TextureObject<float4> &norm,
//==============================================================================
+__global__ void cool_blue_kernel(ftl::cuda::TextureObject<float4> norm,
+ ftl::cuda::TextureObject<uchar4> output,
+ uchar4 colouring, float3x3 pose) {
+ const unsigned int x = blockIdx.x*blockDim.x + threadIdx.x;
+ const unsigned int y = blockIdx.y*blockDim.y + threadIdx.y;
+
+ if(x >= norm.width() || y >= norm.height()) return;
+
+ //output(x,y) = make_uchar4(0,0,0,0);
+ float3 ray = pose * make_float3(0.0f,0.0f,1.0f);
+ ray = ray / length(ray);
+ float3 n = make_float3(norm.tex2D((int)x,(int)y));
+ float l = length(n);
+ if (l == 0) return;
+ n /= l;
+
+ const float d = 1.0f - max(dot(ray, n), 0.0f);
+ uchar4 original = output(x,y); //.tex2D(x,y);
+
+ output(x,y) = make_uchar4(
+ min(255.0f, colouring.x*d + original.x),
+ min(255.0f, colouring.y*d + original.y),
+ min(255.0f, colouring.z*d + original.z), 255);
+}
+
+void ftl::cuda::cool_blue(ftl::cuda::TextureObject<float4> &norm,
+ ftl::cuda::TextureObject<uchar4> &output,
+ const uchar4 &colouring, const float3x3 &pose,
+ cudaStream_t stream) {
+
+ const dim3 gridSize((norm.width() + T_PER_BLOCK - 1)/T_PER_BLOCK, (norm.height() + T_PER_BLOCK - 1)/T_PER_BLOCK);
+ const dim3 blockSize(T_PER_BLOCK, T_PER_BLOCK);
+
+ cool_blue_kernel<<<gridSize, blockSize, 0, stream>>>(norm, output, colouring, pose);
+
+ cudaSafeCall( cudaGetLastError() );
+ #ifdef _DEBUG
+ cudaSafeCall(cudaDeviceSynchronize());
+ //cutilCheckMsg(__FUNCTION__);
+ #endif
+}
+
+//==============================================================================
+
__global__ void filter_normals_kernel(ftl::cuda::TextureObject<float4> norm,
ftl::cuda::TextureObject<float4> output,
ftl::rgbd::Camera camera, float4x4 pose, float thresh) {
diff --git a/components/renderers/cpp/src/splatter.cpp b/components/renderers/cpp/src/splatter.cpp
index 30c47dc1179018a552d8b083245eac71ba56dae2..fad38ba82a1c3925069afd982170761f8d57cd78 100644
--- a/components/renderers/cpp/src/splatter.cpp
+++ b/components/renderers/cpp/src/splatter.cpp
@@ -304,7 +304,7 @@ void Splatter::_renderChannel(
}
}
-bool Splatter::render(ftl::rgbd::VirtualSource *src, ftl::rgbd::Frame &out) {
+bool Splatter::render(ftl::rgbd::VirtualSource *src, ftl::rgbd::Frame &out, const Eigen::Matrix4d &t) {
SHARED_LOCK(scene_->mtx, lk);
if (!src->isReady()) return false;
diff --git a/components/renderers/cpp/src/splatter_cuda.hpp b/components/renderers/cpp/src/splatter_cuda.hpp
index c81977cf38a4380df4143d2cd8d0609c69de8897..53b9f675e9f26144e9f2d31fff9c889b3ef58720 100644
--- a/components/renderers/cpp/src/splatter_cuda.hpp
+++ b/components/renderers/cpp/src/splatter_cuda.hpp
@@ -117,6 +117,11 @@ namespace cuda {
ftl::cuda::TextureObject<uchar4> &colour,
ftl::cuda::TextureObject<int> &mask,
int id, uchar4 style, cudaStream_t stream);
+
+ void merge_convert_depth(
+ ftl::cuda::TextureObject<int> &d1,
+ ftl::cuda::TextureObject<float> &d2,
+ float factor, cudaStream_t stream);
}
}
diff --git a/components/renderers/cpp/src/tri_render.cpp b/components/renderers/cpp/src/tri_render.cpp
index 5912b6480b109db2c3cce960ae5bca6abe6531f8..567a9d139e45678c4d64534e481a8d5c107e3ad5 100644
--- a/components/renderers/cpp/src/tri_render.cpp
+++ b/components/renderers/cpp/src/tri_render.cpp
@@ -206,7 +206,7 @@ void Triangular::__blendChannel(ftl::rgbd::Frame &output, ftl::codecs::Channel i
}*/
template <typename T>
-void Triangular::__reprojectChannel(ftl::rgbd::Frame &output, ftl::codecs::Channel in, ftl::codecs::Channel out, cudaStream_t stream) {
+void Triangular::__reprojectChannel(ftl::rgbd::Frame &output, ftl::codecs::Channel in, ftl::codecs::Channel out, const Eigen::Matrix4d &t, cudaStream_t stream) {
cv::cuda::Stream cvstream = cv::cuda::StreamAccessor::wrapStream(stream);
temp_.create<GpuMat>(
AccumSelector<T>::channel,
@@ -228,7 +228,7 @@ void Triangular::__reprojectChannel(ftl::rgbd::Frame &output, ftl::codecs::Chann
cv::cuda::cvtColor(tmp,col, cv::COLOR_BGR2BGRA);
}
- auto transform = MatrixConversion::toCUDA(s->getPose().cast<float>().inverse()) * params_.m_viewMatrixInverse;
+ auto transform = MatrixConversion::toCUDA(s->getPose().cast<float>().inverse() * t.cast<float>().inverse()) * params_.m_viewMatrixInverse;
auto transformR = MatrixConversion::toCUDA(s->getPose().cast<float>().inverse()).getFloat3x3();
if (mesh_) {
@@ -277,18 +277,18 @@ void Triangular::__reprojectChannel(ftl::rgbd::Frame &output, ftl::codecs::Chann
}
}*/
-void Triangular::_reprojectChannel(ftl::rgbd::Frame &output, ftl::codecs::Channel in, ftl::codecs::Channel out, cudaStream_t stream) {
+void Triangular::_reprojectChannel(ftl::rgbd::Frame &output, ftl::codecs::Channel in, ftl::codecs::Channel out, const Eigen::Matrix4d &t, cudaStream_t stream) {
int type = output.get<GpuMat>(out).type(); // == CV_32F; //ftl::rgbd::isFloatChannel(channel);
switch (type) {
- case CV_32F : __reprojectChannel<float>(output, in, out, stream); break;
- case CV_32FC4 : __reprojectChannel<float4>(output, in, out, stream); break;
- case CV_8UC4 : __reprojectChannel<uchar4>(output, in, out, stream); break;
+ case CV_32F : __reprojectChannel<float>(output, in, out, t, stream); break;
+ case CV_32FC4 : __reprojectChannel<float4>(output, in, out, t, stream); break;
+ case CV_8UC4 : __reprojectChannel<uchar4>(output, in, out, t, stream); break;
default : LOG(ERROR) << "Invalid output channel format";
}
}
-void Triangular::_dibr(ftl::rgbd::Frame &out, cudaStream_t stream) {
+void Triangular::_dibr(ftl::rgbd::Frame &out, const Eigen::Matrix4d &t, cudaStream_t stream) {
cv::cuda::Stream cvstream = cv::cuda::StreamAccessor::wrapStream(stream);
temp_.get<GpuMat>(Channel::Depth2).setTo(cv::Scalar(0x7FFFFFFF), cvstream);
@@ -301,7 +301,7 @@ void Triangular::_dibr(ftl::rgbd::Frame &out, cudaStream_t stream) {
continue;
}
- auto transform = params_.m_viewMatrix * MatrixConversion::toCUDA(s->getPose().cast<float>());
+ auto transform = params_.m_viewMatrix * MatrixConversion::toCUDA(t.cast<float>() * s->getPose().cast<float>());
ftl::cuda::dibr_merge(
f.createTexture<float>(Channel::Depth),
@@ -316,7 +316,7 @@ void Triangular::_dibr(ftl::rgbd::Frame &out, cudaStream_t stream) {
temp_.get<GpuMat>(Channel::Depth2).convertTo(out.get<GpuMat>(Channel::Depth), CV_32F, 1.0f / 100000.0f, cvstream);
}
-void Triangular::_mesh(ftl::rgbd::Frame &out, ftl::rgbd::Source *src, cudaStream_t stream) {
+void Triangular::_mesh(ftl::rgbd::Frame &out, ftl::rgbd::Source *src, const Eigen::Matrix4d &t, cudaStream_t stream) {
cv::cuda::Stream cvstream = cv::cuda::StreamAccessor::wrapStream(stream);
bool do_blend = value("mesh_blend", true);
@@ -338,7 +338,7 @@ void Triangular::_mesh(ftl::rgbd::Frame &out, ftl::rgbd::Source *src, cudaStream
continue;
}
- auto pose = MatrixConversion::toCUDA(s->getPose().cast<float>());
+ auto pose = MatrixConversion::toCUDA(t.cast<float>() * s->getPose().cast<float>());
// Calculate and save virtual view screen position of each source pixel
ftl::cuda::screen_coord(
@@ -374,7 +374,8 @@ void Triangular::_mesh(ftl::rgbd::Frame &out, ftl::rgbd::Source *src, cudaStream
}
// Convert from int depth to float depth
- temp_.get<GpuMat>(Channel::Depth2).convertTo(out.get<GpuMat>(Channel::Depth), CV_32F, 1.0f / 100000.0f, cvstream);
+ //temp_.get<GpuMat>(Channel::Depth2).convertTo(out.get<GpuMat>(Channel::Depth), CV_32F, 1.0f / 100000.0f, cvstream);
+ ftl::cuda::merge_convert_depth(temp_.getTexture<int>(Channel::Depth2), out.createTexture<float>(Channel::Depth), 1.0f / 100000.0f, stream_);
//filters_->filter(out, src, stream);
@@ -388,7 +389,7 @@ void Triangular::_mesh(ftl::rgbd::Frame &out, ftl::rgbd::Source *src, cudaStream
void Triangular::_renderChannel(
ftl::rgbd::Frame &out,
- Channel channel_in, Channel channel_out, cudaStream_t stream)
+ Channel channel_in, Channel channel_out, const Eigen::Matrix4d &t, cudaStream_t stream)
{
if (channel_out == Channel::None || channel_in == Channel::None) return;
cv::cuda::Stream cvstream = cv::cuda::StreamAccessor::wrapStream(stream);
@@ -413,7 +414,7 @@ void Triangular::_renderChannel(
accum_.get<GpuMat>(channel_out).setTo(cv::Scalar(0,0,0,0), cvstream);
}
- _reprojectChannel(out, channel_in, channel_out, stream);
+ _reprojectChannel(out, channel_in, channel_out, t, stream);
}
/*
@@ -463,32 +464,15 @@ static cv::Scalar HSVtoRGB(int H, double S, double V) {
return cv::Scalar((Bs + m) * 255, (Gs + m) * 255, (Rs + m) * 255, 0);
}
-bool Triangular::render(ftl::rgbd::VirtualSource *src, ftl::rgbd::Frame &out) {
+bool Triangular::render(ftl::rgbd::VirtualSource *src, ftl::rgbd::Frame &out, const Eigen::Matrix4d &t) {
SHARED_LOCK(scene_->mtx, lk);
if (!src->isReady()) return false;
//scene_->upload(Channel::Colour + Channel::Depth, stream_);
const auto &camera = src->parameters();
- //cudaSafeCall(cudaSetDevice(scene_->getCUDADevice()));
-
- // Create all the required channels
-
- out.create<GpuMat>(Channel::Depth, Format<float>(camera.width, camera.height));
- out.create<GpuMat>(Channel::Colour, Format<uchar4>(camera.width, camera.height));
- out.createTexture<uchar4>(Channel::Colour, true); // Force interpolated colour
-
-
- if (scene_->frames.size() == 0) return false;
- auto &g = scene_->frames[0].get<GpuMat>(Channel::Colour);
-
- temp_.create<GpuMat>(Channel::Colour, Format<float4>(camera.width, camera.height));
- temp_.create<GpuMat>(Channel::Contribution, Format<float>(camera.width, camera.height));
- temp_.create<GpuMat>(Channel::Depth, Format<int>(camera.width, camera.height));
- temp_.create<GpuMat>(Channel::Depth2, Format<int>(camera.width, camera.height));
- temp_.create<GpuMat>(Channel::Normals, Format<float4>(camera.width, camera.height)); //g.cols, g.rows));
-
cv::cuda::Stream cvstream = cv::cuda::StreamAccessor::wrapStream(stream_);
+ //cudaSafeCall(cudaSetDevice(scene_->getCUDADevice()));
// Parameters object to pass to CUDA describing the camera
SplatParams ¶ms = params_;
@@ -500,20 +484,36 @@ bool Triangular::render(ftl::rgbd::VirtualSource *src, ftl::rgbd::Frame &out) {
params.m_viewMatrix = MatrixConversion::toCUDA(src->getPose().cast<float>().inverse());
params.m_viewMatrixInverse = MatrixConversion::toCUDA(src->getPose().cast<float>());
params.camera = camera;
- // Clear all channels to 0 or max depth
- out.get<GpuMat>(Channel::Depth).setTo(cv::Scalar(1000.0f), cvstream);
+ // Create all the required channels
+
+ if (!out.hasChannel(Channel::Depth)) {
+ out.create<GpuMat>(Channel::Depth, Format<float>(camera.width, camera.height));
+ out.create<GpuMat>(Channel::Colour, Format<uchar4>(camera.width, camera.height));
+ out.createTexture<uchar4>(Channel::Colour, true); // Force interpolated colour
- if (env_image_.empty() || !value("environment_enabled", false)) {
- out.get<GpuMat>(Channel::Colour).setTo(background_, cvstream);
- } else {
- auto pose = params.m_viewMatrixInverse.getFloat3x3();
- ftl::cuda::equirectangular_reproject(
- env_tex_,
- out.createTexture<uchar4>(Channel::Colour, true),
- camera, pose, stream_);
+ out.get<GpuMat>(Channel::Depth).setTo(cv::Scalar(1000.0f), cvstream);
+
+ if (env_image_.empty() || !value("environment_enabled", false)) {
+ out.get<GpuMat>(Channel::Colour).setTo(background_, cvstream);
+ } else {
+ auto pose = params.m_viewMatrixInverse.getFloat3x3();
+ ftl::cuda::equirectangular_reproject(
+ env_tex_,
+ out.createTexture<uchar4>(Channel::Colour, true),
+ camera, pose, stream_);
+ }
}
+ if (scene_->frames.size() == 0) return false;
+ auto &g = scene_->frames[0].get<GpuMat>(Channel::Colour);
+
+ temp_.create<GpuMat>(Channel::Colour, Format<float4>(camera.width, camera.height));
+ temp_.create<GpuMat>(Channel::Contribution, Format<float>(camera.width, camera.height));
+ temp_.create<GpuMat>(Channel::Depth, Format<int>(camera.width, camera.height));
+ temp_.create<GpuMat>(Channel::Depth2, Format<int>(camera.width, camera.height));
+ temp_.create<GpuMat>(Channel::Normals, Format<float4>(camera.width, camera.height)); //g.cols, g.rows));
+
//LOG(INFO) << "Render ready: " << camera.width << "," << camera.height;
bool show_discon = value("show_discontinuity_mask", false);
@@ -601,13 +601,25 @@ bool Triangular::render(ftl::rgbd::VirtualSource *src, ftl::rgbd::Frame &out) {
// Create and render triangles for depth
if (mesh_) {
- _mesh(out, src, stream_);
+ _mesh(out, src, t, stream_);
} else {
- _dibr(out, stream_);
+ _dibr(out, t, stream_);
}
// Reprojection of colours onto surface
- _renderChannel(out, Channel::Colour, Channel::Colour, stream_);
+ _renderChannel(out, Channel::Colour, Channel::Colour, t, stream_);
+
+ if (value("cool_effect", false)) {
+ auto pose = params.m_viewMatrixInverse.getFloat3x3();
+ auto col = parseCUDAColour(value("cool_effect_colour", std::string("#2222ff")));
+
+ ftl::cuda::cool_blue(
+ out.getTexture<float4>(Channel::Normals),
+ out.getTexture<uchar4>(Channel::Colour),
+ col, pose,
+ stream_
+ );
+ }
if (value("show_bad_colour", false)) {
ftl::cuda::show_missing_colour(
@@ -645,7 +657,7 @@ bool Triangular::render(ftl::rgbd::VirtualSource *src, ftl::rgbd::Frame &out) {
else if (chan == Channel::Density) {
out.create<GpuMat>(chan, Format<float>(camera.width, camera.height));
out.get<GpuMat>(chan).setTo(cv::Scalar(0.0f), cvstream);
- _renderChannel(out, Channel::Depth, Channel::Density, stream_);
+ _renderChannel(out, Channel::Depth, Channel::Density, t, stream_);
}
else if (chan == Channel::Right)
{
@@ -670,11 +682,11 @@ bool Triangular::render(ftl::rgbd::VirtualSource *src, ftl::rgbd::Frame &out) {
// Need to re-dibr due to pose change
if (mesh_) {
- _mesh(out, src, stream_);
+ _mesh(out, src, t, stream_);
} else {
- _dibr(out, stream_);
+ _dibr(out, t, stream_);
}
- _renderChannel(out, Channel::Left, Channel::Right, stream_);
+ _renderChannel(out, Channel::Left, Channel::Right, t, stream_);
} else if (chan != Channel::None) {
if (ftl::codecs::isFloatChannel(chan)) {
@@ -684,7 +696,7 @@ bool Triangular::render(ftl::rgbd::VirtualSource *src, ftl::rgbd::Frame &out) {
out.create<GpuMat>(chan, Format<uchar4>(camera.width, camera.height));
out.get<GpuMat>(chan).setTo(background_, cvstream);
}
- _renderChannel(out, chan, chan, stream_);
+ _renderChannel(out, chan, chan, t, stream_);
}
cudaSafeCall(cudaStreamSynchronize(stream_));
diff --git a/components/renderers/cpp/src/triangle_render.cu b/components/renderers/cpp/src/triangle_render.cu
index 7311e50b9bbbd7b7ba78f106e360998b937646d8..2e1966c4252fcf751639cc70a94dbfbdccd43b3b 100644
--- a/components/renderers/cpp/src/triangle_render.cu
+++ b/components/renderers/cpp/src/triangle_render.cu
@@ -165,6 +165,30 @@ void ftl::cuda::triangle_render1(TextureObject<float> &depth_in, TextureObject<i
cudaSafeCall( cudaGetLastError() );
}
+// ==== Merge convert ===========
+
+__global__ void merge_convert_kernel(
+ TextureObject<int> depth_in,
+ TextureObject<float> depth_out,
+ float alpha) {
+ const int x = blockIdx.x*blockDim.x + threadIdx.x;
+ const int y = blockIdx.y*blockDim.y + threadIdx.y;
+
+ if (x < 0 || x >= depth_in.width() || y < 0 || y >= depth_in.height()) return;
+
+ float a = float(depth_in.tex2D(x,y))*alpha;
+ float b = depth_out.tex2D(x,y);
+ depth_out(x,y) = min(a,b);
+}
+
+void ftl::cuda::merge_convert_depth(TextureObject<int> &depth_in, TextureObject<float> &depth_out, float alpha, cudaStream_t stream) {
+ const dim3 gridSize((depth_in.width() + T_PER_BLOCK - 1)/T_PER_BLOCK, (depth_in.height() + T_PER_BLOCK - 1)/T_PER_BLOCK);
+ const dim3 blockSize(T_PER_BLOCK, T_PER_BLOCK);
+
+ merge_convert_kernel<<<gridSize, blockSize, 0, stream>>>(depth_in, depth_out, alpha);
+ cudaSafeCall( cudaGetLastError() );
+}
+
// ==== BLENDER ========
/*
diff --git a/components/rgbd-sources/include/ftl/rgbd/source.hpp b/components/rgbd-sources/include/ftl/rgbd/source.hpp
index 7484788796dba398c525a208ad44708deadf3b70..8958173597a89c00748712a3e1fae69fbc630af0 100644
--- a/components/rgbd-sources/include/ftl/rgbd/source.hpp
+++ b/components/rgbd-sources/include/ftl/rgbd/source.hpp
@@ -31,6 +31,8 @@ class SnapshotReader;
class VirtualSource;
class Player;
+typedef std::function<void(ftl::rgbd::Source*, const ftl::codecs::StreamPacket &spkt, const ftl::codecs::Packet &pkt)> RawCallback;
+
/**
* RGBD Generic data source configurable entity. This class hides the
* internal implementation of an RGBD source by providing accessor functions
@@ -189,12 +191,12 @@ class Source : public ftl::Configurable {
* Currently this only works for a net source since other sources don't
* produce raw encoded data.
*/
- void addRawCallback(const std::function<void(ftl::rgbd::Source*, const ftl::codecs::StreamPacket &spkt, const ftl::codecs::Packet &pkt)> &);
+ void addRawCallback(const RawCallback &);
/**
* THIS DOES NOT WORK CURRENTLY.
*/
- void removeRawCallback(const std::function<void(ftl::rgbd::Source*, const ftl::codecs::StreamPacket &spkt, const ftl::codecs::Packet &pkt)> &);
+ void removeRawCallback(const RawCallback &);
/**
* INTERNAL. Used to send raw data to callbacks.