diff --git a/Doxyfile b/Doxyfile
index 47cb13e46efd68d84723dc80e9d378cdde662d4c..42eb33cc5ec62e06256976f4fccad1db9c757fae 100644
--- a/Doxyfile
+++ b/Doxyfile
@@ -875,7 +875,8 @@ RECURSIVE = YES
EXCLUDE = "components/common/cpp/include/nlohmann/json.hpp" \
"components/common/cpp/include/loguru.hpp" \
- "components/common/cpp/include/ctpl_stl.h"
+ "components/common/cpp/include/ctpl_stl.h" \
+ "build/"
# The EXCLUDE_SYMLINKS tag can be used to select whether or not files or
# directories that are symbolic links (a Unix file system feature) are excluded
diff --git a/applications/gui/src/config_window.cpp b/applications/gui/src/config_window.cpp
index 322f817ae264386ab3a54bfddc72c7d5f588fc36..b3c5ef874961d773f12981c0eee93fd8fd67fa3c 100644
--- a/applications/gui/src/config_window.cpp
+++ b/applications/gui/src/config_window.cpp
@@ -6,6 +6,7 @@
#include <nanogui/entypo.h>
#include <nanogui/formhelper.h>
#include <nanogui/vscrollpanel.h>
+#include <nanogui/opengl.h>
#include <vector>
#include <string>
@@ -15,6 +16,67 @@ using std::string;
using std::vector;
using ftl::config::json_t;
+class SearchBox : public nanogui::TextBox {
+private:
+ std::vector<std::string> configurables_;
+ Widget *buttons_;
+ std::string previous;
+
+ void _setVisible(const std::string &str) {
+ // Check whether the search string has changed to prevent
+ // unnecessary searching.
+ if (str != previous) {
+ for (int i = configurables_.size()-1; i >= 0; --i) {
+ if (configurables_[i].find(mValueTemp) != std::string::npos) {
+ buttons_->childAt(i)->setVisible(true);
+ } else {
+ buttons_->childAt(i)->setVisible(false);
+ }
+ }
+ previous = str;
+ }
+ }
+
+public:
+ SearchBox(Widget *parent, std::vector<std::string> &configurables) : nanogui::TextBox(parent, ""), configurables_(configurables) {
+ setAlignment(TextBox::Alignment::Left);
+ setEditable(true);
+ setPlaceholder("Search");
+ }
+
+ ~SearchBox() {
+ }
+
+ bool keyboardEvent(int key, int scancode, int action, int modifier) {
+ TextBox::keyboardEvent(key, scancode, action, modifier);
+ _setVisible(mValueTemp);
+ return true;
+ }
+
+ void setButtons(Widget *buttons) {
+ buttons_ = buttons;
+ }
+};
+
+static std::string titleForURI(const ftl::URI &uri) {
+ auto *cfg = ftl::config::find(uri.getBaseURI());
+ if (cfg && cfg->get<std::string>("title")) {
+ return *cfg->get<std::string>("title");
+ } else if (uri.getPath().size() > 0) {
+ return uri.getPathSegment(-1);
+ } else {
+ return uri.getHost();
+ }
+}
+
+static std::string genPadding(const std::string &str, size_t count) {
+ std::string res = "";
+ for (size_t i=0; i<count; ++i) {
+ res += str;
+ }
+ return res;
+}
+
ConfigWindow::ConfigWindow(nanogui::Widget *parent, ftl::ctrl::Master *ctrl)
: nanogui::Window(parent, "Settings"), ctrl_(ctrl) {
using namespace nanogui;
@@ -23,17 +85,52 @@ ConfigWindow::ConfigWindow(nanogui::Widget *parent, ftl::ctrl::Master *ctrl)
setPosition(Vector2i(parent->width()/2.0f - 100.0f, parent->height()/2.0f - 100.0f));
//setModal(true);
- configurables_ = ftl::config::list();
+ auto configurables = ftl::config::list();
+ const auto size = configurables.size();
new Label(this, "Select Configurable","sans-bold");
+ auto searchBox = new SearchBox(this, configurables);
+
auto vscroll = new VScrollPanel(this);
vscroll->setFixedHeight(300);
- Widget *buttons = new Widget(vscroll);
+ auto buttons = new Widget(vscroll);
buttons->setLayout(new BoxLayout(Orientation::Vertical, Alignment::Fill));
- for (auto c : configurables_) {
- auto itembutton = new Button(buttons, c);
+ searchBox->setButtons(buttons);
+
+ std::vector<std::string> configurable_titles(size);
+ for (int i = 0; i < size; ++i) {
+ ftl::URI uri(configurables[i]);
+ std::string label = uri.getFragment();
+
+ size_t pos = label.find_last_of("/");
+ if (pos != std::string::npos) label = label.substr(pos+1);
+
+ std::string parentName = configurables[i];
+ size_t pos2 = parentName.find_last_of("/");
+ if (pos2 != std::string::npos) parentName = parentName.substr(0,pos2);
+
+ // FIXME: Does not indicated parent indentation ... needs sorting?
+
+ if (i > 0 && parentName == configurables[i-1]) {
+ ftl::URI uri(configurables[i-1]);
+ configurable_titles[i-1] = std::string("[") + titleForURI(uri) + std::string("] ") + uri.getFragment();
+
+ auto *prev = dynamic_cast<Button*>(buttons->childAt(buttons->childCount()-1));
+ prev->setCaption(configurable_titles[i-1]);
+ prev->setBackgroundColor(nanogui::Color(0.3f,0.3f,0.3f,1.0f));
+ prev->setTextColor(nanogui::Color(1.0f,1.0f,1.0f,1.0f));
+ prev->setIconPosition(Button::IconPosition::Left);
+ prev->setIcon(ENTYPO_ICON_FOLDER);
+ }
+
+ configurable_titles[i] = label;
+
+ auto itembutton = new nanogui::Button(buttons, configurable_titles[i]);
+ std::string c = configurables[i];
+ itembutton->setTooltip(c);
+ itembutton->setBackgroundColor(nanogui::Color(0.9f,0.9f,0.9f,0.9f));
itembutton->setCallback([this,c]() {
LOG(INFO) << "Change configurable: " << c;
_buildForm(c);
@@ -136,3 +233,4 @@ void ConfigWindow::_buildForm(const std::string &suri) {
bool ConfigWindow::exists(const std::string &uri) {
return ftl::config::find(uri) != nullptr;
}
+
diff --git a/applications/gui/src/config_window.hpp b/applications/gui/src/config_window.hpp
index a0fe74155fe5320f983871d8c9237c7571625670..a7acd117116553f3e902bdc6640d4f67e71b1f30 100644
--- a/applications/gui/src/config_window.hpp
+++ b/applications/gui/src/config_window.hpp
@@ -2,10 +2,10 @@
#define _FTL_GUI_CFGWINDOW_HPP_
#include <nanogui/window.h>
+#include <nanogui/formhelper.h>
+
#include <ftl/master.hpp>
#include <ftl/uuid.hpp>
-
-#include <nanogui/formhelper.h>
#include <ftl/net_configurable.hpp>
namespace ftl {
@@ -21,7 +21,6 @@ class ConfigWindow : public nanogui::Window {
private:
ftl::ctrl::Master *ctrl_;
- std::vector<std::string> configurables_;
void _buildForm(const std::string &uri);
void _addElements(nanogui::FormHelper *form, const std::string &suri);
diff --git a/applications/player/src/main.cpp b/applications/player/src/main.cpp
index 2741cac2ef832f3cc60073f7320d443ea9620c9d..6fd7c54ce648a683d46d2f08171e2ad5de040ee7 100644
--- a/applications/player/src/main.cpp
+++ b/applications/player/src/main.cpp
@@ -11,22 +11,35 @@
#include <Eigen/Eigen>
+const static std::string help[] = {
+ "Esc", "close",
+ "0-9", "change source",
+ "D", "toggle depth",
+ "S", "save screenshot",
+};
+
+std::string time_now_string() {
+ char timestamp[18];
+ std::time_t t=std::time(NULL);
+ std::strftime(timestamp, sizeof(timestamp), "%F-%H%M%S", std::localtime(&t));
+ return std::string(timestamp);
+}
+
using ftl::codecs::codec_t;
using ftl::codecs::Channel;
-static ftl::codecs::Decoder *decoder;
-
+static std::map<Channel, ftl::codecs::Decoder*> decoders;
-static void createDecoder(const ftl::codecs::Packet &pkt) {
- if (decoder) {
- if (!decoder->accepts(pkt)) {
- ftl::codecs::free(decoder);
+static void createDecoder(const Channel channel, const ftl::codecs::Packet &pkt) {
+ if (decoders[channel]) {
+ if (!decoders[channel]->accepts(pkt)) {
+ ftl::codecs::free(decoders[channel]);
} else {
return;
}
}
- decoder = ftl::codecs::allocateDecoder(pkt);
+ decoders[channel] = ftl::codecs::allocateDecoder(pkt);
}
static void visualizeDepthMap( const cv::Mat &depth, cv::Mat &out,
@@ -56,28 +69,34 @@ static std::string nameForCodec(ftl::codecs::codec_t c) {
}
int main(int argc, char **argv) {
- std::string filename(argv[1]);
- LOG(INFO) << "Playing: " << filename;
+ std::string filename(argv[1]);
+ LOG(INFO) << "Playing: " << filename;
auto root = ftl::configure(argc, argv, "player_default");
std::ifstream f;
- f.open(filename);
- if (!f.is_open()) LOG(ERROR) << "Could not open file";
+ f.open(filename);
+ if (!f.is_open()) LOG(ERROR) << "Could not open file";
- ftl::codecs::Reader r(f);
- if (!r.begin()) LOG(ERROR) << "Bad ftl file";
+ ftl::codecs::Reader r(f);
+ if (!r.begin()) LOG(ERROR) << "Bad ftl file";
- LOG(INFO) << "Playing...";
+ LOG(INFO) << "Playing...";
- int current_stream = 0;
- int current_channel = 0;
+ int current_stream = 0;
+ int current_channel = 0;
int stream_mask = 0;
+
+ static volatile bool screenshot;
+ static int64_t screenshot_ts;
+ static cv::Mat screenshot_color;
+ static cv::Mat screenshot_depth;
+
std::vector<std::bitset<128>> channel_mask;
ftl::timer::add(ftl::timer::kTimerMain, [¤t_stream,¤t_channel,&r,&stream_mask,&channel_mask](int64_t ts) {
- bool res = r.read(ts, [¤t_stream,¤t_channel,&r,&stream_mask,&channel_mask](const ftl::codecs::StreamPacket &spkt, const ftl::codecs::Packet &pkt) {
+ bool res = r.read(ts, [ts, ¤t_stream,¤t_channel,&r,&stream_mask,&channel_mask](const ftl::codecs::StreamPacket &spkt, const ftl::codecs::Packet &pkt) {
if (!(stream_mask & (1 << spkt.streamID))) {
stream_mask |= 1 << spkt.streamID;
LOG(INFO) << " - Stream found (" << (int)spkt.streamID << ")";
@@ -95,56 +114,95 @@ int main(int argc, char **argv) {
LOG(INFO) << " - Blocks = " << (int)pkt.block_total;
}
- if (spkt.streamID == current_stream) {
+ if (spkt.streamID != current_stream) { return; }
- if (pkt.codec == codec_t::POSE) {
- Eigen::Matrix4d p = Eigen::Map<Eigen::Matrix4d>((double*)pkt.data.data());
- LOG(INFO) << "Have pose: " << p;
- return;
- }
+ if (pkt.codec == codec_t::POSE) {
+ Eigen::Matrix4d p = Eigen::Map<Eigen::Matrix4d>((double*)pkt.data.data());
+ LOG(INFO) << "Have pose: " << p;
+ return;
+ }
- if (pkt.codec == codec_t::CALIBRATION) {
- ftl::rgbd::Camera *camera = (ftl::rgbd::Camera*)pkt.data.data();
- LOG(INFO) << "Have calibration: " << camera->fx;
- return;
- }
-
- if (spkt.channel != static_cast<ftl::codecs::Channel>(current_channel)) return;
+ if (pkt.codec == codec_t::CALIBRATION) {
+ ftl::rgbd::Camera *camera = (ftl::rgbd::Camera*)pkt.data.data();
+ LOG(INFO) << "Have calibration: " << camera->fx;
+ return;
+ }
+
+ auto channel = static_cast<ftl::codecs::Channel>(current_channel);
+ if (pkt.codec == ftl::codecs::codec_t::MSGPACK) { return; }
+ //LOG(INFO) << "Reading packet: (" << (int)spkt.streamID << "," << (int)spkt.channel << ") " << (int)pkt.codec << ", " << (int)pkt.definition;
+
+ cv::cuda::GpuMat gframe(cv::Size(ftl::codecs::getWidth(pkt.definition),ftl::codecs::getHeight(pkt.definition)), (spkt.channel == Channel::Depth) ? CV_32F : CV_8UC3);
+ cv::Mat frame;
+ createDecoder(spkt.channel, pkt);
+
+ try {
+ decoders[spkt.channel]->decode(pkt, gframe);
+ gframe.download(frame);
+ } catch (std::exception &e) {
+ LOG(INFO) << "Decoder exception: " << e.what();
+ }
- //LOG(INFO) << "Reading packet: (" << (int)spkt.streamID << "," << (int)spkt.channel << ") " << (int)pkt.codec << ", " << (int)pkt.definition;
+ if (screenshot) {
+ if (!screenshot_depth.empty() && !screenshot_color.empty()) {
+ std::string fname = time_now_string();
+ LOG(INFO) << "Screenshot saved: " << fname;
+ cv::imwrite(fname + "-depth.tiff", screenshot_depth);
+ cv::imwrite(fname + "-color.tiff", screenshot_color);
+ screenshot = false;
+ screenshot_color = cv::Mat();
+ screenshot_depth = cv::Mat();
+ screenshot_ts = 0;
+ }
+ else {
+ if (screenshot_ts != ts) {
+ screenshot_ts = ts;
+ screenshot_color = cv::Mat();
+ screenshot_depth = cv::Mat();
+ }
+ if (spkt.channel == Channel::Colour) {
+ frame.copyTo(screenshot_color);
+ }
+ else if (spkt.channel == Channel::Depth) {
+ frame.copyTo(screenshot_depth);
+ }
+ }
+ }
- cv::cuda::GpuMat gframe(cv::Size(ftl::codecs::getWidth(pkt.definition),ftl::codecs::getHeight(pkt.definition)), (spkt.channel == Channel::Depth) ? CV_32F : CV_8UC3);
- cv::Mat frame;
- createDecoder(pkt);
+ if (spkt.channel != channel) return;
- try {
- decoder->decode(pkt, gframe);
- gframe.download(frame);
- } catch (std::exception &e) {
- LOG(INFO) << "Decoder exception: " << e.what();
+ if (!frame.empty()) {
+ if (spkt.channel == Channel::Depth) {
+ visualizeDepthMap(frame, frame, 8.0f);
}
+ double time = (double)(spkt.timestamp - r.getStartTime()) / 1000.0;
+ cv::putText(frame, std::string("Time: ") + std::to_string(time) + std::string("s"), cv::Point(10,20), cv::FONT_HERSHEY_PLAIN, 1, cv::Scalar(0,0,255));
- if (!frame.empty()) {
- if (spkt.channel == Channel::Depth) {
- visualizeDepthMap(frame, frame, 8.0f);
- }
- double time = (double)(spkt.timestamp - r.getStartTime()) / 1000.0;
- cv::putText(frame, std::string("Time: ") + std::to_string(time) + std::string("s"), cv::Point(10,20), cv::FONT_HERSHEY_PLAIN, 1, cv::Scalar(0,0,255));
- cv::imshow("Player", frame);
- } else {
- frame.create(cv::Size(600,600), CV_8UC3);
- cv::imshow("Player", frame);
+ // hotkey help text
+ for (int i = 0; i < std::size(help); i += 2) {
+ cv::putText(frame, help[i], cv::Point(10, 40+(i/2)*14), cv::FONT_HERSHEY_PLAIN, 1, cv::Scalar(64,64,255));
+ cv::putText(frame, help[i+1], cv::Point(50, 40+(i/2)*14), cv::FONT_HERSHEY_PLAIN, 1, cv::Scalar(64,64,255));
}
- int key = cv::waitKey(1);
- if (key >= 48 && key <= 57) {
- current_stream = key - 48;
- } else if (key == 'd') {
- current_channel = (current_channel == 0) ? 1 : 0;
- } else if (key == 'r') {
- current_channel = (current_channel == 0) ? 2 : 0;
- } else if (key == 27) {
- ftl::timer::stop(false);
+
+ cv::imshow("Player", frame);
+ } else {
+ frame.create(cv::Size(600,600), CV_8UC3);
+ cv::imshow("Player", frame);
+ }
+ int key = cv::waitKey(1);
+ if (key >= 48 && key <= 57) {
+ int new_stream = key - 48;
+ if ((0 <= new_stream) && (new_stream < channel_mask.size())) {
+ current_stream = new_stream;
}
+ } else if (key == 'd') {
+ current_channel = (current_channel == 0) ? 1 : 0;
+ } else if (key == 'r') {
+ current_channel = (current_channel == 0) ? 2 : 0;
+ } else if (key == 27) {
+ ftl::timer::stop(false);
+ } else if (key == 115) {
+ screenshot = true;
}
});
if (!res) ftl::timer::stop(false);
@@ -153,7 +211,7 @@ int main(int argc, char **argv) {
ftl::timer::start(true);
- r.end();
+ r.end();
ftl::running = false;
return 0;
diff --git a/applications/reconstruct/src/main.cpp b/applications/reconstruct/src/main.cpp
index c3dc37e325bd873bec4e318bafa5ac72a993863b..5cce7a5a236699a58e74a1044dfef35a008bf72e 100644
--- a/applications/reconstruct/src/main.cpp
+++ b/applications/reconstruct/src/main.cpp
@@ -227,6 +227,13 @@ static void run(ftl::Configurable *root) {
});
stream->add(vs);
+ for (auto c : ftl::config::getChildren(root->getID())) {
+ LOG(INFO) << "Tagging configurable: " << c->getID();
+ auto tags = c->value<std::vector<std::string>>("tags", nlohmann::json::array({}));
+ tags.push_back("reconstruction");
+ c->set("tags", tags);
+ }
+
// ---- Recording code -----------------------------------------------------
std::ofstream fileout;
ftl::codecs::Writer writer(fileout);
diff --git a/components/common/cpp/include/ftl/configuration.hpp b/components/common/cpp/include/ftl/configuration.hpp
index 18aaf89f9433911dd71ec3d3519ff2125317e78e..ed7eee95d0e0692242ca5dafd22ef142890561f5 100644
--- a/components/common/cpp/include/ftl/configuration.hpp
+++ b/components/common/cpp/include/ftl/configuration.hpp
@@ -83,6 +83,13 @@ const std::vector<Configurable *> &findByTag(const std::string &tag);
std::vector<std::string> list();
+/**
+ * Recursively get all children of a configurable. The given configurable is
+ * also included in the vector, unless it is null,
+ * in which case an empty vector is returned.
+ */
+const std::vector<Configurable *> getChildren(const std::string &uri);
+
/**
* Adds a Configurable instance to the database of instances so that it can
* then be resolved using find().
diff --git a/components/common/cpp/src/configuration.cpp b/components/common/cpp/src/configuration.cpp
index 9fd9d3081f18781ce0474438a3e1829d3b082b33..c53f50c78decd870c86228de2634588bdb313374 100644
--- a/components/common/cpp/src/configuration.cpp
+++ b/components/common/cpp/src/configuration.cpp
@@ -212,6 +212,17 @@ std::vector<std::string> ftl::config::list() {
return r;
}
+const std::vector<Configurable *> ftl::config::getChildren(const string &uri) {
+ std::vector<Configurable *> children;
+ for (const auto &[curi, c] : config_instance) {
+ auto mismatch = std::mismatch(uri.begin(), uri.end(), curi.begin());
+ if (mismatch.first == uri.end()) {
+ children.push_back(c);
+ }
+ }
+ return children;
+}
+
void ftl::config::registerConfigurable(ftl::Configurable *cfg) {
auto uri = cfg->get<string>("$id");
if (!uri) {
diff --git a/components/operators/src/colours.cpp b/components/operators/src/colours.cpp
index 955bfed58b1ca26c55734eef648eaaec6286409b..267bcb8ca7b9dfa7f748fa3d2d3482bae5c5abe0 100644
--- a/components/operators/src/colours.cpp
+++ b/components/operators/src/colours.cpp
@@ -54,7 +54,7 @@ bool ColourChannels::apply(ftl::rgbd::Frame &in, ftl::rgbd::Frame &out, ftl::rgb
if (in.hasChannel(Channel::Right)) {
auto &right = in.get<cv::cuda::GpuMat>(Channel::Right);
if (depth.size() != right.size()) {
- cv::cuda::resize(right, rbuf_, depth.size(), 0.0, 0.0, cv::INTER_CUBIC, cvstream);
+ cv::cuda::resize(right, rbuf_, depth.size(), 0.0, 0.0, cv::INTER_LINEAR, cvstream);
cv::cuda::swap(right, rbuf_);
}
}
diff --git a/components/operators/src/depth.cpp b/components/operators/src/depth.cpp
index 3d332d6ae397c479e3864967cb47759678dbf7b9..7122513b1e6e90ea98533972391481dffc2f3e7c 100644
--- a/components/operators/src/depth.cpp
+++ b/components/operators/src/depth.cpp
@@ -23,8 +23,8 @@ void DepthChannel::_createPipeline() {
if (pipe_ != nullptr) return;
pipe_ = ftl::config::create<ftl::operators::Graph>(config(), "depth");
- depth_size_ = cv::Size( pipe_->value("width", 1280),
- pipe_->value("height", 720));
+ depth_size_ = cv::Size( config()->value("width", 1280),
+ config()->value("height", 720));
pipe_->append<ftl::operators::ColourChannels>("colour"); // Convert BGR to BGRA
pipe_->append<ftl::operators::FixstarsSGM>("algorithm");
@@ -52,7 +52,7 @@ bool DepthChannel::apply(ftl::rgbd::FrameSet &in, ftl::rgbd::FrameSet &out, cuda
cv::cuda::GpuMat& left = f.get<cv::cuda::GpuMat>(Channel::Left);
cv::cuda::GpuMat& right = f.get<cv::cuda::GpuMat>(Channel::Right);
cv::cuda::GpuMat& depth = f.create<cv::cuda::GpuMat>(Channel::Depth);
- depth.create(left.size(), CV_32FC1);
+ depth.create(depth_size_, CV_32FC1);
if (left.empty() || right.empty()) continue;
diff --git a/components/rgbd-sources/src/sources/stereovideo/local.cpp b/components/rgbd-sources/src/sources/stereovideo/local.cpp
index 03b80ff52f60c95d374cfee289c0b3880bb766bd..f4e11d6debca52b9049fba491c06a9cbd266fb0d 100644
--- a/components/rgbd-sources/src/sources/stereovideo/local.cpp
+++ b/components/rgbd-sources/src/sources/stereovideo/local.cpp
@@ -71,14 +71,19 @@ LocalSource::LocalSource(nlohmann::json &config)
stereo_ = true;
}
+ dwidth_ = value("depth_width", width_);
+ dheight_ = value("depth_height", height_);
+
// Allocate page locked host memory for fast GPU transfer
- left_hm_ = cv::cuda::HostMem(height_, width_, CV_8UC3);
- right_hm_ = cv::cuda::HostMem(height_, width_, CV_8UC3);
+ left_hm_ = cv::cuda::HostMem(dheight_, dwidth_, CV_8UC3);
+ right_hm_ = cv::cuda::HostMem(dheight_, dwidth_, CV_8UC3);
+ hres_hm_ = cv::cuda::HostMem(height_, width_, CV_8UC3);
}
LocalSource::LocalSource(nlohmann::json &config, const string &vid)
: Configurable(config), timestamp_(0.0) {
-
+ LOG(FATAL) << "Stereo video file sources no longer supported";
+ /*
//flip_ = value("flip", false);
//flip_v_ = value("flip_vert", false);
nostereo_ = value("nostereo", false);
@@ -119,11 +124,16 @@ LocalSource::LocalSource(nlohmann::json &config, const string &vid)
stereo_ = false;
}
+ dwidth_ = value("depth_width", width_);
+ dheight_ = value("depth_height", height_);
+
// Allocate page locked host memory for fast GPU transfer
- left_hm_ = cv::cuda::HostMem(height_, width_, CV_8UC3);
- right_hm_ = cv::cuda::HostMem(height_, width_, CV_8UC3);
+ left_hm_ = cv::cuda::HostMem(dheight_, dwidth_, CV_8UC3);
+ right_hm_ = cv::cuda::HostMem(dheight_, dwidth_, CV_8UC3);
+ hres_hm_ = cv::cuda::HostMem(height_, width_, CV_8UC3);
//tps_ = 1.0 / value("max_fps", 25.0);
+ */
}
/*bool LocalSource::left(cv::Mat &l) {
@@ -225,26 +235,32 @@ bool LocalSource::grab() {
return true;
}
-bool LocalSource::get(cv::cuda::GpuMat &l_out, cv::cuda::GpuMat &r_out, Calibrate *c, cv::cuda::Stream &stream) {
- Mat l, r;
+bool LocalSource::get(cv::cuda::GpuMat &l_out, cv::cuda::GpuMat &r_out, cv::cuda::GpuMat &hres_out, Calibrate *c, cv::cuda::Stream &stream) {
+ Mat l, r ,hres;
// Use page locked memory
l = left_hm_.createMatHeader();
r = right_hm_.createMatHeader();
+ hres = hres_hm_.createMatHeader();
+
+ Mat &lfull = (!hasHigherRes()) ? l : hres;
+ Mat &rfull = (!hasHigherRes()) ? r : rtmp_;
if (!camera_a_) return false;
if (camera_b_ || !stereo_) {
- if (!camera_a_->retrieve(l)) {
+ // TODO: Use threads here?
+ if (!camera_a_->retrieve(lfull)) {
LOG(ERROR) << "Unable to read frame from camera A";
return false;
}
- if (camera_b_ && !camera_b_->retrieve(r)) {
+ if (camera_b_ && !camera_b_->retrieve(rfull)) {
LOG(ERROR) << "Unable to read frame from camera B";
return false;
}
} else {
- Mat frame;
+ LOG(FATAL) << "Stereo video no longer supported";
+ /*Mat frame;
if (!camera_a_->retrieve(frame)) {
LOG(ERROR) << "Unable to read frame from video";
return false;
@@ -257,7 +273,7 @@ bool LocalSource::get(cv::cuda::GpuMat &l_out, cv::cuda::GpuMat &r_out, Calibrat
//} else {
l = Mat(frame, Rect(0, 0, resx, frame.rows));
r = Mat(frame, Rect(resx, 0, frame.cols-resx, frame.rows));
- //}
+ //}*/
}
/*if (downsize_ != 1.0f) {
@@ -283,7 +299,18 @@ bool LocalSource::get(cv::cuda::GpuMat &l_out, cv::cuda::GpuMat &r_out, Calibrat
r = tr;
}*/
- c->rectifyStereo(l, r);
+ c->rectifyStereo(lfull, rfull);
+
+ // Need to resize
+ if (hasHigherRes()) {
+ // TODO: Use threads?
+ cv::resize(lfull, l, l.size(), 0.0, 0.0, cv::INTER_CUBIC);
+ cv::resize(rfull, r, r.size(), 0.0, 0.0, cv::INTER_CUBIC);
+ hres_out.upload(hres, stream);
+ //LOG(INFO) << "Early Resize: " << l.size() << " from " << lfull.size();
+ } else {
+ hres_out = cv::cuda::GpuMat();
+ }
l_out.upload(l, stream);
r_out.upload(r, stream);
diff --git a/components/rgbd-sources/src/sources/stereovideo/local.hpp b/components/rgbd-sources/src/sources/stereovideo/local.hpp
index 9f21f5cf79b86f7cdee9455052c55a829eaf0da7..bedd974c9a1a65f275b38322a6f8c88794aefd81 100644
--- a/components/rgbd-sources/src/sources/stereovideo/local.hpp
+++ b/components/rgbd-sources/src/sources/stereovideo/local.hpp
@@ -25,10 +25,15 @@ class LocalSource : public Configurable {
//bool left(cv::Mat &m);
//bool right(cv::Mat &m);
bool grab();
- bool get(cv::cuda::GpuMat &l, cv::cuda::GpuMat &r, Calibrate *c, cv::cuda::Stream &stream);
+ bool get(cv::cuda::GpuMat &l, cv::cuda::GpuMat &r, cv::cuda::GpuMat &h, Calibrate *c, cv::cuda::Stream &stream);
- unsigned int width() const { return width_; }
- unsigned int height() const { return height_; }
+ unsigned int width() const { return dwidth_; }
+ unsigned int height() const { return dheight_; }
+
+ unsigned int fullWidth() const { return width_; }
+ unsigned int fullHeight() const { return height_; }
+
+ inline bool hasHigherRes() const { return dwidth_ != width_; }
//void setFramerate(float fps);
//float getFramerate() const;
@@ -50,9 +55,13 @@ class LocalSource : public Configurable {
cv::VideoCapture *camera_b_;
unsigned int width_;
unsigned int height_;
+ unsigned int dwidth_;
+ unsigned int dheight_;
cv::cuda::HostMem left_hm_;
cv::cuda::HostMem right_hm_;
+ cv::cuda::HostMem hres_hm_;
+ cv::Mat rtmp_;
};
}
diff --git a/components/rgbd-sources/src/sources/stereovideo/stereovideo.cpp b/components/rgbd-sources/src/sources/stereovideo/stereovideo.cpp
index 78bc1dba34ba5c9b8d07cca5b7a11b75d20fd487..8858726c8672a4286eb5fb98a397858d81d3d069 100644
--- a/components/rgbd-sources/src/sources/stereovideo/stereovideo.cpp
+++ b/components/rgbd-sources/src/sources/stereovideo/stereovideo.cpp
@@ -76,7 +76,10 @@ void StereoVideoSource::init(const string &file) {
pipeline_input_->append<ftl::operators::NVOpticalFlow>("optflow");
#endif
- pipeline_depth_ = ftl::config::create<ftl::operators::Graph>(host_, "disparity");
+ //depth_size_ = cv::Size( host_->value("depth_width", 1280),
+ // host_->value("depth_height", 720));
+
+ /*pipeline_depth_ = ftl::config::create<ftl::operators::Graph>(host_, "disparity");
depth_size_ = cv::Size( pipeline_depth_->value("width", color_size_.width),
pipeline_depth_->value("height", color_size_.height));
@@ -90,13 +93,13 @@ void StereoVideoSource::init(const string &file) {
pipeline_depth_->append<ftl::operators::Normals>("normals"); // Estimate surface normals
pipeline_depth_->append<ftl::operators::CrossSupport>("cross");
pipeline_depth_->append<ftl::operators::DiscontinuityMask>("discontinuity_mask");
- pipeline_depth_->append<ftl::operators::AggreMLS>("mls"); // Perform MLS (using smoothing channel)
+ pipeline_depth_->append<ftl::operators::AggreMLS>("mls"); // Perform MLS (using smoothing channel)*/
- calib_ = ftl::create<Calibrate>(host_, "calibration", color_size_, stream_);
+ calib_ = ftl::create<Calibrate>(host_, "calibration", cv::Size(lsrc_->fullWidth(), lsrc_->fullHeight()), stream_);
if (!calib_->isCalibrated()) LOG(WARNING) << "Cameras are not calibrated!";
// Generate camera parameters from camera matrix
- cv::Mat K = calib_->getCameraMatrixLeft(depth_size_);
+ cv::Mat K = calib_->getCameraMatrixLeft(color_size_);
params_ = {
K.at<double>(0,0), // Fx
K.at<double>(1,1), // Fy
@@ -154,9 +157,9 @@ ftl::rgbd::Camera StereoVideoSource::parameters(Channel chan) {
cv::Mat K;
if (chan == Channel::Right) {
- K = calib_->getCameraMatrixRight(depth_size_);
+ K = calib_->getCameraMatrixRight(color_size_);
} else {
- K = calib_->getCameraMatrixLeft(depth_size_);
+ K = calib_->getCameraMatrixLeft(color_size_);
}
// TODO: remove hardcoded values (min/max)
@@ -165,8 +168,8 @@ ftl::rgbd::Camera StereoVideoSource::parameters(Channel chan) {
K.at<double>(1,1), // Fy
-K.at<double>(0,2), // Cx
-K.at<double>(1,2), // Cy
- (unsigned int) depth_size_.width,
- (unsigned int) depth_size_.height,
+ (unsigned int) color_size_.width,
+ (unsigned int) color_size_.height,
0.0f, // 0m min
15.0f, // 15m max
1.0 / calib_->getQ().at<double>(3,2), // Baseline
@@ -187,7 +190,12 @@ bool StereoVideoSource::retrieve() {
frame.reset();
auto &left = frame.create<cv::cuda::GpuMat>(Channel::Left);
auto &right = frame.create<cv::cuda::GpuMat>(Channel::Right);
- lsrc_->get(left, right, calib_, stream2_);
+ cv::cuda::GpuMat dummy;
+ auto &hres = (lsrc_->hasHigherRes()) ? frame.create<cv::cuda::GpuMat>(Channel::ColourHighRes) : dummy;
+
+ lsrc_->get(left, right, hres, calib_, stream2_);
+
+ //LOG(INFO) << "Channel size: " << hres.size();
pipeline_input_->apply(frame, frame, host_, cv::cuda::StreamAccessor::getStream(stream2_));
stream2_.waitForCompletion();
diff --git a/components/rgbd-sources/src/streamer.cpp b/components/rgbd-sources/src/streamer.cpp
index 9525a9fc918294d65cfbfec1922372dfd9978f21..75165ffd1d49dc89a268f9a1c3bb92575b644980 100644
--- a/components/rgbd-sources/src/streamer.cpp
+++ b/components/rgbd-sources/src/streamer.cpp
@@ -535,8 +535,8 @@ void Streamer::_process(ftl::rgbd::FrameSet &fs) {
}
- // TODO: Use ColourHighQuality if available
- if (fs.frames[j].getPackets(Channel::Colour).size() == 0) {
+ auto colChan = (fs.frames[j].hasChannel(Channel::ColourHighRes)) ? Channel::ColourHighRes : Channel::Colour;
+ if (fs.frames[j].getPackets(colChan).size() == 0) {
if (!src->hq_encoder_c1) src->hq_encoder_c1 = ftl::codecs::allocateEncoder(
definition_t::HD1080, hq_devices_, hq_codec_);
auto *enc = src->hq_encoder_c1;
@@ -544,14 +544,14 @@ void Streamer::_process(ftl::rgbd::FrameSet &fs) {
if (enc) {
// TODO: Stagger the reset between nodes... random phasing
if (insert_iframes_ && fs.timestamp % (10*ftl::timer::getInterval()) == 0) enc->reset();
- enc->encode(fs.frames[j].get<cv::cuda::GpuMat>(Channel::Colour), src->hq_bitrate, [this,src,hasChan2](const ftl::codecs::Packet &blk){
+ enc->encode(fs.frames[j].get<cv::cuda::GpuMat>(colChan), src->hq_bitrate, [this,src,hasChan2](const ftl::codecs::Packet &blk){
_transmitPacket(src, blk, Channel::Colour, hasChan2, Quality::High);
});
} else {
LOG(ERROR) << "Insufficient encoder resources";
}
} else {
- const auto &packets = fs.frames[j].getPackets(Channel::Colour);
+ const auto &packets = fs.frames[j].getPackets(colChan);
// FIXME: Adjust block number and total to match number of packets
// Also requires the receiver to decode in block number order.
LOG(INFO) << "Send existing encoding: " << packets.size();
diff --git a/python/ftl/ftlstream.py b/python/ftl/ftlstream.py
index 65514f0ba1109700b338cf30e91b2d046754cc9b..dfbc858c3649c187b12b206478b4e23ca3a23c2b 100644
--- a/python/ftl/ftlstream.py
+++ b/python/ftl/ftlstream.py
@@ -246,21 +246,43 @@ class FTLStreamReader:
warn("frame expected, no image received from decoder")
if ftl.is_float_channel(self._sp.channel):
- # TODO: only supports 8 bits per pixel format and 16 bits
- # ()"old format")
- #
- # NVPipe: (2 * width), high bits in left, low in right
+ if (p.flags & ftl.PacketFlags.MappedDepth):
+ # New format
- high = img[:,(img.shape[1]//2):,0].astype(np.uint32) << 8
- low = img[:,:(img.shape[1]//2),0].astype(np.uint32)
- img = (high|low).astype(np.float)/1000.0
+ # hardcoded constants maxdepth and P
+ maxdepth = 16
+ P = (2.0 * 256.0) / 16384.0
+ # use only 8 bits of 10
+ img = (img >> 2).astype(np.float) / 255
+
+ L = img[:,:,0]
+ Ha = img[:,:,1]
+ Hb = img[:,:,2]
+
+ m = np.floor(4.0 * (L/P) - 0.5).astype(np.int) % 4
+ L0 = L - ((L-(P / 8.0)) % P) + (P / 4.0) * m.astype(np.float) - (P/8.0)
+
+ s = np.zeros(img.shape[:2], dtype=np.float)
+ np.copyto(s, (P/2.0) * Ha, where=m == 0)
+ np.copyto(s, (P/2.0) * Hb, where=m == 1)
+ np.copyto(s, (P/2.0) * (1.0 - Ha), where=m == 2)
+ np.copyto(s, (P/2.0) * (1.0 - Hb), where=m == 3)
+
+ img = (L0+s) * maxdepth
+
+ else:
+ # NvPipe format
+ high = img[:,(img.shape[1]//2):,0].astype(np.uint32) << 8
+ low = img[:,:(img.shape[1]//2),0].astype(np.uint32)
+ img = (high|low).astype(np.float)/1000.0
+ '''
try:
img[img < self._calibration[sp.streamID].min_depth] = 0.0
img[img > self._calibration[sp.streamID].max_depth] = 0.0
except KeyError:
warn("no calibration for received frame")
-
+ '''
self._frame = img
else:
diff --git a/python/ftl/ftltypes.py b/python/ftl/ftltypes.py
index 69eb872489e92bb460ae3c57365b85993a79f387..6c22004140b4e71e6a0ffbf4bc900e09b8a0ad28 100644
--- a/python/ftl/ftltypes.py
+++ b/python/ftl/ftltypes.py
@@ -13,6 +13,10 @@ Packet = namedtuple("Packet", ["codec", "definition", "block_total",
StreamPacket = namedtuple("StreamPacket", ["timestamp", "streamID",
"channel_count", "channel"])
+class PacketFlags:
+ RGB = 0x00000001
+ MappedDepth = 0x00000002
+
# components/codecs/include/ftl/codecs/channels.hpp
class Channel(IntEnum):
None_ = -1
diff --git a/python/ftl/libde265.py b/python/ftl/libde265.py
index c57404eaa00477e8c487d608caecea82a8e57bfa..36a00439be431b2af3ee3b66024d4202d2d7bdca 100644
--- a/python/ftl/libde265.py
+++ b/python/ftl/libde265.py
@@ -1,24 +1,28 @@
-'''
-Python wrapper for libde265. Only decoding is (partly) implemented.
+'''!
+Python wrapper for libde265. Only decoding is implemented.
Requirements:
* libde265 library (libde265.so.0)
* numpy
- * opencv or skimage
+ * opencv (recommended) or skimage
'''
try:
import cv2 as cv
def _resize(img, size):
- return cv.resize(img, dsize=tuple(reversed(size)), interpolation=cv.INTER_CUBIC)
+ dst = np.zeros(size, dtype=img.dtype)
+ cv.resize(img, tuple(reversed(size)), dst, interpolation=cv.INTER_LINEAR)
+ return dst
except ImportError:
+ # seems to be much slower than OpenCV resize()
+
from skimage.transform import resize as resize_skimage
def _resize(img, size):
- # skimage resize() return dtype float64, convert back to uint8
+ # skimage resize() return dtype float64, convert back to original type
# order: 0 nn, 1 bilinear, 3 bicubic
- return (resize_skimage(img, size, order=3, mode="constant", cval=0) * 255).astype(np.uint8)
+ return (resize_skimage(img, size, order=2, mode="constant", cval=0) * np.iinfo(img.dtype).max).astype(img.dtype)
from warnings import warn
@@ -39,6 +43,10 @@ if _threads is None:
_threads = 1
+################################################################################
+# interface and definitions from libde256 api
+################################################################################
+
# error codes copied from header (de265.h)
class _libde265error(IntEnum):
@@ -155,6 +163,8 @@ libde265.de265_get_image_plane.restype = ctypes.POINTER(ctypes.c_char)
libde265.de265_get_number_of_input_bytes_pending.argtypes = [ctypes.c_void_p]
libde265.de265_get_number_of_input_bytes_pending.restype = ctypes.c_int
+################################################################################
+
class libde265Error(Exception):
def __init__(self, code):
super(libde265Error, self).__init__(
@@ -168,7 +178,7 @@ class Decoder:
self._more = ctypes.c_int()
self._out_stride = ctypes.c_int()
self._ctx = libde265.de265_new_decoder()
- self._supress_warnings = False
+ self._disable_warnings = False
err = libde265.de265_start_worker_threads(self._ctx, threads)
@@ -182,28 +192,29 @@ class Decoder:
size = (libde265.de265_get_image_height(de265_image, 0),
libde265.de265_get_image_width(de265_image, 0))
- res = np.zeros((*size, 3), dtype=np.uint8)
+ res = np.zeros((*size, 3), dtype=np.uint16)
# libde265: always 420 (???)
- # chroma_format = libde265.de265_get_chroma_format(de265_image)
+ chroma_format = libde265.de265_get_chroma_format(de265_image)
+ if chroma_format != de265_chroma.de265_chroma_420:
+ raise NotImplementedError("Unsupported chroma format %s" % str(chroma_format))
for c in range(0, 3):
size_channel = (libde265.de265_get_image_height(de265_image, c),
libde265.de265_get_image_width(de265_image, c))
if size_channel[0] > size[0] or size_channel[1] > size[1]:
- # Is this possible?
- print(size, size_channel)
raise Exception("Channel larger than first channel")
bpp = libde265.de265_get_bits_per_pixel(de265_image, c)
- if bpp != 8:
- raise NotImplementedError("%i-bit format not implemented (TODO)" % bpp)
+ if bpp == 8:
+ dtype = np.uint8
+ else:
+ dtype = np.uint16
img_ptr = libde265.de265_get_image_plane(de265_image, c, self._out_stride)
- # libde: how is 10 bits per pixel returned
- ch = np.frombuffer(img_ptr[:size_channel[0] * size_channel[1]], dtype=np.uint8)
+ ch = np.frombuffer(img_ptr[:size_channel[0] * self._out_stride.value], dtype=dtype)
ch.shape = size_channel
res[:,:,c] = _resize(ch, size)
@@ -211,7 +222,7 @@ class Decoder:
return res
def _warning(self):
- if self._supress_warnings:
+ if self._disable_warnings:
return
code = libde265.de265_get_warning(self._ctx)