diff --git a/applications/gui/src/camera.cpp b/applications/gui/src/camera.cpp
index a541d910d9031df9d305cff9ded337f776bcc1e1..f7aa5d74ed22140c8b6f5463cc43efa4b2241538 100644
--- a/applications/gui/src/camera.cpp
+++ b/applications/gui/src/camera.cpp
@@ -271,6 +271,7 @@ void ftl::gui::Camera::_draw(std::vector<ftl::rgbd::FrameSet*> &fss) {
if (!usesFrameset(fs->id)) continue;
// FIXME: Should perhaps remain locked until after end is called?
+ // Definitely: causes flashing if not.
UNIQUE_LOCK(fs->mtx,lk);
renderer_->submit(fs, Channels<0>(channel_), transforms_[fs->id]);
}
diff --git a/applications/gui/src/src_window.cpp b/applications/gui/src/src_window.cpp
index c8adf88195352bd3e1ab543a9855048eed0e8501..8b4090b7c47849d97ad4a290b3f501f0fe4f406a 100644
--- a/applications/gui/src/src_window.cpp
+++ b/applications/gui/src/src_window.cpp
@@ -39,10 +39,21 @@ using ftl::gui::Screen;
using ftl::gui::Scene;
using ftl::rgbd::Source;
using ftl::codecs::Channel;
+using ftl::codecs::Channels;
using std::string;
using std::vector;
using ftl::config::json_t;
+static ftl::rgbd::Generator *createSourceGenerator(const std::vector<ftl::rgbd::Source*> &srcs) {
+
+ auto *grp = new ftl::rgbd::Group();
+ for (auto s : srcs) {
+ s->setChannel(Channel::Depth);
+ grp->addSource(s);
+ }
+ return grp;
+}
+
SourceWindow::SourceWindow(ftl::gui::Screen *screen)
: nanogui::Window(screen, ""), screen_(screen) {
setLayout(new nanogui::BoxLayout(nanogui::Orientation::Vertical, nanogui::Alignment::Fill, 20, 5));
@@ -80,50 +91,7 @@ SourceWindow::SourceWindow(ftl::gui::Screen *screen)
paused_ = false;
cycle_ = 0;
receiver_->onFrameSet([this](ftl::rgbd::FrameSet &fs) {
- // Request the channels required by current camera configuration
- interceptor_->select(fs.id, _aggregateChannels(fs.id));
-
- /*if (fs.id > 0) {
- LOG(INFO) << "Got frameset: " << fs.id;
- return true;
- }*/
-
- // Make sure there are enough framesets allocated
- _checkFrameSets(fs.id);
-
- if (!paused_) {
- // Enforce interpolated colour
- for (int i=0; i<fs.frames.size(); ++i) {
- fs.frames[i].createTexture<uchar4>(Channel::Colour, true);
- }
-
- pre_pipelines_[fs.id]->apply(fs, fs, 0);
-
- fs.swapTo(*framesets_[fs.id]);
- }
-
- /*if (fs.frames[0].hasChannel(Channel::Data)) {
- int data = 0;
- fs.frames[0].get(Channel::Data, data);
- LOG(INFO) << "GOT DATA : " << data;
- }*/
-
- const auto *cstream = interceptor_;
- _createDefaultCameras(*framesets_[fs.id], true); // cstream->available(fs.id).has(Channel::Depth)
-
- //LOG(INFO) << "Channels = " << (unsigned int)cstream->available(fs.id);
-
- int i=0;
- for (auto cam : cameras_) {
- // Only update the camera periodically unless the active camera
- if (screen_->activeCamera() == cam.second.camera ||
- (screen_->activeCamera() == nullptr && cycle_ % cameras_.size() == i++)) cam.second.camera->update(framesets_);
-
- cam.second.camera->update(cstream->available(fs.id));
- }
- ++cycle_;
-
- return true;
+ return _processFrameset(fs, true);
});
speaker_ = ftl::create<ftl::audio::Speaker>(screen_->root(), "speaker_test");
@@ -145,6 +113,7 @@ SourceWindow::SourceWindow(ftl::gui::Screen *screen)
return true;
});
+ // Add network sources
_updateCameras(screen_->control()->getNet()->findAll<string>("list_streams"));
// Also check for a file on command line.
@@ -163,12 +132,78 @@ SourceWindow::SourceWindow(ftl::gui::Screen *screen)
auto *fstream = ftl::create<ftl::stream::File>(screen->root(), std::string("ftlfile-")+std::to_string(ftl_count+1));
fstream->set("filename", path);
stream_->add(fstream, ftl_count++);
+ } else if (path.rfind("device:", 0) == 0) {
+ ftl::URI uri(path);
+ uri.to_json(screen->root()->getConfig()["sources"].emplace_back());
}
}
+ // Finally, check for any device sources configured
+ std::vector<Source*> devices;
+ // Create a vector of all input RGB-Depth sources
+ if (screen->root()->getConfig()["sources"].size() > 0) {
+ devices = ftl::createArray<Source>(screen->root(), "sources", screen->control()->getNet());
+ auto *gen = createSourceGenerator(devices);
+ gen->onFrameSet([this, ftl_count](ftl::rgbd::FrameSet &fs) {
+ fs.id = ftl_count; // Set a frameset id to something unique.
+ return _processFrameset(fs, false);
+ });
+ }
+
stream_->begin();
}
+bool SourceWindow::_processFrameset(ftl::rgbd::FrameSet &fs, bool fromstream) {
+ // Request the channels required by current camera configuration
+ if (fromstream) interceptor_->select(fs.id, _aggregateChannels(fs.id));
+
+ /*if (fs.id > 0) {
+ LOG(INFO) << "Got frameset: " << fs.id;
+ return true;
+ }*/
+
+ // Make sure there are enough framesets allocated
+ _checkFrameSets(fs.id);
+
+ if (!paused_) {
+ // Enforce interpolated colour and GPU upload
+ for (int i=0; i<fs.frames.size(); ++i) {
+ fs.frames[i].createTexture<uchar4>(Channel::Colour, true);
+
+ // TODO: Do all channels. This is a fix for screen capture sources.
+ if (!fs.frames[i].isGPU(Channel::Colour)) fs.frames[i].upload(Channels<0>(Channel::Colour), pre_pipelines_[fs.id]->getStream());
+ }
+
+ pre_pipelines_[fs.id]->apply(fs, fs, 0);
+
+ fs.swapTo(*framesets_[fs.id]);
+ }
+
+ /*if (fs.frames[0].hasChannel(Channel::Data)) {
+ int data = 0;
+ fs.frames[0].get(Channel::Data, data);
+ LOG(INFO) << "GOT DATA : " << data;
+ }*/
+
+ const auto *cstream = interceptor_;
+ _createDefaultCameras(*framesets_[fs.id], true); // cstream->available(fs.id).has(Channel::Depth)
+
+ //LOG(INFO) << "Channels = " << (unsigned int)cstream->available(fs.id);
+
+ int i=0;
+ for (auto cam : cameras_) {
+ // Only update the camera periodically unless the active camera
+ if (screen_->activeCamera() == cam.second.camera ||
+ (screen_->activeCamera() == nullptr && cycle_ % cameras_.size() == i++)) cam.second.camera->update(framesets_);
+
+ if (fromstream) cam.second.camera->update(cstream->available(fs.id));
+ else if (fs.frames.size() > 0) cam.second.camera->update(fs.frames[0].getChannels());
+ }
+ ++cycle_;
+
+ return true;
+}
+
void SourceWindow::_checkFrameSets(int id) {
while (framesets_.size() <= id) {
auto *p = ftl::config::create<ftl::operators::Graph>(screen_->root(), "pre_filters");
diff --git a/applications/gui/src/src_window.hpp b/applications/gui/src/src_window.hpp
index 6eb0236f7e6dec7826381282ef40225a2f0f88f1..da001950efa65ea19883a5bed50fd9d10ecc31df 100644
--- a/applications/gui/src/src_window.hpp
+++ b/applications/gui/src/src_window.hpp
@@ -81,6 +81,7 @@ class SourceWindow : public nanogui::Window {
void _createDefaultCameras(ftl::rgbd::FrameSet &fs, bool makevirtual);
ftl::codecs::Channels<0> _aggregateChannels(int id);
void _checkFrameSets(int id);
+ bool _processFrameset(ftl::rgbd::FrameSet &fs, bool);
};
diff --git a/components/common/cpp/include/ftl/uri.hpp b/components/common/cpp/include/ftl/uri.hpp
index 05325b346088596aac9b723f0b15a44c1f08c68c..24123f168102de184130bb5f1391349b393d877b 100644
--- a/components/common/cpp/include/ftl/uri.hpp
+++ b/components/common/cpp/include/ftl/uri.hpp
@@ -1,6 +1,7 @@
#ifndef _FTL_URI_HPP_
#define _FTL_URI_HPP_
+#include <nlohmann/json_fwd.hpp>
#include <uriparser/Uri.h>
#include <string>
#include <vector>
@@ -62,6 +63,8 @@ namespace ftl {
std::string to_string() const;
+ void to_json(nlohmann::json &);
+
private:
void _parse(uri_t puri);
diff --git a/components/common/cpp/src/uri.cpp b/components/common/cpp/src/uri.cpp
index 90fb33522c1c2d701fac47b0a6d43a5a6afee3a7..39ddb47e107a8a9a8dd499d9d4ab7e7e9ec1ae26 100644
--- a/components/common/cpp/src/uri.cpp
+++ b/components/common/cpp/src/uri.cpp
@@ -1,7 +1,8 @@
#include <ftl/uri.hpp>
+#include <nlohmann/json.hpp>
// #include <filesystem> TODO When available
#include <cstdlib>
-#include <loguru.hpp>
+//#include <loguru.hpp>
#ifndef WIN32
#include <unistd.h>
@@ -176,3 +177,26 @@ void URI::setAttribute(const string &key, const string &value) {
void URI::setAttribute(const string &key, int value) {
m_qmap[key] = std::to_string(value);
}
+
+void URI::to_json(nlohmann::json &json) {
+ std::string uri = getBaseURI();
+ if (m_frag.size() > 0) uri += std::string("#") + getFragment();
+
+ json["uri"] = uri;
+ for (auto i : m_qmap) {
+ auto *current = &json;
+
+ size_t pos = 0;
+ size_t lpos = 0;
+ while ((pos = i.first.find('/', lpos)) != std::string::npos) {
+ current = &((*current)[i.first.substr(lpos, pos-lpos)]);
+ lpos = pos+1;
+ }
+ auto p = nlohmann::json::parse(i.second, nullptr, false);
+ if (!p.is_discarded()) {
+ (*current)[i.first.substr(lpos)] = p;
+ } else {
+ (*current)[i.first.substr(lpos)] = i.second;
+ }
+ }
+}
diff --git a/components/common/cpp/test/uri_unit.cpp b/components/common/cpp/test/uri_unit.cpp
index be94d76c12c6eee26af890b8178def8fd5914b4b..59c5391f35f93050b42a6df8835de6b7ab22adfc 100644
--- a/components/common/cpp/test/uri_unit.cpp
+++ b/components/common/cpp/test/uri_unit.cpp
@@ -1,5 +1,6 @@
#include "catch.hpp"
#include <ftl/uri.hpp>
+#include <nlohmann/json.hpp>
using ftl::URI;
using std::string;
@@ -124,6 +125,36 @@ SCENARIO( "URI::to_string() from a valid URI" ) {
}
}
+SCENARIO( "URI::to_json() from a valid URI" ) {
+ GIVEN( "no query component" ) {
+ URI uri("http://localhost:1000/hello");
+
+ nlohmann::json object;
+ uri.to_json(object);
+
+ REQUIRE( object["uri"].get<std::string>() == "http://localhost:1000/hello" );
+ }
+
+ GIVEN( "one numeric query item" ) {
+ URI uri("http://localhost:1000/hello?a=45");
+
+ nlohmann::json object;
+ uri.to_json(object);
+
+ REQUIRE( object["a"].get<int>() == 45 );
+ }
+
+ GIVEN( "multiple query items" ) {
+ URI uri("http://localhost:1000/hello?a=45&b=world");
+
+ nlohmann::json object;
+ uri.to_json(object);
+
+ REQUIRE( object["a"].get<int>() == 45 );
+ REQUIRE( object["b"].get<std::string>() == "world" );
+ }
+}
+
SCENARIO( "URI::getAttribute() from query" ) {
GIVEN( "a string value" ) {
URI uri("http://localhost:1000/hello?x=world");
diff --git a/components/operators/include/ftl/operators/operator.hpp b/components/operators/include/ftl/operators/operator.hpp
index 79cd008fdba46447dd4a79e51ba6521868604aa9..c8522cb9889b8f2c7a1de3291174917431ab4985 100644
--- a/components/operators/include/ftl/operators/operator.hpp
+++ b/components/operators/include/ftl/operators/operator.hpp
@@ -113,6 +113,8 @@ class Graph : public ftl::Configurable {
bool apply(ftl::rgbd::FrameSet &in, ftl::rgbd::FrameSet &out, cudaStream_t stream=0);
bool apply(ftl::rgbd::FrameSet &in, ftl::rgbd::Frame &out, cudaStream_t stream=0);
+ cudaStream_t getStream() const { return stream_; }
+
private:
std::list<ftl::operators::detail::OperatorNode> operators_;
std::map<std::string, ftl::Configurable*> configs_;
diff --git a/components/rgbd-sources/src/sources/stereovideo/local.cpp b/components/rgbd-sources/src/sources/stereovideo/local.cpp
index b571f35f46c519c60df9745c082db8820c01ab5b..023105bdc7bc7cd8eb69c3c86229477b8aeab099 100644
--- a/components/rgbd-sources/src/sources/stereovideo/local.cpp
+++ b/components/rgbd-sources/src/sources/stereovideo/local.cpp
@@ -66,6 +66,8 @@ LocalSource::LocalSource(nlohmann::json &config)
camera_a_->set(cv::CAP_PROP_FRAME_WIDTH, value("width", 640));
camera_a_->set(cv::CAP_PROP_FRAME_HEIGHT, value("height", 480));
+ //TODO: CAP_PROP_FPS
+ // CAP_PROP_BUFFERSIZE
Mat frame;
camera_a_->grab();
@@ -224,17 +226,6 @@ bool LocalSource::grab() {
return false;
}
- // Record timestamp
- double timestamp = duration_cast<duration<double>>(
- high_resolution_clock::now().time_since_epoch()).count();
-
- // Limit max framerate
- //if (timestamp - timestamp_ < tps_) {
- // sleep_for(milliseconds((int)std::round((tps_ - (timestamp - timestamp_))*1000)));
- //}
-
- timestamp_ = timestamp;
-
return true;
}
diff --git a/components/rgbd-sources/src/sources/stereovideo/stereovideo.cpp b/components/rgbd-sources/src/sources/stereovideo/stereovideo.cpp
index 7d5fb662fe467a07912fd6a4f208af5d5aadc11d..ae78c70aebf2b71b4dd13925d49f0bad93654009 100644
--- a/components/rgbd-sources/src/sources/stereovideo/stereovideo.cpp
+++ b/components/rgbd-sources/src/sources/stereovideo/stereovideo.cpp
@@ -51,6 +51,7 @@ StereoVideoSource::~StereoVideoSource() {
void StereoVideoSource::init(const string &file) {
capabilities_ = kCapVideo | kCapStereo;
+ calibrated_ = false;
if (ftl::is_video(file)) {
// Load video file
@@ -95,6 +96,7 @@ void StereoVideoSource::init(const string &file) {
if (calib_->loadCalibration(fname)) {
calib_->calculateRectificationParameters();
calib_->setRectify(true);
+ calibrated_ = true;
}
}
else {
@@ -135,6 +137,8 @@ void StereoVideoSource::init(const string &file) {
}
}
+ calibrated_ = true; // Means we have intrinsics
+
return true;
});
@@ -199,25 +203,41 @@ void StereoVideoSource::updateParameters() {
// left
- K = calib_->getCameraMatrixLeft(color_size_);
- state_.getLeft() = {
- static_cast<float>(K.at<double>(0,0)), // Fx
- static_cast<float>(K.at<double>(1,1)), // Fy
- static_cast<float>(-K.at<double>(0,2)), // Cx
- static_cast<float>(-K.at<double>(1,2)), // Cy
- (unsigned int) color_size_.width,
- (unsigned int) color_size_.height,
- min_depth,
- max_depth,
- baseline,
- doff
- };
-
- host_->getConfig()["focal"] = params_.fx;
- host_->getConfig()["centre_x"] = params_.cx;
- host_->getConfig()["centre_y"] = params_.cy;
- host_->getConfig()["baseline"] = params_.baseline;
- host_->getConfig()["doffs"] = params_.doffs;
+ // FIXME: Check this change doesn't break anything (adding of calibrated_)
+ if (calibrated_) {
+ K = calib_->getCameraMatrixLeft(color_size_);
+ state_.getLeft() = {
+ static_cast<float>(K.at<double>(0,0)), // Fx
+ static_cast<float>(K.at<double>(1,1)), // Fy
+ static_cast<float>(-K.at<double>(0,2)), // Cx
+ static_cast<float>(-K.at<double>(1,2)), // Cy
+ (unsigned int) color_size_.width,
+ (unsigned int) color_size_.height,
+ min_depth,
+ max_depth,
+ baseline,
+ doff
+ };
+
+ host_->getConfig()["focal"] = params_.fx;
+ host_->getConfig()["centre_x"] = params_.cx;
+ host_->getConfig()["centre_y"] = params_.cy;
+ host_->getConfig()["baseline"] = params_.baseline;
+ host_->getConfig()["doffs"] = params_.doffs;
+ } else {
+ state_.getLeft() = {
+ host_->value("focal", 500.0f), // Fx
+ host_->value("focal", 500.0f), // Fy
+ host_->value("centre_x", -color_size_.width/2.0f), // Cx
+ host_->value("centre_y", -color_size_.height/2.0f), // Cy
+ (unsigned int) color_size_.width,
+ (unsigned int) color_size_.height,
+ min_depth,
+ max_depth,
+ baseline,
+ doff
+ };
+ }
// right
diff --git a/components/rgbd-sources/src/sources/stereovideo/stereovideo.hpp b/components/rgbd-sources/src/sources/stereovideo/stereovideo.hpp
index 4b6b60e63a522c8031f5cc386fe6211eba064b9f..2dfe96c3b789a47b5eb11f71ae554772889130b1 100644
--- a/components/rgbd-sources/src/sources/stereovideo/stereovideo.hpp
+++ b/components/rgbd-sources/src/sources/stereovideo/stereovideo.hpp
@@ -38,6 +38,7 @@ class StereoVideoSource : public detail::Source {
LocalSource *lsrc_;
Calibrate *calib_;
+ bool calibrated_;
cv::Size color_size_;
cv::Size depth_size_;