diff --git a/CMakeLists.txt b/CMakeLists.txt
index bc61a7a6736a252e5b39583dca9246b526719dcb..338ed2cb51b53c80d257ef1228e504c782c4a17f 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -231,6 +231,30 @@ else()
message(WARNING "Portaudio not found - sound disabled")
endif()
+# Assimp library
+#find_library( ASSIMP_LIBRARY NAMES assimp PATHS ${PORTAUDIO_DIR} PATH_SUFFIXES lib)
+#if (ASSIMP_LIBRARY)
+# set(HAVE_ASSIMP TRUE)
+# add_library(assimp UNKNOWN IMPORTED)
+ #set_property(TARGET nanogui PROPERTY INTERFACE_INCLUDE_DIRECTORIES ${NANOGUI_EXTRA_INCS})
+# set_property(TARGET assimp PROPERTY IMPORTED_LOCATION ${ASSIMP_LIBRARY})
+# message(STATUS "Found Assimp: ${ASSIMP_LIBRARY}")
+
+# if(WIN32)
+ # Find include
+# find_path(ASSIMP_INCLUDE_DIRS
+# NAMES assimp/scene.h
+# PATHS "C:/Program Files/Assimp" "C:/Program Files (x86)/Assimp" ${ASSIMP_DIR}
+# PATH_SUFFIXES include
+# )
+# set_property(TARGET assimp PROPERTY INTERFACE_INCLUDE_DIRECTORIES ${ASSIMP_INCLUDE_DIRS})
+# endif()
+#else()
+# set(ASSIMP_LIBRARY "")
+# add_library(assimp INTERFACE)
+# message(WARNING "Assimp not found - no model rendering")
+#endif()
+
find_program( NODE_NPM NAMES npm )
if (NODE_NPM)
message(STATUS "Found NPM: ${NODE_NPM}")
diff --git a/applications/gui/src/camera.cpp b/applications/gui/src/camera.cpp
index 8870944377a742e67fdd0912881187375f12651e..c981c662c8ef2dae9729f239716a20e9f446f459 100644
--- a/applications/gui/src/camera.cpp
+++ b/applications/gui/src/camera.cpp
@@ -12,6 +12,7 @@
#include <ftl/cuda/normals.hpp>
#include <ftl/render/colouriser.hpp>
#include <ftl/cuda/transform.hpp>
+#include <ftl/operators/gt_analysis.hpp>
#include <ftl/render/overlay.hpp>
#include "statsimage.hpp"
@@ -359,6 +360,9 @@ void ftl::gui::Camera::_draw(std::vector<ftl::rgbd::FrameSet*> &fss) {
//}
}
+ renderer_->render();
+ if (isStereo()) renderer2_->render();
+
if (channel_ != Channel::Left && channel_ != Channel::Right && channel_ != Channel::None) {
renderer_->blend(channel_);
if (isStereo()) {
@@ -385,6 +389,7 @@ void ftl::gui::Camera::_draw(std::vector<ftl::rgbd::FrameSet*> &fss) {
if (!post_pipe_) {
post_pipe_ = ftl::config::create<ftl::operators::Graph>(screen_->root(), "post_filters");
post_pipe_->append<ftl::operators::FXAA>("fxaa");
+ post_pipe_->append<ftl::operators::GTAnalysis>("gtanal");
}
post_pipe_->apply(frame_, frame_, 0);
@@ -465,6 +470,11 @@ void ftl::gui::Camera::update(std::vector<ftl::rgbd::FrameSet*> &fss) {
if ((size_t)fid_ >= fs->frames.size()) return;
frame = &fs->frames[fid_];
+ if (frame->hasChannel(Channel::Messages)) {
+ msgs_.clear();
+ frame->get(Channel::Messages, msgs_);
+ }
+
auto n = frame->get<std::string>("name");
if (n) {
name_ = *n;
diff --git a/applications/gui/src/camera.hpp b/applications/gui/src/camera.hpp
index d8c2dda708e9858b4929e4613fb779dc2510a271..764a07b9509c08166c7c2e8839a3bc8f74f6dfea 100644
--- a/applications/gui/src/camera.hpp
+++ b/applications/gui/src/camera.hpp
@@ -114,6 +114,7 @@ class Camera {
//cv::Point3f getNormal(float x, float y) { return getNormal((int) round(x), (int) round(y)); }
void setTransform(const Eigen::Matrix4d &T);
Eigen::Matrix4d getTransform() const;
+ const std::vector<std::string> &getMessages() const { return msgs_; }
#ifdef HAVE_OPENVR
bool isVR() { return vr_mode_; }
@@ -174,6 +175,8 @@ class Camera {
std::string name_;
+ std::vector<std::string> msgs_;
+
int transform_ix_;
std::array<Eigen::Matrix4d,ftl::stream::kMaxStreams> transforms_; // Frameset transforms for virtual cam
Eigen::Matrix4d T_ = Eigen::Matrix4d::Identity();
diff --git a/applications/gui/src/media_panel.cpp b/applications/gui/src/media_panel.cpp
index 63d9d911a7de8ae24a62ff4d5b43159543d50f8f..182324deb9d628f716bac0d515a41d2c9c30c0ed 100644
--- a/applications/gui/src/media_panel.cpp
+++ b/applications/gui/src/media_panel.cpp
@@ -174,7 +174,8 @@ MediaPanel::MediaPanel(ftl::gui::Screen *screen, ftl::gui::SourceWindow *sourceW
popup = popbutton->popup();
popup->setLayout(new GroupLayout());
popup->setTheme(screen->toolbuttheme);
- popup->setAnchorHeight(150);
+ //popup->setAnchorHeight(150);
+ more_button_ = popup;
for (int i=3; i<32; ++i) {
ftl::codecs::Channel c = static_cast<ftl::codecs::Channel>(i);
@@ -251,6 +252,11 @@ void MediaPanel::cameraChanged() {
}
}
+void MediaPanel::performLayout(NVGcontext *ctx) {
+ nanogui::Window::performLayout(ctx);
+ more_button_->setAnchorHeight(more_button_->height()-20);
+}
+
void MediaPanel::recordWindowClosed() {
recordbutton_->setEnabled(true);
}
\ No newline at end of file
diff --git a/applications/gui/src/media_panel.hpp b/applications/gui/src/media_panel.hpp
index 4cf69b5ba204d268ad9143b582d5a71a6d407c67..94700cad073c00fec51bfd5135476938d510de06 100644
--- a/applications/gui/src/media_panel.hpp
+++ b/applications/gui/src/media_panel.hpp
@@ -34,6 +34,8 @@ class MediaPanel : public nanogui::Window {
void recordWindowClosed();
+ void performLayout(NVGcontext *ctx) override;
+
private:
ftl::gui::Screen *screen_;
ftl::gui::SourceWindow *sourceWindow_;
@@ -45,6 +47,7 @@ class MediaPanel : public nanogui::Window {
nanogui::PopupButton *button_channels_;
//nanogui::Button *right_button_;
//nanogui::Button *depth_button_;
+ nanogui::Popup *more_button_;
nanogui::PopupButton *recordbutton_;
std::array<nanogui::Button*,32> channel_buttons_={};
diff --git a/applications/gui/src/screen.cpp b/applications/gui/src/screen.cpp
index 5ad79c9729dc8ddd263207ec303c277da2296277..533da2c4de6aa57531bd4568d38fc2ac836f47ea 100644
--- a/applications/gui/src/screen.cpp
+++ b/applications/gui/src/screen.cpp
@@ -604,6 +604,8 @@ void ftl::gui::Screen::draw(NVGcontext *ctx) {
nvgTextAlign(ctx, NVG_ALIGN_RIGHT);
+ int offset_top = 20;
+
if (root()->value("show_information", true)) {
string msg;
@@ -623,6 +625,18 @@ void ftl::gui::Screen::draw(NVGcontext *ctx) {
nvgText(ctx, screenSize[0]-10, 80, msg.c_str(), NULL);
msg = string("Focal: ") + to_string_with_precision(intrin.fx, 2);
nvgText(ctx, screenSize[0]-10, 100, msg.c_str(), NULL);
+
+ offset_top = 120;
+ } else {
+ offset_top = 80;
+ }
+ }
+
+ if (camera_) {
+ auto &msgs = camera_->getMessages();
+ for (auto &m : msgs) {
+ nvgText(ctx, screenSize[0]-10, offset_top, m.c_str(), NULL);
+ offset_top += 20;
}
}
diff --git a/applications/gui/src/src_window.cpp b/applications/gui/src/src_window.cpp
index b9b9b957f3f8edb65d77f6374cacb6e6ae34f67b..fe3763252538ec1a2ac2597afdb97035c90c8eca 100644
--- a/applications/gui/src/src_window.cpp
+++ b/applications/gui/src/src_window.cpp
@@ -34,6 +34,7 @@
#include <ftl/operators/mvmls.hpp>
#include <ftl/operators/clipping.hpp>
#include <ftl/operators/poser.hpp>
+#include <ftl/operators/gt_analysis.hpp>
#include <nlohmann/json.hpp>
@@ -85,7 +86,7 @@ SourceWindow::SourceWindow(ftl::gui::Screen *screen)
auto vscroll = new VScrollPanel(this);
ipanel_ = new Widget(vscroll);
- ipanel_->setLayout(new GridLayout(nanogui::Orientation::Horizontal, 2,
+ ipanel_->setLayout(new GridLayout(nanogui::Orientation::Horizontal, 3,
nanogui::Alignment::Middle, 0, 5));
screen->net()->onConnect([this](ftl::net::Peer *p) {
@@ -273,6 +274,7 @@ void SourceWindow::_checkFrameSets(size_t id) {
p->append<ftl::operators::CullDiscontinuity>("remove_discontinuity");
p->append<ftl::operators::MultiViewMLS>("mvmls")->value("enabled", false);
p->append<ftl::operators::Poser>("poser")->value("enabled", true);
+ p->append<ftl::operators::GTAnalysis>("gtanal");
pre_pipelines_.push_back(p);
framesets_.push_back(new ftl::rgbd::FrameSet);
diff --git a/applications/tools/CMakeLists.txt b/applications/tools/CMakeLists.txt
index a96555932de3a2389e62059687813ea8f04a216d..0506376d54652e307351142caf71628a4f8d0528 100644
--- a/applications/tools/CMakeLists.txt
+++ b/applications/tools/CMakeLists.txt
@@ -1 +1,7 @@
add_subdirectory(codec_eval)
+
+#if (HAVE_ASSIMP)
+# add_subdirectory(model_truth)
+#endif()
+
+add_subdirectory(middlebury_gen)
diff --git a/applications/tools/middlebury_gen/CMakeLists.txt b/applications/tools/middlebury_gen/CMakeLists.txt
new file mode 100644
index 0000000000000000000000000000000000000000..2dffb172d22cfc554088f3971bb90ff2fd60e8e7
--- /dev/null
+++ b/applications/tools/middlebury_gen/CMakeLists.txt
@@ -0,0 +1,17 @@
+set(MIDSRC
+ src/main.cpp
+)
+
+add_executable(middlebury-gen ${MIDSRC})
+
+target_include_directories(middlebury-gen PUBLIC
+ #$<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/include>
+ #$<INSTALL_INTERFACE:include>
+ PRIVATE src)
+
+if (CUDA_FOUND)
+set_property(TARGET middlebury-gen PROPERTY CUDA_SEPARABLE_COMPILATION ON)
+endif()
+
+#target_include_directories(cv-node PUBLIC ${PROJECT_SOURCE_DIR}/include)
+target_link_libraries(middlebury-gen ftlcommon ftlrgbd Threads::Threads ${OpenCV_LIBS} ftlrender ftloperators ftlstreams)
diff --git a/applications/tools/middlebury_gen/src/main.cpp b/applications/tools/middlebury_gen/src/main.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..043db7d474d304d24f4cf0f5ee3005e8c9281ca4
--- /dev/null
+++ b/applications/tools/middlebury_gen/src/main.cpp
@@ -0,0 +1,335 @@
+#include <ftl/configuration.hpp>
+#include <ftl/streams/filestream.hpp>
+#include <ftl/operators/disparity.hpp>
+#include <ftl/codecs/opencv_encoder.hpp>
+#include <ftl/streams/injectors.hpp>
+
+#include <opencv2/imgcodecs.hpp>
+#include <opencv2/imgproc.hpp>
+#include <opencv2/highgui.hpp>
+#include <nlohmann/json.hpp>
+
+#define LOGURU_REPLACE_GLOG 1
+#include <loguru.hpp>
+
+using ftl::codecs::Channel;
+using ftl::codecs::definition_t;
+using ftl::codecs::codec_t;
+using std::string;
+
+static bool loadMiddleburyCalib(const std::string &filename, ftl::rgbd::Camera &p1, ftl::rgbd::Camera &p2, int &ndisp, double scaling) {
+ FILE* fp = fopen(filename.c_str(), "r");
+
+ float cam0[3][3] = {};
+ float cam1[3][3];
+ float doffs = 0.0f;
+ float baseline = 0.0f;
+ int width = 0;
+ int height = 0;
+ //int ndisp;
+ int isint;
+ int vmin;
+ int vmax;
+ float dyavg;
+ float dymax;
+
+ if (fp != nullptr) {
+ char buff[512];
+ if (fgets(buff, sizeof(buff), fp) != nullptr) sscanf(buff, "cam0 = [%f %f %f; %f %f %f; %f %f %f]\n", &cam0[0][0], &cam0[0][1], &cam0[0][2], &cam0[1][0], &cam0[1][1], &cam0[1][2], &cam0[2][0], &cam0[2][1], &cam0[2][2]);
+ if (fgets(buff, sizeof(buff), fp) != nullptr) sscanf(buff, "cam1 = [%f %f %f; %f %f %f; %f %f %f]\n", &cam1[0][0], &cam1[0][1], &cam1[0][2], &cam1[1][0], &cam1[1][1], &cam1[1][2], &cam1[2][0], &cam1[2][1], &cam1[2][2]);
+ if (fgets(buff, sizeof(buff), fp) != nullptr) sscanf(buff, "doffs = %f\n", &doffs);
+ if (fgets(buff, sizeof(buff), fp) != nullptr) sscanf(buff, "baseline = %f\n", &baseline);
+ if (fgets(buff, sizeof(buff), fp) != nullptr) sscanf(buff, "width = %d\n", &width);
+ if (fgets(buff, sizeof(buff), fp) != nullptr) sscanf(buff, "height = %d\n", &height);
+ if (fgets(buff, sizeof(buff), fp) != nullptr) sscanf(buff, "ndisp = %d\n", &ndisp);
+ if (fgets(buff, sizeof(buff), fp) != nullptr) sscanf(buff, "isint = %d\n", &isint);
+ if (fgets(buff, sizeof(buff), fp) != nullptr) sscanf(buff, "vmin = %d\n", &vmin);
+ if (fgets(buff, sizeof(buff), fp) != nullptr) sscanf(buff, "vmax = %d\n", &vmax);
+ if (fgets(buff, sizeof(buff), fp) != nullptr) sscanf(buff, "dyavg = %f\n", &dyavg);
+ if (fgets(buff, sizeof(buff), fp) != nullptr) sscanf(buff, "dymax = %f\n", &dymax);
+ fclose(fp);
+
+ p1.fx = cam0[0][0] * scaling;
+ p1.fy = p1.fx;
+ p1.cx = -cam0[0][2] * scaling;
+ p1.cy = -cam0[1][2] * scaling;
+ p1.width = width * scaling;
+ p1.height = height * scaling;
+ p1.baseline = baseline / 1000.0f;
+ p1.doffs = doffs * scaling;
+ p1.maxDepth = p1.baseline * p1.fx / (float(vmin) + p1.doffs);
+ p1.minDepth = p1.baseline * p1.fx / (float(vmax) + p1.doffs);
+ p1.doffs = -p1.doffs;
+
+ p2 = p1;
+ p2.fx = cam1[0][0] * scaling;
+ p2.fy = p2.fx;
+ p2.cx = -cam1[0][2] * scaling;
+ p2.cy = -cam1[1][2] * scaling;
+
+ return true;
+ }
+
+ return false;
+}
+
+static void skip_comment(FILE *fp) {
+ // skip comment lines in the headers of pnm files
+
+ char c;
+ while ((c=getc(fp)) == '#')
+ while (getc(fp) != '\n') ;
+ ungetc(c, fp);
+}
+
+static void skip_space(FILE *fp) {
+ // skip white space in the headers or pnm files
+
+ char c;
+ do {
+ c = getc(fp);
+ } while (c == '\n' || c == ' ' || c == '\t' || c == '\r');
+ ungetc(c, fp);
+}
+
+static void read_header(FILE *fp, const char *imtype, char c1, char c2,
+ int *width, int *height, int *nbands, int thirdArg)
+{
+ // read the header of a pnmfile and initialize width and height
+
+ char c;
+
+ if (getc(fp) != c1 || getc(fp) != c2)
+ LOG(FATAL) << "ReadFilePGM: wrong magic code for " << imtype << " file";
+ skip_space(fp);
+ skip_comment(fp);
+ skip_space(fp);
+ if (fscanf(fp, "%d", width) <= 0) {
+ LOG(FATAL) << "PFM file error";
+ };
+ skip_space(fp);
+ if (fscanf(fp, "%d", height) <= 0) {
+ LOG(FATAL) << "PFM file error";
+ }
+ if (thirdArg) {
+ skip_space(fp);
+ if (fscanf(fp, "%d", nbands) <= 0) {
+ LOG(FATAL) << "PFM file error";
+ }
+ }
+ // skip SINGLE newline character after reading image height (or third arg)
+ c = getc(fp);
+ if (c == '\r') // <cr> in some files before newline
+ c = getc(fp);
+ if (c != '\n') {
+ if (c == ' ' || c == '\t' || c == '\r')
+ LOG(FATAL) << "newline expected in file after image height";
+ else
+ LOG(FATAL) << "whitespace expected in file after image height";
+ }
+}
+
+// check whether machine is little endian
+static int littleendian() {
+ int intval = 1;
+ uchar *uval = (uchar *)&intval;
+ return uval[0] == 1;
+}
+
+// 1-band PFM image, see http://netpbm.sourceforge.net/doc/pfm.html
+// 3-band not yet supported
+static void readFilePFM(cv::Mat &img, const string &filename)
+{
+ // Open the file and read the header
+ FILE *fp = fopen(filename.c_str(), "rb");
+ if (fp == 0)
+ LOG(FATAL) << "ReadFilePFM: could not open \"" << filename << "\"";
+
+ int width, height, nBands;
+ read_header(fp, "PFM", 'P', 'f', &width, &height, &nBands, 0);
+
+ skip_space(fp);
+
+ float scalef;
+ if (fscanf(fp, "%f", &scalef) <= 0) { // scale factor (if negative, little endian)
+ LOG(FATAL) << "Invalid PFM file";
+ }
+
+ // skip SINGLE newline character after reading third arg
+ char c = getc(fp);
+ if (c == '\r') // <cr> in some files before newline
+ c = getc(fp);
+ if (c != '\n') {
+ if (c == ' ' || c == '\t' || c == '\r')
+ LOG(FATAL) << "newline expected in file after scale factor";
+ else
+ LOG(FATAL) << "whitespace expected in file after scale factor";
+ }
+
+ // Allocate the image if necessary
+ img = cv::Mat(height, width, CV_32FC1);
+ // Set the image shape
+ //Size sh = img.size();
+
+ int littleEndianFile = (scalef < 0);
+ int littleEndianMachine = littleendian();
+ int needSwap = (littleEndianFile != littleEndianMachine);
+ //printf("endian file = %d, endian machine = %d, need swap = %d\n",
+ // littleEndianFile, littleEndianMachine, needSwap);
+
+ for (int y = height-1; y >= 0; y--) { // PFM stores rows top-to-bottom!!!!
+ int n = width;
+ float* ptr = &img.at<float>(y, 0, 0);
+ if ((int)fread(ptr, sizeof(float), n, fp) != n)
+ LOG(FATAL) << "ReadFilePFM(" << filename << "): file is too short";
+
+ if (needSwap) { // if endianness doesn't agree, swap bytes
+ uchar* ptr = (uchar *)&img.at<uchar>(y, 0, 0);
+ int x = 0;
+ uchar tmp = 0;
+ while (x < n) {
+ tmp = ptr[0]; ptr[0] = ptr[3]; ptr[3] = tmp;
+ tmp = ptr[1]; ptr[1] = ptr[2]; ptr[2] = tmp;
+ ptr += 4;
+ x++;
+ }
+ }
+ }
+ if (fclose(fp))
+ LOG(FATAL) << "ReadFilePGM(" << filename << "): error closing file";
+}
+
+int main(int argc, char **argv) {
+ auto *root = ftl::configure(argc, argv, "tools_default");
+
+ ftl::stream::File *out = ftl::create<ftl::stream::File>(root, "output");
+ out->set("filename", root->value("out", std::string("./out.ftl")));
+ out->setMode(ftl::stream::File::Mode::Write);
+ out->begin(false);
+
+ int height = root->value("height", 1080);
+
+ // For each middlebury test folder
+ auto paths = (*root->get<nlohmann::json>("paths"));
+
+ ftl::rgbd::Frame frame;
+ ftl::rgbd::FrameState state;
+
+ ftl::operators::DisparityToDepth disp2depth(ftl::create<ftl::Configurable>(root, "disparity"));
+
+ ftl::codecs::OpenCVEncoder encoder(ftl::codecs::definition_t::Any, ftl::codecs::definition_t::Any);
+
+ int i=0;
+ for (auto &x : paths.items()) {
+ std::string dir = x.value().get<std::string>();
+
+ std::vector<std::string> dirents = ftl::directory_listing(dir);
+ for (auto &path : dirents) {
+ // Validate the folder as middlebury
+ ftl::rgbd::Camera intrin1;
+ ftl::rgbd::Camera intrin2;
+ int ndisp = 0;
+ if (!loadMiddleburyCalib(path+"/calib.txt", intrin1, intrin2, ndisp, 1.0f)) {
+ LOG(ERROR) << "Could not load middlebury calibration: " << path;
+ continue;
+ }
+
+ frame.reset();
+
+ // Load the colour images into a frame.
+ frame.create<cv::Mat>(Channel::Colour) = cv::imread(path+"/im0.png", cv::IMREAD_COLOR);
+ frame.create<cv::Mat>(Channel::Colour2) = cv::imread(path+"/im1.png", cv::IMREAD_COLOR);
+
+ // Colour convert
+ auto &c1 = frame.get<cv::Mat>(Channel::Colour);
+ auto &c2 = frame.get<cv::Mat>(Channel::Colour2);
+ cv::cvtColor(c1,c1, cv::COLOR_BGR2RGBA);
+ cv::cvtColor(c2,c2, cv::COLOR_BGR2RGBA);
+
+ // Load the ground truth
+ //frame.create<cv::Mat>(Channel::Disparity) = cv::imread(path+"/disp0.pfm", cv::IMREAD_UNCHANGED);
+ readFilePFM(frame.create<cv::Mat>(Channel::Disparity), path+"/disp0.pfm");
+ cv::Mat &disp = frame.get<cv::Mat>(Channel::Disparity);
+ float aspect = float(disp.cols) / float(disp.rows);
+ float scaling = float(height) / float(disp.rows);
+ cv::resize(disp, disp, cv::Size(int(aspect*float(height)),height), 0.0, 0.0, cv::INTER_NEAREST);
+ cv::resize(c1, c1, cv::Size(int(aspect*float(height)),height));
+ cv::resize(c2, c2, cv::Size(int(aspect*float(height)),height));
+
+ int original_width = c1.cols;
+ int desired_width = ftl::codecs::getWidth(ftl::codecs::findDefinition(height));
+ int border_size = (desired_width - c1.cols) / 2;
+ cv::copyMakeBorder(c1, c1, 0, 0, border_size, desired_width - border_size - c1.cols, cv::BORDER_CONSTANT, cv::Scalar(0));
+ cv::copyMakeBorder(c2, c2, 0, 0, border_size, desired_width - border_size - c2.cols, cv::BORDER_CONSTANT, cv::Scalar(0));
+ cv::copyMakeBorder(disp, disp, 0, 0, border_size, desired_width - border_size - disp.cols, cv::BORDER_CONSTANT, cv::Scalar(0));
+
+ // TODO: Adjust principle points (cx)
+
+ LOG(INFO) << "Disparity scaling: " << scaling;
+ LOG(INFO) << "Depth range: " << intrin1.minDepth << " to " << intrin1.maxDepth;
+ LOG(INFO) << "Resolution: " << c1.cols << "x" << c1.rows;
+ disp.convertTo(disp, CV_32F, scaling);
+
+ intrin1 = intrin1.scaled(original_width, c1.rows);
+ intrin2 = intrin2.scaled(original_width, c2.rows);
+ intrin1.cx -= border_size;
+ intrin2.cx -= border_size;
+ intrin1.width = c1.cols;
+ intrin2.width = c2.cols;
+
+ state.setLeft(intrin1);
+ state.setRight(intrin2);
+ frame.setOrigin(&state);
+ ftl::stream::injectCalibration(out, frame, 0, 0, i, false);
+ ftl::stream::injectCalibration(out, frame, 0, 0, i, true);
+
+ // Convert disparity to depth
+ frame.upload(Channel::Disparity + Channel::Colour + Channel::Colour2);
+
+ disp2depth.apply(frame, frame, 0);
+
+ // Encode the frame into the output stream
+ ftl::codecs::StreamPacket spkt;
+ ftl::codecs::Packet pkt;
+
+ spkt.timestamp = 0;
+ spkt.frame_number = i;
+ spkt.streamID = 0;
+ spkt.version = 4;
+ pkt.codec = codec_t::Any;
+ pkt.definition = definition_t::Any;
+ pkt.bitrate = 0;
+ pkt.flags = 0;
+ pkt.frame_count = 1;
+
+ spkt.channel = Channel::Colour;
+ if (!encoder.encode(frame.get<cv::cuda::GpuMat>(Channel::Colour), pkt)) {
+ LOG(ERROR) << "Encode failed for colour";
+ }
+ out->post(spkt, pkt);
+
+ pkt.codec = codec_t::Any;
+ pkt.definition = definition_t::Any;
+ spkt.channel = Channel::Colour2;
+ if (!encoder.encode(frame.get<cv::cuda::GpuMat>(Channel::Colour2), pkt)) {
+ LOG(ERROR) << "Encode failed for colour2";
+ }
+ out->post(spkt, pkt);
+
+ spkt.channel = Channel::GroundTruth;
+ pkt.flags = ftl::codecs::kFlagFloat;
+ pkt.codec = codec_t::Any;
+ pkt.definition = definition_t::Any;
+ if (!encoder.encode(frame.get<cv::cuda::GpuMat>(Channel::Depth), pkt)) {
+ LOG(ERROR) << "Encode failed for depth";
+ }
+ out->post(spkt, pkt);
+
+ ++i;
+ }
+ }
+
+ out->end();
+
+ return 0;
+}
\ No newline at end of file
diff --git a/applications/tools/model_truth/CMakeLists.txt b/applications/tools/model_truth/CMakeLists.txt
new file mode 100644
index 0000000000000000000000000000000000000000..9b386f7ac4152ab2fa169f38f79d94a480d75213
--- /dev/null
+++ b/applications/tools/model_truth/CMakeLists.txt
@@ -0,0 +1,17 @@
+set(GTSRC
+ src/main.cpp
+)
+
+add_executable(model-truth ${GTSRC})
+
+target_include_directories(model-truth PUBLIC
+ #$<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/include>
+ #$<INSTALL_INTERFACE:include>
+ PRIVATE src)
+
+if (CUDA_FOUND)
+set_property(TARGET model-truth PROPERTY CUDA_SEPARABLE_COMPILATION ON)
+endif()
+
+#target_include_directories(cv-node PUBLIC ${PROJECT_SOURCE_DIR}/include)
+target_link_libraries(model-truth ftlcommon ftlrgbd Threads::Threads ${OpenCV_LIBS} ftlctrl ftlnet ftlrender ftloperators ftlstreams assimp)
diff --git a/applications/tools/model_truth/src/main.cpp b/applications/tools/model_truth/src/main.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..1712a76833cf78a51a3424567a4cbdda6162866c
--- /dev/null
+++ b/applications/tools/model_truth/src/main.cpp
@@ -0,0 +1,98 @@
+#include <ftl/configuration.hpp>
+#include <ftl/streams/filestream.hpp>
+#include <ftl/streams/parsers.hpp>
+#include <ftl/render/assimp_render.hpp>
+
+#include <vector>
+
+#define LOGURU_REPLACE_GLOG 1
+#include <loguru.hpp>
+
+#include <GLFW/glfw3.h>
+
+using std::vector;
+using std::string;
+using ftl::codecs::Channel;
+
+struct CameraSpecs {
+ ftl::rgbd::Camera intrinsics1;
+ ftl::rgbd::Camera intrinsics2;
+ Eigen::Matrix4d pose;
+};
+
+int main(int argc, char **argv) {
+ auto *root = ftl::configure(argc, argv, "tools_default");
+
+ vector<CameraSpecs> cameras;
+ vector<ftl::rgbd::Frame> frames;
+
+ // Use existing FTL as pose and intrinsics source
+ ftl::stream::File *inftl = ftl::create<ftl::stream::File>(root, "input_ftl");
+ inftl->set("filename", root->value("in", std::string("")));
+
+ inftl->onPacket([&cameras](const ftl::codecs::StreamPacket &spkt, const ftl::codecs::Packet &pkt) {
+ if (spkt.channel == Channel::Pose) {
+ if (cameras.size() <= spkt.frameNumber()) cameras.resize(spkt.frameNumber()+1);
+ CameraSpecs &spec = cameras[spkt.frameNumber()];
+
+ spec.pose = ftl::stream::parsePose(pkt);
+
+ LOG(INFO) << "HAVE POSE: " << spec.pose;
+ } else if (spkt.channel == Channel::Calibration) {
+ if (cameras.size() <= spkt.frameNumber()) cameras.resize(spkt.frameNumber()+1);
+ CameraSpecs &spec = cameras[spkt.frameNumber()];
+
+ spec.intrinsics1 = ftl::stream::parseCalibration(pkt);
+
+ LOG(INFO) << "HAVE INTRIN: " << spec.intrinsics1.width << "x" << spec.intrinsics1.height;
+ }
+ });
+ inftl->set("looping", false);
+ inftl->begin(false);
+ inftl->tick(ftl::timer::get_time()+10000); // Read 10 seconds of FTL file
+ inftl->end();
+
+ // Load a model using ASSIMP
+ auto *scene = ftl::create<ftl::render::AssimpScene>(root, "scene");
+ scene->set("model", root->value("model", std::string("")));
+ auto *render = ftl::create<ftl::render::AssimpRenderer>(root, "render");
+ render->setScene(scene);
+
+ // Generate OpenGL context and frame buffer
+ GLFWwindow *window;
+ if (!glfwInit()) {
+ LOG(ERROR) << "Could not init GL window";
+ return -1;
+ }
+
+ glfwWindowHint(GLFW_VISIBLE, GL_FALSE);
+ window = glfwCreateWindow(640, 480, "FTL", NULL, NULL);
+
+ if (!window) {
+ glfwTerminate();
+ LOG(ERROR) << "Could not create GL window";
+ return -1;
+ }
+
+ glfwMakeContextCurrent(window);
+
+ scene->load();
+ frames.resize(cameras.size());
+
+ // Render each view into framebuffer
+ for (size_t i=0; i<cameras.size(); ++i) {
+ render->setScene(scene);
+ render->begin(frames[i], Channel::Colour);
+ render->render();
+ render->end();
+ }
+
+ // Download each view and encode as JPG / PNG colour depth pairs
+
+ // Allow options to introduce distortion and noise into depth maps
+
+ // Use a ground truth channel?
+
+ glfwTerminate();
+ return 0;
+}
diff --git a/components/codecs/include/ftl/codecs/channels.hpp b/components/codecs/include/ftl/codecs/channels.hpp
index f2f8a833bfb2f5515a407dbcaf135929ebce4e63..7cd5827f33bc8d2506024e92873754e99e75c007 100644
--- a/components/codecs/include/ftl/codecs/channels.hpp
+++ b/components/codecs/include/ftl/codecs/channels.hpp
@@ -37,6 +37,7 @@ enum struct Channel : int {
RightHighRes = 20, // 8UC3 or 8UC4
Colour2HighRes = 20,
Overlay = 21, // 8UC4
+ GroundTruth = 22, // 32F
Audio = 32,
AudioMono = 32,
@@ -55,7 +56,8 @@ enum struct Channel : int {
Data = 2048, // Custom data, any codec.
Faces = 2049, // Data about detected faces
Transforms = 2050, // Transformation matrices for framesets
- Shapes3D = 2051 // Labeled 3D shapes
+ Shapes3D = 2051, // Labeled 3D shapes
+ Messages = 2052 // Vector of Strings
};
inline bool isVideo(Channel c) { return (int)c < 32; };
@@ -147,6 +149,7 @@ static const Channels kAllChannels(0xFFFFFFFFu);
inline bool isFloatChannel(ftl::codecs::Channel chan) {
switch (chan) {
+ case Channel::GroundTruth:
case Channel::Depth :
//case Channel::Normals :
case Channel::Confidence:
diff --git a/components/codecs/include/ftl/codecs/codecs.hpp b/components/codecs/include/ftl/codecs/codecs.hpp
index 5e594429e5dea928ebf1a92d2fc5c0a50f8fcc41..e2047b54ac1fcc86751a6e1da7b5f941cc19bad3 100644
--- a/components/codecs/include/ftl/codecs/codecs.hpp
+++ b/components/codecs/include/ftl/codecs/codecs.hpp
@@ -61,6 +61,8 @@ enum struct definition_t : uint8_t {
HTC_VIVE = 8,
OLD_SKOOL = 9,
+ MIDDLEBURY = 10,
+ MIDDLEBURY_HD = 11,
hz48000 = 32,
hz44100 = 33,
diff --git a/components/codecs/include/ftl/codecs/packet.hpp b/components/codecs/include/ftl/codecs/packet.hpp
index b0756178529a487e0203d440431ecafd0619a11b..546da9ac9e654757730284d269d6c56e305a3175 100644
--- a/components/codecs/include/ftl/codecs/packet.hpp
+++ b/components/codecs/include/ftl/codecs/packet.hpp
@@ -54,6 +54,8 @@ struct Packet {
MSGPACK_DEFINE(codec, definition, frame_count, bitrate, flags, data);
};
+static constexpr unsigned int kStreamCap_Static = 0x01;
+
/**
* Add timestamp and channel information to a raw encoded frame packet. This
* allows the packet to be located within a larger stream and should be sent
@@ -76,7 +78,9 @@ struct StreamPacket {
inline size_t frameSetID() const { return (version >= 4) ? streamID : 0; }
inline int64_t localTimestamp() const { return timestamp + originClockDelta; }
- int64_t originClockDelta; // Not message packet / saved
+ int64_t originClockDelta; // Not message packet / saved
+ unsigned int hint_capability; // Is this a video stream, for example
+ size_t hint_source_total; // Number of tracks per frame to expect
MSGPACK_DEFINE(timestamp, streamID, frame_number, channel);
diff --git a/components/codecs/src/bitrates.cpp b/components/codecs/src/bitrates.cpp
index b72b437d9b669badfc411c889a11a2db0fc0d7f4..654fb865d5549c91543adfda761bdde703dacf7a 100644
--- a/components/codecs/src/bitrates.cpp
+++ b/components/codecs/src/bitrates.cpp
@@ -24,6 +24,8 @@ static const Resolution resolutions[] = {
0, 0, // ANY
1852, 2056, // HTC_VIVE
640, 480, // Old school 4:3
+ 3000, 1920, // Middlebury
+ 1687, 1080, // Middlebury smaller
0, 0
};
diff --git a/components/codecs/src/channels.cpp b/components/codecs/src/channels.cpp
index 55426cb26513b28289a32c028d3067efa5ac54f5..75dc784fb09a6f9882f1a0156c09314641fdaef4 100644
--- a/components/codecs/src/channels.cpp
+++ b/components/codecs/src/channels.cpp
@@ -29,9 +29,9 @@ static ChannelInfo info[] = {
"ColourHighRes", 0, // 17
"Disparity", 0, // 18
"Smoothing", 0, // 19
- "NoName", 0,
- "NoName", 0,
- "NoName", 0,
+ "Colour2HighRes", 0, // 20
+ "Overlay", 0, // 21
+ "GroundTruth", CV_32F, // 22
"NoName", 0,
"NoName", 0,
"NoName", 0,
diff --git a/components/codecs/src/opencv_decoder.cpp b/components/codecs/src/opencv_decoder.cpp
index 809f716dfc33eb2c6a2f23a52fb4c896e12ef3f0..416b7d90f72a4d8e5f9513de9a1ec9a9c4559db4 100644
--- a/components/codecs/src/opencv_decoder.cpp
+++ b/components/codecs/src/opencv_decoder.cpp
@@ -57,6 +57,8 @@ bool OpenCVDecoder::decode(const ftl::codecs::Packet &pkt, cv::cuda::GpuMat &out
} else if (!tmp_.empty() && tmp_.type() == CV_8UC4 && chunkHead.type() == CV_8UC4) {
//tmp_.copyTo(chunkHead);
chunkHead.upload(tmp_);
+ } else if (!tmp_.empty() && tmp_.type() == CV_16U && chunkHead.type() == CV_16U) {
+ chunkHead.upload(tmp_);
} else {
// Silent ignore?
}
diff --git a/components/codecs/src/opencv_encoder.cpp b/components/codecs/src/opencv_encoder.cpp
index 152f05754c619124b2c99a6f5876dc8016873a3b..b930d04294878f9c3a43d5187a82e9725841a5ca 100644
--- a/components/codecs/src/opencv_encoder.cpp
+++ b/components/codecs/src/opencv_encoder.cpp
@@ -38,14 +38,21 @@ bool OpenCVEncoder::encode(const cv::cuda::GpuMat &in, ftl::codecs::Packet &pkt)
pkt.definition = (pkt.definition == definition_t::Any) ? ftl::codecs::findDefinition(in.cols, in.rows) : pkt.definition;
- if (pkt.definition == definition_t::Invalid || pkt.definition == definition_t::Any) return false;
+ if (pkt.definition == definition_t::Invalid || pkt.definition == definition_t::Any) {
+ LOG(ERROR) << "Invalid definition";
+ return false;
+ }
// Ensure definition does not exceed max
- current_definition_ = ((int)pkt.definition < (int)max_definition) ? max_definition : pkt.definition;
+ current_definition_ = pkt.definition; //((int)pkt.definition < (int)max_definition) ? max_definition : pkt.definition;
in.download(tmp_);
//CHECK(cv::Size(ftl::codecs::getWidth(definition), ftl::codecs::getHeight(definition)) == in.size());
+ if (!is_colour) {
+ tmp_.convertTo(tmp_, CV_16U, 1000.0f);
+ }
+
int width = ftl::codecs::getWidth(current_definition_);
int height = ftl::codecs::getHeight(current_definition_);
@@ -55,7 +62,10 @@ bool OpenCVEncoder::encode(const cv::cuda::GpuMat &in, ftl::codecs::Packet &pkt)
} else {
}*/
- if (width != in.cols || height != in.rows) return false;
+ if (width != in.cols || height != in.rows) {
+ LOG(ERROR) << "Input does not match requested definition";
+ return false;
+ }
if (pkt.codec == codec_t::Any) pkt.codec = (is_colour) ? codec_t::JPG : codec_t::PNG;
diff --git a/components/common/cpp/include/ftl/configuration.hpp b/components/common/cpp/include/ftl/configuration.hpp
index 8ce55cd28198ae78b45a64efb6382c76d228a41f..bdbc4a5904b4161f3ec740bc02441dd1292caeab 100644
--- a/components/common/cpp/include/ftl/configuration.hpp
+++ b/components/common/cpp/include/ftl/configuration.hpp
@@ -22,6 +22,7 @@ bool is_directory(const std::string &path);
bool is_file(const std::string &path);
bool create_directory(const std::string &path);
bool is_video(const std::string &file);
+std::vector<std::string> directory_listing(const std::string &path);
namespace config {
diff --git a/components/common/cpp/src/configuration.cpp b/components/common/cpp/src/configuration.cpp
index 9a1b4e9cdd08f152e0b09647e7da7126d16197d6..0d2b79a97d3d7fb731e6fdb219540ad76dd5e35a 100644
--- a/components/common/cpp/src/configuration.cpp
+++ b/components/common/cpp/src/configuration.cpp
@@ -4,6 +4,7 @@
#ifdef WIN32
#include <windows.h>
+#pragma comment(lib, "User32.lib")
#else
#include <sys/types.h>
#include <sys/stat.h>
@@ -12,6 +13,7 @@
#ifndef WIN32
#include <signal.h>
+#include <dirent.h>
#endif
#include <nlohmann/json.hpp>
@@ -88,6 +90,38 @@ bool ftl::is_file(const std::string &path) {
#endif
}
+std::vector<std::string> ftl::directory_listing(const std::string &path) {
+ std::vector<std::string> res;
+
+#ifdef WIN32
+ std::string path2 = path + "\\*";
+ WIN32_FIND_DATA ffd;
+ HANDLE hFind = FindFirstFile(path2.c_str(), &ffd);
+
+ if (hFind == INVALID_HANDLE_VALUE) return res;
+
+ do {
+ res.push_back(std::string(ffd.cFileName));
+ } while (FindNextFile(hFind, &ffd) != 0);
+
+ FindClose(hFind);
+ return res;
+#else
+ DIR *dir;
+ struct dirent *ent;
+ if ((dir = opendir (path.c_str())) != NULL) {
+ /* print all the files and directories within directory */
+ while ((ent = readdir (dir)) != NULL) {
+ res.push_back(path + std::string(ent->d_name));
+ }
+ closedir (dir);
+ return res;
+ } else {
+ return res;
+ }
+#endif
+}
+
static bool endsWith(const string &s, const string &e) {
return s.size() >= e.size() &&
s.compare(s.size() - e.size(), e.size(), e) == 0;
diff --git a/components/operators/CMakeLists.txt b/components/operators/CMakeLists.txt
index ec80792076b0c4c4ff020e586d0f718b11926233..d0d786884525f45bbf3b4dcb41efda089d0bce89 100644
--- a/components/operators/CMakeLists.txt
+++ b/components/operators/CMakeLists.txt
@@ -31,6 +31,8 @@ set(OPERSRC
src/weighting.cpp
src/weighting.cu
src/poser.cpp
+ src/gt_analysis.cpp
+ src/gt_analysis.cu
)
if (LIBSGM_FOUND)
diff --git a/components/operators/include/ftl/operators/gt_analysis.hpp b/components/operators/include/ftl/operators/gt_analysis.hpp
new file mode 100644
index 0000000000000000000000000000000000000000..9397a96da2ff80985d08d90d7f68697935fc895f
--- /dev/null
+++ b/components/operators/include/ftl/operators/gt_analysis.hpp
@@ -0,0 +1,31 @@
+#ifndef _FTL_OPERATORS_GTANALYSIS_HPP_
+#define _FTL_OPERATORS_GTANALYSIS_HPP_
+
+#include <ftl/operators/operator.hpp>
+#include <ftl/cuda_common.hpp>
+#include <ftl/operators/gt_cuda.hpp>
+
+namespace ftl {
+namespace operators {
+
+/**
+ * Ground Truth analysis, applys indications and metrics that compare any
+ * depth map with a ground truth channel (if both exist).
+ */
+class GTAnalysis : public ftl::operators::Operator {
+ public:
+ explicit GTAnalysis(ftl::Configurable*);
+ ~GTAnalysis();
+
+ inline Operator::Type type() const override { return Operator::Type::OneToOne; }
+
+ bool apply(ftl::rgbd::Frame &in, ftl::rgbd::Frame &out, cudaStream_t stream) override;
+
+ private:
+ ftl::cuda::GTAnalysisData *output_;
+};
+
+}
+}
+
+#endif // _FTL_OPERATORS_GTANALYSIS_HPP_
diff --git a/components/operators/include/ftl/operators/gt_cuda.hpp b/components/operators/include/ftl/operators/gt_cuda.hpp
new file mode 100644
index 0000000000000000000000000000000000000000..cd317a615e21e5dedf30c6d53f054e5564a075fc
--- /dev/null
+++ b/components/operators/include/ftl/operators/gt_cuda.hpp
@@ -0,0 +1,40 @@
+#ifndef _FTL_CUDA_GT_HPP_
+#define _FTL_CUDA_GT_HPP_
+
+#include <ftl/cuda_common.hpp>
+#include <ftl/rgbd/camera.hpp>
+
+namespace ftl {
+namespace cuda {
+
+struct GTAnalysisData {
+ int invalid; // Count of invalid (missing depth)
+ int bad; // Count bad (above x disparity error)
+ float totalerror; // Summed disparity error (of valid values)
+ int masked; // Count of pixels masked.
+};
+
+void gt_analysis(
+ ftl::cuda::TextureObject<uchar4> &colour,
+ ftl::cuda::TextureObject<float> &depth,
+ ftl::cuda::TextureObject<float> >,
+ ftl::cuda::GTAnalysisData *out,
+ const ftl::rgbd::Camera &cam,
+ float threshold,
+ float outmax,
+ cudaStream_t stream
+);
+
+void gt_analysis(
+ ftl::cuda::TextureObject<float> &depth,
+ ftl::cuda::TextureObject<float> >,
+ ftl::cuda::GTAnalysisData *out,
+ const ftl::rgbd::Camera &cam,
+ float threshold,
+ cudaStream_t stream
+);
+
+}
+}
+
+#endif
\ No newline at end of file
diff --git a/components/operators/src/disparity/disp2depth.cu b/components/operators/src/disparity/disp2depth.cu
index 349920dd2d21f8a330c8f2de7ce6aea5e9672185..1e655e2d6429a3c11279dfd4423c0bfdbdff92e1 100644
--- a/components/operators/src/disparity/disp2depth.cu
+++ b/components/operators/src/disparity/disp2depth.cu
@@ -2,6 +2,10 @@
#include <ftl/rgbd/camera.hpp>
#include <opencv2/core/cuda_stream_accessor.hpp>
+#ifndef PINF
+#define PINF __int_as_float(0x7f800000)
+#endif
+
__global__ void d2d_kernel(cv::cuda::PtrStepSz<float> disp, cv::cuda::PtrStepSz<float> depth,
ftl::rgbd::Camera cam) {
diff --git a/components/operators/src/gt_analysis.cpp b/components/operators/src/gt_analysis.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..91d59ca3d362a0f59580773b11fa0a9d7e3ac405
--- /dev/null
+++ b/components/operators/src/gt_analysis.cpp
@@ -0,0 +1,70 @@
+#include <ftl/operators/gt_analysis.hpp>
+#include <ftl/operators/gt_cuda.hpp>
+
+using ftl::operators::GTAnalysis;
+using ftl::codecs::Channel;
+using std::string;
+
+GTAnalysis::GTAnalysis(ftl::Configurable *cfg) : ftl::operators::Operator(cfg) {
+ cudaMalloc(&output_, sizeof(ftl::cuda::GTAnalysisData));
+}
+
+GTAnalysis::~GTAnalysis() {
+ cudaFree(output_);
+}
+
+template <typename T>
+std::string to_string_with_precision(const T a_value, const int n = 6) {
+ std::ostringstream out;
+ out.precision(n);
+ out << std::fixed << a_value;
+ return out.str();
+}
+
+bool GTAnalysis::apply(ftl::rgbd::Frame &in, ftl::rgbd::Frame &out, cudaStream_t stream) {
+ if (in.hasChannel(Channel::Depth) && in.hasChannel(Channel::GroundTruth)) {
+ if (config()->value("show_colour", false)) {
+ ftl::cuda::gt_analysis(
+ in.createTexture<uchar4>(Channel::Colour),
+ in.createTexture<float>(Channel::Depth),
+ in.createTexture<float>(Channel::GroundTruth),
+ output_,
+ in.getLeft(),
+ config()->value("bad_threshold", 2.0f),
+ config()->value("viz_threshold", 5.0f),
+ stream
+ );
+ } else {
+ ftl::cuda::gt_analysis(
+ in.createTexture<float>(Channel::Depth),
+ in.createTexture<float>(Channel::GroundTruth),
+ output_,
+ in.getLeft(),
+ config()->value("bad_threshold", 2.0f),
+ stream
+ );
+ }
+
+ ftl::cuda::GTAnalysisData anal;
+ cudaMemcpy(&anal, output_, sizeof(anal), cudaMemcpyDeviceToHost);
+
+ auto &dmat = in.get<cv::cuda::GpuMat>(Channel::Depth);
+ int totalvalid = dmat.cols*dmat.rows - anal.invalid - anal.masked;
+ //int totaltested = dmat.cols*dmat.rows - anal.masked;
+
+ float pbad = float(anal.bad) / float(totalvalid) * 100.0f;
+ float pinvalid = float(anal.invalid) / float(dmat.cols*dmat.rows - anal.masked) * 100.0f;
+ float avgerr = anal.totalerror / float(totalvalid) * 100.0f;
+
+ std::vector<std::string> msgs;
+ if (in.hasChannel(Channel::Messages)) in.get(Channel::Messages, msgs);
+
+ msgs.push_back(string("Bad %: ") + to_string_with_precision(pbad, 1));
+ msgs.push_back(string("Invalid %: ") + to_string_with_precision(pinvalid,1));
+ msgs.push_back(string("Avg Error: ") + to_string_with_precision(avgerr, 2));
+
+ in.create(Channel::Messages, msgs);
+ }
+
+ return true;
+}
diff --git a/components/operators/src/gt_analysis.cu b/components/operators/src/gt_analysis.cu
new file mode 100644
index 0000000000000000000000000000000000000000..230f2fd20c44dce8e6db125434d6801b69183cef
--- /dev/null
+++ b/components/operators/src/gt_analysis.cu
@@ -0,0 +1,185 @@
+#include <ftl/operators/gt_cuda.hpp>
+
+#ifndef WARP_SIZE
+#define WARP_SIZE 32
+#endif
+
+#define FULL_MASK 0xffffffff
+
+template <bool COLOUR>
+__global__ void gt_anal_kernel(
+ uchar4* __restrict__ colour,
+ int cpitch,
+ int width,
+ int height,
+ const float* __restrict__ depth,
+ int dpitch,
+ const float* __restrict__ gt,
+ int gpitch,
+ ftl::cuda::GTAnalysisData *out,
+ ftl::rgbd::Camera cam,
+ float threshold,
+ float outmax
+) {
+
+ __shared__ int sinvalid;
+ __shared__ int sbad;
+ __shared__ int smasked;
+ __shared__ float serr;
+
+ if (threadIdx.x == 0 && threadIdx.y == 0) {
+ sinvalid = 0;
+ sbad = 0;
+ smasked = 0;
+ serr = 0.0f;
+ }
+ __syncthreads();
+
+ const unsigned int x = blockIdx.x*blockDim.x + threadIdx.x;
+
+ int invalid = 0;
+ int bad = 0;
+ int masked = 0;
+ float err = 0.0f;
+
+ const float numer = cam.baseline*cam.fx;
+
+ if (x < width) {
+ const float* __restrict__ gt_ptr = gt+x;
+ const float* __restrict__ d_ptr = depth+x;
+
+ for (STRIDE_Y(y, height)) {
+ // TODO: Verify gt and depth pitch are same
+ float gtval = gt_ptr[y*dpitch];
+ float dval = d_ptr[y*dpitch];
+
+ const int tmasked = (gtval > cam.minDepth && gtval < cam.maxDepth) ? 0 : 1;
+ const int tinvalid = (tmasked == 0 && (dval <= cam.minDepth || dval >= cam.maxDepth)) ? 1 : 0;
+
+ uchar4 c = make_uchar4((tinvalid==1)?255:0,0,0,255);
+
+ // Convert both to disparity...
+ if (tinvalid == 0 && tmasked == 0) {
+ dval = (numer / dval);
+ gtval = (numer / gtval);
+
+ const float e = fabsf(dval-gtval);
+ bad += (e >= threshold) ? 1 : 0;
+ err += e;
+
+ if (COLOUR) {
+ float nerr = min(1.0f, e / outmax);
+ c.z = min(255.0f, 255.0f * nerr);
+ }
+ }
+
+ invalid += tinvalid;
+ masked += tmasked;
+
+ if (COLOUR) colour[x+y*cpitch] = c;
+ }
+ }
+
+ // Warp aggregate
+ #pragma unroll
+ for (int i = WARP_SIZE/2; i > 0; i /= 2) {
+ bad += __shfl_xor_sync(FULL_MASK, bad, i, WARP_SIZE);
+ invalid += __shfl_xor_sync(FULL_MASK, invalid, i, WARP_SIZE);
+ masked += __shfl_xor_sync(FULL_MASK, masked, i, WARP_SIZE);
+ err += __shfl_xor_sync(FULL_MASK, err, i, WARP_SIZE);
+ }
+
+ // Block aggregate
+ if (threadIdx.x % WARP_SIZE == 0) {
+ atomicAdd(&serr, err);
+ atomicAdd(&sbad, bad);
+ atomicAdd(&sinvalid, invalid);
+ atomicAdd(&smasked, masked);
+ }
+
+ __syncthreads();
+
+ // Global aggregate
+ if (threadIdx.x == 0 && threadIdx.y == 0) {
+ atomicAdd(&out->totalerror, serr);
+ atomicAdd(&out->bad, sbad);
+ atomicAdd(&out->invalid, sinvalid);
+ atomicAdd(&out->masked, smasked);
+ }
+}
+
+void ftl::cuda::gt_analysis(
+ ftl::cuda::TextureObject<uchar4> &colour,
+ ftl::cuda::TextureObject<float> &depth,
+ ftl::cuda::TextureObject<float> >,
+ ftl::cuda::GTAnalysisData *out,
+ const ftl::rgbd::Camera &cam,
+ float threshold,
+ float outmax,
+ cudaStream_t stream
+) {
+ static constexpr int THREADS_X = 128;
+ static constexpr int THREADS_Y = 2;
+
+ const dim3 gridSize((depth.width() + THREADS_X - 1)/THREADS_X,16);
+ const dim3 blockSize(THREADS_X, THREADS_Y);
+
+ cudaMemsetAsync(out, 0, sizeof(ftl::cuda::GTAnalysisData), stream);
+
+ gt_anal_kernel<true><<<gridSize, blockSize, 0, stream>>>(
+ colour.devicePtr(),
+ colour.pixelPitch(),
+ colour.width(),
+ colour.height(),
+ depth.devicePtr(),
+ depth.pixelPitch(),
+ gt.devicePtr(),
+ gt.pixelPitch(),
+ out,
+ cam,
+ threshold,
+ outmax
+ );
+ cudaSafeCall( cudaGetLastError() );
+
+ #ifdef _DEBUG
+ cudaSafeCall(cudaDeviceSynchronize());
+ #endif
+}
+
+void ftl::cuda::gt_analysis(
+ ftl::cuda::TextureObject<float> &depth,
+ ftl::cuda::TextureObject<float> >,
+ ftl::cuda::GTAnalysisData *out,
+ const ftl::rgbd::Camera &cam,
+ float threshold,
+ cudaStream_t stream
+) {
+ static constexpr int THREADS_X = 128;
+ static constexpr int THREADS_Y = 2;
+
+ const dim3 gridSize((depth.width() + THREADS_X - 1)/THREADS_X, 16);
+ const dim3 blockSize(THREADS_X, THREADS_Y);
+
+ cudaMemsetAsync(out, 0, sizeof(ftl::cuda::GTAnalysisData), stream);
+
+ gt_anal_kernel<false><<<gridSize, blockSize, 0, stream>>>(
+ nullptr,
+ 0,
+ depth.width(),
+ depth.height(),
+ depth.devicePtr(),
+ depth.pixelPitch(),
+ gt.devicePtr(),
+ gt.pixelPitch(),
+ out,
+ cam,
+ threshold,
+ 1.0f
+ );
+ cudaSafeCall( cudaGetLastError() );
+
+ #ifdef _DEBUG
+ cudaSafeCall(cudaDeviceSynchronize());
+ #endif
+}
\ No newline at end of file
diff --git a/components/operators/src/weighting.cpp b/components/operators/src/weighting.cpp
index c1c57aa51739b6285248176c5d4ca7db36101917..5445d6786de3bc04bd515315910740a49d35290a 100644
--- a/components/operators/src/weighting.cpp
+++ b/components/operators/src/weighting.cpp
@@ -33,27 +33,29 @@ bool PixelWeights::apply(ftl::rgbd::Frame &in, ftl::rgbd::Frame &out, cudaStream
params.normals = config()->value("use_normals", true);
bool output_normals = config()->value("output_normals", params.normals);
- if (!in.hasChannel(Channel::Depth) || !in.hasChannel(Channel::Support1)) return false;
+ if ((!in.hasChannel(Channel::Depth) && !in.hasChannel(Channel::GroundTruth)) || !in.hasChannel(Channel::Support1)) return false;
+
+ Channel dchan = (in.hasChannel(Channel::Depth)) ? Channel::Depth : Channel::GroundTruth;
if (output_normals) {
ftl::cuda::pixel_weighting(
- out.createTexture<short>(Channel::Weights, ftl::rgbd::Format<short>(in.get<cv::cuda::GpuMat>(Channel::Depth).size())),
- out.createTexture<uint8_t>(Channel::Mask, ftl::rgbd::Format<uint8_t>(in.get<cv::cuda::GpuMat>(Channel::Depth).size())),
- out.createTexture<half4>(Channel::Normals, ftl::rgbd::Format<half4>(in.get<cv::cuda::GpuMat>(Channel::Depth).size())),
+ out.createTexture<short>(Channel::Weights, ftl::rgbd::Format<short>(in.get<cv::cuda::GpuMat>(dchan).size())),
+ out.createTexture<uint8_t>(Channel::Mask, ftl::rgbd::Format<uint8_t>(in.get<cv::cuda::GpuMat>(dchan).size())),
+ out.createTexture<half4>(Channel::Normals, ftl::rgbd::Format<half4>(in.get<cv::cuda::GpuMat>(dchan).size())),
in.createTexture<uchar4>(Channel::Support1),
- in.createTexture<float>(Channel::Depth),
+ in.createTexture<float>(dchan),
in.getLeftCamera(),
- in.get<cv::cuda::GpuMat>(Channel::Depth).size(),
+ in.get<cv::cuda::GpuMat>(dchan).size(),
params, stream
);
} else {
ftl::cuda::pixel_weighting(
- out.createTexture<short>(Channel::Weights, ftl::rgbd::Format<short>(in.get<cv::cuda::GpuMat>(Channel::Depth).size())),
- out.createTexture<uint8_t>(Channel::Mask, ftl::rgbd::Format<uint8_t>(in.get<cv::cuda::GpuMat>(Channel::Depth).size())),
+ out.createTexture<short>(Channel::Weights, ftl::rgbd::Format<short>(in.get<cv::cuda::GpuMat>(dchan).size())),
+ out.createTexture<uint8_t>(Channel::Mask, ftl::rgbd::Format<uint8_t>(in.get<cv::cuda::GpuMat>(dchan).size())),
in.createTexture<uchar4>(Channel::Support1),
- in.createTexture<float>(Channel::Depth),
+ in.createTexture<float>(dchan),
in.getLeftCamera(),
- in.get<cv::cuda::GpuMat>(Channel::Depth).size(),
+ in.get<cv::cuda::GpuMat>(dchan).size(),
params, stream
);
}
diff --git a/components/renderers/cpp/CMakeLists.txt b/components/renderers/cpp/CMakeLists.txt
index 9706be7c62bc97045431b213cf0388c66d3a3b51..8c5c1f7f078de8761c676627012862857fbbab63 100644
--- a/components/renderers/cpp/CMakeLists.txt
+++ b/components/renderers/cpp/CMakeLists.txt
@@ -10,6 +10,8 @@ add_library(ftlrender
src/colouriser.cpp
src/colour_util.cu
src/overlay.cpp
+ #src/assimp_render.cpp
+ #src/assimp_scene.cpp
)
# Various preprocessor definitions have been generated by NanoGUI
diff --git a/components/renderers/cpp/include/ftl/render/CUDARender.hpp b/components/renderers/cpp/include/ftl/render/CUDARender.hpp
index 25f0ffaa608354e4af63c9de128f5a38621d74f4..cfdb4cf4016ef00f6df3b9c37adbd6c26598c813 100644
--- a/components/renderers/cpp/include/ftl/render/CUDARender.hpp
+++ b/components/renderers/cpp/include/ftl/render/CUDARender.hpp
@@ -17,7 +17,7 @@ class Colouriser;
* Generate triangles between connected points and render those. Colour is done
* by weighted reprojection to the original source images.
*/
-class CUDARender : public ftl::render::Renderer {
+class CUDARender : public ftl::render::FSRenderer {
public:
explicit CUDARender(nlohmann::json &config);
~CUDARender();
@@ -28,6 +28,8 @@ class CUDARender : public ftl::render::Renderer {
bool submit(ftl::rgbd::FrameSet *in, ftl::codecs::Channels<0>, const Eigen::Matrix4d &t) override;
//void setOutputDevice(int);
+ void render() override;
+
void blend(ftl::codecs::Channel) override;
void setViewPort(ftl::render::ViewPortMode mode, const ftl::render::ViewPort &vp) {
diff --git a/components/renderers/cpp/include/ftl/render/assimp_render.hpp b/components/renderers/cpp/include/ftl/render/assimp_render.hpp
new file mode 100644
index 0000000000000000000000000000000000000000..b34c93f5dc781c9236b55c3158515c18f773bee1
--- /dev/null
+++ b/components/renderers/cpp/include/ftl/render/assimp_render.hpp
@@ -0,0 +1,50 @@
+#ifndef _FTL_RENDER_ASSIMP_HPP_
+#define _FTL_RENDER_ASSIMP_HPP_
+
+#include <ftl/render/renderer.hpp>
+#include <ftl/render/assimp_scene.hpp>
+//#include <GL/gl.h>
+//#include <GL/glext.h>
+#include <nanogui/glutil.h>
+
+namespace ftl {
+namespace render {
+
+/**
+ * Render Assimp library models using OpenGL into a frame object. The
+ * channels will be OpenGL pixel buffer objects, or should be created as such
+ * before hand. Begin, end, render etc must also be called in a valid OpenGL
+ * context.
+ */
+class AssimpRenderer : public ftl::render::Renderer {
+ public:
+ explicit AssimpRenderer(nlohmann::json &config);
+ ~AssimpRenderer();
+
+ void begin(ftl::rgbd::Frame &, ftl::codecs::Channel) override;
+
+ void end() override;
+
+ void render() override;
+
+ void blend(ftl::codecs::Channel) override;
+
+ void setScene(ftl::render::AssimpScene *);
+
+ /**
+ * Set the pose / model view matix for the scene not the camera.
+ */
+ void setPose(const Eigen::Matrix4d &pose);
+
+ private:
+ AssimpScene *scene_;
+ ftl::rgbd::Frame *out_;
+ ftl::codecs::Channel outchan_;
+ nanogui::GLShader shader_;
+ bool init_;
+};
+
+}
+}
+
+#endif
diff --git a/components/renderers/cpp/include/ftl/render/assimp_scene.hpp b/components/renderers/cpp/include/ftl/render/assimp_scene.hpp
new file mode 100644
index 0000000000000000000000000000000000000000..f07fd424a65d3704a3da8499d3408b7ae2ceed5a
--- /dev/null
+++ b/components/renderers/cpp/include/ftl/render/assimp_scene.hpp
@@ -0,0 +1,50 @@
+#ifndef _FTL_RENDER_ASSIMPSCENE_HPP_
+#define _FTL_RENDER_ASSIMPSCENE_HPP_
+
+#include <ftl/configurable.hpp>
+#include <assimp/scene.h>
+#include <nanogui/glutil.h>
+#include <Eigen/Eigen>
+
+#include <vector>
+#include <map>
+#include <unordered_map>
+
+namespace ftl {
+namespace render {
+
+struct GLMesh {
+ int material;
+ std::vector<int> indices;
+ Eigen::Matrix4d transform;
+};
+
+struct VBOData {
+ float vertex[3];
+ float normal[3];
+ unsigned char colour[4];
+ short uv[2];
+};
+
+class AssimpScene : public ftl::Configurable {
+ public:
+ explicit AssimpScene(nlohmann::json &config);
+ ~AssimpScene();
+
+ void load();
+
+ const aiScene *scene;
+ std::map<std::string, GLuint*> textureIdMap; // map image filenames to textureIds
+ std::vector<GLuint> textureIds;
+ std::vector<VBOData> vbo; // Combined vertex data
+ std::vector<GLMesh> meshes; // map materials to indicies vectors
+
+ private:
+ void _freeTextureIds();
+ bool _loadGLTextures();
+};
+
+}
+}
+
+#endif
diff --git a/components/renderers/cpp/include/ftl/render/renderer.hpp b/components/renderers/cpp/include/ftl/render/renderer.hpp
index 2273220cfcf198067c25d1a3d329a2cb4558b8d1..a7678b92fbb294c91169f28fa117950c6108a499 100644
--- a/components/renderers/cpp/include/ftl/render/renderer.hpp
+++ b/components/renderers/cpp/include/ftl/render/renderer.hpp
@@ -43,7 +43,23 @@ class Renderer : public ftl::Configurable {
*/
virtual void end()=0;
- /**
+ virtual void render()=0;
+
+ virtual void blend(ftl::codecs::Channel)=0;
+
+ protected:
+ Stage stage_;
+};
+
+/**
+ * A renderer specifically for RGB-D framesets.
+ */
+class FSRenderer : public ftl::render::Renderer {
+ public:
+ explicit FSRenderer(nlohmann::json &config) : ftl::render::Renderer(config) {};
+ virtual ~FSRenderer() {};
+
+ /**
* Render all frames of a frameset into the output frame. This can be called
* multiple times between `begin` and `end` to combine multiple framesets.
* Note that the frameset pointer must remain valid until `end` is called,
@@ -55,11 +71,6 @@ class Renderer : public ftl::Configurable {
* to RGB colour appropriately.
*/
virtual bool submit(ftl::rgbd::FrameSet *, ftl::codecs::Channels<0>, const Eigen::Matrix4d &)=0;
-
- virtual void blend(ftl::codecs::Channel)=0;
-
- protected:
- Stage stage_;
};
}
diff --git a/components/renderers/cpp/include/ftl/render/vbo.hpp b/components/renderers/cpp/include/ftl/render/vbo.hpp
new file mode 100644
index 0000000000000000000000000000000000000000..ab7ab6ba1530cb30a703226640fdd43caec1be7c
--- /dev/null
+++ b/components/renderers/cpp/include/ftl/render/vbo.hpp
@@ -0,0 +1,40 @@
+#ifndef _FTL_RENDER_VBO_HPP_
+#define _FTL_RENDER_VBO_HPP_
+
+namespace ftl {
+namespace render {
+
+/**
+ * OpenGL Vertex Buffer Object wrapper.
+ */
+class VBO {
+ public:
+ VBO();
+ ~VBO();
+
+ void begin();
+ void end();
+
+ void bind();
+ void unbind();
+
+ void writeVertex3D(float x, float y, float z);
+ void writeVertex3D(const float3 &v);
+ void writeVertex3D(const Eigen::Vertex3f &v);
+ void writeVertices3D(float *v, size_t c);
+ void writeVertices3D(float3 *v, size_t c);
+
+ void writeColour(uchar r, uchar g, uchar b, uchar a);
+ void writeColour(uchar4 c);
+ void writeColour(float r, float g, float b, float a);
+ void writeColours(float *v, size_t);
+ void writeColours(uchar4 *c, size_t);
+
+ void writeTexCoords(float u, float v);
+ void writeTexCoords(const float2 &uv);
+};
+
+}
+}
+
+#endif
\ No newline at end of file
diff --git a/components/renderers/cpp/src/CUDARender.cpp b/components/renderers/cpp/src/CUDARender.cpp
index 9fb3c045c66bbefa8fa7adaf820d9b8662ba7098..c2673073bf35d959a6c79441be8a67155b94a8e6 100644
--- a/components/renderers/cpp/src/CUDARender.cpp
+++ b/components/renderers/cpp/src/CUDARender.cpp
@@ -31,7 +31,7 @@ using ftl::cuda::Mask;
using ftl::render::parseCUDAColour;
using ftl::render::parseCVColour;
-CUDARender::CUDARender(nlohmann::json &config) : ftl::render::Renderer(config), scene_(nullptr) {
+CUDARender::CUDARender(nlohmann::json &config) : ftl::render::FSRenderer(config), scene_(nullptr) {
/*if (config["clipping"].is_object()) {
auto &c = config["clipping"];
float rx = c.value("pitch", 0.0f);
@@ -139,6 +139,19 @@ void CUDARender::_renderChannel(ftl::rgbd::Frame &output, ftl::codecs::Channel i
f.getLeftCamera(),
transform, transformR, stream
);
+ } else if (f.hasChannel(Channel::GroundTruth)) {
+ ftl::cuda::reproject(
+ texture,
+ f.createTexture<float>(Channel::GroundTruth),
+ output.getTexture<float>(_getDepthChannel()),
+ f.createTexture<short>(Channel::Weights),
+ (mesh_) ? &output.getTexture<half4>(_getNormalsChannel()) : nullptr,
+ accum_,
+ contrib_,
+ params_,
+ f.getLeftCamera(),
+ transform, transformR, stream
+ );
} else {
// Reproject without depth channel or normals
ftl::cuda::reproject(
@@ -254,6 +267,13 @@ void CUDARender::_mesh(ftl::rgbd::Frame &out, const Eigen::Matrix4d &t, cudaStre
screenbuffer,
params_, transform, f.getLeftCamera(), stream
);
+ } else if (f.hasChannel(Channel::GroundTruth)) {
+ ftl::cuda::screen_coord(
+ f.createTexture<float>(Channel::GroundTruth),
+ depthbuffer,
+ screenbuffer,
+ params_, transform, f.getLeftCamera(), stream
+ );
} else {
// Constant depth version
ftl::cuda::screen_coord(
@@ -484,13 +504,19 @@ void CUDARender::begin(ftl::rgbd::Frame &out, ftl::codecs::Channel chan) {
stage_ = Stage::ReadySubmit;
}
-void CUDARender::blend(Channel c) {
- if (stage_ == Stage::Finished) {
- throw FTL_Error("Cannot call blend at this time");
- } else if (stage_ == Stage::ReadySubmit) {
+void CUDARender::render() {
+ if (stage_ != Stage::ReadySubmit) {
+ throw FTL_Error("Cannot call render at this time");
+ } else {
stage_ = Stage::Blending;
_endSubmit();
}
+}
+
+void CUDARender::blend(Channel c) {
+ if (stage_ != Stage::Blending) {
+ throw FTL_Error("Cannot call blend at this time");
+ }
cv::cuda::Stream cvstream = cv::cuda::StreamAccessor::wrapStream(stream_);
diff --git a/components/renderers/cpp/src/assimp_render.cpp b/components/renderers/cpp/src/assimp_render.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..c199ca9160061556518840fac4280749d18bab23
--- /dev/null
+++ b/components/renderers/cpp/src/assimp_render.cpp
@@ -0,0 +1,112 @@
+#include <ftl/render/assimp_render.hpp>
+
+#include <assimp/Importer.hpp>
+#include <assimp/postprocess.h>
+#include <assimp/scene.h>
+#include <GL/gl.h>
+
+using ftl::render::AssimpRenderer;
+
+namespace {
+ constexpr char const *const assimpVertexShader =
+ R"(#version 330
+ in vec3 vertex;
+ uniform float focal;
+ uniform float width;
+ uniform float height;
+ uniform float far;
+ uniform float near;
+ uniform mat4 pose;
+ uniform vec3 scale;
+
+ void main() {
+ vec4 vert = pose*(vec4(scale*vertex,1.0));
+ vert = vert / vert.w;
+ //vec4 pos = vec4(-vert.x*focal / -vert.z / (width/2.0),
+ // vert.y*focal / -vert.z / (height/2.0),
+ // (vert.z-near) / (far-near) * 2.0 - 1.0, 1.0);
+
+ vec4 pos = vec4(
+ vert.x*focal / (width/2.0),
+ -vert.y*focal / (height/2.0),
+ -vert.z * ((far+near) / (far-near)) + (2.0 * near * far / (far-near)),
+ //((vert.z - near) / (far - near) * 2.0 - 1.0) * vert.z,
+ vert.z
+ );
+
+ gl_Position = pos;
+ })";
+
+ constexpr char const *const assimpFragmentShader =
+ R"(#version 330
+ uniform vec4 blockColour;
+ out vec4 color;
+
+ void main() {
+ color = blockColour;
+ })";
+}
+
+AssimpRenderer::AssimpRenderer(nlohmann::json &config) : ftl::render::Renderer(config) {
+ init_ = false;
+}
+
+AssimpRenderer::~AssimpRenderer() {
+
+}
+
+void AssimpRenderer::begin(ftl::rgbd::Frame &f, ftl::codecs::Channel c) {
+ if (!scene_) {
+ throw FTL_Error("No Assimp scene");
+ }
+
+ out_ = &f;
+ outchan_ = c;
+
+ if (!init_) {
+ shader_.init("AssimpShader", assimpVertexShader, assimpFragmentShader);
+ shader_.bind();
+ init_ = true;
+ } else {
+ shader_.bind();
+ }
+
+ const auto &intrin = f.getLeftCamera();
+
+ glViewport(0, 0, intrin.width, intrin.height); // Reset The Current Viewport
+
+ glMatrixMode(GL_PROJECTION); // Select The Projection Matrix
+ glLoadIdentity(); // Reset The Projection Matrix
+
+ //gluPerspective(2.0f*atan(0.5f*float(intrin.height) / intrin.fy),(GLfloat)intrin.width/(GLfloat)intrin.height,intrin.minDepth,intrin.maxDepth);
+
+ glMatrixMode(GL_MODELVIEW); // Select The Modelview Matrix
+ glLoadIdentity();
+
+ glEnable(GL_TEXTURE_2D);
+ glShadeModel(GL_SMOOTH); // Enables Smooth Shading
+ glClearColor(1.0f, 1.0f, 1.0f, 0.0f);
+ glClearDepth(1.0f); // Depth Buffer Setup
+ glEnable(GL_DEPTH_TEST); // Enables Depth Testing
+ glDepthFunc(GL_LEQUAL); // The Type Of Depth Test To Do
+
+ glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT);
+
+ scene_->load();
+}
+
+void AssimpRenderer::end() {
+ glFlush();
+}
+
+void AssimpRenderer::render() {
+
+}
+
+void AssimpRenderer::blend(ftl::codecs::Channel) {
+ throw FTL_Error("Blend not supported in Assimp render");
+}
+
+void AssimpRenderer::setScene(ftl::render::AssimpScene *scene) {
+ scene_ = scene;
+}
diff --git a/components/renderers/cpp/src/assimp_scene.cpp b/components/renderers/cpp/src/assimp_scene.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..9b2adf294a82912994c72faa9f5ef75276215f96
--- /dev/null
+++ b/components/renderers/cpp/src/assimp_scene.cpp
@@ -0,0 +1,133 @@
+#include <ftl/render/assimp_scene.hpp>
+
+#include <assimp/Importer.hpp>
+#include <assimp/postprocess.h>
+#include <assimp/scene.h>
+
+#include <opencv2/imgproc.hpp>
+#include <opencv2/imgcodecs.hpp>
+
+using ftl::render::AssimpScene;
+using std::string;
+
+static std::string getBasePath(const std::string& path) {
+ size_t pos = path.find_last_of("\\/");
+ return (std::string::npos == pos) ? "" : path.substr(0, pos + 1);
+}
+
+AssimpScene::AssimpScene(nlohmann::json &config) : ftl::Configurable(config) {
+
+}
+
+AssimpScene::~AssimpScene() {
+ _freeTextureIds();
+}
+
+void AssimpScene::load() {
+ if (scene) return;
+
+ Assimp::Importer importer;
+ scene = importer.ReadFile(value("model", std::string("")), aiProcessPreset_TargetRealtime_Quality);
+
+ if (!scene) {
+ throw FTL_Error("Could not load model: " << value("model", string("")));
+ }
+
+ _loadGLTextures();
+
+ for (size_t n=0; n<scene->mNumMeshes; ++n) {
+ const aiMesh* mesh = scene->mMeshes[n];
+
+ vbo.reserve(vbo.size()+mesh->mNumVertices);
+ for (size_t i=0; i<mesh->mNumVertices; ++i) {
+ //vertices.push_back(mesh->mVertices)
+ }
+ }
+}
+
+void AssimpScene::_freeTextureIds() {
+ textureIdMap.clear();
+}
+
+bool AssimpScene::_loadGLTextures() {
+ _freeTextureIds();
+
+ if (scene->HasTextures()) return true;
+
+ /* getTexture Filenames and Numb of Textures */
+ for (unsigned int m=0; m<scene->mNumMaterials; m++)
+ {
+ int texIndex = 0;
+ aiReturn texFound = AI_SUCCESS;
+
+ aiString path; // filename
+
+ while (texFound == AI_SUCCESS)
+ {
+ texFound = scene->mMaterials[m]->GetTexture(aiTextureType_DIFFUSE, texIndex, &path);
+ textureIdMap[path.data] = NULL; //fill map with textures, pointers still NULL yet
+ texIndex++;
+ }
+ }
+
+ const size_t numTextures = textureIdMap.size();
+
+ /* create and fill array with GL texture ids */
+ //textureIds = new GLuint[numTextures];
+ textureIds.resize(numTextures);
+ glGenTextures(static_cast<GLsizei>(numTextures), textureIds.data()); /* Texture name generation */
+
+ /* get iterator */
+ std::map<std::string, GLuint*>::iterator itr = textureIdMap.begin();
+
+ std::string basepath = getBasePath(value("model", std::string("")));
+ for (size_t i=0; i<numTextures; i++) {
+
+ //save IL image ID
+ std::string filename = (*itr).first; // get filename
+ (*itr).second = &textureIds[i]; // save texture id for filename in map
+ ++itr; // next texture
+
+
+ //ilBindImage(imageIds[i]); /* Binding of DevIL image name */
+ std::string fileloc = basepath + filename; /* Loading of image */
+ //success = ilLoadImage(fileloc.c_str());
+ int x, y, n;
+ cv::Mat img = cv::imread(fileloc);
+ unsigned char *data = img.data;
+
+ if (!img.empty()) {
+ // Convert every colour component into unsigned byte.If your image contains
+ // alpha channel you can replace IL_RGB with IL_RGBA
+ //success = ilConvertImage(IL_RGB, IL_UNSIGNED_BYTE);
+ /*if (!success)
+ {
+ abortGLInit("Couldn't convert image");
+ return -1;
+ }*/
+ // Binding of texture name
+ glBindTexture(GL_TEXTURE_2D, textureIds[i]);
+ // redefine standard texture values
+ // We will use linear interpolation for magnification filter
+ glTexParameteri( GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR );
+ // We will use linear interpolation for minifying filter
+ glTexParameteri( GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR );
+ // Texture specification
+ glTexImage2D(GL_TEXTURE_2D, 0, n, x, y, 0, GL_RGBA, GL_UNSIGNED_BYTE, data);// Texture specification.
+
+ // we also want to be able to deal with odd texture dimensions
+ glPixelStorei( GL_UNPACK_ALIGNMENT, 1 );
+ glPixelStorei( GL_UNPACK_ROW_LENGTH, 0 );
+ glPixelStorei( GL_UNPACK_SKIP_PIXELS, 0 );
+ glPixelStorei( GL_UNPACK_SKIP_ROWS, 0 );
+ }
+ else
+ {
+ /* Error occurred */
+ //MessageBox(NULL, UTFConverter("Couldn't load Image: " + fileloc).c_wstr(), TEXT("ERROR"), MB_OK | MB_ICONEXCLAMATION);
+ throw FTL_Error("Could no load texture image: " << fileloc);
+ }
+ }
+
+ return true;
+}
\ No newline at end of file
diff --git a/components/renderers/cpp/src/colouriser.cpp b/components/renderers/cpp/src/colouriser.cpp
index 69ad566a20ea391c32fea468ab5e7cb38ef63027..be92ed854f5699bf2b20883ce6d8424094371659 100644
--- a/components/renderers/cpp/src/colouriser.cpp
+++ b/components/renderers/cpp/src/colouriser.cpp
@@ -118,6 +118,7 @@ TextureObject<uchar4> &Colouriser::colourise(ftl::rgbd::Frame &f, Channel c, cud
case Channel::ColourHighRes :
case Channel::Colour :
case Channel::Colour2 : return _processColour(f,c,stream);
+ case Channel::GroundTruth :
case Channel::Depth :
case Channel::Depth2 : return _processFloat(f,c, value("depth_min", f.getLeft().minDepth), value("depth_max", f.getLeft().maxDepth), stream);
case Channel::Normals :
diff --git a/components/rgbd-sources/include/ftl/rgbd/frame.hpp b/components/rgbd-sources/include/ftl/rgbd/frame.hpp
index 6cbfcec5599b3eecee7a68e115d5874027eae13f..8fc747cd7f968bef73a09161619774bb9c607ed6 100644
--- a/components/rgbd-sources/include/ftl/rgbd/frame.hpp
+++ b/components/rgbd-sources/include/ftl/rgbd/frame.hpp
@@ -76,7 +76,11 @@ class Frame : public ftl::data::Frame<0,32,ftl::rgbd::FrameState,VideoData> {
public:
using ftl::data::Frame<0,32,ftl::rgbd::FrameState,VideoData>::create;
- Frame() {}
+ Frame();
+ Frame(Frame &&f);
+ ~Frame();
+
+ Frame &operator=(Frame &&f);
// Prevent frame copy, instead use a move.
//Frame(const Frame &)=delete;
diff --git a/components/rgbd-sources/src/frame.cpp b/components/rgbd-sources/src/frame.cpp
index c5200dbca2e3988400155a6708dfda0c449b106f..8c809c026650015ca28b3a2074faaa9ab4fffb29 100644
--- a/components/rgbd-sources/src/frame.cpp
+++ b/components/rgbd-sources/src/frame.cpp
@@ -1,6 +1,9 @@
#include <ftl/rgbd/frame.hpp>
+#define LOGURU_REPLACE_GLOG 1
+#include <loguru.hpp>
+
using ftl::rgbd::Frame;
using ftl::rgbd::FrameState;
using ftl::codecs::Channels;
@@ -9,6 +12,7 @@ using ftl::rgbd::VideoData;
static cv::Mat none;
static cv::cuda::GpuMat noneGPU;
+static std::atomic<int> frame_count = 0;
template <>
cv::Mat &VideoData::as<cv::Mat>() {
@@ -72,6 +76,24 @@ cv::cuda::GpuMat &VideoData::make<cv::cuda::GpuMat>() {
}
}*/
+Frame::Frame() {
+ ++frame_count;
+ //LOG(INFO) << "Frames: " << frame_count;
+}
+
+Frame::Frame(Frame &&f) : ftl::data::Frame<0,32,ftl::rgbd::FrameState,VideoData>(std::move(f)) {
+
+}
+
+Frame &Frame::operator=(Frame &&f) {
+ ftl::data::Frame<0,32,ftl::rgbd::FrameState,VideoData>::operator=(std::move(f));
+ return *this;
+}
+
+Frame::~Frame() {
+ --frame_count;
+}
+
void Frame::download(Channel c, cv::cuda::Stream stream) {
download(Channels(c), stream);
}
diff --git a/components/streams/include/ftl/streams/filestream.hpp b/components/streams/include/ftl/streams/filestream.hpp
index a5d75b9477d07ad464ac28dc007100a34c337bdc..bd6a580ea03d7358b8254ae5f4ad7cce76814466 100644
--- a/components/streams/include/ftl/streams/filestream.hpp
+++ b/components/streams/include/ftl/streams/filestream.hpp
@@ -70,6 +70,7 @@ class File : public Stream {
bool active_;
int version_;
ftl::timer::TimerHandle timer_;
+ bool is_video_;
StreamCallback cb_;
MUTEX mutex_;
diff --git a/components/streams/src/injectors.hpp b/components/streams/include/ftl/streams/injectors.hpp
similarity index 100%
rename from components/streams/src/injectors.hpp
rename to components/streams/include/ftl/streams/injectors.hpp
diff --git a/components/streams/src/parsers.hpp b/components/streams/include/ftl/streams/parsers.hpp
similarity index 100%
rename from components/streams/src/parsers.hpp
rename to components/streams/include/ftl/streams/parsers.hpp
diff --git a/components/streams/src/filestream.cpp b/components/streams/src/filestream.cpp
index 189a970f4674f051a2fec32d10923c1822855211..75de7bdc3f2ddd79476acff00c97c0d071cfa9b0 100644
--- a/components/streams/src/filestream.cpp
+++ b/components/streams/src/filestream.cpp
@@ -71,7 +71,10 @@ bool File::_checkFile() {
checked_ = true;
+ is_video_ = count < 9;
+
LOG(INFO) << " -- Frame rate = " << (1000 / min_ts_diff);
+ if (!is_video_) LOG(INFO) << " -- Static image";
interval_ = min_ts_diff;
return true;
}
@@ -222,6 +225,7 @@ bool File::tick(int64_t ts) {
// Adjust timestamp
// FIXME: A potential bug where multiple times are merged into one?
std::get<0>(data).timestamp = (((std::get<0>(data).timestamp) - first_ts_) / interval_) * interval_ + timestart_;
+ std::get<0>(data).hint_capability = (is_video_) ? 0 : ftl::codecs::kStreamCap_Static;
// Maintain availability of channels.
available(0) += std::get<0>(data).channel;
diff --git a/components/streams/src/injectors.cpp b/components/streams/src/injectors.cpp
index 2a58e5eac0fe174233b39a4a7ba501f3a54e3795..01dcbef368a8b642abbdf91b25aa31e3c8ee857c 100644
--- a/components/streams/src/injectors.cpp
+++ b/components/streams/src/injectors.cpp
@@ -1,4 +1,4 @@
-#include "injectors.hpp"
+#include <ftl/streams/injectors.hpp>
#include <ftl/utility/vectorbuffer.hpp>
using ftl::codecs::Channel;
diff --git a/components/streams/src/netstream.cpp b/components/streams/src/netstream.cpp
index a7cef8a2d5c225cdeece3e3971f55aafaeb15a28..0cba3f4dc533a3cfd71b116e0da9709718fbe3da 100644
--- a/components/streams/src/netstream.cpp
+++ b/components/streams/src/netstream.cpp
@@ -154,6 +154,8 @@ bool Net::begin() {
// FIXME: see #335
//spkt.timestamp -= clock_adjust_;
spkt.originClockDelta = clock_adjust_;
+ spkt.hint_capability = 0;
+ spkt.hint_source_total = 0;
//LOG(INFO) << "LATENCY: " << ftl::timer::get_time() - spkt.localTimestamp() << " : " << spkt.timestamp << " - " << clock_adjust_;
spkt.version = 4;
@@ -275,7 +277,10 @@ bool Net::_sendRequest(Channel c, uint8_t frameset, uint8_t frames, uint8_t coun
ftl::timer::get_time(),
frameset,
frames,
- c
+ c,
+ 0,
+ 0,
+ 0
};
net_->send(peer_, uri_, (short)0, spkt, pkt);
diff --git a/components/streams/src/parsers.cpp b/components/streams/src/parsers.cpp
index 95c521da670eaead6ac6723e4636e9b1440d8b2e..29a3350ec1550f26a38856cfe3137758998c1032 100644
--- a/components/streams/src/parsers.cpp
+++ b/components/streams/src/parsers.cpp
@@ -1,4 +1,4 @@
-#include "parsers.hpp"
+#include <ftl/streams/parsers.hpp>
#include <loguru.hpp>
diff --git a/components/streams/src/receiver.cpp b/components/streams/src/receiver.cpp
index d34c256bf8b9287ed4ec94947274c25df4687f21..ac61f171786ded4a1e2f2c54a4c2c3dacd881e2a 100644
--- a/components/streams/src/receiver.cpp
+++ b/components/streams/src/receiver.cpp
@@ -4,8 +4,8 @@
#include <opencv2/cudaimgproc.hpp>
-#include "parsers.hpp"
-#include "injectors.hpp"
+#include <ftl/streams/parsers.hpp>
+#include <ftl/streams/injectors.hpp>
#define LOGURU_REPLACE_GLOG 1
#include <loguru.hpp>
@@ -205,6 +205,8 @@ void Receiver::_processVideo(const StreamPacket &spkt, const Packet &pkt) {
// Allocate a decode surface, this is a tiled image to be split later
surface.create(height*ty, width*tx, ((isFloatChannel(spkt.channel)) ? ((pkt.flags & 0x2) ? CV_16UC4 : CV_16U) : CV_8UC4));
+ bool is_static = ividstate.decoders[channum] && (spkt.hint_capability & ftl::codecs::kStreamCap_Static);
+
// Find or create the decoder
_createDecoder(ividstate, channum, pkt);
auto *decoder = ividstate.decoders[channum];
@@ -214,15 +216,17 @@ void Receiver::_processVideo(const StreamPacket &spkt, const Packet &pkt) {
}
// Do the actual decode into the surface buffer
- try {
- FTL_Profile("Decode", 0.015);
- if (!decoder->decode(pkt, surface)) {
- LOG(ERROR) << "Decode failed on channel " << (int)spkt.channel;
+ if (!is_static) {
+ try {
+ FTL_Profile("Decode", 0.015);
+ if (!decoder->decode(pkt, surface)) {
+ LOG(ERROR) << "Decode failed on channel " << (int)spkt.channel;
+ return;
+ }
+ } catch (std::exception &e) {
+ LOG(ERROR) << "Decode failed for " << spkt.timestamp << ": " << e.what();
return;
}
- } catch (std::exception &e) {
- LOG(ERROR) << "Decode failed for " << spkt.timestamp << ": " << e.what();
- return;
}
auto cvstream = cv::cuda::StreamAccessor::wrapStream(decoder->stream());
diff --git a/components/streams/src/sender.cpp b/components/streams/src/sender.cpp
index b85625fa0e56acc107ef840dc6d36792989792a0..18717bb4becda1344c265f7df5bb454c5edcf233 100644
--- a/components/streams/src/sender.cpp
+++ b/components/streams/src/sender.cpp
@@ -4,7 +4,7 @@
#include <opencv2/cudaimgproc.hpp>
-#include "injectors.hpp"
+#include <ftl/streams/injectors.hpp>
#define LOGURU_REPLACE_GLOG 1
#include <loguru.hpp>
diff --git a/components/streams/test/receiver_unit.cpp b/components/streams/test/receiver_unit.cpp
index 2aadaa8e4cc4d51ba2598c75a7f8752d8a9aea91..755c55c2293717c3490171813f08abebec735343 100644
--- a/components/streams/test/receiver_unit.cpp
+++ b/components/streams/test/receiver_unit.cpp
@@ -2,7 +2,7 @@
#include <ftl/streams/receiver.hpp>
#include <ftl/codecs/nvpipe_encoder.hpp>
-#include "../src/injectors.hpp"
+#include <ftl/streams/injectors.hpp>
#include <nlohmann/json.hpp>
diff --git a/components/structures/include/ftl/data/frame.hpp b/components/structures/include/ftl/data/frame.hpp
index 88f31c4b8dcd22f57ea953e9978fdc5679e57bd5..637621169847822ae835abdf0049dfee8a9d1b07 100644
--- a/components/structures/include/ftl/data/frame.hpp
+++ b/components/structures/include/ftl/data/frame.hpp
@@ -54,10 +54,20 @@ class Frame {
public:
Frame() : origin_(nullptr) {}
+ Frame(Frame &&f) {
+ f.swapTo(*this);
+ f.reset();
+ }
+
+ Frame &operator=(Frame &&f) {
+ f.swapTo(*this);
+ f.reset();
+ return *this;
+ }
// Prevent frame copy, instead use a move.
- //Frame(const Frame &)=delete;
- //Frame &operator=(const Frame &)=delete;
+ Frame(const Frame &)=delete;
+ Frame &operator=(const Frame &)=delete;
/**
* Perform a buffer swap of the selected channels. This is intended to be