From 3fcab6e281bfe3e138301505042eb364e2a2760f Mon Sep 17 00:00:00 2001
From: Nicolas Pope <nwpope@utu.fi>
Date: Mon, 9 Nov 2020 10:38:47 +0200
Subject: [PATCH] Fix some bugs
---
SDK/CPP/private/pipeline_impl.cpp | 4 +-
SDK/CPP/private/room_impl.cpp | 4 +-
SDK/CPP/public/CMakeLists.txt | 1 +
SDK/CPP/public/samples/common/cmd_args.cpp | 35 +++++++++++
SDK/CPP/public/samples/common/cmd_args.hpp | 6 ++
.../public/samples/fusion_evaluator/main.cpp | 59 +++++++++++++++++--
components/codecs/src/opencv_decoder.cpp | 2 +
.../include/ftl/operators/fusion.hpp | 2 +-
.../src/analysis/evaluation/gt_analysis.cpp | 11 ++++
.../operators/src/fusion/carving/carver.cu | 1 +
components/operators/src/fusion/fusion.cpp | 13 +++-
.../fusion/smoothing/mls_multi_weighted.cu | 2 +-
components/rgbd-sources/src/frame.cpp | 2 +-
components/streams/src/feed.cpp | 8 ++-
components/streams/src/receiver.cpp | 2 +-
15 files changed, 136 insertions(+), 16 deletions(-)
create mode 100644 SDK/CPP/public/samples/common/cmd_args.cpp
create mode 100644 SDK/CPP/public/samples/common/cmd_args.hpp
diff --git a/SDK/CPP/private/pipeline_impl.cpp b/SDK/CPP/private/pipeline_impl.cpp
index 114f73ad4..cdc52fce3 100644
--- a/SDK/CPP/private/pipeline_impl.cpp
+++ b/SDK/CPP/private/pipeline_impl.cpp
@@ -6,6 +6,8 @@
#include <ftl/operators/fusion.hpp>
#include <ftl/operators/gt_analysis.hpp>
+#include <loguru.hpp>
+
using voltu::internal::PipelineImpl;
PipelineImpl::PipelineImpl(ftl::Configurable *root)
@@ -28,7 +30,7 @@ void PipelineImpl::submit(const voltu::FramePtr &frame)
const auto &sets = fimp->getInternalFrameSets();
- if (sets.size() > 1) throw voltu::exceptions::IncompatibleOperation();
+ if (sets.size() > 1 || sets.size() == 0) throw voltu::exceptions::IncompatibleOperation();
for (const auto &fs : sets)
{
diff --git a/SDK/CPP/private/room_impl.cpp b/SDK/CPP/private/room_impl.cpp
index 76ab1b92b..764b57403 100644
--- a/SDK/CPP/private/room_impl.cpp
+++ b/SDK/CPP/private/room_impl.cpp
@@ -20,7 +20,7 @@ bool RoomImpl::waitNextFrame(int64_t timeout)
{
if (!filter_)
{
- filter_ = feed_->filter(fsids_, {ftl::codecs::Channel::Colour, ftl::codecs::Channel::Depth});
+ filter_ = feed_->filter(fsids_, {ftl::codecs::Channel::Colour, ftl::codecs::Channel::Depth, ftl::codecs::Channel::GroundTruth});
filter_->on([this](const std::shared_ptr<ftl::data::FrameSet> &fs)
{
UNIQUE_LOCK(mutex_, lk);
@@ -36,7 +36,7 @@ bool RoomImpl::waitNextFrame(int64_t timeout)
{
if (timeout > 0)
{
- cv_.wait_for(lk, std::chrono::seconds(timeout), [this] {
+ cv_.wait_for(lk, std::chrono::milliseconds(timeout), [this] {
return last_read_ < last_seen_;
});
diff --git a/SDK/CPP/public/CMakeLists.txt b/SDK/CPP/public/CMakeLists.txt
index a0eb7eaf5..b3a4d7dc2 100644
--- a/SDK/CPP/public/CMakeLists.txt
+++ b/SDK/CPP/public/CMakeLists.txt
@@ -64,6 +64,7 @@ target_link_libraries(voltu_basic_virtual_cam voltu_sdk)
add_executable(voltu_fusion_evaluator
samples/fusion_evaluator/main.cpp
+ samples/common/cmd_args.cpp
)
target_link_libraries(voltu_fusion_evaluator voltu_sdk)
diff --git a/SDK/CPP/public/samples/common/cmd_args.cpp b/SDK/CPP/public/samples/common/cmd_args.cpp
new file mode 100644
index 000000000..7219fb215
--- /dev/null
+++ b/SDK/CPP/public/samples/common/cmd_args.cpp
@@ -0,0 +1,35 @@
+#include "cmd_args.hpp"
+
+std::map<std::string, std::string> read_options(char ***argv, int *argc)
+{
+ std::map<std::string, std::string> opts;
+
+ (*argc)--; // Remove application path
+ (*argv)++;
+
+ while (*argc > 0) {
+ std::string cmd((*argv)[0]);
+
+ size_t p;
+ if (cmd[0] != '-' || (p = cmd.find("=")) == std::string::npos) {
+ opts[cmd.substr(0)] = "true";
+ } else {
+ auto val = cmd.substr(p+1);
+#ifdef WIN32
+ if ((val[0] >= 48 && val[0] <= 57) || val == "true" || val == "false" || val == "null") {
+#else
+ if (std::isdigit(val[0]) || val == "true" || val == "false" || val == "null") {
+#endif
+ opts[cmd.substr(0, p-2)] = val;
+ } else {
+ if (val[0] == '\\') opts[cmd.substr(2, p-2)] = val;
+ else opts[cmd.substr(0, p-2)] = "\""+val+"\"";
+ }
+ }
+
+ (*argc)--;
+ (*argv)++;
+ }
+
+ return opts;
+}
diff --git a/SDK/CPP/public/samples/common/cmd_args.hpp b/SDK/CPP/public/samples/common/cmd_args.hpp
new file mode 100644
index 000000000..a42486b42
--- /dev/null
+++ b/SDK/CPP/public/samples/common/cmd_args.hpp
@@ -0,0 +1,6 @@
+#pragma once
+
+#include <map>
+#include <string>
+
+std::map<std::string, std::string> read_options(char ***argv, int *argc);
diff --git a/SDK/CPP/public/samples/fusion_evaluator/main.cpp b/SDK/CPP/public/samples/fusion_evaluator/main.cpp
index 908d0925e..7eebe3461 100644
--- a/SDK/CPP/public/samples/fusion_evaluator/main.cpp
+++ b/SDK/CPP/public/samples/fusion_evaluator/main.cpp
@@ -4,6 +4,8 @@
#include <thread>
#include <chrono>
+#include "../common/cmd_args.hpp"
+
#include <opencv2/highgui.hpp>
using std::cout;
@@ -12,19 +14,51 @@ using std::string;
int main(int argc, char **argv)
{
- if (argc != 2) return -1;
+ bool do_fusion = true;
+ bool do_eval = true;
+ voltu::Channel display_channel = voltu::Channel::kColour;
+ std::list<std::string> paths;
+
+ auto opts = read_options(&argv, &argc);
+
+ for (const auto &s : opts)
+ {
+ if (s.first == "--no-fusion")
+ {
+ do_fusion = false;
+ }
+ else if (s.first == "--display")
+ {
+ if (s.second == "normals")
+ {
+ display_channel = voltu::Channel::kNormals;
+ }
+ }
+ else if (s.first == "--no-eval")
+ {
+ do_eval = false;
+ }
+ else if (s.first[0] != '-')
+ {
+ paths.push_back(s.first);
+ }
+ }
auto vtu = voltu::instance();
- if (!vtu->open(argv[1]))
+ for (const auto &p : paths)
{
- cout << "Could not open source" << endl;
- return -1;
+ if (!vtu->open(p))
+ {
+ cout << "Could not open source" << endl;
+ return -1;
+ }
}
while (vtu->listRooms().size() == 0)
{
std::this_thread::sleep_for(std::chrono::milliseconds(100));
+ cout << "Wait room..." << endl;
}
auto room = vtu->getRoom(vtu->listRooms().front());
@@ -34,18 +68,31 @@ int main(int argc, char **argv)
return -1;
}
+ //room->waitNextFrame(5000);
auto frame = room->getFrame();
auto pipe = vtu->createPipeline();
auto op1 = pipe->appendOperator(voltu::OperatorId::kFusion);
auto op2 = pipe->appendOperator(voltu::OperatorId::kGTEvaluator);
+ op1->property("enabled")->setBool(do_fusion);
+ op2->property("enabled")->setBool(do_eval);
op2->property("show_colour")->setBool(true);
pipe->submit(frame);
- pipe->waitCompletion(1000);
+ if (!pipe->waitCompletion(3000))
+ {
+ cout << "Pipeline timeout" << endl;
+ return -1;
+ }
+
+ auto imgset = frame->getImageSet(display_channel);
- auto imgset = frame->getImageSet(voltu::Channel::kColour);
+ if (imgset.size() == 0)
+ {
+ cout << "No images!" << endl;
+ return -1;
+ }
for (auto img : imgset)
{
diff --git a/components/codecs/src/opencv_decoder.cpp b/components/codecs/src/opencv_decoder.cpp
index 6e094a0e5..9133d3944 100644
--- a/components/codecs/src/opencv_decoder.cpp
+++ b/components/codecs/src/opencv_decoder.cpp
@@ -36,6 +36,8 @@ bool OpenCVDecoder::decode(const ftl::codecs::Packet &pkt, cv::cuda::GpuMat &out
if (tmp2_.type() == CV_8UC3) {
cv::cvtColor(tmp2_, tmp_, cv::COLOR_RGB2BGRA);
+ } else if (tmp2_.type() == CV_8U) {
+ tmp_ = tmp2_;
} else {
if (pkt.flags & ftl::codecs::kFlagFlipRGB) {
cv::cvtColor(tmp2_, tmp_, cv::COLOR_RGBA2BGRA);
diff --git a/components/operators/include/ftl/operators/fusion.hpp b/components/operators/include/ftl/operators/fusion.hpp
index 0062f1b64..fb0dec87e 100644
--- a/components/operators/include/ftl/operators/fusion.hpp
+++ b/components/operators/include/ftl/operators/fusion.hpp
@@ -17,7 +17,7 @@ class Fusion : public ftl::operators::Operator {
bool apply(ftl::rgbd::FrameSet &in, ftl::rgbd::FrameSet &out, cudaStream_t stream) override;
- static void configuration(ftl::Configurable*) {}
+ static void configuration(ftl::Configurable*);
private:
ftl::cuda::MLSMultiIntensity mls_;
diff --git a/components/operators/src/analysis/evaluation/gt_analysis.cpp b/components/operators/src/analysis/evaluation/gt_analysis.cpp
index c9559084e..b80d2beba 100644
--- a/components/operators/src/analysis/evaluation/gt_analysis.cpp
+++ b/components/operators/src/analysis/evaluation/gt_analysis.cpp
@@ -1,6 +1,8 @@
#include <ftl/operators/gt_analysis.hpp>
#include <ftl/operators/cuda/gt.hpp>
+#include <opencv2/core/cuda_stream_accessor.hpp>
+
using ftl::operators::GTAnalysis;
using ftl::codecs::Channel;
using std::string;
@@ -10,6 +12,7 @@ GTAnalysis::GTAnalysis(ftl::operators::Graph *g, ftl::Configurable *cfg) : ftl::
}
void GTAnalysis::configuration(ftl::Configurable *cfg) {
+ cfg->value("enabled", true);
cfg->value("use_disparity", true);
cfg->value("show_colour", false);
}
@@ -72,6 +75,14 @@ bool GTAnalysis::apply(ftl::rgbd::Frame &in, ftl::rgbd::Frame &out, cudaStream_t
const float npixels = dmat.rows * dmat.cols;
ftl::cuda::GTAnalysisData err;
+ if (!in.hasChannel(Channel::Mask)) {
+ cv::cuda::Stream cvstream = cv::cuda::StreamAccessor::wrapStream(stream);
+
+ auto &m = in.create<cv::cuda::GpuMat>(Channel::Mask);
+ m.create(dmat.size(), CV_8UC1);
+ m.setTo(cv::Scalar(0), cvstream);
+ }
+
for (const auto &o : (use_disp ? options_disparity : options_depth)) {
if (config()->value("show_colour", false)) {
ftl::cuda::gt_analysis(
diff --git a/components/operators/src/fusion/carving/carver.cu b/components/operators/src/fusion/carving/carver.cu
index d1dd480ef..1fc2013e0 100644
--- a/components/operators/src/fusion/carving/carver.cu
+++ b/components/operators/src/fusion/carving/carver.cu
@@ -95,6 +95,7 @@ __global__ void reverse_check_kernel(
// TODO: Threshold comes from depth error characteristics
// If the value is significantly further then carve. Depth error
// is not always easy to calculate, depends on source.
+ // FIXME: Use length between 3D points, not depth
if (!(d2 < ointrin.maxDepth && d2 - campos.z > d2*d2*err_coef)) {
match = fabsf(campos.z - d2) < d2*d2*err_coef; break;
}
diff --git a/components/operators/src/fusion/fusion.cpp b/components/operators/src/fusion/fusion.cpp
index 5e0a624b4..606df89d6 100644
--- a/components/operators/src/fusion/fusion.cpp
+++ b/components/operators/src/fusion/fusion.cpp
@@ -12,6 +12,13 @@ using ftl::operators::Fusion;
using ftl::codecs::Channel;
using cv::cuda::GpuMat;
+void Fusion::configuration(ftl::Configurable *cfg) {
+ cfg->value("enabled", true);
+ cfg->value("mls_smoothing", 2.0f);
+ cfg->value("mls_iterations", 2);
+ cfg->value("visibility_carving", true);
+}
+
Fusion::Fusion(ftl::operators::Graph *g, ftl::Configurable *cfg) : ftl::operators::Operator(g, cfg), mls_(3) {
}
@@ -35,7 +42,11 @@ bool Fusion::apply(ftl::rgbd::FrameSet &in, ftl::rgbd::FrameSet &out, cudaStream
const GpuMat &d = in.frames[i].get<GpuMat>(Channel::Depth);
cv::cuda::cvtColor(col, temp_, cv::COLOR_BGRA2GRAY, 0, cvstream);
- cv::cuda::resize(temp_, temp2_, d.size(), 0, 0, cv::INTER_LINEAR, cvstream);
+ if (temp_.size() != d.size()) {
+ cv::cuda::resize(temp_, temp2_, d.size(), 0, 0, cv::INTER_LINEAR, cvstream);
+ } else {
+ temp2_ = temp_;
+ }
// TODO: Not the best since the mean is entirely lost here.
// Perhaps check mean also with greater smoothing value
diff --git a/components/operators/src/fusion/smoothing/mls_multi_weighted.cu b/components/operators/src/fusion/smoothing/mls_multi_weighted.cu
index b9fbec08e..11248e93b 100644
--- a/components/operators/src/fusion/smoothing/mls_multi_weighted.cu
+++ b/components/operators/src/fusion/smoothing/mls_multi_weighted.cu
@@ -372,7 +372,7 @@ __global__ void mean_subtract_kernel(
mean /= float((2*RADIUS+1)*(2*RADIUS+1));
float diff = float(intensity[x+y*pitch]) - mean;
- contrast[x+y*pitch] = make_uchar2(max(0, min(254, int(diff)+127)), int(mean));
+ contrast[x+y*cpitch] = make_uchar2(max(0, min(254, int(diff)+127)), int(mean));
}
}
diff --git a/components/rgbd-sources/src/frame.cpp b/components/rgbd-sources/src/frame.cpp
index 496c48807..255d438c0 100644
--- a/components/rgbd-sources/src/frame.cpp
+++ b/components/rgbd-sources/src/frame.cpp
@@ -71,7 +71,7 @@ cv::cuda::GpuMat &VideoFrame::createGPU(const ftl::rgbd::FormatBase &f) {
}
const cv::Mat &VideoFrame::getCPU() const {
- if (!validhost) {
+ if (!validhost && !gpu.empty()) {
// TODO: Use stream and page locked mem.
gpu.download(host);
validhost = true;
diff --git a/components/streams/src/feed.cpp b/components/streams/src/feed.cpp
index 2cd767af9..b51cc2ba9 100644
--- a/components/streams/src/feed.cpp
+++ b/components/streams/src/feed.cpp
@@ -534,7 +534,7 @@ void Feed::_createPipeline(uint32_t fsid) {
p->append<ftl::operators::BorderMask>("border_mask");
p->append<ftl::operators::CullDiscontinuity>("remove_discontinuity");
p->append<ftl::operators::MultiViewMLS>("mvmls")->value("enabled", false);
- p->append<ftl::operators::Fusion>("fusion")->value("enabled", false);
+ p->append<ftl::operators::Fusion>("fusion")->set("enabled", false);
p->append<ftl::operators::DisplayMask>("display_mask")->value("enabled", false);
p->append<ftl::operators::Poser>("poser")->value("enabled", true);
p->append<ftl::operators::GTAnalysis>("gtanalyse");
@@ -1115,8 +1115,12 @@ std::string Feed::getSourceURI(ftl::data::FrameID id) {
std::vector<unsigned int> Feed::listFrameSets() {
SHARED_LOCK(mtx_, lk);
+
+ cudaDeviceSynchronize();
+ cudaSafeCall( cudaGetLastError() );
+
std::vector<unsigned int> result;
- result.reserve(fsid_lookup_.size());
+ result.reserve(latest_.size());
for (const auto [k, fs] : latest_) {
if (fs) {
result.push_back(k);
diff --git a/components/streams/src/receiver.cpp b/components/streams/src/receiver.cpp
index 97c58c7dc..be3d01d85 100644
--- a/components/streams/src/receiver.cpp
+++ b/components/streams/src/receiver.cpp
@@ -343,7 +343,7 @@ void Receiver::_processVideo(const StreamPacket &spkt, const Packet &pkt) {
int cvtype = ftl::codecs::type(spkt.channel);
if (surface.type() != cvtype) {
- LOG(ERROR) << "Invalid video format received";
+ LOG(ERROR) << "Invalid video format received: " << cvtype << " for " << (int)spkt.channel;
_terminateVideoPacket(spkt, pkt);
return;
}
--
GitLab