diff --git a/applications/reconstruct2/src/main.cpp b/applications/reconstruct2/src/main.cpp index d79ac84eedede22737fa45a90f49465e88bb6db4..a7e94f5012f4abced96a65933ee868bbecbd11e4 100644 --- a/applications/reconstruct2/src/main.cpp +++ b/applications/reconstruct2/src/main.cpp @@ -5,6 +5,19 @@ #include <nlohmann/json.hpp> #include <loguru.hpp> +#include "ftl/operators/smoothing.hpp" +#include "ftl/operators/colours.hpp" +#include "ftl/operators/normals.hpp" +#include "ftl/operators/filling.hpp" +#include "ftl/operators/segmentation.hpp" +#include "ftl/operators/mask.hpp" +#include "ftl/operators/antialiasing.hpp" +#include "ftl/operators/mvmls.hpp" +#include "ftl/operators/clipping.hpp" +#include <ftl/operators/disparity.hpp> +#include <ftl/operators/poser.hpp> +#include <ftl/operators/detectandtrack.hpp> + using ftl::net::Universe; using ftl::stream::Feed; using ftl::codecs::Channel; @@ -73,6 +86,28 @@ static void run(ftl::Configurable *root) { auto *filter = feed->filter({Channel::Colour, Channel::Depth}); feed->set("uri", root->value("uri", std::string("ftl://ftlab.utu.fi/reconstruction"))); + feed->setPipelineCreator([](ftl::operators::Graph *pipeline) { + pipeline->append<ftl::operators::DepthChannel>("depth"); // Ensure there is a depth channel + pipeline->append<ftl::operators::DisparityBilateralFilter>("bilateral_filter")->value("enabled", false); + pipeline->append<ftl::operators::DisparityToDepth>("calculate_depth")->value("enabled", false); + pipeline->append<ftl::operators::ColourChannels>("colour"); // Convert BGR to BGRA + pipeline->append<ftl::operators::ClipScene>("clipping")->value("enabled", false); + pipeline->append<ftl::operators::DetectAndTrack>("facedetection")->value("enabled", false); + pipeline->append<ftl::operators::ArUco>("aruco")->value("enabled", false); + //pipeline_->append<ftl::operators::HFSmoother>("hfnoise"); // Remove high-frequency noise + pipeline->append<ftl::operators::Normals>("normals"); // Estimate surface normals + //pipeline_->append<ftl::operators::SmoothChannel>("smoothing"); // Generate a smoothing channel + //pipeline_->append<ftl::operators::ScanFieldFill>("filling"); // Generate a smoothing channel + pipeline->append<ftl::operators::CrossSupport>("cross"); + pipeline->append<ftl::operators::DiscontinuityMask>("discontinuity"); + pipeline->append<ftl::operators::CrossSupport>("cross2")->value("discon_support", true); + pipeline->append<ftl::operators::BorderMask>("border_mask")->value("enabled", false); + pipeline->append<ftl::operators::CullDiscontinuity>("remove_discontinuity")->set("enabled", false); + //pipeline_->append<ftl::operators::AggreMLS>("mls"); // Perform MLS (using smoothing channel) + pipeline->append<ftl::operators::VisCrossSupport>("viscross")->value("enabled", false); + pipeline->append<ftl::operators::MultiViewMLS>("mvmls"); + pipeline->append<ftl::operators::Poser>("poser")->value("enabled", false); + }); //feed->lowLatencyMode(); feed->startStreaming(filter); diff --git a/components/operators/src/fusion/mvmls.cpp b/components/operators/src/fusion/mvmls.cpp index a444d8d1a9d2b1deb1471ce420e9fc36b1e940fd..c16b3ab43097c4bda940a2f361592cd1fb5784b3 100644 --- a/components/operators/src/fusion/mvmls.cpp +++ b/components/operators/src/fusion/mvmls.cpp @@ -47,8 +47,19 @@ bool MultiViewMLS::apply(ftl::rgbd::FrameSet &in, ftl::rgbd::FrameSet &out, cuda bool show_consistency = config()->value("show_consistency", false); bool show_adjustment = config()->value("show_adjustment", false); - if (in.frames.size() < 1) return false; - auto size = in.firstFrame().get<GpuMat>(Channel::Depth).size(); + if (in.frames.size() < 1 || in.count == 0) return false; + cv::Size size(0,0); + for (auto &f : in.frames) { + if (f.hasChannel(Channel::Depth)) { + size = f.get<GpuMat>(Channel::Depth).size(); + break; + } + } + + if (size.width == 0) { + in.firstFrame().message(ftl::data::Message::Warning_MISSING_CHANNEL, "Missing Depth Channel in MVMLS operator"); + return false; + } // Make sure we have enough buffers while (normals_horiz_.size() < in.frames.size()) { diff --git a/components/operators/src/normals.cpp b/components/operators/src/normals.cpp index 0e89ef3ed1dc2dda26627e13f5d9639f0a62d775..09618fa4805a4e171c59c32f37b54df3298d5daa 100644 --- a/components/operators/src/normals.cpp +++ b/components/operators/src/normals.cpp @@ -19,7 +19,8 @@ Normals::~Normals() { bool Normals::apply(ftl::rgbd::Frame &in, ftl::rgbd::Frame &out, cudaStream_t stream) { if (!in.hasChannel(Channel::Depth)) { out.message(ftl::data::Message::Warning_MISSING_CHANNEL, "Missing Depth Channel in Normals operator"); - throw FTL_Error("Missing depth channel in Normals operator"); + //throw FTL_Error("Missing depth channel in Normals operator"); + return false; } if (out.hasChannel(Channel::Normals)) { @@ -49,7 +50,8 @@ NormalDot::~NormalDot() { bool NormalDot::apply(ftl::rgbd::Frame &in, ftl::rgbd::Frame &out, cudaStream_t stream) { if (!in.hasChannel(Channel::Depth)) { out.message(ftl::data::Message::Warning_MISSING_CHANNEL, "Missing Depth Channel in Normals operator"); - throw FTL_Error("Missing depth channel in Normals operator"); + //throw FTL_Error("Missing depth channel in Normals operator"); + return false; } if (out.hasChannel(Channel::Normals)) { diff --git a/components/streams/include/ftl/streams/feed.hpp b/components/streams/include/ftl/streams/feed.hpp index 307fe36d2a143e0d5cca5306f51783d2aa1eec9f..2b2d22ed0e2e65ddeafbe8147eec6ace3f9e6ad6 100644 --- a/components/streams/include/ftl/streams/feed.hpp +++ b/components/streams/include/ftl/streams/feed.hpp @@ -135,6 +135,8 @@ public: cv::Mat getThumbnail(const std::string &uri); std::string getName(const std::string &uri); + void setPipelineCreator(const std::function<void(ftl::operators::Graph*)> &cb); + ftl::operators::Graph* addPipeline(const std::string &name); ftl::operators::Graph* addPipeline(uint32_t fsid); /** Returns pointer to filter object. Pointers will be invalid after Feed @@ -189,6 +191,7 @@ private: std::unordered_map<uint32_t, ftl::render::Source*> renderers_; std::unordered_map<uint32_t, ftl::operators::Graph*> pre_pipelines_; std::list<ftl::streams::ManualSourceBuilder*> render_builders_; + std::function<void(ftl::operators::Graph*)> pipe_creator_; std::vector<std::string> netcams_; ftl::Handler<const std::vector<std::string> &> new_sources_cb_; diff --git a/components/streams/src/feed.cpp b/components/streams/src/feed.cpp index 31e3ba4a6bc82b982cfb9f4a5d60c6cc46a12c11..873f94f3840efab03c9f2d54ac63c2114aea2f1b 100644 --- a/components/streams/src/feed.cpp +++ b/components/streams/src/feed.cpp @@ -444,22 +444,31 @@ void Feed::_createPipeline(uint32_t fsid) { LOG(INFO) << "Creating pipeline"; auto *p = _addPipeline(fsid); - p->append<ftl::operators::DepthChannel>("depth")->value("enabled", false); - p->append<ftl::operators::ClipScene>("clipping")->value("enabled", false); - p->append<ftl::operators::ColourChannels>("colour"); // Convert BGR to BGRA - p->append<ftl::operators::DetectAndTrack>("facedetection")->value("enabled", false); - p->append<ftl::operators::ArUco>("aruco")->value("enabled", false); - //p->append<ftl::operators::HFSmoother>("hfnoise"); - p->append<ftl::operators::CrossSupport>("cross"); - p->append<ftl::operators::PixelWeights>("weights"); - p->append<ftl::operators::CullWeight>("remove_weights")->value("enabled", false); - p->append<ftl::operators::DegradeWeight>("degrade"); - p->append<ftl::operators::VisCrossSupport>("viscross")->set("enabled", false); - p->append<ftl::operators::BorderMask>("border_mask"); - p->append<ftl::operators::CullDiscontinuity>("remove_discontinuity"); - p->append<ftl::operators::MultiViewMLS>("mvmls")->value("enabled", false); - p->append<ftl::operators::Poser>("poser")->value("enabled", true); - p->append<ftl::operators::GTAnalysis>("gtanalyse"); + if (pipe_creator_) { + pipe_creator_(p); + } else { + p->append<ftl::operators::DepthChannel>("depth")->value("enabled", false); + p->append<ftl::operators::ClipScene>("clipping")->value("enabled", false); + p->append<ftl::operators::ColourChannels>("colour"); // Convert BGR to BGRA + p->append<ftl::operators::DetectAndTrack>("facedetection")->value("enabled", false); + p->append<ftl::operators::ArUco>("aruco")->value("enabled", false); + //p->append<ftl::operators::HFSmoother>("hfnoise"); + p->append<ftl::operators::CrossSupport>("cross"); + p->append<ftl::operators::PixelWeights>("weights"); + p->append<ftl::operators::CullWeight>("remove_weights")->value("enabled", false); + p->append<ftl::operators::DegradeWeight>("degrade"); + p->append<ftl::operators::VisCrossSupport>("viscross")->set("enabled", false); + p->append<ftl::operators::BorderMask>("border_mask"); + p->append<ftl::operators::CullDiscontinuity>("remove_discontinuity"); + p->append<ftl::operators::MultiViewMLS>("mvmls")->value("enabled", false); + p->append<ftl::operators::Poser>("poser")->value("enabled", true); + p->append<ftl::operators::GTAnalysis>("gtanalyse"); + } +} + +void Feed::setPipelineCreator(const std::function<void(ftl::operators::Graph*)> &cb) { + UNIQUE_LOCK(mtx_, lk); + pipe_creator_ = cb; } void Feed::removeFilter(Feed::Filter* filter) {