diff --git a/applications/reconstruct/CMakeLists.txt b/applications/reconstruct/CMakeLists.txt
index b089d4a1aeae9654b50865c7400ca0e381f874e6..5dc0ef5dbce517fb88cdf0fb0cc9fdc7b84a778a 100644
--- a/applications/reconstruct/CMakeLists.txt
+++ b/applications/reconstruct/CMakeLists.txt
@@ -20,6 +20,7 @@ set(REPSRC
src/ilw/fill.cu
src/ilw/discontinuity.cu
src/ilw/correspondence.cu
+ src/reconstruction.cpp
)
add_executable(ftl-reconstruct ${REPSRC})
diff --git a/applications/reconstruct/src/main.cpp b/applications/reconstruct/src/main.cpp
index 26d2e6e42dfbf890e5dc2056f09d08e9df034296..c4905e1e5c9713d8c89f7d926548c439797bc627 100644
--- a/applications/reconstruct/src/main.cpp
+++ b/applications/reconstruct/src/main.cpp
@@ -18,6 +18,8 @@
#include <ftl/codecs/writer.hpp>
#include <ftl/codecs/reader.hpp>
+#include "reconstruction.hpp"
+
#include "ilw/ilw.hpp"
#include <ftl/render/tri_render.hpp>
@@ -246,53 +248,43 @@ static void run(ftl::Configurable *root) {
}
}
- std::vector<ftl::rgbd::FrameSet> scene_A(sourcecounts.size()); // Output of align process
- std::vector<ftl::rgbd::FrameSet> scene_B(sourcecounts.size()); // Input of render process
+ ftl::rgbd::FrameSet fs_out;
//ftl::voxhash::SceneRep *scene = ftl::create<ftl::voxhash::SceneRep>(root, "voxelhash");
ftl::rgbd::Streamer *stream = ftl::create<ftl::rgbd::Streamer>(root, "stream", net);
- ftl::rgbd::VirtualSource *virt = ftl::create<ftl::rgbd::VirtualSource>(root, "virtual");
+ ftl::rgbd::VirtualSource *vs = ftl::create<ftl::rgbd::VirtualSource>(root, "virtual");
//root->set("tags", nlohmann::json::array({ root->getID()+"/virtual" }));
- ftl::render::Triangular *splat = ftl::create<ftl::render::Triangular>(root, "renderer", &(scene_B[0]));
- std::vector<ftl::rgbd::Group *> groups;
- ftl::ILW *align = ftl::create<ftl::ILW>(root, "merge");
int o = root->value("origin_pose", 0) % sources.size();
- virt->setPose(sources[o]->getPose());
+ vs->setPose(sources[o]->getPose());
- auto *renderpipe = ftl::config::create<ftl::operators::Graph>(root, "render_pipe");
- renderpipe->append<ftl::operators::ColourChannels>("colour"); // Generate interpolation texture...
- renderpipe->append<ftl::operators::FXAA>("antialiasing");
-
- // Generate virtual camera render when requested by streamer
- virt->onRender([splat,virt,&scene_B,align,renderpipe](ftl::rgbd::Frame &out) {
- //virt->setTimestamp(scene_B.timestamp);
- // Do we need to convert Lab to BGR?
- if (align->isLabColour()) {
- for (auto &f : scene_B[0].frames) {
- auto &col = f.get<cv::cuda::GpuMat>(Channel::Colour);
- cv::cuda::cvtColor(col,col, cv::COLOR_Lab2BGR); // TODO: Add stream
- }
- }
- splat->render(virt, out);
- renderpipe->apply(out, out, virt, 0);
- });
- stream->add(virt);
+ vector<ftl::Reconstruction*> groups;
size_t cumulative = 0;
for (auto c : sourcecounts) {
- auto group = new ftl::rgbd::Group;
+ std::string id = std::to_string(cumulative);
+ auto reconstr = ftl::create<ftl::Reconstruction>(root, id, id);
for (size_t i=cumulative; i<cumulative+c; i++) {
- Source *in = sources[i];
- in->setChannel(Channel::Depth);
- group->addSource(in);
+ reconstr->addSource(sources[i]);
}
- groups.push_back(group);
+ groups.push_back(reconstr);
cumulative += c;
}
- // ---- Recording code -----------------------------------------------------
+ auto *renderpipe = ftl::config::create<ftl::operators::Graph>(root, "render_pipe");
+ renderpipe->append<ftl::operators::ColourChannels>("colour"); // Generate interpolation texture...
+ renderpipe->append<ftl::operators::FXAA>("antialiasing");
+ vs->onRender([vs, &groups, &renderpipe](ftl::rgbd::Frame &out) {
+ for (auto &reconstr : groups) {
+ reconstr->render(vs, out);
+ }
+ renderpipe->apply(out, out, vs, 0);
+ });
+ stream->add(vs);
+
+ // ---- Recording code -----------------------------------------------------
+ /*
std::ofstream fileout;
ftl::codecs::Writer writer(fileout);
auto recorder = [&writer,&groups](ftl::rgbd::Source *src, const ftl::codecs::StreamPacket &spkt, const ftl::codecs::Packet &pkt) {
@@ -331,73 +323,13 @@ static void run(ftl::Configurable *root) {
fileout.close();
}
});
-
+ */
// -------------------------------------------------------------------------
stream->setLatency(6); // FIXME: This depends on source!?
//stream->add(group);
stream->run();
- bool busy = false;
-
- // Create the source depth map pipeline
- auto *pipeline1 = ftl::config::create<ftl::operators::Graph>(root, "pre_filters");
- pipeline1->append<ftl::operators::ClipScene>("clipping");
- pipeline1->append<ftl::operators::ColourChannels>("colour"); // Convert BGR to BGRA
- //pipeline1->append<ftl::operators::HFSmoother>("hfnoise"); // Remove high-frequency noise
- pipeline1->append<ftl::operators::Normals>("normals"); // Estimate surface normals
- //pipeline1->append<ftl::operators::SmoothChannel>("smoothing"); // Generate a smoothing channel
- //pipeline1->append<ftl::operators::ScanFieldFill>("filling"); // Generate a smoothing channel
- pipeline1->append<ftl::operators::CrossSupport>("cross");
- pipeline1->append<ftl::operators::DiscontinuityMask>("discontinuity");
- pipeline1->append<ftl::operators::CrossSupport>("cross2")->set("discon_support", true);
- pipeline1->append<ftl::operators::CullDiscontinuity>("remove_discontinuity");
- //pipeline1->append<ftl::operators::AggreMLS>("mls"); // Perform MLS (using smoothing channel)
- pipeline1->append<ftl::operators::VisCrossSupport>("viscross")->set("enabled", false);
- pipeline1->append<ftl::operators::MultiViewMLS>("mvmls");
- // Alignment
-
- size_t size = groups.size();
- for (size_t i = 0; i < size; ++i) {
- groups[i]->setLatency(4);
- groups[i]->setName("ReconGroup-" + std::to_string(i));
- groups[i]->sync([splat,virt,&busy,&slave,&scene_A,&scene_B,&align,controls,pipeline1,i](ftl::rgbd::FrameSet &fs) -> bool {
- //cudaSetDevice(scene->getCUDADevice());
-
- //if (slave.isPaused()) return true;
- if (controls->value("paused", false)) return true;
-
- // TODO: busy needs to be an array now.
- if (busy) {
- LOG(INFO) << "Group frameset dropped: " << fs.timestamp;
- return true;
- }
- busy = true;
-
- // Swap the entire frameset to allow rapid return
- fs.swapTo(scene_A[i]);
-
- ftl::pool.push([&scene_B,&scene_A,&busy,&slave,&align,pipeline1,i](int id) {
- //cudaSetDevice(scene->getCUDADevice());
- // TODO: Release frameset here...
- //cudaSafeCall(cudaStreamSynchronize(scene->getIntegrationStream()));
-
- UNIQUE_LOCK(scene_A[i].mtx, lk);
-
- pipeline1->apply(scene_A[i], scene_A[i], 0);
- //align->process(scene_A[i]);
-
-
- // TODO: To use second GPU, could do a download, swap, device change,
- // then upload to other device. Or some direct device-2-device copy.
- scene_A[i].swapTo(scene_B[i]);
- LOG(INFO) << "Align complete... " << scene_A[i].timestamp;
- busy = false;
- });
- return true;
- });
- }
-
LOG(INFO) << "Start timer";
ftl::timer::start(true);
@@ -411,10 +343,8 @@ static void run(ftl::Configurable *root) {
LOG(INFO) << "Deleting...";
- delete align;
- delete splat;
delete stream;
- delete virt;
+ delete vs;
delete net;
for (auto g : groups) {
delete g;
diff --git a/applications/reconstruct/src/reconstruction.cpp b/applications/reconstruct/src/reconstruction.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..bbaf7f8b5e6414140233aa9480ccc1239c7de262
--- /dev/null
+++ b/applications/reconstruct/src/reconstruction.cpp
@@ -0,0 +1,77 @@
+#include "reconstruction.hpp"
+
+#include "ftl/operators/smoothing.hpp"
+#include "ftl/operators/colours.hpp"
+#include "ftl/operators/normals.hpp"
+#include "ftl/operators/filling.hpp"
+#include "ftl/operators/segmentation.hpp"
+#include "ftl/operators/mask.hpp"
+#include "ftl/operators/antialiasing.hpp"
+#include "ftl/operators/mvmls.hpp"
+#include "ftl/operators/clipping.hpp"
+
+using ftl::Reconstruction;
+using ftl::codecs::Channel;
+
+Reconstruction::Reconstruction(nlohmann::json &config, const std::string name) :
+ ftl::Configurable(config), busy_(false), fs_render_(), fs_align_() {
+ group_ = new ftl::rgbd::Group;
+ group_->setName("ReconGroup-" + name);
+ group_->setLatency(4);
+
+ renderer_ = ftl::create<ftl::render::Triangular>(this, "renderer", &fs_render_);
+
+ pipeline_ = ftl::config::create<ftl::operators::Graph>(this, "pre_filters");
+ pipeline_->append<ftl::operators::ClipScene>("clipping")->set("enabled", false);
+ pipeline_->append<ftl::operators::ColourChannels>("colour"); // Convert BGR to BGRA
+ //pipeline_->append<ftl::operators::HFSmoother>("hfnoise"); // Remove high-frequency noise
+ pipeline_->append<ftl::operators::Normals>("normals"); // Estimate surface normals
+ //pipeline_->append<ftl::operators::SmoothChannel>("smoothing"); // Generate a smoothing channel
+ //pipeline_->append<ftl::operators::ScanFieldFill>("filling"); // Generate a smoothing channel
+ pipeline_->append<ftl::operators::CrossSupport>("cross");
+ pipeline_->append<ftl::operators::DiscontinuityMask>("discontinuity");
+ pipeline_->append<ftl::operators::CrossSupport>("cross2")->set("discon_support", true);
+ pipeline_->append<ftl::operators::CullDiscontinuity>("remove_discontinuity");
+ //pipeline_->append<ftl::operators::AggreMLS>("mls"); // Perform MLS (using smoothing channel)
+ pipeline_->append<ftl::operators::VisCrossSupport>("viscross")->set("enabled", false);
+ pipeline_->append<ftl::operators::MultiViewMLS>("mvmls");
+
+ group_->sync([this](ftl::rgbd::FrameSet &fs) -> bool {
+ // TODO: pause
+
+ if (busy_) {
+ LOG(INFO) << "Group frameset dropped: " << fs.timestamp;
+ return true;
+ }
+ busy_ = true;
+
+ // Swap the entire frameset to allow rapid return
+ fs.swapTo(fs_align_);
+
+ ftl::pool.push([this](int id) {
+ UNIQUE_LOCK(fs_align_.mtx, lk);
+ pipeline_->apply(fs_align_, fs_align_, 0);
+
+ // TODO: To use second GPU, could do a download, swap, device change,
+ // then upload to other device. Or some direct device-2-device copy.
+ fs_align_.swapTo(fs_render_);
+
+ LOG(INFO) << "Align complete... " << fs_align_.timestamp;
+ busy_ = false;
+ });
+ return true;
+ });
+}
+
+Reconstruction::~Reconstruction() {
+ // TODO delete
+}
+
+void Reconstruction::addSource(ftl::rgbd::Source *src) {
+ src->setChannel(Channel::Depth);
+ group_->addSource(src); // TODO: check if source is already in group?
+}
+
+void Reconstruction::render(ftl::rgbd::VirtualSource *vs, ftl::rgbd::Frame &out) {
+ renderer_->render(vs, out);
+}
\ No newline at end of file
diff --git a/applications/reconstruct/src/reconstruction.hpp b/applications/reconstruct/src/reconstruction.hpp
index bde04000706607bb752ab9447015fd3259c344a0..340e65bf2f47650400d4c6850c1ae1310a0458aa 100644
--- a/applications/reconstruct/src/reconstruction.hpp
+++ b/applications/reconstruct/src/reconstruction.hpp
@@ -1,11 +1,19 @@
#ifndef _FTL_RECONSTRUCTION_HPP_
#define _FTL_RECONSTRUCTION_HPP_
+#include "ftl/configurable.hpp"
+#include "ftl/rgbd/source.hpp"
+#include "ftl/rgbd/frame.hpp"
+#include "ftl/rgbd/group.hpp"
+#include "ftl/rgbd/frameset.hpp"
+#include "ftl/operators/operator.hpp"
+#include "ftl/render/tri_render.hpp"
+
namespace ftl {
class Reconstruction : public ftl::Configurable {
public:
- Reconstruction();
+ Reconstruction(nlohmann::json &config, const std::string name);
~Reconstruction();
void addSource(ftl::rgbd::Source *);
@@ -16,8 +24,10 @@ class Reconstruction : public ftl::Configurable {
void render(ftl::rgbd::VirtualSource *vs, ftl::rgbd::Frame &out);
private:
+ bool busy_;
+ ftl::rgbd::FrameSet fs_render_;
+ ftl::rgbd::FrameSet fs_align_;
ftl::rgbd::Group *group_;
- ftl::rgbd::FrameSet fs_;
ftl::operators::Graph *pipeline_;
ftl::render::Triangular *renderer_;
};