diff --git a/applications/reconstruct/src/main.cpp b/applications/reconstruct/src/main.cpp index d7ce5f9d4f84ee172c23edc9656f1c1d30d7c0b8..c2341e7de37abd2fd03195b519666dbab5f6eda8 100644 --- a/applications/reconstruct/src/main.cpp +++ b/applications/reconstruct/src/main.cpp @@ -145,6 +145,7 @@ static void run(ftl::Configurable *root) { // TODO: To use second GPU, could do a download, swap, device change, // then upload to other device. Or some direct device-2-device copy. scene_A.swapTo(scene_B); + LOG(INFO) << "Align complete... " << scene_A.timestamp; busy = false; }); return true; diff --git a/components/renderers/cpp/src/splat_render.cpp b/components/renderers/cpp/src/splat_render.cpp index 2097ae284655b6e56155d641d49252f744994bfc..9d84f2f821e52d05ba158f5f055f2b5b641737f9 100644 --- a/components/renderers/cpp/src/splat_render.cpp +++ b/components/renderers/cpp/src/splat_render.cpp @@ -7,6 +7,7 @@ using ftl::render::Splatter; using ftl::rgbd::Channel; +using ftl::rgbd::Channels; using ftl::rgbd::Format; using cv::cuda::GpuMat; @@ -21,6 +22,8 @@ Splatter::~Splatter() { bool Splatter::render(ftl::rgbd::VirtualSource *src, ftl::rgbd::Frame &out, cudaStream_t stream) { if (!src->isReady()) return false; + LOG(INFO) << "Render ready2"; + const auto &camera = src->parameters(); //cudaSafeCall(cudaSetDevice(scene_->getCUDADevice())); @@ -38,6 +41,8 @@ bool Splatter::render(ftl::rgbd::VirtualSource *src, ftl::rgbd::Frame &out, cuda cv::cuda::Stream cvstream = cv::cuda::StreamAccessor::wrapStream(stream); + LOG(INFO) << "Render ready1"; + // Create buffers if they don't exist /*if ((unsigned int)depth1_.width() != camera.width || (unsigned int)depth1_.height() != camera.height) { depth1_ = ftl::cuda::TextureObject<int>(camera.width, camera.height); @@ -77,13 +82,27 @@ bool Splatter::render(ftl::rgbd::VirtualSource *src, ftl::rgbd::Frame &out, cuda out.get<GpuMat>(Channel::Depth).setTo(cv::Scalar(1000.0f), cvstream); out.get<GpuMat>(Channel::Colour).setTo(cv::Scalar(0,0,0), cvstream); + LOG(INFO) << "Render ready"; + // Render each camera into virtual view - for (auto &f : scene_->frames) { + for (size_t i=0; i<scene_->frames.size(); ++i) { + auto &f = scene_->frames[i]; + auto *s = scene_->sources[i]; + + if (!f.hasChannel(Channel::Depth) || f.isCPU(Channel::Depth)) { + LOG(ERROR) << "Missing required Depth channel"; + return false; + } + // Needs to create points channel first? if (!f.hasChannel(Channel::Points)) { + LOG(INFO) << "Creating points..."; + auto &t = f.createTexture<float4>(Channel::Points, Format<float4>(f.get<GpuMat>(Channel::Colour).size())); - auto pose = MatrixConversion::toCUDA(f.source()->getPose().cast<float>().inverse()); - ftl::cuda::point_cloud(t, f.createTexture<float>(Channel::Depth), f.source()->parameters(), pose, stream); + auto pose = MatrixConversion::toCUDA(s->getPose().cast<float>().inverse()); + ftl::cuda::point_cloud(t, f.createTexture<float>(Channel::Depth), s->parameters(), pose, stream); + + LOG(INFO) << "POINTS Added"; } ftl::cuda::dibr_merge( @@ -91,6 +110,8 @@ bool Splatter::render(ftl::rgbd::VirtualSource *src, ftl::rgbd::Frame &out, cuda temp_.createTexture<int>(Channel::Depth), params, stream ); + + LOG(INFO) << "DIBR DONE"; } //ftl::cuda::dibr(depth1_, colour1_, normal1_, depth2_, colour_tmp_, depth3_, scene_->cameraCount(), params, stream); diff --git a/components/rgbd-sources/src/virtual.cpp b/components/rgbd-sources/src/virtual.cpp index 63ee5ddbb712db1d9306f27a14e1165edb6576ef..c38a3da528b532fd10b105acc1f696da24419228 100644 --- a/components/rgbd-sources/src/virtual.cpp +++ b/components/rgbd-sources/src/virtual.cpp @@ -2,9 +2,57 @@ using ftl::rgbd::VirtualSource; using ftl::rgbd::Source; +using ftl::rgbd::Channel; -VirtualSource::VirtualSource(ftl::config::json_t &cfg) : Source(cfg) { +class VirtualImpl : public ftl::rgbd::detail::Source { + public: + explicit VirtualImpl(ftl::rgbd::Source *host) : ftl::rgbd::detail::Source(host) { + + } + + ~VirtualImpl() { + + } + + bool capture(int64_t ts) override { + return true; + } + + bool retrieve() override { + return true; + } + + bool compute(int n, int b) override { + if (callback) { + frame.reset(); + + try { + callback(frame); + } catch (std::exception &e) { + LOG(ERROR) << "Exception in render callback: " << e.what(); + } catch (...) { + LOG(ERROR) << "Unknown exception in render callback"; + } + + if (frame.hasChannel(Channel::Colour) && frame.hasChannel(Channel::Depth)) { + frame.download(Channel::Colour + Channel::Depth); + cv::swap(frame.get<cv::Mat>(Channel::Colour), rgb_); + cv::swap(frame.get<cv::Mat>(Channel::Depth), depth_); + } else { + LOG(ERROR) << "Missing colour or depth frame in rendering"; + } + } + return true; + } + bool isReady() override { return true; } + + std::function<void(ftl::rgbd::Frame &)> callback; + ftl::rgbd::Frame frame; +}; + +VirtualSource::VirtualSource(ftl::config::json_t &cfg) : Source(cfg) { + impl_ = new VirtualImpl(this); } VirtualSource::~VirtualSource() { @@ -12,7 +60,7 @@ VirtualSource::~VirtualSource() { } void VirtualSource::onRender(const std::function<void(ftl::rgbd::Frame &)> &f) { - + dynamic_cast<VirtualImpl*>(impl_)->callback = f; } /*