#include <loguru.hpp> #include "ftl/operators/disparity.hpp" #include <ftl/operators/cuda/disparity.hpp> #include <ftl/utility/image_debug.hpp> #include <opencv2/cudaimgproc.hpp> #include <opencv2/cudaarithm.hpp> #include <opencv2/cudafilters.hpp> #include <opencv2/cudawarping.hpp> using cv::Size; using cv::cuda::GpuMat; using ftl::rgbd::Format; using ftl::codecs::Channel; using ftl::rgbd::Frame; using ftl::rgbd::Source; using ftl::operators::FixstarsSGM; using ftl::operators::Buffer; void FixstarsSGM::_variance_mask(cv::InputArray in, cv::OutputArray out, int wsize, cv::cuda::Stream &cvstream) { if (in.isGpuMat() && out.isGpuMat()) { mean_.create(in.size(), CV_32FC1); mean2_.create(in.size(), CV_32FC1); im2_.create(in.size(), CV_32FC1); in.getGpuMat().convertTo(im_, CV_32FC1, cvstream); cv::cuda::multiply(im_, im_, im2_, 1.0, CV_32FC1, cvstream); if (!filter_) filter_ = cv::cuda::createBoxFilter(CV_32FC1, CV_32FC1, cv::Size(wsize,wsize)); filter_->apply(im_, mean_, cvstream); // E[X] filter_->apply(im2_, mean2_, cvstream); // E[X^2] cv::cuda::multiply(mean_, mean_, mean_, 1.0, -1, cvstream); // (E[X])^2 // NOTE: floating point accuracy in subtraction // (cv::cuda::createBoxFilter only supports float and 8 bit integer types) cv::cuda::subtract(mean2_, mean_, out.getGpuMatRef(), cv::noArray(), -1, cvstream); // E[X^2] - (E[X])^2 } else { throw std::exception(); /* todo CPU version */ } } void FixstarsSGM::computeP2(cudaStream_t &stream) { const int P3 = config()->value("P3", P2_); auto cvstream = cv::cuda::StreamAccessor::wrapStream(stream); P2_map_.setTo(P3, cvstream); if (config()->value("use_P2_map", false)) { edges_.create(size_, CV_8UC1); auto ptr = canny_; ptr->detect(lbw_, edges_, cvstream); P2_map_.setTo(P2_, edges_, cvstream); } } FixstarsSGM::FixstarsSGM(ftl::operators::Graph *g, ftl::Configurable* cfg) : ftl::operators::Operator(g, cfg) { ssgm_ = nullptr; size_ = Size(0, 0); uniqueness_ = cfg->value("uniqueness", 0.95f); P1_ = cfg->value("P1", 10); P2_ = cfg->value("P2", 120); max_disp_ = cfg->value("num_disp", 256); ct_shape_ = static_cast<sgm::CensusShape>(cfg->value("ct_shape", 2)); if (uniqueness_ < 0.0 || uniqueness_ > 1.0) { uniqueness_ = 1.0; LOG(WARNING) << "Invalid uniqueness, using default (1.0)"; } if (P1_ <= 0 ) { P1_ = 10; LOG(WARNING) << "Invalid value for P1, using default (10)"; } if (P2_ < P1_) { P2_ = P1_; LOG(WARNING) << "Invalid value for P2, using value of P1 (" << P1_ << ")"; } if (!(max_disp_ == 256 || max_disp_ == 128 || max_disp_ == 192)) { max_disp_ = 256; LOG(WARNING) << "Invalid value for max_disp, using default value (256)"; } cfg->on("P1", [this, cfg]() { int P1 = cfg->value("P1", 0); if (P1 <= 0) { LOG(WARNING) << "Invalid value for P1 (" << P1 << ")"; } else { P1_ = P1; updateParameters(); } }); cfg->on("P2", [this, cfg]() { int P2 = cfg->value("P2", 0); if (P2 < P1_) { LOG(WARNING) << "Invalid value for P2 (" << P2 << ")"; } else { P2_ = P2; updateParameters(); } }); cfg->on("uniqueness", [this, cfg]() { double uniqueness = cfg->value("uniqueness", 0.0); if (uniqueness < 0.0 || uniqueness > 1.0) { LOG(WARNING) << "Invalid value for uniqueness (" << uniqueness << ")"; } else { uniqueness_ = uniqueness; updateParameters(); } }); cfg->on("ct_shape", [this, cfg]() { ct_shape_ = static_cast<sgm::CensusShape>(cfg->value("ct_shape", 2)); updateParameters(); }); updateP2Parameters(); cfg->on("canny_low", [this, cfg]() { updateP2Parameters(); }); cfg->on("canny_high", [this, cfg]() { updateP2Parameters(); }); } FixstarsSGM::~FixstarsSGM() { if (ssgm_) { delete ssgm_; } } bool FixstarsSGM::init() { if (size_ == Size(0, 0)) { return false; } if (ssgm_) { delete ssgm_; } lbw_.create(size_, CV_8UC1); rbw_.create(size_, CV_8UC1); disp_int_.create(size_, CV_16SC1); weights_.create(size_, CV_32FC1); weights_.setTo(1.0); LOG(INFO) << "INIT FIXSTARS"; ssgm_ = new sgm::StereoSGM(size_.width, size_.height, max_disp_, 8, 16, lbw_.step, disp_int_.step / sizeof(short), sgm::EXECUTE_INOUT_CUDA2CUDA, sgm::StereoSGM::Parameters(P1_, P2_, uniqueness_, true) ); P2_map_.create(size_, CV_8UC1); return true; } bool FixstarsSGM::updateParameters() { if (ssgm_ == nullptr) { return false; } return this->ssgm_->updateParameters( sgm::StereoSGM::Parameters(P1_, P2_, uniqueness_, true, ct_shape_)); } bool FixstarsSGM::updateP2Parameters() { canny_ = cv::cuda::createCannyEdgeDetector( config()->value("canny_low", 30.0), config()->value("canny_high", 120.0)); return true; } bool FixstarsSGM::apply(Frame &in, Frame &out, cudaStream_t stream) { if (!in.hasChannel(Channel::Left) || !in.hasChannel(Channel::Right)) { LOG(ERROR) << "Fixstars is missing Left or Right channel"; return false; } auto &l = in.get<GpuMat>(Channel::Left); const auto &r = in.get<GpuMat>(Channel::Right); const auto &intrin = in.getLeft(); if (l.empty() || r.empty() || intrin.width == 0) { LOG(ERROR) << "Missing data for Fixstars"; return false; } if (size_.width != intrin.width) { size_ = cv::Size(intrin.width, intrin.height); if (!init()) { return false; } } bool has_estimate = graph()->hasBuffer(Buffer::Disparity, in.source()); //in.hasChannel(Channel::Disparity); auto &disp = graph()->createBuffer(Buffer::Disparity, in.source()); disp.create(size_, CV_16SC1); auto cvstream = cv::cuda::StreamAccessor::wrapStream(stream); cv::cuda::cvtColor(l, lbw_full_, cv::COLOR_BGRA2GRAY, 0, cvstream); cv::cuda::cvtColor(r, rbw_full_, cv::COLOR_BGRA2GRAY, 0, cvstream); if (l.size() != size_) { cv::cuda::resize(lbw_full_, lbw_, size_, 0, 0, cv::INTER_CUBIC, cvstream); cv::cuda::resize(rbw_full_, rbw_, size_, 0, 0, cv::INTER_CUBIC, cvstream); } else { lbw_ = lbw_full_; rbw_ = rbw_full_; } //cvstream.waitForCompletion(); computeP2(stream); bool use_variance = config()->value("use_variance", true); if (use_variance) { _variance_mask(lbw_, weightsF_, config()->value("var_wsize", 11), cvstream); float minweight = std::min(1.0f, std::max(0.0f, config()->value("var_minweight", 0.5f))); cv::cuda::normalize(weightsF_, weightsF_, minweight, 1.0, cv::NORM_MINMAX, -1, cv::noArray(), cvstream); weightsF_.convertTo(weights_, CV_8UC1, 255.0f, cvstream); //ftl::utility::show_image(weights_, "VarWeight", 0.5f, ftl::utility::ImageVisualisation::HEAT_MAPPED); //if ((int)P2_map_.step != P2_map_.cols) LOG(ERROR) << "P2 map step error: " << P2_map_.cols << "," << P2_map_.step; ssgm_->execute(lbw_.data, rbw_.data, disp_int_.data, P2_map_.data, (uint8_t*) weights_.data, weights_.step1(), config()->value("min_disp", 60), stream); } else { ssgm_->execute(lbw_.data, rbw_.data, disp_int_.data, P2_map_.data, nullptr, 0, config()->value("min_disp", 60), stream); } // GpuMat left_pixels(dispt_, cv::Rect(0, 0, max_disp_, dispt_.rows)); // left_pixels.setTo(0); if (config()->value("merge_estimates", false) && has_estimate) { ftl::cuda::merge_disparities(disp_int_, disp, stream); } cv::cuda::threshold(disp_int_, disp, 16383.0f, 0.0f, cv::THRESH_TOZERO_INV, cvstream); if (config()->value("check_reprojection", false)) { ftl::cuda::check_reprojection(disp, in.getTexture<uchar4>(Channel::Colour), in.createTexture<uchar4>(Channel::Colour2, true), stream); } if (config()->value("show_P2_map", false)) { cv::cuda::cvtColor(P2_map_, out.get<GpuMat>(Channel::Colour), cv::COLOR_GRAY2BGRA, 0, cvstream); } if (config()->value("show_rpe", false)) { ftl::cuda::show_rpe(disp, in.set<GpuMat>(Channel::Left), r, 100.0f, stream); } if (config()->value("show_disp_density", false)) { ftl::cuda::show_disp_density(disp, in.set<GpuMat>(Channel::Left), 100.0f, stream); } //disp_int_.convertTo(disp, CV_32F, 1.0f / 16.0f, cvstream); return true; }