/* Copyright 2019 Nicolas Pope */ #include "fixstars_sgm.hpp" #include <loguru.hpp> #include <opencv2/cudastereo.hpp> using ftl::algorithms::FixstarsSGM; using cv::Mat; using cv::cuda::GpuMat; //static ftl::Disparity::Register fixstarssgm("libsgm", FixstarsSGM::create); FixstarsSGM::FixstarsSGM(nlohmann::json &config) : Disparity(config) { ssgm_ = nullptr; const int width = size_.width; const int height = size_.height; CHECK((width >= 480) && (height >= 360)); uniqueness_ = value("uniqueness", 0.95f); P1_ = value("P1", 10); P2_ = value("P2", 120); CHECK((uniqueness_ >= 0.0) && (uniqueness_ <= 1.0)); CHECK(P1_ >= 0); CHECK(P2_ > P1_); use_filter_ = value("use_filter", false); if (use_filter_) { int radius = value("filter_radius", 25); int iter = value("filter_iter", 1); CHECK(radius > 0) << "filter_radius must be greater than 0"; CHECK(iter > 0) << "filter_iter must be greater than 0"; filter_ = cv::cuda::createDisparityBilateralFilter(max_disp_ << 4, radius, iter); } #ifdef HAVE_OPTFLOW use_off_ = value("use_off", false); if (use_off_) { int off_size = value("off_size", 9); double off_threshold = value("off_threshold", 0.9); off_ = ftl::rgbd::OFDisparityFilter(size_, off_size, off_threshold); } #endif init(size_); } void FixstarsSGM::init(const cv::Size size) { if (ssgm_) { delete ssgm_; } lbw_ = GpuMat(size, CV_8UC1); rbw_ = GpuMat(size, CV_8UC1); dispt_ = GpuMat(size, CV_16SC1); ssgm_ = new sgm::StereoSGM(size.width, size.height, max_disp_, 8, 16, lbw_.step, dispt_.step / sizeof(short), sgm::EXECUTE_INOUT_CUDA2CUDA, sgm::StereoSGM::Parameters(P1_, P2_, uniqueness_, true) ); } void FixstarsSGM::compute(ftl::rgbd::Frame &frame, cv::cuda::Stream &stream) { /*if (!frame.hasChannel(ftl::rgbd::kChanLeftGray)) { auto &rgb = frame.getChannel<GpuMat>(ftl::rgbd::kChanLeft, stream); auto &gray = frame.setChannel<GpuMat>(ftl::rgbd::kChanLeftGray); cv::cuda::cvtColor(rgb, gray, cv::COLOR_BGR2GRAY, 0, stream); } if (!frame.hasChannel(ftl::rgbd::kChanRightGray)) { auto &rgb = frame.getChannel<GpuMat>(ftl::rgbd::kChanRight, stream); auto &gray = frame.setChannel<GpuMat>(ftl::rgbd::kChanRightGray); cv::cuda::cvtColor(rgb, gray, cv::COLOR_BGR2GRAY, 0, stream); }*/ const auto &l = frame.getChannel<GpuMat>(ftl::rgbd::kChanLeft, stream); const auto &r = frame.getChannel<GpuMat>(ftl::rgbd::kChanRight, stream); auto &disp = frame.setChannel<GpuMat>(ftl::rgbd::kChanDisparity); if (disp.size() != l.size()) { disp = GpuMat(l.size(), CV_32FC1); } GpuMat l_scaled; if (l.size() != size_) { GpuMat _r; scaleInput(l, r, l_scaled, _r, stream); cv::cuda::cvtColor(l_scaled, lbw_, cv::COLOR_BGR2GRAY, 0, stream); cv::cuda::cvtColor(_r, rbw_, cv::COLOR_BGR2GRAY, 0, stream); } else { cv::cuda::cvtColor(l, lbw_, cv::COLOR_BGR2GRAY, 0, stream); cv::cuda::cvtColor(r, rbw_, cv::COLOR_BGR2GRAY, 0, stream); } stream.waitForCompletion(); ssgm_->execute(lbw_.data, rbw_.data, dispt_.data); GpuMat left_pixels(dispt_, cv::Rect(0, 0, max_disp_, dispt_.rows)); left_pixels.setTo(0); cv::cuda::threshold(dispt_, dispt_, 4096.0f, 0.0f, cv::THRESH_TOZERO_INV, stream); // TODO: filter could be applied after upscaling (to the upscaled disparity image) if (use_filter_) { filter_->apply( dispt_, (l.size() == size_) ? l : l_scaled, dispt_, stream ); } GpuMat dispt_scaled; if (l.size() != size_) { scaleDisparity(l.size(), dispt_, dispt_scaled, stream); } else { dispt_scaled = dispt_; } dispt_scaled.convertTo(disp, CV_32F, 1.0f / 16.0f, stream); #ifdef HAVE_OPTFLOW if (use_off_) { off_.filter(frame, stream); } #endif } void FixstarsSGM::setMask(Mat &mask) { return; // TODO(Nick) Not needed, but also code below does not work with new GPU pipeline CHECK(mask.type() == CV_8UC1) << "mask type must be CV_8U"; if (!ssgm_) { init(size_); } mask_l_ = GpuMat(mask); ssgm_->setMask((uint8_t*)mask.data, mask.cols); }