/* * Copyright 2019 Nicolas Pope */ #include <loguru.hpp> #include <ftl/config.h> #include <ftl/configuration.hpp> #include <ftl/threads.hpp> #include <iostream> #include <sstream> #include <string> #include <ctime> #include "calibrate.hpp" #include <opencv2/core.hpp> #include <opencv2/core/utility.hpp> #include <opencv2/imgproc.hpp> #include <opencv2/calib3d.hpp> using ftl::rgbd::detail::Calibrate; using cv::FileStorage; using cv::INTER_LINEAR; using cv::FileNode; using cv::FileNodeIterator; using cv::Mat; using cv::cuda::GpuMat; using cv::cuda::Stream; using cv::Size; using cv::Point2f; using cv::Point3f; using cv::Matx33d; using cv::Scalar; using std::string; using std::vector; Calibrate::Calibrate(nlohmann::json &config, cv::Size image_size, cv::cuda::Stream &stream) : ftl::Configurable(config) { std::pair<Mat, Mat> map1, map2; calibrated_ = _loadCalibration(image_size, map1, map2); if (calibrated_) { _updateIntrinsics(); LOG(INFO) << "Calibration loaded from file"; } else { LOG(WARNING) << "Calibration not loaded"; } this->on("use_intrinsics", [this](const ftl::config::Event &e) { _updateIntrinsics(); }); } bool Calibrate::_loadCalibration(cv::Size img_size, std::pair<Mat, Mat> &map1, std::pair<Mat, Mat> &map2) { FileStorage fs; // reading intrinsic parameters auto ifile = ftl::locateFile(value("intrinsics", std::string("intrinsics.yml"))); if (ifile) { fs.open((*ifile).c_str(), FileStorage::READ); if (!fs.isOpened()) { LOG(WARNING) << "Could not open intrinsics file"; return false; } LOG(INFO) << "Intrinsics from: " << *ifile; } else { LOG(WARNING) << "Calibration intrinsics file not found"; return false; } { vector<Mat> K, D; fs["K"] >> K; fs["D"] >> D; K[0].copyTo(M1_); K[1].copyTo(M2_); D[0].copyTo(D1_); D[1].copyTo(D2_); } CHECK(M1_.size() == Size(3, 3)); CHECK(M2_.size() == Size(3, 3)); CHECK(D1_.size() == Size(5, 1)); CHECK(D2_.size() == Size(5, 1)); auto efile = ftl::locateFile(value("extrinsics", std::string("extrinsics.yml"))); if (efile) { fs.open((*efile).c_str(), FileStorage::READ); if (!fs.isOpened()) { LOG(WARNING) << "Could not open extrinsics file"; return false; } LOG(INFO) << "Extrinsics from: " << *efile; } else { LOG(WARNING) << "Calibration extrinsics file not found"; return false; } fs["R"] >> R_; fs["T"] >> T_; fs["R1"] >> R1_; fs["R2"] >> R2_; fs["P1"] >> P1_; fs["P2"] >> P2_; fs["Q"] >> Q_; img_size_ = img_size; // TODO: normalize calibration double scale_x = 1.0 / 1280.0; double scale_y = 1.0 / 720.0; Mat scale(cv::Size(3, 3), CV_64F, 0.0); scale.at<double>(0, 0) = (double) img_size.width * scale_x; scale.at<double>(1, 1) = (double) img_size.height * scale_y; scale.at<double>(2, 2) = 1.0; M1_ = scale * M1_; M2_ = scale * M2_; P1_ = scale * P1_; P2_ = scale * P2_; Q_.at<double>(0, 3) = Q_.at<double>(3, 2) * (double) img_size.width * scale_x; Q_.at<double>(1, 3) = Q_.at<double>(3, 2) * (double) img_size.height * scale_y; Q_.at<double>(3, 3) = Q_.at<double>(3, 3) * (double) img_size.width * scale_x; // cv::cuda::remap() works only with CV_32FC1 initUndistortRectifyMap(M1_, D1_, R1_, P1_, img_size_, CV_32FC1, map1.first, map2.first); initUndistortRectifyMap(M2_, D2_, R2_, P2_, img_size_, CV_32FC1, map1.second, map2.second); return true; } void Calibrate::updateCalibration(const ftl::rgbd::Camera &p) { std::pair<Mat, Mat> map1, map2; Q_.at<double>(3, 2) = 1.0 / p.baseline; Q_.at<double>(2, 3) = p.fx; Q_.at<double>(0, 3) = p.cx; Q_.at<double>(1, 3) = p.cy; // FIXME:(Nick) Update camera matrix also... _updateIntrinsics(); } void Calibrate::_updateIntrinsics() { // TODO: pass parameters? Mat R1, R2, P1, P2; std::pair<Mat, Mat> map1, map2; ftl::rgbd::Camera params(); if (this->value("use_intrinsics", true)) { // rectify R1 = R1_; R2 = R2_; P1 = P1_; P2 = P2_; } else { // no rectification R1 = Mat::eye(Size(3, 3), CV_64FC1); R2 = R1; P1 = M1_; P2 = M2_; } // Set correct camera matrices for // getCameraMatrix(), getCameraMatrixLeft(), getCameraMatrixRight() C_l_ = P1; C_r_ = P2; initUndistortRectifyMap(M1_, D1_, R1, P1, img_size_, CV_32FC1, map1.first, map2.first); initUndistortRectifyMap(M2_, D2_, R2, P2, img_size_, CV_32FC1, map1.second, map2.second); // CHECK Is this thread safe!!!! map1_gpu_.first.upload(map1.first); map1_gpu_.second.upload(map1.second); map2_gpu_.first.upload(map2.first); map2_gpu_.second.upload(map2.second); } void Calibrate::rectifyStereo(GpuMat &l, GpuMat &r, Stream &stream) { // cv::cuda::remap() can not use same Mat for input and output GpuMat l_tmp(l.size(), l.type()); GpuMat r_tmp(r.size(), r.type()); cv::cuda::remap(l, l_tmp, map1_gpu_.first, map2_gpu_.first, cv::INTER_LINEAR, 0, cv::Scalar(), stream); cv::cuda::remap(r, r_tmp, map1_gpu_.second, map2_gpu_.second, cv::INTER_LINEAR, 0, cv::Scalar(), stream); stream.waitForCompletion(); l = l_tmp; r = r_tmp; } bool Calibrate::isCalibrated() { return calibrated_; }