diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 30514de41c5a7d803c61d6ff44eadd558e268643..b9712ca6487b4ddda8db7a98f509716c935ffd5a 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -6,7 +6,7 @@ variables: GIT_SUBMODULE_STRATEGY: recursive - CMAKE_ARGS_WINDOWS: '-DCMAKE_GENERATOR_PLATFORM=x64 -DPORTAUDIO_DIR="D:/Build/portaudio" -DNVPIPE_DIR="D:/Build/NvPipe" -DEigen3_DIR="C:/Program Files (x86)/Eigen3/share/eigen3/cmake" -DOpenCV_DIR="D:/Build/opencv-4.1.1" -DCUDA_TOOLKIT_ROOT_DIR="C:/Program Files/NVIDIA GPU Computing Toolkit/CUDA/v10.1" -DWITH_OPENVR=TRUE -DOPENVR_DIR="D:/Build/OpenVRSDK"' + CMAKE_ARGS_WINDOWS: '-DCMAKE_GENERATOR_PLATFORM=x64 -DPORTAUDIO_DIR="D:/Build/portaudio" -DNVPIPE_DIR="D:/Build/NvPipe" -DEigen3_DIR="C:/Program Files (x86)/Eigen3/share/eigen3/cmake" -DOpenCV_DIR="D:/Build/opencv-4.1.1" -DCUDA_TOOLKIT_ROOT_DIR="C:/Program Files/NVIDIA GPU Computing Toolkit/CUDA/v10.1" -DWITH_OPENVR=TRUE -DOPENVR_DIR="D:/Build/OpenVRSDK" -DWITH_CERES=FALSE' stages: - all diff --git a/CMakeLists.txt b/CMakeLists.txt index 46783116d8b5670caf45b5fb317b81dcd79c2323..ce1b438e4e2a5659f2166b178585128989816063 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -13,7 +13,8 @@ enable_testing() option(WITH_NVPIPE "Use NvPipe for compression if available" ON) option(WITH_OPTFLOW "Use NVIDIA Optical Flow if available" OFF) option(WITH_OPENVR "Build with OpenVR support" OFF) -option(WITH_FIXSTARS "Use Fixstars libSGM if available" ON) +option(WITH_FIXSTARS "Use Fixstars libSGM" ON) +option(WITH_CERES "Use Ceres solver" ON) option(USE_CPPCHECK "Apply cppcheck during build" ON) option(BUILD_VISION "Enable the vision component" ON) option(BUILD_RECONSTRUCT "Enable the reconstruction component" ON) @@ -24,6 +25,10 @@ option(BUILD_TOOLS "Compile developer and research tools" ON) option(BUILD_TESTS "Compile all unit and integration tests" ON) option(ENABLE_PROFILER "Enable builtin performance profiling" OFF) +if (BUILD_TESTS) + include_directories(lib/catch) +endif() + set(THREADS_PREFER_PTHREAD_FLAG ON) set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_SOURCE_DIR}/cmake/") @@ -31,13 +36,13 @@ include(Findglog) MACRO( VERSION_STR_TO_INTS major minor patch version ) - STRING( REGEX REPLACE "([0-9]+).[0-9]+.[0-9]+" "\\1" ${major} ${version} ) - STRING( REGEX REPLACE "[0-9]+.([0-9]+).[0-9]+" "\\1" ${minor} ${version} ) - STRING( REGEX REPLACE "[0-9]+.[0-9]+.([0-9]+)" "\\1" ${patch} ${version} ) + STRING( REGEX REPLACE "([0-9]+).[0-9]+.[0-9]+" "\\1" ${major} ${version} ) + STRING( REGEX REPLACE "[0-9]+.([0-9]+).[0-9]+" "\\1" ${minor} ${version} ) + STRING( REGEX REPLACE "[0-9]+.[0-9]+.([0-9]+)" "\\1" ${patch} ${version} ) ENDMACRO( VERSION_STR_TO_INTS ) -find_package( OpenCV REQUIRED COMPONENTS core imgproc highgui cudaimgproc calib3d imgcodecs videoio aruco cudaarithm cudastereo cudaoptflow face tracking quality) +find_package( OpenCV REQUIRED COMPONENTS core imgproc highgui cudaimgproc calib3d imgcodecs videoio aruco cudaarithm cudastereo cudaoptflow face tracking quality xfeatures2d) find_package( Threads REQUIRED ) find_package( URIParser REQUIRED ) find_package( MsgPack REQUIRED ) @@ -111,12 +116,19 @@ endif() # ============================================================================== if (WITH_FIXSTARS) - find_package( LibSGM ) + find_package(LibSGM REQUIRED) if (LibSGM_FOUND) set(HAVE_LIBSGM true) endif() endif() +if (WITH_CERES) + find_package(Ceres REQUIRED) + set(HAVE_CERES true) +else() + add_library(ceres INTERFACE) +endif() + if(${CMAKE_VERSION} VERSION_GREATER "3.12.0") cmake_policy(SET CMP0074 NEW) endif() @@ -326,6 +338,7 @@ SET(CMAKE_USE_RELATIVE_PATHS ON) set(CMAKE_WINDOWS_EXPORT_ALL_SYMBOLS ON) add_subdirectory(components/common/cpp) +add_subdirectory(applications/calibration) add_subdirectory(components/codecs) add_subdirectory(components/structures) add_subdirectory(components/net) @@ -334,7 +347,7 @@ add_subdirectory(components/control/cpp) add_subdirectory(components/operators) add_subdirectory(components/streams) add_subdirectory(components/audio) -add_subdirectory(applications/calibration) +add_subdirectory(components/calibration) #add_subdirectory(applications/groupview) #add_subdirectory(applications/player) #add_subdirectory(applications/recorder) @@ -355,7 +368,10 @@ if (BUILD_VISION) endif() if (BUILD_CALIBRATION) - find_package(Ceres REQUIRED) + if (NOT HAVE_CERES) + message(ERROR "Ceres is required") + endif() + add_subdirectory(applications/calibration-ceres) add_subdirectory(applications/calibration-multi) endif() diff --git a/applications/calibration-ceres/CMakeLists.txt b/applications/calibration-ceres/CMakeLists.txt index f10bc716360aee2bfede7152c6d763192258ae7f..d994acaa990d35355f4f891f8a539dd5335782ed 100644 --- a/applications/calibration-ceres/CMakeLists.txt +++ b/applications/calibration-ceres/CMakeLists.txt @@ -3,11 +3,10 @@ add_executable (ftl-calibration-ceres src/calibration_data.cpp src/visibility.cpp src/calibration.cpp - src/optimization.cpp ) target_include_directories(ftl-calibration-ceres PRIVATE src/) target_include_directories(ftl-calibration-ceres PUBLIC ${OpenCV_INCLUDE_DIRS}) -target_link_libraries(ftl-calibration-ceres pthread ftlcommon Eigen3::Eigen ceres) +target_link_libraries(ftl-calibration-ceres ftlcalibration Threads::Threads ftlcommon Eigen3::Eigen ceres) add_subdirectory(test) diff --git a/applications/calibration-ceres/src/calibration.cpp b/applications/calibration-ceres/src/calibration.cpp index 2c09538e3672742b9874b287994cb3dde8d92e37..ab0f9412f366fad9675192a9082b25ce3e053ae6 100644 --- a/applications/calibration-ceres/src/calibration.cpp +++ b/applications/calibration-ceres/src/calibration.cpp @@ -1,5 +1,5 @@ #include "calibration.hpp" -#include "optimization.hpp" +#include "ftl/calibration/optimize.hpp" #include "loguru.hpp" @@ -20,27 +20,6 @@ using ftl::calibration::BundleAdjustment; using namespace ftl::calibration; -double ftl::calibration::reprojectionError(const vector<Point2d>& points_im, - const vector<Point3d>& points, const Mat& K, const Mat& D, const Mat& R, - const Mat& t) { - - Mat rvec; - if (R.size() == Size(3, 3)) { cv::Rodrigues(R, rvec); } - else { rvec = R; } - - vector<Point2d> points_reprojected; - cv::projectPoints(points, rvec, t, K, D, points_reprojected); - - double err = 0.0; - int npoints = points_im.size(); - - for (int i = 0; i < npoints; i++) { - err += norm(points_im[i] - points_reprojected[i]); - } - - return err / ((double) npoints); -} - int ftl::calibration::recoverPose(const Mat &E, const vector<Point2d> &_points1, const vector<Point2d> &_points2, const Mat &_cameraMatrix1, const Mat &_cameraMatrix2, Mat &_R, Mat &_t, double distanceThresh, diff --git a/applications/calibration-ceres/src/calibration.hpp b/applications/calibration-ceres/src/calibration.hpp index 5151d1a96766a7d06d044b7818d4105123815421..11bb36d920da1e638ce49ad699a955fe5032acec 100644 --- a/applications/calibration-ceres/src/calibration.hpp +++ b/applications/calibration-ceres/src/calibration.hpp @@ -9,75 +9,6 @@ namespace ftl { namespace calibration { -namespace transform { - -/** - * @brief Get rotation matrix and translation vector from transformation matrix. - * @param T transformation matrix (4x4) or (3x4) - * @param R (out) rotation matrix (3x3) - * @param t (out) translation vector (3x1) - */ -inline void getRotationAndTranslation(const cv::Mat &T, cv::Mat &R, cv::Mat &t) { - T(cv::Rect(0, 0, 3, 3)).copyTo(R); - T(cv::Rect(3, 0, 1, 3)).copyTo(t); -} - -/** - * @brief Inverse transform inplace - * @param R rotation matrix (3x3) - * @param t translation vector (3x1) - */ -inline void inverse(cv::Mat &R, cv::Mat &t) { - cv::Mat t_(3, 1, CV_64FC1); - - t_.at<double>(0) = -R.col(0).dot(t); - t_.at<double>(1) = -R.col(1).dot(t); - t_.at<double>(2) = -R.col(2).dot(t); - - cv::swap(t_, t); - R = R.t(); -} - -/** - * @brief Inverse transform inplace - * @param T transformation matrix (4x4) - */ -inline void inverse(cv::Mat &T) { - cv::Mat rmat; - cv::Mat tvec; - getRotationAndTranslation(T, rmat, tvec); - T = cv::Mat::eye(4, 4, CV_64FC1); - - T(cv::Rect(3, 0, 1, 1)) = -rmat.col(0).dot(tvec); - T(cv::Rect(3, 1, 1, 1)) = -rmat.col(1).dot(tvec); - T(cv::Rect(3, 2, 1, 1)) = -rmat.col(2).dot(tvec); - T(cv::Rect(0, 0, 3, 3)) = rmat.t(); -} - -inline cv::Point3d apply(const cv::Point3d& point, const cv::InputArray& R, const cv::InputArray& t) { - cv::Mat R_ = R.getMat(); - cv::Mat t_ = t.getMat(); - cv::Mat p_new = R_ * cv::Mat(point) + t_; - return cv::Point3d(p_new.at<double>(0), p_new.at<double>(1), p_new.at<double>(2)); -} - -inline cv::Point3d apply(const cv::Point3d& point, const cv::InputArray& T) { - cv::Mat T_ = T.getMat(); - cv::Mat rmat; - cv::Mat tvec; - getRotationAndTranslation(T_, rmat, tvec); - return apply(point, rmat, tvec); -} - -} // namespace transform - -/** - * @brief Calculate MSE reprojection error - */ -double reprojectionError(const std::vector<cv::Point2d>& points_im, - const std::vector<cv::Point3d>& points, const cv::Mat& K, const cv::Mat& D, - const cv::Mat& R, const cv::Mat& t); - /** * Same as OpenCV's recoverPose(), but does not assume same intrinsic paramters * for both cameras. diff --git a/applications/calibration-ceres/src/calibration_data.cpp b/applications/calibration-ceres/src/calibration_data.cpp index 28b77fa8b6b21a07abb8b7e32f46dab0f4363505..1e634a3921330ba6b2b8a04d75ecdd5352596409 100644 --- a/applications/calibration-ceres/src/calibration_data.cpp +++ b/applications/calibration-ceres/src/calibration_data.cpp @@ -15,135 +15,7 @@ using cv::Rect; using cv::Size; using ftl::calibration::CalibrationData; -using ftl::calibration::Camera; -//////////////////////////////////////////////////////////////////////////////// - -void Camera::setRotation(const Mat& R) { - if (((R.size() != Size(3, 3)) && - (R.size() != Size(3, 1)) && - (R.size() != Size(1, 3))) || - (R.type() != CV_64FC1)) { throw std::exception(); } - - Mat rvec; - if (R.size() == cv::Size(3, 3)) { cv::Rodrigues(R, rvec); } - else { rvec = R; } - - data[Parameter::RX] = rvec.at<double>(0); - data[Parameter::RY] = rvec.at<double>(1); - data[Parameter::RZ] = rvec.at<double>(2); -} - -void Camera::setTranslation(const Mat& t) { - if ((t.type() != CV_64FC1) || - (t.size() != cv::Size(1, 3))) { throw std::exception(); } - - data[Parameter::TX] = t.at<double>(0); - data[Parameter::TY] = t.at<double>(1); - data[Parameter::TZ] = t.at<double>(2); -} - - -void Camera::setIntrinsic(const Mat& K) { - if ((K.type() != CV_64FC1) || (K.size() != cv::Size(3, 3))) { - throw std::exception(); - } - - data[Parameter::F] = K.at<double>(0, 0); - data[Parameter::CX] = K.at<double>(0, 2); - data[Parameter::CY] = K.at<double>(1, 2); -} - -void Camera::setDistortion(const Mat& D) { - if ((D.type() != CV_64FC1)) { throw std::exception(); } - switch(D.total()) { - case 12: - /* - data[Parameter::S1] = D.at<double>(8); - data[Parameter::S2] = D.at<double>(9); - data[Parameter::S3] = D.at<double>(10); - data[Parameter::S4] = D.at<double>(11); - */ - [[fallthrough]]; - - case 8: - /* - data[Parameter::K4] = D.at<double>(5); - data[Parameter::K5] = D.at<double>(6); - data[Parameter::K6] = D.at<double>(7); - */ - [[fallthrough]]; - - case 5: - data[Parameter::K3] = D.at<double>(4); - [[fallthrough]]; - - default: - data[Parameter::K1] = D.at<double>(0); - data[Parameter::K2] = D.at<double>(1); - /* - data[Parameter::P1] = D.at<double>(2); - data[Parameter::P2] = D.at<double>(3); - */ - } -} - -Camera::Camera(const Mat &K, const Mat &D, const Mat &R, const Mat &tvec) { - setIntrinsic(K, D); - if (!R.empty()) { setRotation(R); } - if (!tvec.empty()) { setTranslation(tvec); } -} - -Mat Camera::intrinsicMatrix() const { - Mat K = Mat::eye(3, 3, CV_64FC1); - K.at<double>(0, 0) = data[Parameter::F]; - K.at<double>(1, 1) = data[Parameter::F]; - K.at<double>(0, 2) = data[Parameter::CX]; - K.at<double>(1, 2) = data[Parameter::CY]; - return K; -} - -Mat Camera::distortionCoefficients() const { - Mat D; - // OpenCV distortion parameter sizes: 4, 5, 8, 12, 14 - if (_NDISTORTION_PARAMETERS <= 4) { D = Mat::zeros(4, 1, CV_64FC1); } - else if (_NDISTORTION_PARAMETERS <= 5) { D = Mat::zeros(5, 1, CV_64FC1); } - else if (_NDISTORTION_PARAMETERS <= 8) { D = Mat::zeros(8, 1, CV_64FC1); } - else if (_NDISTORTION_PARAMETERS <= 12) { D = Mat::zeros(12, 1, CV_64FC1); } - else if (_NDISTORTION_PARAMETERS <= 14) { D = Mat::zeros(14, 1, CV_64FC1); } - - for (int i = 0; i < _NDISTORTION_PARAMETERS; i++) { - D.at<double>(i) = data[Parameter::DISTORTION+i]; - } - return D; -} - -Mat Camera::rvec() const { - return Mat(Vec3d(data[Parameter::RX], data[Parameter::RY], data[Parameter::RZ])); -} - -Mat Camera::tvec() const { - return Mat(Vec3d(data[Parameter::TX], data[Parameter::TY], data[Parameter::TZ])); -} - -Mat Camera::rmat() const { - Mat R; - cv::Rodrigues(rvec(), R); - return R; -} - -Mat Camera::extrinsicMatrix() const { - Mat T = Mat::eye(4, 4, CV_64FC1); - rmat().copyTo(T(Rect(0, 0, 3, 3))); - tvec().copyTo(T(Rect(3, 0, 1, 3))); - return T; -} - -Mat Camera::extrinsicMatrixInverse() const { - return extrinsicMatrix().inv(); -} - -//////////////////////////////////////////////////////////////////////////////// CalibrationData::CalibrationData(int n_cameras) { init(n_cameras); } diff --git a/applications/calibration-ceres/src/calibration_data.hpp b/applications/calibration-ceres/src/calibration_data.hpp index e98ed3d3c0f86a97e78961eeeb2e102581da9e14..9e5871bf2ce6407ec58a938c0574dd4d6374f6a7 100644 --- a/applications/calibration-ceres/src/calibration_data.hpp +++ b/applications/calibration-ceres/src/calibration_data.hpp @@ -4,6 +4,7 @@ #include <vector> #include <opencv2/core/core.hpp> +#include <ftl/calibration/optimize.hpp> #define _NDISTORTION_PARAMETERS 3 #define _NCAMERA_PARAMETERS (9 + _NDISTORTION_PARAMETERS) @@ -11,58 +12,6 @@ namespace ftl { namespace calibration { -/** - * Camera paramters - */ -struct Camera { - Camera() {} - Camera(const cv::Mat& K, const cv::Mat& D, const cv::Mat& R, const cv::Mat& tvec); - - void setRotation(const cv::Mat& R); - void setTranslation(const cv::Mat& tvec); - void setExtrinsic(const cv::Mat& R, const cv::Mat& t) { - setRotation(R); - setTranslation(t); - } - - void setIntrinsic(const cv::Mat& K); - void setDistortion(const cv::Mat &D); - void setIntrinsic(const cv::Mat& K, const cv::Mat& D) { - setIntrinsic(K); - setDistortion(D); - } - - cv::Mat intrinsicMatrix() const; - cv::Mat distortionCoefficients() const; - - cv::Mat rvec() const; - cv::Mat tvec() const; - cv::Mat rmat() const; - - cv::Mat extrinsicMatrix() const; - cv::Mat extrinsicMatrixInverse() const; - - double data[_NCAMERA_PARAMETERS] = {0.0}; - - enum Parameter { - ROTATION = 0, - RX = 0, - RY = 1, - RZ = 2, - TRANSLATION = 3, - TX = 3, - TY = 4, - TZ = 5, - F = 6, - CX = 7, - CY = 8, - DISTORTION = 9, - K1 = 9, - K2 = 10, - K3 = 11 - }; -}; - class CalibrationData { public: CalibrationData() {}; diff --git a/applications/calibration-ceres/src/main.cpp b/applications/calibration-ceres/src/main.cpp index 087c3283ba303eceab82ac565c626ec0051e403e..9020d0e7737f43a7eff44406ad7853243161a4e5 100644 --- a/applications/calibration-ceres/src/main.cpp +++ b/applications/calibration-ceres/src/main.cpp @@ -10,7 +10,9 @@ #include <ceres/ceres.h> #include <ceres/rotation.h> -#include "optimization.hpp" +#include <ftl/calibration.hpp> + +#include "calibration_data.hpp" #include "visibility.hpp" #include "calibration.hpp" diff --git a/applications/calibration-ceres/test/CMakeLists.txt b/applications/calibration-ceres/test/CMakeLists.txt index 5f76122ccdb66a999d22974cab932b6ec655b5e9..6c90f1d58ff1aa6e8c4d2c7bac0c2fd60f53917b 100644 --- a/applications/calibration-ceres/test/CMakeLists.txt +++ b/applications/calibration-ceres/test/CMakeLists.txt @@ -1,7 +1,3 @@ - -#add_library(Catch INTERFACE) -#target_include_directories(Catch INTERFACE ${CATCH_INCLUDE_DIR}) - add_executable(visibility_unit ./tests.cpp ../src/visibility.cpp @@ -9,21 +5,21 @@ add_executable(visibility_unit ) target_include_directories(visibility_unit PUBLIC ../src/) -target_link_libraries(visibility_unit pthread dl ftlcommon) +target_link_libraries(visibility_unit Threads::Threads dl ftlcommon) add_test(VisibilityTest visibility_unit) ################################################################################ -add_executable(calibration_unit - ./tests.cpp - ../src/calibration.cpp - ../src/optimization.cpp - ../src/calibration_data.cpp - ./calibration_unit.cpp -) +#add_executable(calibration_unit +# ./tests.cpp +# ../src/calibration.cpp +# ../src/optimization.cpp +# ../src/calibration_data.cpp +# ./calibration_unit.cpp +#) -target_include_directories(calibration_unit PUBLIC ../src/) -target_link_libraries(calibration_unit pthread dl ftlcommon Eigen3::Eigen ceres) +#target_include_directories(calibration_unit PUBLIC ../src/) +#target_link_libraries(calibration_unit Threads::Threads dl ftlcommon Eigen3::Eigen ceres) -add_test(CalibrationTest calibration_unit WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}) +#add_test(CalibrationTest calibration_unit WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}) diff --git a/applications/calibration-multi/CMakeLists.txt b/applications/calibration-multi/CMakeLists.txt index 135d48adfea1a7114868baa0d3fc7bfe7680160a..d8c9c5b40a711dbd3778187163bfd4bba327bf7a 100644 --- a/applications/calibration-multi/CMakeLists.txt +++ b/applications/calibration-multi/CMakeLists.txt @@ -3,13 +3,11 @@ set(CALIBMULTI src/visibility.cpp src/util.cpp src/multicalibrate.cpp - ../calibration-ceres/src/optimization.cpp ../calibration-ceres/src/calibration.cpp - ../calibration-ceres/src/calibration_data.cpp ) add_executable(ftl-calibrate-multi ${CALIBMULTI}) target_include_directories(ftl-calibrate-multi PRIVATE src ../calibration-ceres/src) -target_link_libraries(ftl-calibrate-multi ftlcommon ftlnet ftlrgbd ftlstreams Threads::Threads ${OpenCV_LIBS} ceres) +target_link_libraries(ftl-calibrate-multi ftlcalibration ftlcommon ftlnet ftlrgbd ftlstreams Threads::Threads ${OpenCV_LIBS} ceres) diff --git a/applications/calibration-multi/src/multicalibrate.cpp b/applications/calibration-multi/src/multicalibrate.cpp index b9627ce252211ef9075e165270d6d88376038814..836a90fde0d0b091ad8207315463ffab6a20e295 100644 --- a/applications/calibration-multi/src/multicalibrate.cpp +++ b/applications/calibration-multi/src/multicalibrate.cpp @@ -2,7 +2,8 @@ #include "calibration_data.hpp" #include "calibration.hpp" -#include "optimization.hpp" + +#include <ftl/calibration/optimize.hpp> #include <opencv2/core.hpp> #include <opencv2/calib3d.hpp> diff --git a/applications/calibration/CMakeLists.txt b/applications/calibration/CMakeLists.txt index 0efa6e95c7c66e4480e2965e0cccffae39d5cb09..2452f392a9cb6d0dbcac4737ba99ff106996f699 100644 --- a/applications/calibration/CMakeLists.txt +++ b/applications/calibration/CMakeLists.txt @@ -10,6 +10,6 @@ add_executable(ftl-calibrate ${CALIBSRC}) target_include_directories(ftl-calibrate PRIVATE src) -target_link_libraries(ftl-calibrate ftlcommon Threads::Threads ${OpenCV_LIBS}) +target_link_libraries(ftl-calibrate ftlcalibration ftlcommon Threads::Threads ${OpenCV_LIBS}) diff --git a/applications/calibration/src/lens.cpp b/applications/calibration/src/lens.cpp index b9a01581582b4ef12482932d839a0caccaf1367e..ffa3cd078593a3533b6cfa105ff3eae7f21e5f5e 100644 --- a/applications/calibration/src/lens.cpp +++ b/applications/calibration/src/lens.cpp @@ -2,7 +2,7 @@ #include "lens.hpp" #include <ftl/config.h> -#include <ftl/calibration.hpp> +#include <ftl/calibration/parameters.hpp> #include <loguru.hpp> diff --git a/components/calibration/CMakeLists.txt b/components/calibration/CMakeLists.txt new file mode 100644 index 0000000000000000000000000000000000000000..00faed0b6a4c0cd5292048d8580fc6cbc0657d63 --- /dev/null +++ b/components/calibration/CMakeLists.txt @@ -0,0 +1,24 @@ +set(CALIBSRC + src/parameters.cpp +) + +if (WITH_CERES) + list(APPEND CALIBSRC src/optimize.cpp) +endif() + +add_library(ftlcalibration ${CALIBSRC}) + +target_include_directories(ftlcalibration + PUBLIC + $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/include> + $<INSTALL_INTERFACE:include> + PRIVATE + src/ + ${OpenCV_INCLUDE_DIRS} +) + +target_link_libraries(ftlcalibration ftlcommon Threads::Threads ${OpenCV_LIBS} Eigen3::Eigen ceres) + +if (BUILD_TESTS) + ADD_SUBDIRECTORY(test) +endif() diff --git a/components/calibration/include/ftl/calibration.hpp b/components/calibration/include/ftl/calibration.hpp new file mode 100644 index 0000000000000000000000000000000000000000..be7533631af984486a99ed30fe14a141b9f28195 --- /dev/null +++ b/components/calibration/include/ftl/calibration.hpp @@ -0,0 +1,2 @@ +#include "calibration/parameters.hpp" +#include "calibration/optimize.hpp" diff --git a/applications/calibration-ceres/src/optimization.hpp b/components/calibration/include/ftl/calibration/optimize.hpp similarity index 96% rename from applications/calibration-ceres/src/optimization.hpp rename to components/calibration/include/ftl/calibration/optimize.hpp index 3b0716c7b6b73ab6cc25bdcc2cedf65b33a5da22..ea5908fce66b418d49710d608b723f0b731c7e3a 100644 --- a/applications/calibration-ceres/src/optimization.hpp +++ b/components/calibration/include/ftl/calibration/optimize.hpp @@ -1,13 +1,14 @@ #pragma once -#ifndef _FTL_OPTIMIZATION_HPP_ -#define _FTL_OPTIMIZATION_HPP_ - -#include "optimization.hpp" -#include "calibration_data.hpp" +#ifndef _FTL_CALIBRATION_OPTIMIZE_HPP_ +#define _FTL_CALIBRATION_OPTIMIZE_HPP_ #include <vector> #include <functional> +#include "parameters.hpp" + +#include <ftl/config.h> + #include <ceres/ceres.h> #include <opencv2/core/core.hpp> @@ -20,7 +21,7 @@ static_assert(std::is_standard_layout<cv::Point3d>()); namespace ftl { namespace calibration { -static_assert(std::is_standard_layout<Camera>()); +#ifdef HAVE_CERES /** * @brief Optimize scale. @@ -165,6 +166,8 @@ private: std::vector<BundleAdjustment::ConstraintObject> constraints_object_; }; +#endif + } } diff --git a/components/calibration/include/ftl/calibration/parameters.hpp b/components/calibration/include/ftl/calibration/parameters.hpp new file mode 100644 index 0000000000000000000000000000000000000000..f261506570921b8c243855e6ace9ea6b2fcfdd95 --- /dev/null +++ b/components/calibration/include/ftl/calibration/parameters.hpp @@ -0,0 +1,184 @@ +#pragma once +#ifndef _FTL_CALIBRATION_PARAMETERS_HPP_ +#define _FTL_CALIBRATION_PARAMETERS_HPP_ + +#include <opencv2/core/core.hpp> + +namespace ftl { +namespace calibration { + +/** + * Camera paramters + */ +struct Camera { + Camera() {} + Camera(const cv::Mat& K, const cv::Mat& D, const cv::Mat& R, const cv::Mat& tvec); + + void setRotation(const cv::Mat& R); + void setTranslation(const cv::Mat& tvec); + void setExtrinsic(const cv::Mat& R, const cv::Mat& t) { + setRotation(R); + setTranslation(t); + } + + void setIntrinsic(const cv::Mat& K); + void setDistortion(const cv::Mat &D); + void setIntrinsic(const cv::Mat& K, const cv::Mat& D) { + setIntrinsic(K); + setDistortion(D); + } + + cv::Mat intrinsicMatrix() const; + cv::Mat distortionCoefficients() const; + + cv::Mat rvec() const; + cv::Mat tvec() const; + cv::Mat rmat() const; + + cv::Mat extrinsicMatrix() const; + cv::Mat extrinsicMatrixInverse() const; + + const static int n_parameters = 12; + const static int n_distortion_parameters = 3; + + double data[n_parameters] = {0.0}; + + enum Parameter { + ROTATION = 0, + RX = 0, + RY = 1, + RZ = 2, + TRANSLATION = 3, + TX = 3, + TY = 4, + TZ = 5, + F = 6, + CX = 7, + CY = 8, + DISTORTION = 9, + K1 = 9, + K2 = 10, + K3 = 11 + }; +}; + +namespace validate { + +/** + * @brief Valid translation for stereo camera. + */ +bool translationStereo(const cv::Mat &t); + +bool rotationMatrix(const cv::Mat &M); + +bool pose(const cv::Mat &M); + +bool cameraMatrix(const cv::Mat &M); + +/** + * @brief Check if D contains valid distortion coefficients. + * @param D distortion coefficients + * @param size resolution + * @note Tangential and prism distortion coefficients are not validated. + * + * Radial distortion is always monotonic for real lenses and distortion + * function has to be bijective. This is verified by evaluating the distortion + * function for integer values from 0 to sqrt(width^2+height^2). + * + * Camera model documented in + * https://docs.opencv.org/master/d9/d0c/group__calib3d.html#details + */ +bool distortionCoefficients(const cv::Mat &D, cv::Size size); + +} + + +namespace transform { + +// TODO: Some of the methods can be directly replace with OpenCV +// (opencv2/calib3d.hpp) + +/** + * @brief Get rotation matrix and translation vector from transformation matrix. + * @param T transformation matrix (4x4) or (3x4) + * @param R (out) rotation matrix (3x3) + * @param t (out) translation vector (3x1) + */ +inline void getRotationAndTranslation(const cv::Mat &T, cv::Mat &R, cv::Mat &t) { + T(cv::Rect(0, 0, 3, 3)).copyTo(R); + T(cv::Rect(3, 0, 1, 3)).copyTo(t); +} + +/** + * @brief Inverse transform inplace + * @param R rotation matrix (3x3) + * @param t translation vector (3x1) + */ +inline void inverse(cv::Mat &R, cv::Mat &t) { + cv::Mat t_(3, 1, CV_64FC1); + + t_.at<double>(0) = -R.col(0).dot(t); + t_.at<double>(1) = -R.col(1).dot(t); + t_.at<double>(2) = -R.col(2).dot(t); + + cv::swap(t_, t); + R = R.t(); +} + +/** + * @brief Inverse transform inplace + * @param T transformation matrix (4x4) + */ +inline void inverse(cv::Mat &T) { + cv::Mat rmat; + cv::Mat tvec; + getRotationAndTranslation(T, rmat, tvec); + T = cv::Mat::eye(4, 4, CV_64FC1); + + T(cv::Rect(3, 0, 1, 1)) = -rmat.col(0).dot(tvec); + T(cv::Rect(3, 1, 1, 1)) = -rmat.col(1).dot(tvec); + T(cv::Rect(3, 2, 1, 1)) = -rmat.col(2).dot(tvec); + T(cv::Rect(0, 0, 3, 3)) = rmat.t(); +} + +inline cv::Point3d apply(const cv::Point3d& point, const cv::InputArray& R, const cv::InputArray& t) { + cv::Mat R_ = R.getMat(); + cv::Mat t_ = t.getMat(); + cv::Mat p_new = R_ * cv::Mat(point) + t_; + return cv::Point3d(p_new.at<double>(0), p_new.at<double>(1), p_new.at<double>(2)); +} + +inline cv::Point3d apply(const cv::Point3d& point, const cv::InputArray& T) { + cv::Mat T_ = T.getMat(); + cv::Mat rmat; + cv::Mat tvec; + getRotationAndTranslation(T_, rmat, tvec); + return apply(point, rmat, tvec); +} + +} // namespace transform + +/** + * @brief Scale camera intrinsic matrix + * @param size_new New resolution + * @param size_old Original (camera matrix) resolution + */ +cv::Mat scaleCameraMatrix(const cv::Mat &K, const cv::Size &size_new, const cv::Size &size_old); + +/** + * @brief Calculate MSE reprojection error + * @param points_im points in pixel coordinates + * @param points points in camera coordinates + * @param K intrinsic matrix + * @param D distortion coefficients + * @param R rotation matrix or vector + * @param t translation vector + */ +double reprojectionError(const std::vector<cv::Point2d>& points_im, + const std::vector<cv::Point3d>& points, const cv::Mat& K, const cv::Mat& D, + const cv::Mat& R, const cv::Mat& t); + +} +} + +#endif diff --git a/applications/calibration-ceres/src/optimization.cpp b/components/calibration/src/optimize.cpp similarity index 98% rename from applications/calibration-ceres/src/optimization.cpp rename to components/calibration/src/optimize.cpp index aafe2a38a960c3349521a0d2f88369cafb0f8597..9ea70c4425091acb08a8348f164e8fa406ea1780 100644 --- a/applications/calibration-ceres/src/optimization.cpp +++ b/components/calibration/src/optimize.cpp @@ -1,5 +1,5 @@ -#include "optimization.hpp" -#include "calibration.hpp" +#include "ftl/calibration/optimize.hpp" +#include "ftl/calibration/parameters.hpp" #include "loguru.hpp" @@ -86,12 +86,12 @@ struct ReprojectionError { static ceres::CostFunction* Create( const double observed_x, const double observed_y) { - return (new ceres::AutoDiffCostFunction<ReprojectionError, 2, _NCAMERA_PARAMETERS, 3>( + return (new ceres::AutoDiffCostFunction<ReprojectionError, 2, Camera::n_parameters, 3>( new ReprojectionError(observed_x, observed_y))); } static ceres::CostFunction* Create( const Point2d &observed) { - return (new ceres::AutoDiffCostFunction<ReprojectionError, 2, _NCAMERA_PARAMETERS, 3>( + return (new ceres::AutoDiffCostFunction<ReprojectionError, 2, Camera::n_parameters, 3>( new ReprojectionError(observed.x, observed.y))); } @@ -333,14 +333,14 @@ void BundleAdjustment::_buildBundleAdjustmentProblem(ceres::Problem &problem, co vector<int> params(fixed_parameters.begin(), fixed_parameters.end()); - if (params.size() == _NCAMERA_PARAMETERS) { + if (params.size() == Camera::n_parameters) { // Ceres crashes if all parameters are set constant using // SubsetParameterization() problem.SetParameterBlockConstant(getCameraPtr(i)); } else if (params.size() > 0) { problem.SetParameterization(getCameraPtr(i), - new ceres::SubsetParameterization(_NCAMERA_PARAMETERS, params)); + new ceres::SubsetParameterization(Camera::n_parameters, params)); } } } diff --git a/components/calibration/src/parameters.cpp b/components/calibration/src/parameters.cpp new file mode 100644 index 0000000000000000000000000000000000000000..a947771378f592e2398bb3e278e7214d213e8159 --- /dev/null +++ b/components/calibration/src/parameters.cpp @@ -0,0 +1,304 @@ +#include "ftl/calibration/parameters.hpp" + +#include <opencv2/calib3d/calib3d.hpp> + +using cv::Mat; +using cv::Size; +using cv::Point2d; +using cv::Point3d; +using cv::Vec3d; +using cv::Rect; + +using std::vector; + +using namespace ftl::calibration; + +using ftl::calibration::Camera; + +//////////////////////////////////////////////////////////////////////////////// + +void Camera::setRotation(const Mat& R) { + if (((R.size() != Size(3, 3)) && + (R.size() != Size(3, 1)) && + (R.size() != Size(1, 3))) || + (R.type() != CV_64FC1)) { throw std::exception(); } + + Mat rvec; + if (R.size() == cv::Size(3, 3)) { cv::Rodrigues(R, rvec); } + else { rvec = R; } + + data[Parameter::RX] = rvec.at<double>(0); + data[Parameter::RY] = rvec.at<double>(1); + data[Parameter::RZ] = rvec.at<double>(2); +} + +void Camera::setTranslation(const Mat& t) { + if ((t.type() != CV_64FC1) || + (t.size() != cv::Size(1, 3))) { throw std::exception(); } + + data[Parameter::TX] = t.at<double>(0); + data[Parameter::TY] = t.at<double>(1); + data[Parameter::TZ] = t.at<double>(2); +} + + +void Camera::setIntrinsic(const Mat& K) { + if ((K.type() != CV_64FC1) || (K.size() != cv::Size(3, 3))) { + throw std::exception(); + } + + data[Parameter::F] = K.at<double>(0, 0); + data[Parameter::CX] = K.at<double>(0, 2); + data[Parameter::CY] = K.at<double>(1, 2); +} + +void Camera::setDistortion(const Mat& D) { + if ((D.type() != CV_64FC1)) { throw std::exception(); } + switch(D.total()) { + case 12: + /* + data[Parameter::S1] = D.at<double>(8); + data[Parameter::S2] = D.at<double>(9); + data[Parameter::S3] = D.at<double>(10); + data[Parameter::S4] = D.at<double>(11); + */ + [[fallthrough]]; + + case 8: + /* + data[Parameter::K4] = D.at<double>(5); + data[Parameter::K5] = D.at<double>(6); + data[Parameter::K6] = D.at<double>(7); + */ + [[fallthrough]]; + + case 5: + data[Parameter::K3] = D.at<double>(4); + [[fallthrough]]; + + default: + data[Parameter::K1] = D.at<double>(0); + data[Parameter::K2] = D.at<double>(1); + /* + data[Parameter::P1] = D.at<double>(2); + data[Parameter::P2] = D.at<double>(3); + */ + } +} + +Camera::Camera(const Mat &K, const Mat &D, const Mat &R, const Mat &tvec) { + setIntrinsic(K, D); + if (!R.empty()) { setRotation(R); } + if (!tvec.empty()) { setTranslation(tvec); } +} + +Mat Camera::intrinsicMatrix() const { + Mat K = Mat::eye(3, 3, CV_64FC1); + K.at<double>(0, 0) = data[Parameter::F]; + K.at<double>(1, 1) = data[Parameter::F]; + K.at<double>(0, 2) = data[Parameter::CX]; + K.at<double>(1, 2) = data[Parameter::CY]; + return K; +} + +Mat Camera::distortionCoefficients() const { + Mat D; + // OpenCV distortion parameter sizes: 4, 5, 8, 12, 14 + if (Camera::n_distortion_parameters <= 4) { D = Mat::zeros(4, 1, CV_64FC1); } + else if (Camera::n_distortion_parameters <= 5) { D = Mat::zeros(5, 1, CV_64FC1); } + else if (Camera::n_distortion_parameters <= 8) { D = Mat::zeros(8, 1, CV_64FC1); } + else if (Camera::n_distortion_parameters <= 12) { D = Mat::zeros(12, 1, CV_64FC1); } + else if (Camera::n_distortion_parameters <= 14) { D = Mat::zeros(14, 1, CV_64FC1); } + + for (int i = 0; i < Camera::n_distortion_parameters; i++) { + D.at<double>(i) = data[Parameter::DISTORTION+i]; + } + return D; +} + +Mat Camera::rvec() const { + return Mat(Vec3d(data[Parameter::RX], data[Parameter::RY], data[Parameter::RZ])); +} + +Mat Camera::tvec() const { + return Mat(Vec3d(data[Parameter::TX], data[Parameter::TY], data[Parameter::TZ])); +} + +Mat Camera::rmat() const { + Mat R; + cv::Rodrigues(rvec(), R); + return R; +} + +Mat Camera::extrinsicMatrix() const { + Mat T = Mat::eye(4, 4, CV_64FC1); + rmat().copyTo(T(Rect(0, 0, 3, 3))); + tvec().copyTo(T(Rect(3, 0, 1, 3))); + return T; +} + +Mat Camera::extrinsicMatrixInverse() const { + return extrinsicMatrix().inv(); +} + +//////////////////////////////////////////////////////////////////////////////// + +bool validate::translationStereo(const Mat &t) { + if (t.type() != CV_64F) { return false; } + if (t.channels() != 1) { return false; } + if (t.size() != Size(1, 3)) { return false; } + if (cv::norm(t, cv::NORM_L2) == 0) { return false; } + return true; +} + +bool validate::rotationMatrix(const Mat &M) { + if (M.type() != CV_64F) { return false; } + if (M.channels() != 1) { return false; } + if (M.size() != Size(3, 3)) { return false; } + + double det = cv::determinant(M); + if (abs(abs(det)-1.0) > 0.00001) { return false; } + + // TODO: floating point errors (result not exactly identity matrix) + // rotation matrix is orthogonal: M.T * M == M * M.T == I + //if (cv::countNonZero((M.t() * M) != Mat::eye(Size(3, 3), CV_64FC1)) != 0) + // { return false; } + + return true; +} + +bool validate::pose(const Mat &M) { + if (M.size() != Size(4, 4)) { return false; } + if (!validate::rotationMatrix(M(cv::Rect(0 , 0, 3, 3)))) + { return false; } + if (!( (M.at<double>(3, 0) == 0.0) && + (M.at<double>(3, 1) == 0.0) && + (M.at<double>(3, 2) == 0.0) && + (M.at<double>(3, 3) == 1.0))) { return false; } + + return true; +} + +bool validate::cameraMatrix(const Mat &M) { + if (M.type() != CV_64F) { return false; } + if (M.channels() != 1) { return false; } + if (M.size() != Size(3, 3)) { return false; } + + if (!( (M.at<double>(2, 0) == 0.0) && + (M.at<double>(2, 1) == 0.0) && + (M.at<double>(2, 2) == 1.0))) { return false; } + + return true; +} + +bool ftl::calibration::validate::distortionCoefficients(const Mat &D, Size size) { + if (D.type() != CV_64FC1) { return false; } + if (!( + (D.total() == 4) || + (D.total() == 5) || + (D.total() == 8) || + (D.total() == 12))) { return false; } + + for (int i = 0; i < D.total(); i++) { + if (!std::isfinite(D.at<double>(i))) { return false; } + } + + double k[6] = {0.0}; + //double p[2] = {0.0}; + //double s[4] = {0.0}; + + switch(D.total()) { + case 12: + /* + s[0] = D.at<double>(8); + s[1] = D.at<double>(9); + s[2] = D.at<double>(10); + s[3] = D.at<double>(11); + */ + [[fallthrough]]; + + case 8: + k[3] = D.at<double>(5); + k[4] = D.at<double>(6); + k[5] = D.at<double>(7); + [[fallthrough]]; + + case 5: + k[2] = D.at<double>(4); + [[fallthrough]]; + + default: + k[0] = D.at<double>(0); + k[1] = D.at<double>(1); + /* + p[0] = D.at<double>(2); + p[1] = D.at<double>(3); + */ + } + + int diagonal = sqrt(size.width*size.width+size.height*size.height) + 1.0; + + bool is_n = true; + bool is_p = true; + + double dist_prev_n = 0; + double dist_prev_p = 0; + + for (int r = 0; r < diagonal; r++) { + double r2 = r*r; + double r4 = r2*r2; + double r6 = r4*r2; + + double rdist = 1.0 + k[0]*r2 + k[1]*r4 + k[2]*r6; + double irdist2 = 1./(1.0 + k[3]*r2 + k[4]*r4 + k[5]*r6); + double dist = r2*rdist*irdist2; // + s[0]*r2 + s[1]*r4; + + if (is_n) { + if (r2 == 0) {} + else if (!(dist < dist_prev_n)) { is_n = false; } + dist_prev_n = dist; + } + + if (is_p) { + if (r2 == 0) {} + else if (!(dist > dist_prev_p)) { is_p = false; } + dist_prev_p = dist; + } + + if (!is_n && !is_p) { return false; } + } + + return true; +} + +Mat ftl::calibration::scaleCameraMatrix(const Mat &K, const Size &size_new, const Size &size_old) { + Mat S(cv::Size(3, 3), CV_64F, 0.0); + double scale_x = ((double) size_new.width) / ((double) size_old.width); + double scale_y = ((double) size_new.height) / ((double) size_old.height); + + S.at<double>(0, 0) = scale_x; + S.at<double>(1, 1) = scale_y; + S.at<double>(2, 2) = 1.0; + return (S*K); +} + +double ftl::calibration::reprojectionError(const vector<Point2d>& points_im, + const vector<Point3d>& points, const Mat& K, const Mat& D, const Mat& R, + const Mat& t) { + + Mat rvec; + if (R.size() == Size(3, 3)) { cv::Rodrigues(R, rvec); } + else { rvec = R; } + + vector<Point2d> points_reprojected; + cv::projectPoints(points, rvec, t, K, D, points_reprojected); + + double err = 0.0; + int npoints = points_im.size(); + + for (int i = 0; i < npoints; i++) { + err += norm(points_im[i] - points_reprojected[i]); + } + + return err / ((double) npoints); +} diff --git a/components/calibration/test/CMakeLists.txt b/components/calibration/test/CMakeLists.txt new file mode 100644 index 0000000000000000000000000000000000000000..0bd1f1f198347c38c74973b14a5253b4bcd93d09 --- /dev/null +++ b/components/calibration/test/CMakeLists.txt @@ -0,0 +1,10 @@ +### Calibration ################################################################ +add_executable(calibration_parameters_unit + ./tests.cpp + ./test_parameters.cpp + ../src/parameters.cpp +) + +target_include_directories(calibration_parameters_unit PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}/../include") +target_link_libraries(calibration_parameters_unit ftlcommon Threads::Threads ${OS_LIBS} ${OpenCV_LIBS}) +add_test(CalibrationValidateTest calibration_parameters_unit) \ No newline at end of file diff --git a/components/common/cpp/test/calibration_common_unit.cpp b/components/calibration/test/test_parameters.cpp similarity index 96% rename from components/common/cpp/test/calibration_common_unit.cpp rename to components/calibration/test/test_parameters.cpp index 5610c169d3b78c1755fd1f9279e7e9bf94bc46c4..7d19b9d757587ff7e73cde565c3643f8b8d5937e 100644 --- a/components/common/cpp/test/calibration_common_unit.cpp +++ b/components/calibration/test/test_parameters.cpp @@ -1,5 +1,5 @@ #include "catch.hpp" -#include "ftl/calibration.hpp" +#include "ftl/calibration/parameters.hpp" using cv::Size; using cv::Mat; diff --git a/components/calibration/test/tests.cpp b/components/calibration/test/tests.cpp new file mode 100644 index 0000000000000000000000000000000000000000..178916eab8b9c7aabb87ff99894b48443ad6ecb6 --- /dev/null +++ b/components/calibration/test/tests.cpp @@ -0,0 +1,3 @@ +#define CATCH_CONFIG_MAIN +#include "catch.hpp" + diff --git a/components/common/cpp/CMakeLists.txt b/components/common/cpp/CMakeLists.txt index 49eaac5738ce8caff4d102d15bc8030c6798721c..57ec51ace3ab13823105f2c5936e419e020809a6 100644 --- a/components/common/cpp/CMakeLists.txt +++ b/components/common/cpp/CMakeLists.txt @@ -11,7 +11,6 @@ set(COMMONSRC src/ctpl_stl.cpp src/timer.cpp src/profiler.cpp - src/calibration.cpp ) check_function_exists(uriParseSingleUriA HAVE_URIPARSESINGLE) diff --git a/components/common/cpp/include/ftl/calibration.hpp b/components/common/cpp/include/ftl/calibration.hpp deleted file mode 100644 index 00662685237cdabf5e9d8a855ced5d1493f4a4e5..0000000000000000000000000000000000000000 --- a/components/common/cpp/include/ftl/calibration.hpp +++ /dev/null @@ -1,50 +0,0 @@ -#pragma once -#ifndef _FTL_COMMON_CALIBRATION_HPP_ -#define _FTL_COMMON_CALIBRATION_HPP_ - -#include <opencv2/core/core.hpp> - -namespace ftl { -namespace calibration { - -namespace validate { - -/** - * @brief Valid translation for stereo camera. - */ -bool translationStereo(const cv::Mat &t); - -bool rotationMatrix(const cv::Mat &M); - -bool pose(const cv::Mat &M); - -bool cameraMatrix(const cv::Mat &M); - -/** - * @brief Check if D contains valid distortion coefficients. - * @param D distortion coefficients - * @param size resolution - * @note Tangential and prism distortion coefficients are not validated. - * - * Radial distortion is always monotonic for real lenses and distortion - * function has to be bijective. This is verified by evaluating the distortion - * function for integer values from 0 to sqrt(width^2+height^2). - * - * Camera model documented in - * https://docs.opencv.org/master/d9/d0c/group__calib3d.html#details - */ -bool distortionCoefficients(const cv::Mat &D, cv::Size size); - -} - -/** - * @brief Scale camera intrinsic matrix - * @param size_new New resolution - * @param size_old Original (camera matrix) resolution - */ -cv::Mat scaleCameraMatrix(const cv::Mat &K, const cv::Size &size_new, const cv::Size &size_old); - -} -} - -#endif diff --git a/components/common/cpp/include/ftl/config.h.in b/components/common/cpp/include/ftl/config.h.in index 5b54fea8e957b5fb3f7e9d403e6f45aec41c7c51..ecf9e79f798b333a91f68b814583f5edf46615fc 100644 --- a/components/common/cpp/include/ftl/config.h.in +++ b/components/common/cpp/include/ftl/config.h.in @@ -19,6 +19,7 @@ #cmakedefine HAVE_OPTFLOW #cmakedefine HAVE_RENDER #cmakedefine HAVE_LIBSGM +#cmakedefine HAVE_CERES #cmakedefine HAVE_REALSENSE #cmakedefine HAVE_NANOGUI #cmakedefine HAVE_LIBARCHIVE diff --git a/components/common/cpp/src/calibration.cpp b/components/common/cpp/src/calibration.cpp deleted file mode 100644 index 6a6bd8fa31197b72bec301a510a603c8b45213f1..0000000000000000000000000000000000000000 --- a/components/common/cpp/src/calibration.cpp +++ /dev/null @@ -1,145 +0,0 @@ -#include "ftl/calibration.hpp" - -using cv::Mat; -using cv::Size; - -using namespace ftl::calibration; - -bool validate::translationStereo(const Mat &t) { - if (t.type() != CV_64F) { return false; } - if (t.channels() != 1) { return false; } - if (t.size() != Size(1, 3)) { return false; } - if (cv::norm(t, cv::NORM_L2) == 0) { return false; } - return true; -} - -bool validate::rotationMatrix(const Mat &M) { - if (M.type() != CV_64F) { return false; } - if (M.channels() != 1) { return false; } - if (M.size() != Size(3, 3)) { return false; } - - double det = cv::determinant(M); - if (abs(abs(det)-1.0) > 0.00001) { return false; } - - // TODO: floating point errors (result not exactly identity matrix) - // rotation matrix is orthogonal: M.T * M == M * M.T == I - //if (cv::countNonZero((M.t() * M) != Mat::eye(Size(3, 3), CV_64FC1)) != 0) - // { return false; } - - return true; -} - -bool validate::pose(const Mat &M) { - if (M.size() != Size(4, 4)) { return false; } - if (!validate::rotationMatrix(M(cv::Rect(0 , 0, 3, 3)))) - { return false; } - if (!( (M.at<double>(3, 0) == 0.0) && - (M.at<double>(3, 1) == 0.0) && - (M.at<double>(3, 2) == 0.0) && - (M.at<double>(3, 3) == 1.0))) { return false; } - - return true; -} - -bool validate::cameraMatrix(const Mat &M) { - if (M.type() != CV_64F) { return false; } - if (M.channels() != 1) { return false; } - if (M.size() != Size(3, 3)) { return false; } - - if (!( (M.at<double>(2, 0) == 0.0) && - (M.at<double>(2, 1) == 0.0) && - (M.at<double>(2, 2) == 1.0))) { return false; } - - return true; -} - -bool ftl::calibration::validate::distortionCoefficients(const Mat &D, Size size) { - if (D.type() != CV_64FC1) { return false; } - if (!( - (D.total() == 4) || - (D.total() == 5) || - (D.total() == 8) || - (D.total() == 12))) { return false; } - - for (int i = 0; i < D.total(); i++) { - if (!std::isfinite(D.at<double>(i))) { return false; } - } - - double k[6] = {0.0}; - //double p[2] = {0.0}; - //double s[4] = {0.0}; - - switch(D.total()) { - case 12: - /* - s[0] = D.at<double>(8); - s[1] = D.at<double>(9); - s[2] = D.at<double>(10); - s[3] = D.at<double>(11); - */ - [[fallthrough]]; - - case 8: - k[3] = D.at<double>(5); - k[4] = D.at<double>(6); - k[5] = D.at<double>(7); - [[fallthrough]]; - - case 5: - k[2] = D.at<double>(4); - [[fallthrough]]; - - default: - k[0] = D.at<double>(0); - k[1] = D.at<double>(1); - /* - p[0] = D.at<double>(2); - p[1] = D.at<double>(3); - */ - } - - int diagonal = sqrt(size.width*size.width+size.height*size.height) + 1.0; - - bool is_n = true; - bool is_p = true; - - double dist_prev_n = 0; - double dist_prev_p = 0; - - for (int r = 0; r < diagonal; r++) { - double r2 = r*r; - double r4 = r2*r2; - double r6 = r4*r2; - - double rdist = 1.0 + k[0]*r2 + k[1]*r4 + k[2]*r6; - double irdist2 = 1./(1.0 + k[3]*r2 + k[4]*r4 + k[5]*r6); - double dist = r2*rdist*irdist2; // + s[0]*r2 + s[1]*r4; - - if (is_n) { - if (r2 == 0) {} - else if (!(dist < dist_prev_n)) { is_n = false; } - dist_prev_n = dist; - } - - if (is_p) { - if (r2 == 0) {} - else if (!(dist > dist_prev_p)) { is_p = false; } - dist_prev_p = dist; - } - - if (!is_n && !is_p) { return false; } - } - - return true; -} - -Mat ftl::calibration::scaleCameraMatrix(const Mat &K, const Size &size_new, const Size &size_old) { - Mat S(cv::Size(3, 3), CV_64F, 0.0); - double scale_x = ((double) size_new.width) / ((double) size_old.width); - double scale_y = ((double) size_new.height) / ((double) size_old.height); - - S.at<double>(0, 0) = scale_x; - S.at<double>(1, 1) = scale_y; - S.at<double>(2, 2) = 1.0; - return (S*K); -} \ No newline at end of file diff --git a/components/common/cpp/test/CMakeLists.txt b/components/common/cpp/test/CMakeLists.txt index c5d394ac48e429e46b9a41204a25aa4448250702..de71019fb70424cbdf6243c1dcf7e9f5071db504 100644 --- a/components/common/cpp/test/CMakeLists.txt +++ b/components/common/cpp/test/CMakeLists.txt @@ -39,13 +39,4 @@ target_link_libraries(msgpack_unit Threads::Threads ${OS_LIBS} ${OpenCV_LIBS}) add_test(ConfigurableUnitTest configurable_unit) add_test(URIUnitTest uri_unit) add_test(MsgpackUnitTest msgpack_unit) -# add_test(TimerUnitTest timer_unit) CI server can't achieve this - -### Calibration ################################################################ -add_executable(calibration_common_unit - $<TARGET_OBJECTS:CatchTest> - ./calibration_common_unit.cpp -) -target_include_directories(calibration_common_unit PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}/../include") -target_link_libraries(calibration_common_unit ftlcommon Threads::Threads ${OS_LIBS} ${OpenCV_LIBS}) -add_test(CalibrationCommonTest calibration_common_unit) \ No newline at end of file +# add_test(TimerUnitTest timer_unit) CI server can't achieve this \ No newline at end of file diff --git a/components/rgbd-sources/CMakeLists.txt b/components/rgbd-sources/CMakeLists.txt index 9e5647e88ec11abed63497d2d6366e0bddbd40fa..fafceea695332cc0f0daf1c5bedffc649c3ad727 100644 --- a/components/rgbd-sources/CMakeLists.txt +++ b/components/rgbd-sources/CMakeLists.txt @@ -38,7 +38,7 @@ if (CUDA_FOUND) set_property(TARGET ftlrgbd PROPERTY CUDA_SEPARABLE_COMPILATION OFF) endif() -target_link_libraries(ftlrgbd ftlcommon ${OpenCV_LIBS} ${LIBSGM_LIBRARIES} ${CUDA_LIBRARIES} Eigen3::Eigen realsense ftlnet ${LibArchive_LIBRARIES} ftlcodecs ftloperators ftldata ${X11_X11_LIB} ${X11_Xext_LIB}) +target_link_libraries(ftlrgbd ftlcalibration ftlcommon ${OpenCV_LIBS} ${LIBSGM_LIBRARIES} ${CUDA_LIBRARIES} Eigen3::Eigen realsense ftlnet ${LibArchive_LIBRARIES} ftlcodecs ftloperators ftldata ${X11_X11_LIB} ${X11_Xext_LIB}) if (BUILD_TESTS) add_subdirectory(test) diff --git a/components/rgbd-sources/src/sources/stereovideo/calibrate.cpp b/components/rgbd-sources/src/sources/stereovideo/calibrate.cpp index f67262c9e71c743bcd6d6cc9e740f60f29219a49..7851fb1541e591c65ee5b17167a22ecfaa6baea8 100644 --- a/components/rgbd-sources/src/sources/stereovideo/calibrate.cpp +++ b/components/rgbd-sources/src/sources/stereovideo/calibrate.cpp @@ -6,7 +6,7 @@ #include <ftl/config.h> #include <ftl/configuration.hpp> #include <ftl/threads.hpp> -#include <ftl/calibration.hpp> +#include <ftl/calibration/parameters.hpp> #include "calibrate.hpp" #include "ftl/exception.hpp" diff --git a/applications/calibration-ceres/test/catch.hpp b/lib/catch/catch.hpp similarity index 100% rename from applications/calibration-ceres/test/catch.hpp rename to lib/catch/catch.hpp