diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 30514de41c5a7d803c61d6ff44eadd558e268643..b9712ca6487b4ddda8db7a98f509716c935ffd5a 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -6,7 +6,7 @@
variables:
GIT_SUBMODULE_STRATEGY: recursive
- CMAKE_ARGS_WINDOWS: '-DCMAKE_GENERATOR_PLATFORM=x64 -DPORTAUDIO_DIR="D:/Build/portaudio" -DNVPIPE_DIR="D:/Build/NvPipe" -DEigen3_DIR="C:/Program Files (x86)/Eigen3/share/eigen3/cmake" -DOpenCV_DIR="D:/Build/opencv-4.1.1" -DCUDA_TOOLKIT_ROOT_DIR="C:/Program Files/NVIDIA GPU Computing Toolkit/CUDA/v10.1" -DWITH_OPENVR=TRUE -DOPENVR_DIR="D:/Build/OpenVRSDK"'
+ CMAKE_ARGS_WINDOWS: '-DCMAKE_GENERATOR_PLATFORM=x64 -DPORTAUDIO_DIR="D:/Build/portaudio" -DNVPIPE_DIR="D:/Build/NvPipe" -DEigen3_DIR="C:/Program Files (x86)/Eigen3/share/eigen3/cmake" -DOpenCV_DIR="D:/Build/opencv-4.1.1" -DCUDA_TOOLKIT_ROOT_DIR="C:/Program Files/NVIDIA GPU Computing Toolkit/CUDA/v10.1" -DWITH_OPENVR=TRUE -DOPENVR_DIR="D:/Build/OpenVRSDK" -DWITH_CERES=FALSE'
stages:
- all
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 46783116d8b5670caf45b5fb317b81dcd79c2323..ce1b438e4e2a5659f2166b178585128989816063 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -13,7 +13,8 @@ enable_testing()
option(WITH_NVPIPE "Use NvPipe for compression if available" ON)
option(WITH_OPTFLOW "Use NVIDIA Optical Flow if available" OFF)
option(WITH_OPENVR "Build with OpenVR support" OFF)
-option(WITH_FIXSTARS "Use Fixstars libSGM if available" ON)
+option(WITH_FIXSTARS "Use Fixstars libSGM" ON)
+option(WITH_CERES "Use Ceres solver" ON)
option(USE_CPPCHECK "Apply cppcheck during build" ON)
option(BUILD_VISION "Enable the vision component" ON)
option(BUILD_RECONSTRUCT "Enable the reconstruction component" ON)
@@ -24,6 +25,10 @@ option(BUILD_TOOLS "Compile developer and research tools" ON)
option(BUILD_TESTS "Compile all unit and integration tests" ON)
option(ENABLE_PROFILER "Enable builtin performance profiling" OFF)
+if (BUILD_TESTS)
+ include_directories(lib/catch)
+endif()
+
set(THREADS_PREFER_PTHREAD_FLAG ON)
set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_SOURCE_DIR}/cmake/")
@@ -31,13 +36,13 @@ include(Findglog)
MACRO( VERSION_STR_TO_INTS major minor patch version )
- STRING( REGEX REPLACE "([0-9]+).[0-9]+.[0-9]+" "\\1" ${major} ${version} )
- STRING( REGEX REPLACE "[0-9]+.([0-9]+).[0-9]+" "\\1" ${minor} ${version} )
- STRING( REGEX REPLACE "[0-9]+.[0-9]+.([0-9]+)" "\\1" ${patch} ${version} )
+ STRING( REGEX REPLACE "([0-9]+).[0-9]+.[0-9]+" "\\1" ${major} ${version} )
+ STRING( REGEX REPLACE "[0-9]+.([0-9]+).[0-9]+" "\\1" ${minor} ${version} )
+ STRING( REGEX REPLACE "[0-9]+.[0-9]+.([0-9]+)" "\\1" ${patch} ${version} )
ENDMACRO( VERSION_STR_TO_INTS )
-find_package( OpenCV REQUIRED COMPONENTS core imgproc highgui cudaimgproc calib3d imgcodecs videoio aruco cudaarithm cudastereo cudaoptflow face tracking quality)
+find_package( OpenCV REQUIRED COMPONENTS core imgproc highgui cudaimgproc calib3d imgcodecs videoio aruco cudaarithm cudastereo cudaoptflow face tracking quality xfeatures2d)
find_package( Threads REQUIRED )
find_package( URIParser REQUIRED )
find_package( MsgPack REQUIRED )
@@ -111,12 +116,19 @@ endif()
# ==============================================================================
if (WITH_FIXSTARS)
- find_package( LibSGM )
+ find_package(LibSGM REQUIRED)
if (LibSGM_FOUND)
set(HAVE_LIBSGM true)
endif()
endif()
+if (WITH_CERES)
+ find_package(Ceres REQUIRED)
+ set(HAVE_CERES true)
+else()
+ add_library(ceres INTERFACE)
+endif()
+
if(${CMAKE_VERSION} VERSION_GREATER "3.12.0")
cmake_policy(SET CMP0074 NEW)
endif()
@@ -326,6 +338,7 @@ SET(CMAKE_USE_RELATIVE_PATHS ON)
set(CMAKE_WINDOWS_EXPORT_ALL_SYMBOLS ON)
add_subdirectory(components/common/cpp)
+add_subdirectory(applications/calibration)
add_subdirectory(components/codecs)
add_subdirectory(components/structures)
add_subdirectory(components/net)
@@ -334,7 +347,7 @@ add_subdirectory(components/control/cpp)
add_subdirectory(components/operators)
add_subdirectory(components/streams)
add_subdirectory(components/audio)
-add_subdirectory(applications/calibration)
+add_subdirectory(components/calibration)
#add_subdirectory(applications/groupview)
#add_subdirectory(applications/player)
#add_subdirectory(applications/recorder)
@@ -355,7 +368,10 @@ if (BUILD_VISION)
endif()
if (BUILD_CALIBRATION)
- find_package(Ceres REQUIRED)
+ if (NOT HAVE_CERES)
+ message(ERROR "Ceres is required")
+ endif()
+
add_subdirectory(applications/calibration-ceres)
add_subdirectory(applications/calibration-multi)
endif()
diff --git a/applications/calibration-ceres/CMakeLists.txt b/applications/calibration-ceres/CMakeLists.txt
index f10bc716360aee2bfede7152c6d763192258ae7f..d994acaa990d35355f4f891f8a539dd5335782ed 100644
--- a/applications/calibration-ceres/CMakeLists.txt
+++ b/applications/calibration-ceres/CMakeLists.txt
@@ -3,11 +3,10 @@ add_executable (ftl-calibration-ceres
src/calibration_data.cpp
src/visibility.cpp
src/calibration.cpp
- src/optimization.cpp
)
target_include_directories(ftl-calibration-ceres PRIVATE src/)
target_include_directories(ftl-calibration-ceres PUBLIC ${OpenCV_INCLUDE_DIRS})
-target_link_libraries(ftl-calibration-ceres pthread ftlcommon Eigen3::Eigen ceres)
+target_link_libraries(ftl-calibration-ceres ftlcalibration Threads::Threads ftlcommon Eigen3::Eigen ceres)
add_subdirectory(test)
diff --git a/applications/calibration-ceres/src/calibration.cpp b/applications/calibration-ceres/src/calibration.cpp
index 2c09538e3672742b9874b287994cb3dde8d92e37..ab0f9412f366fad9675192a9082b25ce3e053ae6 100644
--- a/applications/calibration-ceres/src/calibration.cpp
+++ b/applications/calibration-ceres/src/calibration.cpp
@@ -1,5 +1,5 @@
#include "calibration.hpp"
-#include "optimization.hpp"
+#include "ftl/calibration/optimize.hpp"
#include "loguru.hpp"
@@ -20,27 +20,6 @@ using ftl::calibration::BundleAdjustment;
using namespace ftl::calibration;
-double ftl::calibration::reprojectionError(const vector<Point2d>& points_im,
- const vector<Point3d>& points, const Mat& K, const Mat& D, const Mat& R,
- const Mat& t) {
-
- Mat rvec;
- if (R.size() == Size(3, 3)) { cv::Rodrigues(R, rvec); }
- else { rvec = R; }
-
- vector<Point2d> points_reprojected;
- cv::projectPoints(points, rvec, t, K, D, points_reprojected);
-
- double err = 0.0;
- int npoints = points_im.size();
-
- for (int i = 0; i < npoints; i++) {
- err += norm(points_im[i] - points_reprojected[i]);
- }
-
- return err / ((double) npoints);
-}
-
int ftl::calibration::recoverPose(const Mat &E, const vector<Point2d> &_points1,
const vector<Point2d> &_points2, const Mat &_cameraMatrix1,
const Mat &_cameraMatrix2, Mat &_R, Mat &_t, double distanceThresh,
diff --git a/applications/calibration-ceres/src/calibration.hpp b/applications/calibration-ceres/src/calibration.hpp
index 5151d1a96766a7d06d044b7818d4105123815421..11bb36d920da1e638ce49ad699a955fe5032acec 100644
--- a/applications/calibration-ceres/src/calibration.hpp
+++ b/applications/calibration-ceres/src/calibration.hpp
@@ -9,75 +9,6 @@
namespace ftl {
namespace calibration {
-namespace transform {
-
-/**
- * @brief Get rotation matrix and translation vector from transformation matrix.
- * @param T transformation matrix (4x4) or (3x4)
- * @param R (out) rotation matrix (3x3)
- * @param t (out) translation vector (3x1)
- */
-inline void getRotationAndTranslation(const cv::Mat &T, cv::Mat &R, cv::Mat &t) {
- T(cv::Rect(0, 0, 3, 3)).copyTo(R);
- T(cv::Rect(3, 0, 1, 3)).copyTo(t);
-}
-
-/**
- * @brief Inverse transform inplace
- * @param R rotation matrix (3x3)
- * @param t translation vector (3x1)
- */
-inline void inverse(cv::Mat &R, cv::Mat &t) {
- cv::Mat t_(3, 1, CV_64FC1);
-
- t_.at<double>(0) = -R.col(0).dot(t);
- t_.at<double>(1) = -R.col(1).dot(t);
- t_.at<double>(2) = -R.col(2).dot(t);
-
- cv::swap(t_, t);
- R = R.t();
-}
-
-/**
- * @brief Inverse transform inplace
- * @param T transformation matrix (4x4)
- */
-inline void inverse(cv::Mat &T) {
- cv::Mat rmat;
- cv::Mat tvec;
- getRotationAndTranslation(T, rmat, tvec);
- T = cv::Mat::eye(4, 4, CV_64FC1);
-
- T(cv::Rect(3, 0, 1, 1)) = -rmat.col(0).dot(tvec);
- T(cv::Rect(3, 1, 1, 1)) = -rmat.col(1).dot(tvec);
- T(cv::Rect(3, 2, 1, 1)) = -rmat.col(2).dot(tvec);
- T(cv::Rect(0, 0, 3, 3)) = rmat.t();
-}
-
-inline cv::Point3d apply(const cv::Point3d& point, const cv::InputArray& R, const cv::InputArray& t) {
- cv::Mat R_ = R.getMat();
- cv::Mat t_ = t.getMat();
- cv::Mat p_new = R_ * cv::Mat(point) + t_;
- return cv::Point3d(p_new.at<double>(0), p_new.at<double>(1), p_new.at<double>(2));
-}
-
-inline cv::Point3d apply(const cv::Point3d& point, const cv::InputArray& T) {
- cv::Mat T_ = T.getMat();
- cv::Mat rmat;
- cv::Mat tvec;
- getRotationAndTranslation(T_, rmat, tvec);
- return apply(point, rmat, tvec);
-}
-
-} // namespace transform
-
-/**
- * @brief Calculate MSE reprojection error
- */
-double reprojectionError(const std::vector<cv::Point2d>& points_im,
- const std::vector<cv::Point3d>& points, const cv::Mat& K, const cv::Mat& D,
- const cv::Mat& R, const cv::Mat& t);
-
/**
* Same as OpenCV's recoverPose(), but does not assume same intrinsic paramters
* for both cameras.
diff --git a/applications/calibration-ceres/src/calibration_data.cpp b/applications/calibration-ceres/src/calibration_data.cpp
index 28b77fa8b6b21a07abb8b7e32f46dab0f4363505..1e634a3921330ba6b2b8a04d75ecdd5352596409 100644
--- a/applications/calibration-ceres/src/calibration_data.cpp
+++ b/applications/calibration-ceres/src/calibration_data.cpp
@@ -15,135 +15,7 @@ using cv::Rect;
using cv::Size;
using ftl::calibration::CalibrationData;
-using ftl::calibration::Camera;
-////////////////////////////////////////////////////////////////////////////////
-
-void Camera::setRotation(const Mat& R) {
- if (((R.size() != Size(3, 3)) &&
- (R.size() != Size(3, 1)) &&
- (R.size() != Size(1, 3))) ||
- (R.type() != CV_64FC1)) { throw std::exception(); }
-
- Mat rvec;
- if (R.size() == cv::Size(3, 3)) { cv::Rodrigues(R, rvec); }
- else { rvec = R; }
-
- data[Parameter::RX] = rvec.at<double>(0);
- data[Parameter::RY] = rvec.at<double>(1);
- data[Parameter::RZ] = rvec.at<double>(2);
-}
-
-void Camera::setTranslation(const Mat& t) {
- if ((t.type() != CV_64FC1) ||
- (t.size() != cv::Size(1, 3))) { throw std::exception(); }
-
- data[Parameter::TX] = t.at<double>(0);
- data[Parameter::TY] = t.at<double>(1);
- data[Parameter::TZ] = t.at<double>(2);
-}
-
-
-void Camera::setIntrinsic(const Mat& K) {
- if ((K.type() != CV_64FC1) || (K.size() != cv::Size(3, 3))) {
- throw std::exception();
- }
-
- data[Parameter::F] = K.at<double>(0, 0);
- data[Parameter::CX] = K.at<double>(0, 2);
- data[Parameter::CY] = K.at<double>(1, 2);
-}
-
-void Camera::setDistortion(const Mat& D) {
- if ((D.type() != CV_64FC1)) { throw std::exception(); }
- switch(D.total()) {
- case 12:
- /*
- data[Parameter::S1] = D.at<double>(8);
- data[Parameter::S2] = D.at<double>(9);
- data[Parameter::S3] = D.at<double>(10);
- data[Parameter::S4] = D.at<double>(11);
- */
- [[fallthrough]];
-
- case 8:
- /*
- data[Parameter::K4] = D.at<double>(5);
- data[Parameter::K5] = D.at<double>(6);
- data[Parameter::K6] = D.at<double>(7);
- */
- [[fallthrough]];
-
- case 5:
- data[Parameter::K3] = D.at<double>(4);
- [[fallthrough]];
-
- default:
- data[Parameter::K1] = D.at<double>(0);
- data[Parameter::K2] = D.at<double>(1);
- /*
- data[Parameter::P1] = D.at<double>(2);
- data[Parameter::P2] = D.at<double>(3);
- */
- }
-}
-
-Camera::Camera(const Mat &K, const Mat &D, const Mat &R, const Mat &tvec) {
- setIntrinsic(K, D);
- if (!R.empty()) { setRotation(R); }
- if (!tvec.empty()) { setTranslation(tvec); }
-}
-
-Mat Camera::intrinsicMatrix() const {
- Mat K = Mat::eye(3, 3, CV_64FC1);
- K.at<double>(0, 0) = data[Parameter::F];
- K.at<double>(1, 1) = data[Parameter::F];
- K.at<double>(0, 2) = data[Parameter::CX];
- K.at<double>(1, 2) = data[Parameter::CY];
- return K;
-}
-
-Mat Camera::distortionCoefficients() const {
- Mat D;
- // OpenCV distortion parameter sizes: 4, 5, 8, 12, 14
- if (_NDISTORTION_PARAMETERS <= 4) { D = Mat::zeros(4, 1, CV_64FC1); }
- else if (_NDISTORTION_PARAMETERS <= 5) { D = Mat::zeros(5, 1, CV_64FC1); }
- else if (_NDISTORTION_PARAMETERS <= 8) { D = Mat::zeros(8, 1, CV_64FC1); }
- else if (_NDISTORTION_PARAMETERS <= 12) { D = Mat::zeros(12, 1, CV_64FC1); }
- else if (_NDISTORTION_PARAMETERS <= 14) { D = Mat::zeros(14, 1, CV_64FC1); }
-
- for (int i = 0; i < _NDISTORTION_PARAMETERS; i++) {
- D.at<double>(i) = data[Parameter::DISTORTION+i];
- }
- return D;
-}
-
-Mat Camera::rvec() const {
- return Mat(Vec3d(data[Parameter::RX], data[Parameter::RY], data[Parameter::RZ]));
-}
-
-Mat Camera::tvec() const {
- return Mat(Vec3d(data[Parameter::TX], data[Parameter::TY], data[Parameter::TZ]));
-}
-
-Mat Camera::rmat() const {
- Mat R;
- cv::Rodrigues(rvec(), R);
- return R;
-}
-
-Mat Camera::extrinsicMatrix() const {
- Mat T = Mat::eye(4, 4, CV_64FC1);
- rmat().copyTo(T(Rect(0, 0, 3, 3)));
- tvec().copyTo(T(Rect(3, 0, 1, 3)));
- return T;
-}
-
-Mat Camera::extrinsicMatrixInverse() const {
- return extrinsicMatrix().inv();
-}
-
-////////////////////////////////////////////////////////////////////////////////
CalibrationData::CalibrationData(int n_cameras) { init(n_cameras); }
diff --git a/applications/calibration-ceres/src/calibration_data.hpp b/applications/calibration-ceres/src/calibration_data.hpp
index e98ed3d3c0f86a97e78961eeeb2e102581da9e14..9e5871bf2ce6407ec58a938c0574dd4d6374f6a7 100644
--- a/applications/calibration-ceres/src/calibration_data.hpp
+++ b/applications/calibration-ceres/src/calibration_data.hpp
@@ -4,6 +4,7 @@
#include <vector>
#include <opencv2/core/core.hpp>
+#include <ftl/calibration/optimize.hpp>
#define _NDISTORTION_PARAMETERS 3
#define _NCAMERA_PARAMETERS (9 + _NDISTORTION_PARAMETERS)
@@ -11,58 +12,6 @@
namespace ftl {
namespace calibration {
-/**
- * Camera paramters
- */
-struct Camera {
- Camera() {}
- Camera(const cv::Mat& K, const cv::Mat& D, const cv::Mat& R, const cv::Mat& tvec);
-
- void setRotation(const cv::Mat& R);
- void setTranslation(const cv::Mat& tvec);
- void setExtrinsic(const cv::Mat& R, const cv::Mat& t) {
- setRotation(R);
- setTranslation(t);
- }
-
- void setIntrinsic(const cv::Mat& K);
- void setDistortion(const cv::Mat &D);
- void setIntrinsic(const cv::Mat& K, const cv::Mat& D) {
- setIntrinsic(K);
- setDistortion(D);
- }
-
- cv::Mat intrinsicMatrix() const;
- cv::Mat distortionCoefficients() const;
-
- cv::Mat rvec() const;
- cv::Mat tvec() const;
- cv::Mat rmat() const;
-
- cv::Mat extrinsicMatrix() const;
- cv::Mat extrinsicMatrixInverse() const;
-
- double data[_NCAMERA_PARAMETERS] = {0.0};
-
- enum Parameter {
- ROTATION = 0,
- RX = 0,
- RY = 1,
- RZ = 2,
- TRANSLATION = 3,
- TX = 3,
- TY = 4,
- TZ = 5,
- F = 6,
- CX = 7,
- CY = 8,
- DISTORTION = 9,
- K1 = 9,
- K2 = 10,
- K3 = 11
- };
-};
-
class CalibrationData {
public:
CalibrationData() {};
diff --git a/applications/calibration-ceres/src/main.cpp b/applications/calibration-ceres/src/main.cpp
index 087c3283ba303eceab82ac565c626ec0051e403e..9020d0e7737f43a7eff44406ad7853243161a4e5 100644
--- a/applications/calibration-ceres/src/main.cpp
+++ b/applications/calibration-ceres/src/main.cpp
@@ -10,7 +10,9 @@
#include <ceres/ceres.h>
#include <ceres/rotation.h>
-#include "optimization.hpp"
+#include <ftl/calibration.hpp>
+
+#include "calibration_data.hpp"
#include "visibility.hpp"
#include "calibration.hpp"
diff --git a/applications/calibration-ceres/test/CMakeLists.txt b/applications/calibration-ceres/test/CMakeLists.txt
index 5f76122ccdb66a999d22974cab932b6ec655b5e9..6c90f1d58ff1aa6e8c4d2c7bac0c2fd60f53917b 100644
--- a/applications/calibration-ceres/test/CMakeLists.txt
+++ b/applications/calibration-ceres/test/CMakeLists.txt
@@ -1,7 +1,3 @@
-
-#add_library(Catch INTERFACE)
-#target_include_directories(Catch INTERFACE ${CATCH_INCLUDE_DIR})
-
add_executable(visibility_unit
./tests.cpp
../src/visibility.cpp
@@ -9,21 +5,21 @@ add_executable(visibility_unit
)
target_include_directories(visibility_unit PUBLIC ../src/)
-target_link_libraries(visibility_unit pthread dl ftlcommon)
+target_link_libraries(visibility_unit Threads::Threads dl ftlcommon)
add_test(VisibilityTest visibility_unit)
################################################################################
-add_executable(calibration_unit
- ./tests.cpp
- ../src/calibration.cpp
- ../src/optimization.cpp
- ../src/calibration_data.cpp
- ./calibration_unit.cpp
-)
+#add_executable(calibration_unit
+# ./tests.cpp
+# ../src/calibration.cpp
+# ../src/optimization.cpp
+# ../src/calibration_data.cpp
+# ./calibration_unit.cpp
+#)
-target_include_directories(calibration_unit PUBLIC ../src/)
-target_link_libraries(calibration_unit pthread dl ftlcommon Eigen3::Eigen ceres)
+#target_include_directories(calibration_unit PUBLIC ../src/)
+#target_link_libraries(calibration_unit Threads::Threads dl ftlcommon Eigen3::Eigen ceres)
-add_test(CalibrationTest calibration_unit WORKING_DIRECTORY ${CMAKE_SOURCE_DIR})
+#add_test(CalibrationTest calibration_unit WORKING_DIRECTORY ${CMAKE_SOURCE_DIR})
diff --git a/applications/calibration-multi/CMakeLists.txt b/applications/calibration-multi/CMakeLists.txt
index 135d48adfea1a7114868baa0d3fc7bfe7680160a..d8c9c5b40a711dbd3778187163bfd4bba327bf7a 100644
--- a/applications/calibration-multi/CMakeLists.txt
+++ b/applications/calibration-multi/CMakeLists.txt
@@ -3,13 +3,11 @@ set(CALIBMULTI
src/visibility.cpp
src/util.cpp
src/multicalibrate.cpp
- ../calibration-ceres/src/optimization.cpp
../calibration-ceres/src/calibration.cpp
- ../calibration-ceres/src/calibration_data.cpp
)
add_executable(ftl-calibrate-multi ${CALIBMULTI})
target_include_directories(ftl-calibrate-multi PRIVATE src ../calibration-ceres/src)
-target_link_libraries(ftl-calibrate-multi ftlcommon ftlnet ftlrgbd ftlstreams Threads::Threads ${OpenCV_LIBS} ceres)
+target_link_libraries(ftl-calibrate-multi ftlcalibration ftlcommon ftlnet ftlrgbd ftlstreams Threads::Threads ${OpenCV_LIBS} ceres)
diff --git a/applications/calibration-multi/src/multicalibrate.cpp b/applications/calibration-multi/src/multicalibrate.cpp
index b9627ce252211ef9075e165270d6d88376038814..836a90fde0d0b091ad8207315463ffab6a20e295 100644
--- a/applications/calibration-multi/src/multicalibrate.cpp
+++ b/applications/calibration-multi/src/multicalibrate.cpp
@@ -2,7 +2,8 @@
#include "calibration_data.hpp"
#include "calibration.hpp"
-#include "optimization.hpp"
+
+#include <ftl/calibration/optimize.hpp>
#include <opencv2/core.hpp>
#include <opencv2/calib3d.hpp>
diff --git a/applications/calibration/CMakeLists.txt b/applications/calibration/CMakeLists.txt
index 0efa6e95c7c66e4480e2965e0cccffae39d5cb09..2452f392a9cb6d0dbcac4737ba99ff106996f699 100644
--- a/applications/calibration/CMakeLists.txt
+++ b/applications/calibration/CMakeLists.txt
@@ -10,6 +10,6 @@ add_executable(ftl-calibrate ${CALIBSRC})
target_include_directories(ftl-calibrate PRIVATE src)
-target_link_libraries(ftl-calibrate ftlcommon Threads::Threads ${OpenCV_LIBS})
+target_link_libraries(ftl-calibrate ftlcalibration ftlcommon Threads::Threads ${OpenCV_LIBS})
diff --git a/applications/calibration/src/lens.cpp b/applications/calibration/src/lens.cpp
index b9a01581582b4ef12482932d839a0caccaf1367e..ffa3cd078593a3533b6cfa105ff3eae7f21e5f5e 100644
--- a/applications/calibration/src/lens.cpp
+++ b/applications/calibration/src/lens.cpp
@@ -2,7 +2,7 @@
#include "lens.hpp"
#include <ftl/config.h>
-#include <ftl/calibration.hpp>
+#include <ftl/calibration/parameters.hpp>
#include <loguru.hpp>
diff --git a/components/audio/src/speaker.cpp b/components/audio/src/speaker.cpp
index 64c333851a1cd95e37902b8dd672423bc69dc79b..f9ebd5d6e84451cdc61cb5aa4f48de1685328ad1 100644
--- a/components/audio/src/speaker.cpp
+++ b/components/audio/src/speaker.cpp
@@ -83,6 +83,7 @@ void Speaker::_open(int fsize, int sample, int channels) {
buffer_ = new ftl::audio::MonoBuffer16<2000>(sample);
}
+ #ifdef HAVE_PORTAUDIO
auto err = Pa_OpenDefaultStream(
&stream_,
0,
@@ -111,6 +112,10 @@ void Speaker::_open(int fsize, int sample, int channels) {
}
LOG(INFO) << "Speaker ready.";
+ #else
+ LOG(INFO) << "Built without portaudio (no sound)";
+ #endif
+
}
void Speaker::queue(int64_t ts, ftl::audio::Frame &frame) {
diff --git a/components/calibration/CMakeLists.txt b/components/calibration/CMakeLists.txt
new file mode 100644
index 0000000000000000000000000000000000000000..00faed0b6a4c0cd5292048d8580fc6cbc0657d63
--- /dev/null
+++ b/components/calibration/CMakeLists.txt
@@ -0,0 +1,24 @@
+set(CALIBSRC
+ src/parameters.cpp
+)
+
+if (WITH_CERES)
+ list(APPEND CALIBSRC src/optimize.cpp)
+endif()
+
+add_library(ftlcalibration ${CALIBSRC})
+
+target_include_directories(ftlcalibration
+ PUBLIC
+ $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/include>
+ $<INSTALL_INTERFACE:include>
+ PRIVATE
+ src/
+ ${OpenCV_INCLUDE_DIRS}
+)
+
+target_link_libraries(ftlcalibration ftlcommon Threads::Threads ${OpenCV_LIBS} Eigen3::Eigen ceres)
+
+if (BUILD_TESTS)
+ ADD_SUBDIRECTORY(test)
+endif()
diff --git a/components/calibration/include/ftl/calibration.hpp b/components/calibration/include/ftl/calibration.hpp
new file mode 100644
index 0000000000000000000000000000000000000000..be7533631af984486a99ed30fe14a141b9f28195
--- /dev/null
+++ b/components/calibration/include/ftl/calibration.hpp
@@ -0,0 +1,2 @@
+#include "calibration/parameters.hpp"
+#include "calibration/optimize.hpp"
diff --git a/applications/calibration-ceres/src/optimization.hpp b/components/calibration/include/ftl/calibration/optimize.hpp
similarity index 90%
rename from applications/calibration-ceres/src/optimization.hpp
rename to components/calibration/include/ftl/calibration/optimize.hpp
index 3b0716c7b6b73ab6cc25bdcc2cedf65b33a5da22..1009a97b21aa9d763801b7c512a4e1d13456b4a3 100644
--- a/applications/calibration-ceres/src/optimization.hpp
+++ b/components/calibration/include/ftl/calibration/optimize.hpp
@@ -1,13 +1,14 @@
#pragma once
-#ifndef _FTL_OPTIMIZATION_HPP_
-#define _FTL_OPTIMIZATION_HPP_
-
-#include "optimization.hpp"
-#include "calibration_data.hpp"
+#ifndef _FTL_CALIBRATION_OPTIMIZE_HPP_
+#define _FTL_CALIBRATION_OPTIMIZE_HPP_
#include <vector>
#include <functional>
+#include "parameters.hpp"
+
+#include <ftl/config.h>
+
#include <ceres/ceres.h>
#include <opencv2/core/core.hpp>
@@ -20,7 +21,7 @@ static_assert(std::is_standard_layout<cv::Point3d>());
namespace ftl {
namespace calibration {
-static_assert(std::is_standard_layout<Camera>());
+#ifdef HAVE_CERES
/**
* @brief Optimize scale.
@@ -118,17 +119,21 @@ public:
*/
void run();
- /** @brief Calculate MSE error (for one camera)
+ /** @brief Calculate RMS error (for one camera)
*/
- double reprojectionError(int camera);
+ double reprojectionError(const int camera) const;
- /** @brief Calculate MSE error for all cameras
+ /** @brief Calculate RMS error for all cameras
*/
- double reprojectionError();
+ double reprojectionError() const;
protected:
double* getCameraPtr(int i) { return cameras_[i]->data; }
+ /** @brief Calculate MSE error
+ */
+ void _reprojectionErrorMSE(const int camera, double &error, double &npoints) const;
+
void _buildProblem(ceres::Problem& problem, const BundleAdjustment::Options& options);
void _buildBundleAdjustmentProblem(ceres::Problem& problem, const BundleAdjustment::Options& options);
void _buildLengthProblem(ceres::Problem& problem, const BundleAdjustment::Options& options);
@@ -165,6 +170,8 @@ private:
std::vector<BundleAdjustment::ConstraintObject> constraints_object_;
};
+#endif
+
}
}
diff --git a/components/calibration/include/ftl/calibration/parameters.hpp b/components/calibration/include/ftl/calibration/parameters.hpp
new file mode 100644
index 0000000000000000000000000000000000000000..f261506570921b8c243855e6ace9ea6b2fcfdd95
--- /dev/null
+++ b/components/calibration/include/ftl/calibration/parameters.hpp
@@ -0,0 +1,184 @@
+#pragma once
+#ifndef _FTL_CALIBRATION_PARAMETERS_HPP_
+#define _FTL_CALIBRATION_PARAMETERS_HPP_
+
+#include <opencv2/core/core.hpp>
+
+namespace ftl {
+namespace calibration {
+
+/**
+ * Camera paramters
+ */
+struct Camera {
+ Camera() {}
+ Camera(const cv::Mat& K, const cv::Mat& D, const cv::Mat& R, const cv::Mat& tvec);
+
+ void setRotation(const cv::Mat& R);
+ void setTranslation(const cv::Mat& tvec);
+ void setExtrinsic(const cv::Mat& R, const cv::Mat& t) {
+ setRotation(R);
+ setTranslation(t);
+ }
+
+ void setIntrinsic(const cv::Mat& K);
+ void setDistortion(const cv::Mat &D);
+ void setIntrinsic(const cv::Mat& K, const cv::Mat& D) {
+ setIntrinsic(K);
+ setDistortion(D);
+ }
+
+ cv::Mat intrinsicMatrix() const;
+ cv::Mat distortionCoefficients() const;
+
+ cv::Mat rvec() const;
+ cv::Mat tvec() const;
+ cv::Mat rmat() const;
+
+ cv::Mat extrinsicMatrix() const;
+ cv::Mat extrinsicMatrixInverse() const;
+
+ const static int n_parameters = 12;
+ const static int n_distortion_parameters = 3;
+
+ double data[n_parameters] = {0.0};
+
+ enum Parameter {
+ ROTATION = 0,
+ RX = 0,
+ RY = 1,
+ RZ = 2,
+ TRANSLATION = 3,
+ TX = 3,
+ TY = 4,
+ TZ = 5,
+ F = 6,
+ CX = 7,
+ CY = 8,
+ DISTORTION = 9,
+ K1 = 9,
+ K2 = 10,
+ K3 = 11
+ };
+};
+
+namespace validate {
+
+/**
+ * @brief Valid translation for stereo camera.
+ */
+bool translationStereo(const cv::Mat &t);
+
+bool rotationMatrix(const cv::Mat &M);
+
+bool pose(const cv::Mat &M);
+
+bool cameraMatrix(const cv::Mat &M);
+
+/**
+ * @brief Check if D contains valid distortion coefficients.
+ * @param D distortion coefficients
+ * @param size resolution
+ * @note Tangential and prism distortion coefficients are not validated.
+ *
+ * Radial distortion is always monotonic for real lenses and distortion
+ * function has to be bijective. This is verified by evaluating the distortion
+ * function for integer values from 0 to sqrt(width^2+height^2).
+ *
+ * Camera model documented in
+ * https://docs.opencv.org/master/d9/d0c/group__calib3d.html#details
+ */
+bool distortionCoefficients(const cv::Mat &D, cv::Size size);
+
+}
+
+
+namespace transform {
+
+// TODO: Some of the methods can be directly replace with OpenCV
+// (opencv2/calib3d.hpp)
+
+/**
+ * @brief Get rotation matrix and translation vector from transformation matrix.
+ * @param T transformation matrix (4x4) or (3x4)
+ * @param R (out) rotation matrix (3x3)
+ * @param t (out) translation vector (3x1)
+ */
+inline void getRotationAndTranslation(const cv::Mat &T, cv::Mat &R, cv::Mat &t) {
+ T(cv::Rect(0, 0, 3, 3)).copyTo(R);
+ T(cv::Rect(3, 0, 1, 3)).copyTo(t);
+}
+
+/**
+ * @brief Inverse transform inplace
+ * @param R rotation matrix (3x3)
+ * @param t translation vector (3x1)
+ */
+inline void inverse(cv::Mat &R, cv::Mat &t) {
+ cv::Mat t_(3, 1, CV_64FC1);
+
+ t_.at<double>(0) = -R.col(0).dot(t);
+ t_.at<double>(1) = -R.col(1).dot(t);
+ t_.at<double>(2) = -R.col(2).dot(t);
+
+ cv::swap(t_, t);
+ R = R.t();
+}
+
+/**
+ * @brief Inverse transform inplace
+ * @param T transformation matrix (4x4)
+ */
+inline void inverse(cv::Mat &T) {
+ cv::Mat rmat;
+ cv::Mat tvec;
+ getRotationAndTranslation(T, rmat, tvec);
+ T = cv::Mat::eye(4, 4, CV_64FC1);
+
+ T(cv::Rect(3, 0, 1, 1)) = -rmat.col(0).dot(tvec);
+ T(cv::Rect(3, 1, 1, 1)) = -rmat.col(1).dot(tvec);
+ T(cv::Rect(3, 2, 1, 1)) = -rmat.col(2).dot(tvec);
+ T(cv::Rect(0, 0, 3, 3)) = rmat.t();
+}
+
+inline cv::Point3d apply(const cv::Point3d& point, const cv::InputArray& R, const cv::InputArray& t) {
+ cv::Mat R_ = R.getMat();
+ cv::Mat t_ = t.getMat();
+ cv::Mat p_new = R_ * cv::Mat(point) + t_;
+ return cv::Point3d(p_new.at<double>(0), p_new.at<double>(1), p_new.at<double>(2));
+}
+
+inline cv::Point3d apply(const cv::Point3d& point, const cv::InputArray& T) {
+ cv::Mat T_ = T.getMat();
+ cv::Mat rmat;
+ cv::Mat tvec;
+ getRotationAndTranslation(T_, rmat, tvec);
+ return apply(point, rmat, tvec);
+}
+
+} // namespace transform
+
+/**
+ * @brief Scale camera intrinsic matrix
+ * @param size_new New resolution
+ * @param size_old Original (camera matrix) resolution
+ */
+cv::Mat scaleCameraMatrix(const cv::Mat &K, const cv::Size &size_new, const cv::Size &size_old);
+
+/**
+ * @brief Calculate MSE reprojection error
+ * @param points_im points in pixel coordinates
+ * @param points points in camera coordinates
+ * @param K intrinsic matrix
+ * @param D distortion coefficients
+ * @param R rotation matrix or vector
+ * @param t translation vector
+ */
+double reprojectionError(const std::vector<cv::Point2d>& points_im,
+ const std::vector<cv::Point3d>& points, const cv::Mat& K, const cv::Mat& D,
+ const cv::Mat& R, const cv::Mat& t);
+
+}
+}
+
+#endif
diff --git a/applications/calibration-ceres/src/optimization.cpp b/components/calibration/src/optimize.cpp
similarity index 93%
rename from applications/calibration-ceres/src/optimization.cpp
rename to components/calibration/src/optimize.cpp
index aafe2a38a960c3349521a0d2f88369cafb0f8597..e5319e3fc190284d82f250cc8fc58a2df0fa1de5 100644
--- a/applications/calibration-ceres/src/optimization.cpp
+++ b/components/calibration/src/optimize.cpp
@@ -1,5 +1,5 @@
-#include "optimization.hpp"
-#include "calibration.hpp"
+#include "ftl/calibration/optimize.hpp"
+#include "ftl/calibration/parameters.hpp"
#include "loguru.hpp"
@@ -86,12 +86,12 @@ struct ReprojectionError {
static ceres::CostFunction* Create( const double observed_x,
const double observed_y) {
- return (new ceres::AutoDiffCostFunction<ReprojectionError, 2, _NCAMERA_PARAMETERS, 3>(
+ return (new ceres::AutoDiffCostFunction<ReprojectionError, 2, Camera::n_parameters, 3>(
new ReprojectionError(observed_x, observed_y)));
}
static ceres::CostFunction* Create( const Point2d &observed) {
- return (new ceres::AutoDiffCostFunction<ReprojectionError, 2, _NCAMERA_PARAMETERS, 3>(
+ return (new ceres::AutoDiffCostFunction<ReprojectionError, 2, Camera::n_parameters, 3>(
new ReprojectionError(observed.x, observed.y)));
}
@@ -333,14 +333,14 @@ void BundleAdjustment::_buildBundleAdjustmentProblem(ceres::Problem &problem, co
vector<int> params(fixed_parameters.begin(), fixed_parameters.end());
- if (params.size() == _NCAMERA_PARAMETERS) {
+ if (params.size() == Camera::n_parameters) {
// Ceres crashes if all parameters are set constant using
// SubsetParameterization()
problem.SetParameterBlockConstant(getCameraPtr(i));
}
else if (params.size() > 0) {
problem.SetParameterization(getCameraPtr(i),
- new ceres::SubsetParameterization(_NCAMERA_PARAMETERS, params));
+ new ceres::SubsetParameterization(Camera::n_parameters, params));
}
}
}
@@ -418,7 +418,7 @@ void BundleAdjustment::run() {
run(options);
}
-double BundleAdjustment::reprojectionError(int camera) {
+void BundleAdjustment::_reprojectionErrorMSE(const int camera, double &error, double &npoints) const {
vector<Point2d> observations;
vector<Point3d> points;
@@ -435,11 +435,24 @@ double BundleAdjustment::reprojectionError(int camera) {
auto rvec = cameras_[camera]->rvec();
auto tvec = cameras_[camera]->tvec();
- return ftl::calibration::reprojectionError(observations, points, K, Mat::zeros(1, 5, CV_64FC1), rvec, tvec);
+ error = ftl::calibration::reprojectionError(observations, points, K, Mat::zeros(1, 5, CV_64FC1), rvec, tvec);
+ npoints = points.size();
}
-double BundleAdjustment::reprojectionError() {
+double BundleAdjustment::reprojectionError(const int camera) const {
+ double error, ncameras;
+ _reprojectionErrorMSE(camera, ncameras, error);
+ return sqrt(error);
+}
+
+double BundleAdjustment::reprojectionError() const {
double error = 0.0;
- for (size_t i = 0; i < cameras_.size(); i++) { error += reprojectionError(i); }
- return error * (1.0 / (double) cameras_.size());
+ double npoints = 0.0;
+ for (size_t i = 0; i < cameras_.size(); i++) {
+ double e, n;
+ _reprojectionErrorMSE(i, e, n);
+ error += e * n;
+ npoints += n;
+ }
+ return sqrt(error / npoints);
}
diff --git a/components/calibration/src/parameters.cpp b/components/calibration/src/parameters.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..a947771378f592e2398bb3e278e7214d213e8159
--- /dev/null
+++ b/components/calibration/src/parameters.cpp
@@ -0,0 +1,304 @@
+#include "ftl/calibration/parameters.hpp"
+
+#include <opencv2/calib3d/calib3d.hpp>
+
+using cv::Mat;
+using cv::Size;
+using cv::Point2d;
+using cv::Point3d;
+using cv::Vec3d;
+using cv::Rect;
+
+using std::vector;
+
+using namespace ftl::calibration;
+
+using ftl::calibration::Camera;
+
+////////////////////////////////////////////////////////////////////////////////
+
+void Camera::setRotation(const Mat& R) {
+ if (((R.size() != Size(3, 3)) &&
+ (R.size() != Size(3, 1)) &&
+ (R.size() != Size(1, 3))) ||
+ (R.type() != CV_64FC1)) { throw std::exception(); }
+
+ Mat rvec;
+ if (R.size() == cv::Size(3, 3)) { cv::Rodrigues(R, rvec); }
+ else { rvec = R; }
+
+ data[Parameter::RX] = rvec.at<double>(0);
+ data[Parameter::RY] = rvec.at<double>(1);
+ data[Parameter::RZ] = rvec.at<double>(2);
+}
+
+void Camera::setTranslation(const Mat& t) {
+ if ((t.type() != CV_64FC1) ||
+ (t.size() != cv::Size(1, 3))) { throw std::exception(); }
+
+ data[Parameter::TX] = t.at<double>(0);
+ data[Parameter::TY] = t.at<double>(1);
+ data[Parameter::TZ] = t.at<double>(2);
+}
+
+
+void Camera::setIntrinsic(const Mat& K) {
+ if ((K.type() != CV_64FC1) || (K.size() != cv::Size(3, 3))) {
+ throw std::exception();
+ }
+
+ data[Parameter::F] = K.at<double>(0, 0);
+ data[Parameter::CX] = K.at<double>(0, 2);
+ data[Parameter::CY] = K.at<double>(1, 2);
+}
+
+void Camera::setDistortion(const Mat& D) {
+ if ((D.type() != CV_64FC1)) { throw std::exception(); }
+ switch(D.total()) {
+ case 12:
+ /*
+ data[Parameter::S1] = D.at<double>(8);
+ data[Parameter::S2] = D.at<double>(9);
+ data[Parameter::S3] = D.at<double>(10);
+ data[Parameter::S4] = D.at<double>(11);
+ */
+ [[fallthrough]];
+
+ case 8:
+ /*
+ data[Parameter::K4] = D.at<double>(5);
+ data[Parameter::K5] = D.at<double>(6);
+ data[Parameter::K6] = D.at<double>(7);
+ */
+ [[fallthrough]];
+
+ case 5:
+ data[Parameter::K3] = D.at<double>(4);
+ [[fallthrough]];
+
+ default:
+ data[Parameter::K1] = D.at<double>(0);
+ data[Parameter::K2] = D.at<double>(1);
+ /*
+ data[Parameter::P1] = D.at<double>(2);
+ data[Parameter::P2] = D.at<double>(3);
+ */
+ }
+}
+
+Camera::Camera(const Mat &K, const Mat &D, const Mat &R, const Mat &tvec) {
+ setIntrinsic(K, D);
+ if (!R.empty()) { setRotation(R); }
+ if (!tvec.empty()) { setTranslation(tvec); }
+}
+
+Mat Camera::intrinsicMatrix() const {
+ Mat K = Mat::eye(3, 3, CV_64FC1);
+ K.at<double>(0, 0) = data[Parameter::F];
+ K.at<double>(1, 1) = data[Parameter::F];
+ K.at<double>(0, 2) = data[Parameter::CX];
+ K.at<double>(1, 2) = data[Parameter::CY];
+ return K;
+}
+
+Mat Camera::distortionCoefficients() const {
+ Mat D;
+ // OpenCV distortion parameter sizes: 4, 5, 8, 12, 14
+ if (Camera::n_distortion_parameters <= 4) { D = Mat::zeros(4, 1, CV_64FC1); }
+ else if (Camera::n_distortion_parameters <= 5) { D = Mat::zeros(5, 1, CV_64FC1); }
+ else if (Camera::n_distortion_parameters <= 8) { D = Mat::zeros(8, 1, CV_64FC1); }
+ else if (Camera::n_distortion_parameters <= 12) { D = Mat::zeros(12, 1, CV_64FC1); }
+ else if (Camera::n_distortion_parameters <= 14) { D = Mat::zeros(14, 1, CV_64FC1); }
+
+ for (int i = 0; i < Camera::n_distortion_parameters; i++) {
+ D.at<double>(i) = data[Parameter::DISTORTION+i];
+ }
+ return D;
+}
+
+Mat Camera::rvec() const {
+ return Mat(Vec3d(data[Parameter::RX], data[Parameter::RY], data[Parameter::RZ]));
+}
+
+Mat Camera::tvec() const {
+ return Mat(Vec3d(data[Parameter::TX], data[Parameter::TY], data[Parameter::TZ]));
+}
+
+Mat Camera::rmat() const {
+ Mat R;
+ cv::Rodrigues(rvec(), R);
+ return R;
+}
+
+Mat Camera::extrinsicMatrix() const {
+ Mat T = Mat::eye(4, 4, CV_64FC1);
+ rmat().copyTo(T(Rect(0, 0, 3, 3)));
+ tvec().copyTo(T(Rect(3, 0, 1, 3)));
+ return T;
+}
+
+Mat Camera::extrinsicMatrixInverse() const {
+ return extrinsicMatrix().inv();
+}
+
+////////////////////////////////////////////////////////////////////////////////
+
+bool validate::translationStereo(const Mat &t) {
+ if (t.type() != CV_64F) { return false; }
+ if (t.channels() != 1) { return false; }
+ if (t.size() != Size(1, 3)) { return false; }
+ if (cv::norm(t, cv::NORM_L2) == 0) { return false; }
+ return true;
+}
+
+bool validate::rotationMatrix(const Mat &M) {
+ if (M.type() != CV_64F) { return false; }
+ if (M.channels() != 1) { return false; }
+ if (M.size() != Size(3, 3)) { return false; }
+
+ double det = cv::determinant(M);
+ if (abs(abs(det)-1.0) > 0.00001) { return false; }
+
+ // TODO: floating point errors (result not exactly identity matrix)
+ // rotation matrix is orthogonal: M.T * M == M * M.T == I
+ //if (cv::countNonZero((M.t() * M) != Mat::eye(Size(3, 3), CV_64FC1)) != 0)
+ // { return false; }
+
+ return true;
+}
+
+bool validate::pose(const Mat &M) {
+ if (M.size() != Size(4, 4)) { return false; }
+ if (!validate::rotationMatrix(M(cv::Rect(0 , 0, 3, 3))))
+ { return false; }
+ if (!( (M.at<double>(3, 0) == 0.0) &&
+ (M.at<double>(3, 1) == 0.0) &&
+ (M.at<double>(3, 2) == 0.0) &&
+ (M.at<double>(3, 3) == 1.0))) { return false; }
+
+ return true;
+}
+
+bool validate::cameraMatrix(const Mat &M) {
+ if (M.type() != CV_64F) { return false; }
+ if (M.channels() != 1) { return false; }
+ if (M.size() != Size(3, 3)) { return false; }
+
+ if (!( (M.at<double>(2, 0) == 0.0) &&
+ (M.at<double>(2, 1) == 0.0) &&
+ (M.at<double>(2, 2) == 1.0))) { return false; }
+
+ return true;
+}
+
+bool ftl::calibration::validate::distortionCoefficients(const Mat &D, Size size) {
+ if (D.type() != CV_64FC1) { return false; }
+ if (!(
+ (D.total() == 4) ||
+ (D.total() == 5) ||
+ (D.total() == 8) ||
+ (D.total() == 12))) { return false; }
+
+ for (int i = 0; i < D.total(); i++) {
+ if (!std::isfinite(D.at<double>(i))) { return false; }
+ }
+
+ double k[6] = {0.0};
+ //double p[2] = {0.0};
+ //double s[4] = {0.0};
+
+ switch(D.total()) {
+ case 12:
+ /*
+ s[0] = D.at<double>(8);
+ s[1] = D.at<double>(9);
+ s[2] = D.at<double>(10);
+ s[3] = D.at<double>(11);
+ */
+ [[fallthrough]];
+
+ case 8:
+ k[3] = D.at<double>(5);
+ k[4] = D.at<double>(6);
+ k[5] = D.at<double>(7);
+ [[fallthrough]];
+
+ case 5:
+ k[2] = D.at<double>(4);
+ [[fallthrough]];
+
+ default:
+ k[0] = D.at<double>(0);
+ k[1] = D.at<double>(1);
+ /*
+ p[0] = D.at<double>(2);
+ p[1] = D.at<double>(3);
+ */
+ }
+
+ int diagonal = sqrt(size.width*size.width+size.height*size.height) + 1.0;
+
+ bool is_n = true;
+ bool is_p = true;
+
+ double dist_prev_n = 0;
+ double dist_prev_p = 0;
+
+ for (int r = 0; r < diagonal; r++) {
+ double r2 = r*r;
+ double r4 = r2*r2;
+ double r6 = r4*r2;
+
+ double rdist = 1.0 + k[0]*r2 + k[1]*r4 + k[2]*r6;
+ double irdist2 = 1./(1.0 + k[3]*r2 + k[4]*r4 + k[5]*r6);
+ double dist = r2*rdist*irdist2; // + s[0]*r2 + s[1]*r4;
+
+ if (is_n) {
+ if (r2 == 0) {}
+ else if (!(dist < dist_prev_n)) { is_n = false; }
+ dist_prev_n = dist;
+ }
+
+ if (is_p) {
+ if (r2 == 0) {}
+ else if (!(dist > dist_prev_p)) { is_p = false; }
+ dist_prev_p = dist;
+ }
+
+ if (!is_n && !is_p) { return false; }
+ }
+
+ return true;
+}
+
+Mat ftl::calibration::scaleCameraMatrix(const Mat &K, const Size &size_new, const Size &size_old) {
+ Mat S(cv::Size(3, 3), CV_64F, 0.0);
+ double scale_x = ((double) size_new.width) / ((double) size_old.width);
+ double scale_y = ((double) size_new.height) / ((double) size_old.height);
+
+ S.at<double>(0, 0) = scale_x;
+ S.at<double>(1, 1) = scale_y;
+ S.at<double>(2, 2) = 1.0;
+ return (S*K);
+}
+
+double ftl::calibration::reprojectionError(const vector<Point2d>& points_im,
+ const vector<Point3d>& points, const Mat& K, const Mat& D, const Mat& R,
+ const Mat& t) {
+
+ Mat rvec;
+ if (R.size() == Size(3, 3)) { cv::Rodrigues(R, rvec); }
+ else { rvec = R; }
+
+ vector<Point2d> points_reprojected;
+ cv::projectPoints(points, rvec, t, K, D, points_reprojected);
+
+ double err = 0.0;
+ int npoints = points_im.size();
+
+ for (int i = 0; i < npoints; i++) {
+ err += norm(points_im[i] - points_reprojected[i]);
+ }
+
+ return err / ((double) npoints);
+}
diff --git a/components/calibration/test/CMakeLists.txt b/components/calibration/test/CMakeLists.txt
new file mode 100644
index 0000000000000000000000000000000000000000..0bd1f1f198347c38c74973b14a5253b4bcd93d09
--- /dev/null
+++ b/components/calibration/test/CMakeLists.txt
@@ -0,0 +1,10 @@
+### Calibration ################################################################
+add_executable(calibration_parameters_unit
+ ./tests.cpp
+ ./test_parameters.cpp
+ ../src/parameters.cpp
+)
+
+target_include_directories(calibration_parameters_unit PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}/../include")
+target_link_libraries(calibration_parameters_unit ftlcommon Threads::Threads ${OS_LIBS} ${OpenCV_LIBS})
+add_test(CalibrationValidateTest calibration_parameters_unit)
\ No newline at end of file
diff --git a/components/common/cpp/test/calibration_common_unit.cpp b/components/calibration/test/test_parameters.cpp
similarity index 96%
rename from components/common/cpp/test/calibration_common_unit.cpp
rename to components/calibration/test/test_parameters.cpp
index 5610c169d3b78c1755fd1f9279e7e9bf94bc46c4..7d19b9d757587ff7e73cde565c3643f8b8d5937e 100644
--- a/components/common/cpp/test/calibration_common_unit.cpp
+++ b/components/calibration/test/test_parameters.cpp
@@ -1,5 +1,5 @@
#include "catch.hpp"
-#include "ftl/calibration.hpp"
+#include "ftl/calibration/parameters.hpp"
using cv::Size;
using cv::Mat;
diff --git a/components/calibration/test/tests.cpp b/components/calibration/test/tests.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..0c7c351f437f5f43f3bb62beb254a9f1ecbec5a0
--- /dev/null
+++ b/components/calibration/test/tests.cpp
@@ -0,0 +1,2 @@
+#define CATCH_CONFIG_MAIN
+#include "catch.hpp"
diff --git a/components/common/cpp/CMakeLists.txt b/components/common/cpp/CMakeLists.txt
index 49eaac5738ce8caff4d102d15bc8030c6798721c..57ec51ace3ab13823105f2c5936e419e020809a6 100644
--- a/components/common/cpp/CMakeLists.txt
+++ b/components/common/cpp/CMakeLists.txt
@@ -11,7 +11,6 @@ set(COMMONSRC
src/ctpl_stl.cpp
src/timer.cpp
src/profiler.cpp
- src/calibration.cpp
)
check_function_exists(uriParseSingleUriA HAVE_URIPARSESINGLE)
diff --git a/components/common/cpp/include/ftl/calibration.hpp b/components/common/cpp/include/ftl/calibration.hpp
deleted file mode 100644
index 00662685237cdabf5e9d8a855ced5d1493f4a4e5..0000000000000000000000000000000000000000
--- a/components/common/cpp/include/ftl/calibration.hpp
+++ /dev/null
@@ -1,50 +0,0 @@
-#pragma once
-#ifndef _FTL_COMMON_CALIBRATION_HPP_
-#define _FTL_COMMON_CALIBRATION_HPP_
-
-#include <opencv2/core/core.hpp>
-
-namespace ftl {
-namespace calibration {
-
-namespace validate {
-
-/**
- * @brief Valid translation for stereo camera.
- */
-bool translationStereo(const cv::Mat &t);
-
-bool rotationMatrix(const cv::Mat &M);
-
-bool pose(const cv::Mat &M);
-
-bool cameraMatrix(const cv::Mat &M);
-
-/**
- * @brief Check if D contains valid distortion coefficients.
- * @param D distortion coefficients
- * @param size resolution
- * @note Tangential and prism distortion coefficients are not validated.
- *
- * Radial distortion is always monotonic for real lenses and distortion
- * function has to be bijective. This is verified by evaluating the distortion
- * function for integer values from 0 to sqrt(width^2+height^2).
- *
- * Camera model documented in
- * https://docs.opencv.org/master/d9/d0c/group__calib3d.html#details
- */
-bool distortionCoefficients(const cv::Mat &D, cv::Size size);
-
-}
-
-/**
- * @brief Scale camera intrinsic matrix
- * @param size_new New resolution
- * @param size_old Original (camera matrix) resolution
- */
-cv::Mat scaleCameraMatrix(const cv::Mat &K, const cv::Size &size_new, const cv::Size &size_old);
-
-}
-}
-
-#endif
diff --git a/components/common/cpp/include/ftl/config.h.in b/components/common/cpp/include/ftl/config.h.in
index 5b54fea8e957b5fb3f7e9d403e6f45aec41c7c51..ecf9e79f798b333a91f68b814583f5edf46615fc 100644
--- a/components/common/cpp/include/ftl/config.h.in
+++ b/components/common/cpp/include/ftl/config.h.in
@@ -19,6 +19,7 @@
#cmakedefine HAVE_OPTFLOW
#cmakedefine HAVE_RENDER
#cmakedefine HAVE_LIBSGM
+#cmakedefine HAVE_CERES
#cmakedefine HAVE_REALSENSE
#cmakedefine HAVE_NANOGUI
#cmakedefine HAVE_LIBARCHIVE
diff --git a/components/common/cpp/src/calibration.cpp b/components/common/cpp/src/calibration.cpp
deleted file mode 100644
index 6a6bd8fa31197b72bec301a510a603c8b45213f1..0000000000000000000000000000000000000000
--- a/components/common/cpp/src/calibration.cpp
+++ /dev/null
@@ -1,145 +0,0 @@
-#include "ftl/calibration.hpp"
-
-using cv::Mat;
-using cv::Size;
-
-using namespace ftl::calibration;
-
-bool validate::translationStereo(const Mat &t) {
- if (t.type() != CV_64F) { return false; }
- if (t.channels() != 1) { return false; }
- if (t.size() != Size(1, 3)) { return false; }
- if (cv::norm(t, cv::NORM_L2) == 0) { return false; }
- return true;
-}
-
-bool validate::rotationMatrix(const Mat &M) {
- if (M.type() != CV_64F) { return false; }
- if (M.channels() != 1) { return false; }
- if (M.size() != Size(3, 3)) { return false; }
-
- double det = cv::determinant(M);
- if (abs(abs(det)-1.0) > 0.00001) { return false; }
-
- // TODO: floating point errors (result not exactly identity matrix)
- // rotation matrix is orthogonal: M.T * M == M * M.T == I
- //if (cv::countNonZero((M.t() * M) != Mat::eye(Size(3, 3), CV_64FC1)) != 0)
- // { return false; }
-
- return true;
-}
-
-bool validate::pose(const Mat &M) {
- if (M.size() != Size(4, 4)) { return false; }
- if (!validate::rotationMatrix(M(cv::Rect(0 , 0, 3, 3))))
- { return false; }
- if (!( (M.at<double>(3, 0) == 0.0) &&
- (M.at<double>(3, 1) == 0.0) &&
- (M.at<double>(3, 2) == 0.0) &&
- (M.at<double>(3, 3) == 1.0))) { return false; }
-
- return true;
-}
-
-bool validate::cameraMatrix(const Mat &M) {
- if (M.type() != CV_64F) { return false; }
- if (M.channels() != 1) { return false; }
- if (M.size() != Size(3, 3)) { return false; }
-
- if (!( (M.at<double>(2, 0) == 0.0) &&
- (M.at<double>(2, 1) == 0.0) &&
- (M.at<double>(2, 2) == 1.0))) { return false; }
-
- return true;
-}
-
-bool ftl::calibration::validate::distortionCoefficients(const Mat &D, Size size) {
- if (D.type() != CV_64FC1) { return false; }
- if (!(
- (D.total() == 4) ||
- (D.total() == 5) ||
- (D.total() == 8) ||
- (D.total() == 12))) { return false; }
-
- for (int i = 0; i < D.total(); i++) {
- if (!std::isfinite(D.at<double>(i))) { return false; }
- }
-
- double k[6] = {0.0};
- //double p[2] = {0.0};
- //double s[4] = {0.0};
-
- switch(D.total()) {
- case 12:
- /*
- s[0] = D.at<double>(8);
- s[1] = D.at<double>(9);
- s[2] = D.at<double>(10);
- s[3] = D.at<double>(11);
- */
- [[fallthrough]];
-
- case 8:
- k[3] = D.at<double>(5);
- k[4] = D.at<double>(6);
- k[5] = D.at<double>(7);
- [[fallthrough]];
-
- case 5:
- k[2] = D.at<double>(4);
- [[fallthrough]];
-
- default:
- k[0] = D.at<double>(0);
- k[1] = D.at<double>(1);
- /*
- p[0] = D.at<double>(2);
- p[1] = D.at<double>(3);
- */
- }
-
- int diagonal = sqrt(size.width*size.width+size.height*size.height) + 1.0;
-
- bool is_n = true;
- bool is_p = true;
-
- double dist_prev_n = 0;
- double dist_prev_p = 0;
-
- for (int r = 0; r < diagonal; r++) {
- double r2 = r*r;
- double r4 = r2*r2;
- double r6 = r4*r2;
-
- double rdist = 1.0 + k[0]*r2 + k[1]*r4 + k[2]*r6;
- double irdist2 = 1./(1.0 + k[3]*r2 + k[4]*r4 + k[5]*r6);
- double dist = r2*rdist*irdist2; // + s[0]*r2 + s[1]*r4;
-
- if (is_n) {
- if (r2 == 0) {}
- else if (!(dist < dist_prev_n)) { is_n = false; }
- dist_prev_n = dist;
- }
-
- if (is_p) {
- if (r2 == 0) {}
- else if (!(dist > dist_prev_p)) { is_p = false; }
- dist_prev_p = dist;
- }
-
- if (!is_n && !is_p) { return false; }
- }
-
- return true;
-}
-
-Mat ftl::calibration::scaleCameraMatrix(const Mat &K, const Size &size_new, const Size &size_old) {
- Mat S(cv::Size(3, 3), CV_64F, 0.0);
- double scale_x = ((double) size_new.width) / ((double) size_old.width);
- double scale_y = ((double) size_new.height) / ((double) size_old.height);
-
- S.at<double>(0, 0) = scale_x;
- S.at<double>(1, 1) = scale_y;
- S.at<double>(2, 2) = 1.0;
- return (S*K);
-}
\ No newline at end of file
diff --git a/components/common/cpp/test/CMakeLists.txt b/components/common/cpp/test/CMakeLists.txt
index c5d394ac48e429e46b9a41204a25aa4448250702..de71019fb70424cbdf6243c1dcf7e9f5071db504 100644
--- a/components/common/cpp/test/CMakeLists.txt
+++ b/components/common/cpp/test/CMakeLists.txt
@@ -39,13 +39,4 @@ target_link_libraries(msgpack_unit Threads::Threads ${OS_LIBS} ${OpenCV_LIBS})
add_test(ConfigurableUnitTest configurable_unit)
add_test(URIUnitTest uri_unit)
add_test(MsgpackUnitTest msgpack_unit)
-# add_test(TimerUnitTest timer_unit) CI server can't achieve this
-
-### Calibration ################################################################
-add_executable(calibration_common_unit
- $<TARGET_OBJECTS:CatchTest>
- ./calibration_common_unit.cpp
-)
-target_include_directories(calibration_common_unit PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}/../include")
-target_link_libraries(calibration_common_unit ftlcommon Threads::Threads ${OS_LIBS} ${OpenCV_LIBS})
-add_test(CalibrationCommonTest calibration_common_unit)
\ No newline at end of file
+# add_test(TimerUnitTest timer_unit) CI server can't achieve this
\ No newline at end of file
diff --git a/components/rgbd-sources/CMakeLists.txt b/components/rgbd-sources/CMakeLists.txt
index 9e5647e88ec11abed63497d2d6366e0bddbd40fa..fafceea695332cc0f0daf1c5bedffc649c3ad727 100644
--- a/components/rgbd-sources/CMakeLists.txt
+++ b/components/rgbd-sources/CMakeLists.txt
@@ -38,7 +38,7 @@ if (CUDA_FOUND)
set_property(TARGET ftlrgbd PROPERTY CUDA_SEPARABLE_COMPILATION OFF)
endif()
-target_link_libraries(ftlrgbd ftlcommon ${OpenCV_LIBS} ${LIBSGM_LIBRARIES} ${CUDA_LIBRARIES} Eigen3::Eigen realsense ftlnet ${LibArchive_LIBRARIES} ftlcodecs ftloperators ftldata ${X11_X11_LIB} ${X11_Xext_LIB})
+target_link_libraries(ftlrgbd ftlcalibration ftlcommon ${OpenCV_LIBS} ${LIBSGM_LIBRARIES} ${CUDA_LIBRARIES} Eigen3::Eigen realsense ftlnet ${LibArchive_LIBRARIES} ftlcodecs ftloperators ftldata ${X11_X11_LIB} ${X11_Xext_LIB})
if (BUILD_TESTS)
add_subdirectory(test)
diff --git a/components/rgbd-sources/src/sources/stereovideo/calibrate.cpp b/components/rgbd-sources/src/sources/stereovideo/calibrate.cpp
index f67262c9e71c743bcd6d6cc9e740f60f29219a49..7851fb1541e591c65ee5b17167a22ecfaa6baea8 100644
--- a/components/rgbd-sources/src/sources/stereovideo/calibrate.cpp
+++ b/components/rgbd-sources/src/sources/stereovideo/calibrate.cpp
@@ -6,7 +6,7 @@
#include <ftl/config.h>
#include <ftl/configuration.hpp>
#include <ftl/threads.hpp>
-#include <ftl/calibration.hpp>
+#include <ftl/calibration/parameters.hpp>
#include "calibrate.hpp"
#include "ftl/exception.hpp"
diff --git a/applications/calibration-ceres/test/catch.hpp b/lib/catch/catch.hpp
similarity index 100%
rename from applications/calibration-ceres/test/catch.hpp
rename to lib/catch/catch.hpp