diff --git a/applications/calibration-multi/src/main.cpp b/applications/calibration-multi/src/main.cpp
index 0901be79c5c488f71dc436f97f18cb20b85886ab..f9c258d7665933b510e6dc5af08e6c499e16ec91 100644
--- a/applications/calibration-multi/src/main.cpp
+++ b/applications/calibration-multi/src/main.cpp
@@ -204,8 +204,9 @@ void calibrate( MultiCameraCalibrationNew &calib, vector<string> &uri_cameras,
if (params.reference_camera < 0) {
reference_camera = calib.getOptimalReferenceCamera();
reference_camera -= (reference_camera & 1);
+ LOG(INFO) << "optimal camera (automatic): " << reference_camera;
}
- LOG(INFO) << "optimal camera: " << reference_camera;
+ LOG(INFO) << "reference camera: " << reference_camera;
if (params.optimize_intrinsic) calib.setFixIntrinsic(0);
@@ -536,11 +537,13 @@ int main(int argc, char **argv) {
const int min_visible = root->value<int>("min_visible", 3);
// minimum for how many times pattern is seen per camera
const int n_views = root->value<int>("n_views", 500);
+ // reference camera, -1 for automatic
+ const int ref_camera = root->value<int>("reference_camera", -1);
// registration file path
const string registration_file = root->value<string>("registration_file", FTL_LOCAL_CONFIG_ROOT "/registration.json");
// location where extrinsic calibration files saved
const string output_directory = root->value<string>("output_directory", "./");
-
+
CalibrationParams params;
params.save_extrinsic = save_extrinsic;
params.save_intrinsic = save_intrinsic;
@@ -560,6 +563,7 @@ int main(int argc, char **argv) {
<< "\n calibration_data_file: " << calibration_data_file
<< "\n min_visible: " << min_visible
<< "\n n_views: " << n_views
+ << "\n reference_camera: " << ref_camera << (ref_camera != -1 ? "" : " (automatic)")
<< "\n registration_file: " << registration_file
<< "\n output_directory: " << output_directory
<< "\n";
diff --git a/applications/calibration-multi/src/multicalibrate.cpp b/applications/calibration-multi/src/multicalibrate.cpp
index 074e778ed8323fed3f7a8625288b37f62e43f612..2292d178d70f0f7c68b79cbbf1cb9172cdd173be 100644
--- a/applications/calibration-multi/src/multicalibrate.cpp
+++ b/applications/calibration-multi/src/multicalibrate.cpp
@@ -393,6 +393,13 @@ double MultiCameraCalibrationNew::calibratePair(size_t camera_from, size_t camer
vector<uchar> inliers;
Mat F, E;
F = cv::findFundamentalMat(points1, points2, fm_method_, fm_ransac_threshold_, fm_confidence_, inliers);
+
+ if (F.empty())
+ {
+ LOG(ERROR) << "Fundamental matrix estimation failed. Possibly degenerate configuration?";
+ return INFINITY;
+ }
+
E = K2.t() * F * K1;
// Only include inliers
diff --git a/applications/groupview/src/main.cpp b/applications/groupview/src/main.cpp
index 3a821a01323fbe51dbc8b6fdef98597f0bd20853..d1dab2607ce15ea1ec5defe63aa4ff2b48ee95b6 100644
--- a/applications/groupview/src/main.cpp
+++ b/applications/groupview/src/main.cpp
@@ -90,15 +90,17 @@ void modeLeftRight(ftl::Configurable *root) {
group.sync([&mutex, &new_frames, &rgb_new](ftl::rgbd::FrameSet &frames) {
mutex.lock();
bool good = true;
- for (size_t i = 0; i < frames.channel1.size(); i ++) {
- if (frames.channel1[i].empty()) good = false;
- if (frames.channel1[i].empty()) good = false;
- if (frames.channel1[i].channels() != 3) good = false; // ASSERT
- if (frames.channel2[i].channels() != 3) good = false;
+ for (size_t i = 0; i < frames.frames.size(); i ++) {
+ auto &chan1 = frames.frames[i].getChannel<cv::Mat>(ftl::rgbd::kChanColour);
+ auto &chan2 = frames.frames[i].getChannel<cv::Mat>(frames.sources[i]->getChannel());
+ if (chan1.empty()) good = false;
+ if (chan2.empty()) good = false;
+ if (chan1.channels() != 3) good = false; // ASSERT
+ if (chan2.channels() != 3) good = false;
if (!good) break;
- frames.channel1[i].copyTo(rgb_new[2 * i]);
- frames.channel2[i].copyTo(rgb_new[2 * i + 1]);
+ chan1.copyTo(rgb_new[2 * i]);
+ chan2.copyTo(rgb_new[2 * i + 1]);
}
new_frames = good;
@@ -180,14 +182,19 @@ void modeFrame(ftl::Configurable *root, int frames=1) {
//LOG(INFO) << "Complete set: " << fs.timestamp;
if (!ftl::running) { return false; }
+ std::vector<cv::Mat> frames;
for (size_t i=0; i<fs.sources.size(); ++i) {
- if (fs.channel1[i].empty() || fs.channel2[i].empty()) return true;
+ auto &chan1 = fs.frames[i].getChannel<cv::Mat>(ftl::rgbd::kChanColour);
+ auto &chan2 = fs.frames[i].getChannel<cv::Mat>(fs.sources[i]->getChannel());
+ if (chan1.empty() || chan2.empty()) return true;
+
+ frames.push_back(chan1);
}
cv::Mat show;
- stack(fs.channel1, show);
+ stack(frames, show);
cv::resize(show, show, cv::Size(1280,720));
cv::namedWindow("Cameras", cv::WINDOW_KEEPRATIO | cv::WINDOW_NORMAL);
@@ -206,9 +213,12 @@ void modeFrame(ftl::Configurable *root, int frames=1) {
auto writer = ftl::rgbd::SnapshotWriter(std::string(timestamp) + ".tar.gz");
for (size_t i=0; i<fs.sources.size(); ++i) {
+ auto &chan1 = fs.frames[i].getChannel<cv::Mat>(ftl::rgbd::kChanColour);
+ auto &chan2 = fs.frames[i].getChannel<cv::Mat>(fs.sources[i]->getChannel());
+
writer.addSource(fs.sources[i]->getURI(), fs.sources[i]->parameters(), fs.sources[i]->getPose());
- LOG(INFO) << "SAVE: " << fs.channel1[i].cols << ", " << fs.channel2[i].type();
- writer.addRGBD(i, fs.channel1[i], fs.channel2[i]);
+ //LOG(INFO) << "SAVE: " << fs.channel1[i].cols << ", " << fs.channel2[i].type();
+ writer.addRGBD(i, chan1, chan2);
}
}
#endif // HAVE_LIBARCHIVE
diff --git a/applications/reconstruct/src/voxel_scene.cpp b/applications/reconstruct/src/voxel_scene.cpp
index cba6e61e9e9134845e59d71dd6e69d4ad7ae8beb..ab8b2c1dd25b530e1ffb4fcf4a29c42bd86dfbb6 100644
--- a/applications/reconstruct/src/voxel_scene.cpp
+++ b/applications/reconstruct/src/voxel_scene.cpp
@@ -233,6 +233,8 @@ int SceneRep::upload(ftl::rgbd::FrameSet &fs) {
for (size_t i=0; i<cameras_.size(); ++i) {
auto &cam = cameras_[i];
+ auto &chan1 = fs.frames[i].getChannel<cv::Mat>(ftl::rgbd::kChanColour);
+ auto &chan2 = fs.frames[i].getChannel<cv::Mat>(fs.sources[i]->getChannel());
// Get the RGB-Depth frame from input
Source *input = cam.source;
@@ -247,12 +249,12 @@ int SceneRep::upload(ftl::rgbd::FrameSet &fs) {
// Must be in RGBA for GPU
Mat rgbt, rgba;
- cv::cvtColor(fs.channel1[i],rgbt, cv::COLOR_BGR2Lab);
+ cv::cvtColor(chan1,rgbt, cv::COLOR_BGR2Lab);
cv::cvtColor(rgbt,rgba, cv::COLOR_BGR2BGRA);
// Send to GPU and merge view into scene
//cam.gpu.updateParams(cam.params);
- cam.gpu.updateData(fs.channel2[i], rgba, cam.stream);
+ cam.gpu.updateData(chan2, rgba, cam.stream);
//setLastRigidTransform(input->getPose().cast<float>());
diff --git a/components/rgbd-sources/include/ftl/rgbd/frame.hpp b/components/rgbd-sources/include/ftl/rgbd/frame.hpp
index 227913e5de056f15380390201da96b66ec23e27f..c07fc1494a32405777bb7b06a465b54e7845bfc6 100644
--- a/components/rgbd-sources/include/ftl/rgbd/frame.hpp
+++ b/components/rgbd-sources/include/ftl/rgbd/frame.hpp
@@ -12,6 +12,7 @@ namespace rgbd {
typedef unsigned int channel_t;
static const channel_t kChanNone = 0;
+static const channel_t kChanColour = 0x0001;
static const channel_t kChanLeft = 0x0001; // CV_8UC3
static const channel_t kChanDepth = 0x0002; // CV_32FC1
static const channel_t kChanRight = 0x0004; // CV_8UC3
@@ -58,7 +59,11 @@ public:
*/
bool hasChannel(const ftl::rgbd::channel_t& channel)
{
+<<<<<<< HEAD
return available_[_channelIdx(channel)];
+=======
+ return (channel == ftl::rgbd::kChanNone) ? true : available_[_channelIdx(channel)];
+>>>>>>> master
}
/* @brief Method to get reference to the channel content
diff --git a/components/rgbd-sources/include/ftl/rgbd/frameset.hpp b/components/rgbd-sources/include/ftl/rgbd/frameset.hpp
new file mode 100644
index 0000000000000000000000000000000000000000..93839efda9c6dc8cfe53e6775293949d5c1747e8
--- /dev/null
+++ b/components/rgbd-sources/include/ftl/rgbd/frameset.hpp
@@ -0,0 +1,32 @@
+#ifndef _FTL_RGBD_FRAMESET_HPP_
+#define _FTL_RGBD_FRAMESET_HPP_
+
+#include <ftl/rgbd/frame.hpp>
+
+#include <opencv2/opencv.hpp>
+#include <vector>
+
+namespace ftl {
+namespace rgbd {
+
+class Source;
+
+/**
+ * Represents a set of synchronised frames, each with two channels. This is
+ * used to collect all frames from multiple computers that have the same
+ * timestamp.
+ */
+struct FrameSet {
+ int64_t timestamp; // Millisecond timestamp of all frames
+ std::vector<Source*> sources; // All source objects involved.
+ std::vector<ftl::rgbd::Frame> frames;
+ std::atomic<int> count; // Number of valid frames
+ std::atomic<unsigned int> mask; // Mask of all sources that contributed
+ bool stale; // True if buffers have been invalidated
+ SHARED_MUTEX mtx;
+};
+
+}
+}
+
+#endif // _FTL_RGBD_FRAMESET_HPP_
diff --git a/components/rgbd-sources/include/ftl/rgbd/group.hpp b/components/rgbd-sources/include/ftl/rgbd/group.hpp
index 000eea0ae60303d816e09298613ddc30f0a8146a..e4ffb5984e625b1b9859a3f193a7f4b44c956684 100644
--- a/components/rgbd-sources/include/ftl/rgbd/group.hpp
+++ b/components/rgbd-sources/include/ftl/rgbd/group.hpp
@@ -3,6 +3,8 @@
#include <ftl/threads.hpp>
#include <ftl/timer.hpp>
+#include <ftl/rgbd/frame.hpp>
+#include <ftl/rgbd/frameset.hpp>
#include <opencv2/opencv.hpp>
#include <vector>
@@ -12,22 +14,6 @@ namespace rgbd {
class Source;
-/**
- * Represents a set of synchronised frames, each with two channels. This is
- * used to collect all frames from multiple computers that have the same
- * timestamp.
- */
-struct FrameSet {
- int64_t timestamp; // Millisecond timestamp of all frames
- std::vector<Source*> sources; // All source objects involved.
- std::vector<cv::Mat> channel1; // RGB
- std::vector<cv::Mat> channel2; // Depth (usually)
- std::atomic<int> count; // Number of valid frames
- std::atomic<unsigned int> mask; // Mask of all sources that contributed
- bool stale; // True if buffers have been invalidated
- SHARED_MUTEX mtx;
-};
-
// Allows a latency of 20 frames maximum
static const size_t kFrameBufferSize = 20;
diff --git a/components/rgbd-sources/src/group.cpp b/components/rgbd-sources/src/group.cpp
index e61289220ad92fff8e278905a075f01d93b6ad9a..57d770ba12db4f4b15aebf48cedd33aa2b880fa5 100644
--- a/components/rgbd-sources/src/group.cpp
+++ b/components/rgbd-sources/src/group.cpp
@@ -52,6 +52,8 @@ void Group::addSource(ftl::rgbd::Source *src) {
src->setCallback([this,ix,src](int64_t timestamp, cv::Mat &rgb, cv::Mat &depth) {
if (timestamp == 0) return;
+ auto chan = src->getChannel();
+
//LOG(INFO) << "SRC CB: " << timestamp << " (" << framesets_[head_].timestamp << ")";
UNIQUE_LOCK(mutex_, lk);
@@ -73,11 +75,15 @@ void Group::addSource(ftl::rgbd::Source *src) {
//LOG(INFO) << "Adding frame: " << ix << " for " << timestamp;
// Ensure channels match source mat format
- fs.channel1[ix].create(rgb.size(), rgb.type());
- fs.channel2[ix].create(depth.size(), depth.type());
+ //fs.channel1[ix].create(rgb.size(), rgb.type());
+ //fs.channel2[ix].create(depth.size(), depth.type());
+ fs.frames[ix].setChannel<cv::Mat>(ftl::rgbd::kChanColour).create(rgb.size(), rgb.type());
+ fs.frames[ix].setChannel<cv::Mat>(chan).create(depth.size(), depth.type());
- cv::swap(rgb, fs.channel1[ix]);
- cv::swap(depth, fs.channel2[ix]);
+ //cv::swap(rgb, fs.channel1[ix]);
+ //cv::swap(depth, fs.channel2[ix]);
+ cv::swap(rgb, fs.frames[ix].setChannel<cv::Mat>(ftl::rgbd::kChanColour));
+ cv::swap(depth, fs.frames[ix].setChannel<cv::Mat>(chan));
++fs.count;
fs.mask |= (1 << ix);
@@ -271,8 +277,9 @@ void Group::_addFrameset(int64_t timestamp) {
framesets_[head_].count = 0;
framesets_[head_].mask = 0;
framesets_[head_].stale = false;
- framesets_[head_].channel1.resize(sources_.size());
- framesets_[head_].channel2.resize(sources_.size());
+ //framesets_[head_].channel1.resize(sources_.size());
+ //framesets_[head_].channel2.resize(sources_.size());
+ framesets_[head_].frames.resize(sources_.size());
if (framesets_[head_].sources.size() != sources_.size()) {
framesets_[head_].sources.clear();
@@ -301,8 +308,9 @@ void Group::_addFrameset(int64_t timestamp) {
framesets_[head_].count = 0;
framesets_[head_].mask = 0;
framesets_[head_].stale = false;
- framesets_[head_].channel1.resize(sources_.size());
- framesets_[head_].channel2.resize(sources_.size());
+ //framesets_[head_].channel1.resize(sources_.size());
+ //framesets_[head_].channel2.resize(sources_.size());
+ framesets_[head_].frames.resize(sources_.size());
if (framesets_[head_].sources.size() != sources_.size()) {
framesets_[head_].sources.clear();
diff --git a/components/rgbd-sources/src/stereovideo.cpp b/components/rgbd-sources/src/stereovideo.cpp
index d718d1f7ed1f29bac101f639317671acf420a882..b5b7e3bd6a7bea857aee8e3cfddc16a7d14e2560 100644
--- a/components/rgbd-sources/src/stereovideo.cpp
+++ b/components/rgbd-sources/src/stereovideo.cpp
@@ -192,9 +192,15 @@ bool StereoVideoSource::retrieve() {
if (frames_[1].hasChannel(kChanLeftGray))
{
+<<<<<<< HEAD
auto &left_gray_prev = frames_[1].getChannel<cv::cuda::GpuMat>(kChanLeftGray, stream2_);
auto &optflow = frame.setChannel<cv::cuda::GpuMat>(kChanFlow);
nvof_->calc(left_gray, left_gray_prev, optflow, stream2_);
+=======
+ auto &left_gray_prev = frame.getChannel<cv::cuda::GpuMat>(kChanLeftGray, stream2_);
+ auto &optflow = frame.setChannel<cv::cuda::GpuMat>(kChanFlow);
+ nvof_->calc(left_gray, left_gray_prev, optflow_, stream2_);
+>>>>>>> master
// nvof_->upSampler() isn't implemented with CUDA
// cv::cuda::resize() does not work wiht 2-channel input
// cv::cuda::resize(optflow_, optflow, left.size(), 0.0, 0.0, cv::INTER_NEAREST, stream2_);
diff --git a/components/rgbd-sources/src/streamer.cpp b/components/rgbd-sources/src/streamer.cpp
index 4a9ccf529fbfcfac285312e1651d5a4b292fb849..c2676ac6a07b2c47914da9affffde5823d0fb9ca 100644
--- a/components/rgbd-sources/src/streamer.cpp
+++ b/components/rgbd-sources/src/streamer.cpp
@@ -365,7 +365,8 @@ void Streamer::_process(ftl::rgbd::FrameSet &fs) {
if (!src) continue;
if (!fs.sources[j]->isReady()) continue;
if (src->clientCount == 0) continue;
- if (fs.channel1[j].empty() || (fs.sources[j]->getChannel() != ftl::rgbd::kChanNone && fs.channel2[j].empty())) continue;
+ //if (fs.channel1[j].empty() || (fs.sources[j]->getChannel() != ftl::rgbd::kChanNone && fs.channel2[j].empty())) continue;
+ if (!fs.frames[j].hasChannel(ftl::rgbd::kChanColour) || !fs.frames[j].hasChannel(fs.sources[j]->getChannel())) continue;
bool hasChan2 = fs.sources[j]->getChannel() != ftl::rgbd::kChanNone;
@@ -387,14 +388,14 @@ void Streamer::_process(ftl::rgbd::FrameSet &fs) {
// Receiver only waits for channel 1 by default
// TODO: Each encode could be done in own thread
if (hasChan2) {
- enc2->encode(fs.channel2[j], src->hq_bitrate, [this,src,hasChan2](const ftl::codecs::Packet &blk){
+ enc2->encode(fs.frames[j].getChannel<cv::Mat>(fs.sources[j]->getChannel()), src->hq_bitrate, [this,src,hasChan2](const ftl::codecs::Packet &blk){
_transmitPacket(src, blk, 1, hasChan2, true);
});
} else {
if (enc2) enc2->reset();
}
- enc1->encode(fs.channel1[j], src->hq_bitrate, [this,src,hasChan2](const ftl::codecs::Packet &blk){
+ enc1->encode(fs.frames[j].getChannel<cv::Mat>(ftl::rgbd::kChanColour), src->hq_bitrate, [this,src,hasChan2](const ftl::codecs::Packet &blk){
_transmitPacket(src, blk, 0, hasChan2, true);
});
}
@@ -415,14 +416,14 @@ void Streamer::_process(ftl::rgbd::FrameSet &fs) {
// Important to send channel 2 first if needed...
// Receiver only waits for channel 1 by default
if (hasChan2) {
- enc2->encode(fs.channel2[j], src->lq_bitrate, [this,src,hasChan2](const ftl::codecs::Packet &blk){
+ enc2->encode(fs.frames[j].getChannel<cv::Mat>(fs.sources[j]->getChannel()), src->lq_bitrate, [this,src,hasChan2](const ftl::codecs::Packet &blk){
_transmitPacket(src, blk, 1, hasChan2, false);
});
} else {
if (enc2) enc2->reset();
}
- enc1->encode(fs.channel1[j], src->lq_bitrate, [this,src,hasChan2](const ftl::codecs::Packet &blk){
+ enc1->encode(fs.frames[j].getChannel<cv::Mat>(ftl::rgbd::kChanColour), src->lq_bitrate, [this,src,hasChan2](const ftl::codecs::Packet &blk){
_transmitPacket(src, blk, 0, hasChan2, false);
});
}
diff --git a/config/config_nick.jsonc b/config/config_nick.jsonc
index 2dac067a2115b634ec53fa73d58ef702f8d4f1ca..53673a170693bd7ffd6d4386977417897fa55b8a 100644
--- a/config/config_nick.jsonc
+++ b/config/config_nick.jsonc
@@ -128,7 +128,15 @@
"SDFUseGradients": false,
"showBlockBorders": false
},
- "baseline": 0.5,
+ "baseline": 0.05,
+ "focal": 700,
+ "width": 1280,
+ "height": 720,
+ "maxDepth": 15.0,
+ "minDepth": 0.05,
+ "splatting": true,
+ "texturing": true,
+ "upsampling": false,
"uri": "device:virtual"
}
},
@@ -150,7 +158,7 @@
"confidenceThreshold": 0.0,
"mls": true,
"voxels": false,
- "clipping": true,
+ "clipping": false,
"bbox_x_max": 0.6,
"bbox_x_min": -0.6,
"bbox_y_max": 3.0,