diff --git a/applications/groupview/src/main.cpp b/applications/groupview/src/main.cpp
index 3a821a01323fbe51dbc8b6fdef98597f0bd20853..d1dab2607ce15ea1ec5defe63aa4ff2b48ee95b6 100644
--- a/applications/groupview/src/main.cpp
+++ b/applications/groupview/src/main.cpp
@@ -90,15 +90,17 @@ void modeLeftRight(ftl::Configurable *root) {
group.sync([&mutex, &new_frames, &rgb_new](ftl::rgbd::FrameSet &frames) {
mutex.lock();
bool good = true;
- for (size_t i = 0; i < frames.channel1.size(); i ++) {
- if (frames.channel1[i].empty()) good = false;
- if (frames.channel1[i].empty()) good = false;
- if (frames.channel1[i].channels() != 3) good = false; // ASSERT
- if (frames.channel2[i].channels() != 3) good = false;
+ for (size_t i = 0; i < frames.frames.size(); i ++) {
+ auto &chan1 = frames.frames[i].getChannel<cv::Mat>(ftl::rgbd::kChanColour);
+ auto &chan2 = frames.frames[i].getChannel<cv::Mat>(frames.sources[i]->getChannel());
+ if (chan1.empty()) good = false;
+ if (chan2.empty()) good = false;
+ if (chan1.channels() != 3) good = false; // ASSERT
+ if (chan2.channels() != 3) good = false;
if (!good) break;
- frames.channel1[i].copyTo(rgb_new[2 * i]);
- frames.channel2[i].copyTo(rgb_new[2 * i + 1]);
+ chan1.copyTo(rgb_new[2 * i]);
+ chan2.copyTo(rgb_new[2 * i + 1]);
}
new_frames = good;
@@ -180,14 +182,19 @@ void modeFrame(ftl::Configurable *root, int frames=1) {
//LOG(INFO) << "Complete set: " << fs.timestamp;
if (!ftl::running) { return false; }
+ std::vector<cv::Mat> frames;
for (size_t i=0; i<fs.sources.size(); ++i) {
- if (fs.channel1[i].empty() || fs.channel2[i].empty()) return true;
+ auto &chan1 = fs.frames[i].getChannel<cv::Mat>(ftl::rgbd::kChanColour);
+ auto &chan2 = fs.frames[i].getChannel<cv::Mat>(fs.sources[i]->getChannel());
+ if (chan1.empty() || chan2.empty()) return true;
+
+ frames.push_back(chan1);
}
cv::Mat show;
- stack(fs.channel1, show);
+ stack(frames, show);
cv::resize(show, show, cv::Size(1280,720));
cv::namedWindow("Cameras", cv::WINDOW_KEEPRATIO | cv::WINDOW_NORMAL);
@@ -206,9 +213,12 @@ void modeFrame(ftl::Configurable *root, int frames=1) {
auto writer = ftl::rgbd::SnapshotWriter(std::string(timestamp) + ".tar.gz");
for (size_t i=0; i<fs.sources.size(); ++i) {
+ auto &chan1 = fs.frames[i].getChannel<cv::Mat>(ftl::rgbd::kChanColour);
+ auto &chan2 = fs.frames[i].getChannel<cv::Mat>(fs.sources[i]->getChannel());
+
writer.addSource(fs.sources[i]->getURI(), fs.sources[i]->parameters(), fs.sources[i]->getPose());
- LOG(INFO) << "SAVE: " << fs.channel1[i].cols << ", " << fs.channel2[i].type();
- writer.addRGBD(i, fs.channel1[i], fs.channel2[i]);
+ //LOG(INFO) << "SAVE: " << fs.channel1[i].cols << ", " << fs.channel2[i].type();
+ writer.addRGBD(i, chan1, chan2);
}
}
#endif // HAVE_LIBARCHIVE
diff --git a/applications/reconstruct/src/voxel_scene.cpp b/applications/reconstruct/src/voxel_scene.cpp
index cba6e61e9e9134845e59d71dd6e69d4ad7ae8beb..ab8b2c1dd25b530e1ffb4fcf4a29c42bd86dfbb6 100644
--- a/applications/reconstruct/src/voxel_scene.cpp
+++ b/applications/reconstruct/src/voxel_scene.cpp
@@ -233,6 +233,8 @@ int SceneRep::upload(ftl::rgbd::FrameSet &fs) {
for (size_t i=0; i<cameras_.size(); ++i) {
auto &cam = cameras_[i];
+ auto &chan1 = fs.frames[i].getChannel<cv::Mat>(ftl::rgbd::kChanColour);
+ auto &chan2 = fs.frames[i].getChannel<cv::Mat>(fs.sources[i]->getChannel());
// Get the RGB-Depth frame from input
Source *input = cam.source;
@@ -247,12 +249,12 @@ int SceneRep::upload(ftl::rgbd::FrameSet &fs) {
// Must be in RGBA for GPU
Mat rgbt, rgba;
- cv::cvtColor(fs.channel1[i],rgbt, cv::COLOR_BGR2Lab);
+ cv::cvtColor(chan1,rgbt, cv::COLOR_BGR2Lab);
cv::cvtColor(rgbt,rgba, cv::COLOR_BGR2BGRA);
// Send to GPU and merge view into scene
//cam.gpu.updateParams(cam.params);
- cam.gpu.updateData(fs.channel2[i], rgba, cam.stream);
+ cam.gpu.updateData(chan2, rgba, cam.stream);
//setLastRigidTransform(input->getPose().cast<float>());
diff --git a/components/rgbd-sources/include/ftl/rgbd/frame.hpp b/components/rgbd-sources/include/ftl/rgbd/frame.hpp
index 227913e5de056f15380390201da96b66ec23e27f..a1822fd163174bf9a55a697fa7736c0ac3d9a5bb 100644
--- a/components/rgbd-sources/include/ftl/rgbd/frame.hpp
+++ b/components/rgbd-sources/include/ftl/rgbd/frame.hpp
@@ -12,6 +12,7 @@ namespace rgbd {
typedef unsigned int channel_t;
static const channel_t kChanNone = 0;
+static const channel_t kChanColour = 0x0001;
static const channel_t kChanLeft = 0x0001; // CV_8UC3
static const channel_t kChanDepth = 0x0002; // CV_32FC1
static const channel_t kChanRight = 0x0004; // CV_8UC3
@@ -58,7 +59,7 @@ public:
*/
bool hasChannel(const ftl::rgbd::channel_t& channel)
{
- return available_[_channelIdx(channel)];
+ return (channel == ftl::rgbd::kChanNone) ? true : available_[_channelIdx(channel)];
}
/* @brief Method to get reference to the channel content
diff --git a/components/rgbd-sources/include/ftl/rgbd/frameset.hpp b/components/rgbd-sources/include/ftl/rgbd/frameset.hpp
new file mode 100644
index 0000000000000000000000000000000000000000..93839efda9c6dc8cfe53e6775293949d5c1747e8
--- /dev/null
+++ b/components/rgbd-sources/include/ftl/rgbd/frameset.hpp
@@ -0,0 +1,32 @@
+#ifndef _FTL_RGBD_FRAMESET_HPP_
+#define _FTL_RGBD_FRAMESET_HPP_
+
+#include <ftl/rgbd/frame.hpp>
+
+#include <opencv2/opencv.hpp>
+#include <vector>
+
+namespace ftl {
+namespace rgbd {
+
+class Source;
+
+/**
+ * Represents a set of synchronised frames, each with two channels. This is
+ * used to collect all frames from multiple computers that have the same
+ * timestamp.
+ */
+struct FrameSet {
+ int64_t timestamp; // Millisecond timestamp of all frames
+ std::vector<Source*> sources; // All source objects involved.
+ std::vector<ftl::rgbd::Frame> frames;
+ std::atomic<int> count; // Number of valid frames
+ std::atomic<unsigned int> mask; // Mask of all sources that contributed
+ bool stale; // True if buffers have been invalidated
+ SHARED_MUTEX mtx;
+};
+
+}
+}
+
+#endif // _FTL_RGBD_FRAMESET_HPP_
diff --git a/components/rgbd-sources/include/ftl/rgbd/group.hpp b/components/rgbd-sources/include/ftl/rgbd/group.hpp
index 000eea0ae60303d816e09298613ddc30f0a8146a..e4ffb5984e625b1b9859a3f193a7f4b44c956684 100644
--- a/components/rgbd-sources/include/ftl/rgbd/group.hpp
+++ b/components/rgbd-sources/include/ftl/rgbd/group.hpp
@@ -3,6 +3,8 @@
#include <ftl/threads.hpp>
#include <ftl/timer.hpp>
+#include <ftl/rgbd/frame.hpp>
+#include <ftl/rgbd/frameset.hpp>
#include <opencv2/opencv.hpp>
#include <vector>
@@ -12,22 +14,6 @@ namespace rgbd {
class Source;
-/**
- * Represents a set of synchronised frames, each with two channels. This is
- * used to collect all frames from multiple computers that have the same
- * timestamp.
- */
-struct FrameSet {
- int64_t timestamp; // Millisecond timestamp of all frames
- std::vector<Source*> sources; // All source objects involved.
- std::vector<cv::Mat> channel1; // RGB
- std::vector<cv::Mat> channel2; // Depth (usually)
- std::atomic<int> count; // Number of valid frames
- std::atomic<unsigned int> mask; // Mask of all sources that contributed
- bool stale; // True if buffers have been invalidated
- SHARED_MUTEX mtx;
-};
-
// Allows a latency of 20 frames maximum
static const size_t kFrameBufferSize = 20;
diff --git a/components/rgbd-sources/src/group.cpp b/components/rgbd-sources/src/group.cpp
index e61289220ad92fff8e278905a075f01d93b6ad9a..57d770ba12db4f4b15aebf48cedd33aa2b880fa5 100644
--- a/components/rgbd-sources/src/group.cpp
+++ b/components/rgbd-sources/src/group.cpp
@@ -52,6 +52,8 @@ void Group::addSource(ftl::rgbd::Source *src) {
src->setCallback([this,ix,src](int64_t timestamp, cv::Mat &rgb, cv::Mat &depth) {
if (timestamp == 0) return;
+ auto chan = src->getChannel();
+
//LOG(INFO) << "SRC CB: " << timestamp << " (" << framesets_[head_].timestamp << ")";
UNIQUE_LOCK(mutex_, lk);
@@ -73,11 +75,15 @@ void Group::addSource(ftl::rgbd::Source *src) {
//LOG(INFO) << "Adding frame: " << ix << " for " << timestamp;
// Ensure channels match source mat format
- fs.channel1[ix].create(rgb.size(), rgb.type());
- fs.channel2[ix].create(depth.size(), depth.type());
+ //fs.channel1[ix].create(rgb.size(), rgb.type());
+ //fs.channel2[ix].create(depth.size(), depth.type());
+ fs.frames[ix].setChannel<cv::Mat>(ftl::rgbd::kChanColour).create(rgb.size(), rgb.type());
+ fs.frames[ix].setChannel<cv::Mat>(chan).create(depth.size(), depth.type());
- cv::swap(rgb, fs.channel1[ix]);
- cv::swap(depth, fs.channel2[ix]);
+ //cv::swap(rgb, fs.channel1[ix]);
+ //cv::swap(depth, fs.channel2[ix]);
+ cv::swap(rgb, fs.frames[ix].setChannel<cv::Mat>(ftl::rgbd::kChanColour));
+ cv::swap(depth, fs.frames[ix].setChannel<cv::Mat>(chan));
++fs.count;
fs.mask |= (1 << ix);
@@ -271,8 +277,9 @@ void Group::_addFrameset(int64_t timestamp) {
framesets_[head_].count = 0;
framesets_[head_].mask = 0;
framesets_[head_].stale = false;
- framesets_[head_].channel1.resize(sources_.size());
- framesets_[head_].channel2.resize(sources_.size());
+ //framesets_[head_].channel1.resize(sources_.size());
+ //framesets_[head_].channel2.resize(sources_.size());
+ framesets_[head_].frames.resize(sources_.size());
if (framesets_[head_].sources.size() != sources_.size()) {
framesets_[head_].sources.clear();
@@ -301,8 +308,9 @@ void Group::_addFrameset(int64_t timestamp) {
framesets_[head_].count = 0;
framesets_[head_].mask = 0;
framesets_[head_].stale = false;
- framesets_[head_].channel1.resize(sources_.size());
- framesets_[head_].channel2.resize(sources_.size());
+ //framesets_[head_].channel1.resize(sources_.size());
+ //framesets_[head_].channel2.resize(sources_.size());
+ framesets_[head_].frames.resize(sources_.size());
if (framesets_[head_].sources.size() != sources_.size()) {
framesets_[head_].sources.clear();
diff --git a/components/rgbd-sources/src/streamer.cpp b/components/rgbd-sources/src/streamer.cpp
index 4a9ccf529fbfcfac285312e1651d5a4b292fb849..c2676ac6a07b2c47914da9affffde5823d0fb9ca 100644
--- a/components/rgbd-sources/src/streamer.cpp
+++ b/components/rgbd-sources/src/streamer.cpp
@@ -365,7 +365,8 @@ void Streamer::_process(ftl::rgbd::FrameSet &fs) {
if (!src) continue;
if (!fs.sources[j]->isReady()) continue;
if (src->clientCount == 0) continue;
- if (fs.channel1[j].empty() || (fs.sources[j]->getChannel() != ftl::rgbd::kChanNone && fs.channel2[j].empty())) continue;
+ //if (fs.channel1[j].empty() || (fs.sources[j]->getChannel() != ftl::rgbd::kChanNone && fs.channel2[j].empty())) continue;
+ if (!fs.frames[j].hasChannel(ftl::rgbd::kChanColour) || !fs.frames[j].hasChannel(fs.sources[j]->getChannel())) continue;
bool hasChan2 = fs.sources[j]->getChannel() != ftl::rgbd::kChanNone;
@@ -387,14 +388,14 @@ void Streamer::_process(ftl::rgbd::FrameSet &fs) {
// Receiver only waits for channel 1 by default
// TODO: Each encode could be done in own thread
if (hasChan2) {
- enc2->encode(fs.channel2[j], src->hq_bitrate, [this,src,hasChan2](const ftl::codecs::Packet &blk){
+ enc2->encode(fs.frames[j].getChannel<cv::Mat>(fs.sources[j]->getChannel()), src->hq_bitrate, [this,src,hasChan2](const ftl::codecs::Packet &blk){
_transmitPacket(src, blk, 1, hasChan2, true);
});
} else {
if (enc2) enc2->reset();
}
- enc1->encode(fs.channel1[j], src->hq_bitrate, [this,src,hasChan2](const ftl::codecs::Packet &blk){
+ enc1->encode(fs.frames[j].getChannel<cv::Mat>(ftl::rgbd::kChanColour), src->hq_bitrate, [this,src,hasChan2](const ftl::codecs::Packet &blk){
_transmitPacket(src, blk, 0, hasChan2, true);
});
}
@@ -415,14 +416,14 @@ void Streamer::_process(ftl::rgbd::FrameSet &fs) {
// Important to send channel 2 first if needed...
// Receiver only waits for channel 1 by default
if (hasChan2) {
- enc2->encode(fs.channel2[j], src->lq_bitrate, [this,src,hasChan2](const ftl::codecs::Packet &blk){
+ enc2->encode(fs.frames[j].getChannel<cv::Mat>(fs.sources[j]->getChannel()), src->lq_bitrate, [this,src,hasChan2](const ftl::codecs::Packet &blk){
_transmitPacket(src, blk, 1, hasChan2, false);
});
} else {
if (enc2) enc2->reset();
}
- enc1->encode(fs.channel1[j], src->lq_bitrate, [this,src,hasChan2](const ftl::codecs::Packet &blk){
+ enc1->encode(fs.frames[j].getChannel<cv::Mat>(ftl::rgbd::kChanColour), src->lq_bitrate, [this,src,hasChan2](const ftl::codecs::Packet &blk){
_transmitPacket(src, blk, 0, hasChan2, false);
});
}