Skip to content
Snippets Groups Projects
Commit 59cd828f authored by Nicolas Pope's avatar Nicolas Pope
Browse files

Merge branch 'feature/174/useframe' into 'master'

Implements #174 to use new Frame class

Closes #174

See merge request nicolas.pope/ftl!104
parents ff2b59dc d8e8db29
No related branches found
No related tags found
1 merge request!104Implements #174 to use new Frame class
Pipeline #13530 passed
...@@ -90,15 +90,17 @@ void modeLeftRight(ftl::Configurable *root) { ...@@ -90,15 +90,17 @@ void modeLeftRight(ftl::Configurable *root) {
group.sync([&mutex, &new_frames, &rgb_new](ftl::rgbd::FrameSet &frames) { group.sync([&mutex, &new_frames, &rgb_new](ftl::rgbd::FrameSet &frames) {
mutex.lock(); mutex.lock();
bool good = true; bool good = true;
for (size_t i = 0; i < frames.channel1.size(); i ++) { for (size_t i = 0; i < frames.frames.size(); i ++) {
if (frames.channel1[i].empty()) good = false; auto &chan1 = frames.frames[i].getChannel<cv::Mat>(ftl::rgbd::kChanColour);
if (frames.channel1[i].empty()) good = false; auto &chan2 = frames.frames[i].getChannel<cv::Mat>(frames.sources[i]->getChannel());
if (frames.channel1[i].channels() != 3) good = false; // ASSERT if (chan1.empty()) good = false;
if (frames.channel2[i].channels() != 3) good = false; if (chan2.empty()) good = false;
if (chan1.channels() != 3) good = false; // ASSERT
if (chan2.channels() != 3) good = false;
if (!good) break; if (!good) break;
frames.channel1[i].copyTo(rgb_new[2 * i]); chan1.copyTo(rgb_new[2 * i]);
frames.channel2[i].copyTo(rgb_new[2 * i + 1]); chan2.copyTo(rgb_new[2 * i + 1]);
} }
new_frames = good; new_frames = good;
...@@ -180,14 +182,19 @@ void modeFrame(ftl::Configurable *root, int frames=1) { ...@@ -180,14 +182,19 @@ void modeFrame(ftl::Configurable *root, int frames=1) {
//LOG(INFO) << "Complete set: " << fs.timestamp; //LOG(INFO) << "Complete set: " << fs.timestamp;
if (!ftl::running) { return false; } if (!ftl::running) { return false; }
std::vector<cv::Mat> frames;
for (size_t i=0; i<fs.sources.size(); ++i) { for (size_t i=0; i<fs.sources.size(); ++i) {
if (fs.channel1[i].empty() || fs.channel2[i].empty()) return true; auto &chan1 = fs.frames[i].getChannel<cv::Mat>(ftl::rgbd::kChanColour);
auto &chan2 = fs.frames[i].getChannel<cv::Mat>(fs.sources[i]->getChannel());
if (chan1.empty() || chan2.empty()) return true;
frames.push_back(chan1);
} }
cv::Mat show; cv::Mat show;
stack(fs.channel1, show); stack(frames, show);
cv::resize(show, show, cv::Size(1280,720)); cv::resize(show, show, cv::Size(1280,720));
cv::namedWindow("Cameras", cv::WINDOW_KEEPRATIO | cv::WINDOW_NORMAL); cv::namedWindow("Cameras", cv::WINDOW_KEEPRATIO | cv::WINDOW_NORMAL);
...@@ -206,9 +213,12 @@ void modeFrame(ftl::Configurable *root, int frames=1) { ...@@ -206,9 +213,12 @@ void modeFrame(ftl::Configurable *root, int frames=1) {
auto writer = ftl::rgbd::SnapshotWriter(std::string(timestamp) + ".tar.gz"); auto writer = ftl::rgbd::SnapshotWriter(std::string(timestamp) + ".tar.gz");
for (size_t i=0; i<fs.sources.size(); ++i) { for (size_t i=0; i<fs.sources.size(); ++i) {
auto &chan1 = fs.frames[i].getChannel<cv::Mat>(ftl::rgbd::kChanColour);
auto &chan2 = fs.frames[i].getChannel<cv::Mat>(fs.sources[i]->getChannel());
writer.addSource(fs.sources[i]->getURI(), fs.sources[i]->parameters(), fs.sources[i]->getPose()); writer.addSource(fs.sources[i]->getURI(), fs.sources[i]->parameters(), fs.sources[i]->getPose());
LOG(INFO) << "SAVE: " << fs.channel1[i].cols << ", " << fs.channel2[i].type(); //LOG(INFO) << "SAVE: " << fs.channel1[i].cols << ", " << fs.channel2[i].type();
writer.addRGBD(i, fs.channel1[i], fs.channel2[i]); writer.addRGBD(i, chan1, chan2);
} }
} }
#endif // HAVE_LIBARCHIVE #endif // HAVE_LIBARCHIVE
......
...@@ -233,6 +233,8 @@ int SceneRep::upload(ftl::rgbd::FrameSet &fs) { ...@@ -233,6 +233,8 @@ int SceneRep::upload(ftl::rgbd::FrameSet &fs) {
for (size_t i=0; i<cameras_.size(); ++i) { for (size_t i=0; i<cameras_.size(); ++i) {
auto &cam = cameras_[i]; auto &cam = cameras_[i];
auto &chan1 = fs.frames[i].getChannel<cv::Mat>(ftl::rgbd::kChanColour);
auto &chan2 = fs.frames[i].getChannel<cv::Mat>(fs.sources[i]->getChannel());
// Get the RGB-Depth frame from input // Get the RGB-Depth frame from input
Source *input = cam.source; Source *input = cam.source;
...@@ -247,12 +249,12 @@ int SceneRep::upload(ftl::rgbd::FrameSet &fs) { ...@@ -247,12 +249,12 @@ int SceneRep::upload(ftl::rgbd::FrameSet &fs) {
// Must be in RGBA for GPU // Must be in RGBA for GPU
Mat rgbt, rgba; Mat rgbt, rgba;
cv::cvtColor(fs.channel1[i],rgbt, cv::COLOR_BGR2Lab); cv::cvtColor(chan1,rgbt, cv::COLOR_BGR2Lab);
cv::cvtColor(rgbt,rgba, cv::COLOR_BGR2BGRA); cv::cvtColor(rgbt,rgba, cv::COLOR_BGR2BGRA);
// Send to GPU and merge view into scene // Send to GPU and merge view into scene
//cam.gpu.updateParams(cam.params); //cam.gpu.updateParams(cam.params);
cam.gpu.updateData(fs.channel2[i], rgba, cam.stream); cam.gpu.updateData(chan2, rgba, cam.stream);
//setLastRigidTransform(input->getPose().cast<float>()); //setLastRigidTransform(input->getPose().cast<float>());
......
...@@ -12,6 +12,7 @@ namespace rgbd { ...@@ -12,6 +12,7 @@ namespace rgbd {
typedef unsigned int channel_t; typedef unsigned int channel_t;
static const channel_t kChanNone = 0; static const channel_t kChanNone = 0;
static const channel_t kChanColour = 0x0001;
static const channel_t kChanLeft = 0x0001; // CV_8UC3 static const channel_t kChanLeft = 0x0001; // CV_8UC3
static const channel_t kChanDepth = 0x0002; // CV_32FC1 static const channel_t kChanDepth = 0x0002; // CV_32FC1
static const channel_t kChanRight = 0x0004; // CV_8UC3 static const channel_t kChanRight = 0x0004; // CV_8UC3
...@@ -58,7 +59,7 @@ public: ...@@ -58,7 +59,7 @@ public:
*/ */
bool hasChannel(const ftl::rgbd::channel_t& channel) bool hasChannel(const ftl::rgbd::channel_t& channel)
{ {
return available_[_channelIdx(channel)]; return (channel == ftl::rgbd::kChanNone) ? true : available_[_channelIdx(channel)];
} }
/* @brief Method to get reference to the channel content /* @brief Method to get reference to the channel content
......
#ifndef _FTL_RGBD_FRAMESET_HPP_
#define _FTL_RGBD_FRAMESET_HPP_
#include <ftl/rgbd/frame.hpp>
#include <opencv2/opencv.hpp>
#include <vector>
namespace ftl {
namespace rgbd {
class Source;
/**
* Represents a set of synchronised frames, each with two channels. This is
* used to collect all frames from multiple computers that have the same
* timestamp.
*/
struct FrameSet {
int64_t timestamp; // Millisecond timestamp of all frames
std::vector<Source*> sources; // All source objects involved.
std::vector<ftl::rgbd::Frame> frames;
std::atomic<int> count; // Number of valid frames
std::atomic<unsigned int> mask; // Mask of all sources that contributed
bool stale; // True if buffers have been invalidated
SHARED_MUTEX mtx;
};
}
}
#endif // _FTL_RGBD_FRAMESET_HPP_
...@@ -3,6 +3,8 @@ ...@@ -3,6 +3,8 @@
#include <ftl/threads.hpp> #include <ftl/threads.hpp>
#include <ftl/timer.hpp> #include <ftl/timer.hpp>
#include <ftl/rgbd/frame.hpp>
#include <ftl/rgbd/frameset.hpp>
#include <opencv2/opencv.hpp> #include <opencv2/opencv.hpp>
#include <vector> #include <vector>
...@@ -12,22 +14,6 @@ namespace rgbd { ...@@ -12,22 +14,6 @@ namespace rgbd {
class Source; class Source;
/**
* Represents a set of synchronised frames, each with two channels. This is
* used to collect all frames from multiple computers that have the same
* timestamp.
*/
struct FrameSet {
int64_t timestamp; // Millisecond timestamp of all frames
std::vector<Source*> sources; // All source objects involved.
std::vector<cv::Mat> channel1; // RGB
std::vector<cv::Mat> channel2; // Depth (usually)
std::atomic<int> count; // Number of valid frames
std::atomic<unsigned int> mask; // Mask of all sources that contributed
bool stale; // True if buffers have been invalidated
SHARED_MUTEX mtx;
};
// Allows a latency of 20 frames maximum // Allows a latency of 20 frames maximum
static const size_t kFrameBufferSize = 20; static const size_t kFrameBufferSize = 20;
......
...@@ -52,6 +52,8 @@ void Group::addSource(ftl::rgbd::Source *src) { ...@@ -52,6 +52,8 @@ void Group::addSource(ftl::rgbd::Source *src) {
src->setCallback([this,ix,src](int64_t timestamp, cv::Mat &rgb, cv::Mat &depth) { src->setCallback([this,ix,src](int64_t timestamp, cv::Mat &rgb, cv::Mat &depth) {
if (timestamp == 0) return; if (timestamp == 0) return;
auto chan = src->getChannel();
//LOG(INFO) << "SRC CB: " << timestamp << " (" << framesets_[head_].timestamp << ")"; //LOG(INFO) << "SRC CB: " << timestamp << " (" << framesets_[head_].timestamp << ")";
UNIQUE_LOCK(mutex_, lk); UNIQUE_LOCK(mutex_, lk);
...@@ -73,11 +75,15 @@ void Group::addSource(ftl::rgbd::Source *src) { ...@@ -73,11 +75,15 @@ void Group::addSource(ftl::rgbd::Source *src) {
//LOG(INFO) << "Adding frame: " << ix << " for " << timestamp; //LOG(INFO) << "Adding frame: " << ix << " for " << timestamp;
// Ensure channels match source mat format // Ensure channels match source mat format
fs.channel1[ix].create(rgb.size(), rgb.type()); //fs.channel1[ix].create(rgb.size(), rgb.type());
fs.channel2[ix].create(depth.size(), depth.type()); //fs.channel2[ix].create(depth.size(), depth.type());
fs.frames[ix].setChannel<cv::Mat>(ftl::rgbd::kChanColour).create(rgb.size(), rgb.type());
fs.frames[ix].setChannel<cv::Mat>(chan).create(depth.size(), depth.type());
cv::swap(rgb, fs.channel1[ix]); //cv::swap(rgb, fs.channel1[ix]);
cv::swap(depth, fs.channel2[ix]); //cv::swap(depth, fs.channel2[ix]);
cv::swap(rgb, fs.frames[ix].setChannel<cv::Mat>(ftl::rgbd::kChanColour));
cv::swap(depth, fs.frames[ix].setChannel<cv::Mat>(chan));
++fs.count; ++fs.count;
fs.mask |= (1 << ix); fs.mask |= (1 << ix);
...@@ -271,8 +277,9 @@ void Group::_addFrameset(int64_t timestamp) { ...@@ -271,8 +277,9 @@ void Group::_addFrameset(int64_t timestamp) {
framesets_[head_].count = 0; framesets_[head_].count = 0;
framesets_[head_].mask = 0; framesets_[head_].mask = 0;
framesets_[head_].stale = false; framesets_[head_].stale = false;
framesets_[head_].channel1.resize(sources_.size()); //framesets_[head_].channel1.resize(sources_.size());
framesets_[head_].channel2.resize(sources_.size()); //framesets_[head_].channel2.resize(sources_.size());
framesets_[head_].frames.resize(sources_.size());
if (framesets_[head_].sources.size() != sources_.size()) { if (framesets_[head_].sources.size() != sources_.size()) {
framesets_[head_].sources.clear(); framesets_[head_].sources.clear();
...@@ -301,8 +308,9 @@ void Group::_addFrameset(int64_t timestamp) { ...@@ -301,8 +308,9 @@ void Group::_addFrameset(int64_t timestamp) {
framesets_[head_].count = 0; framesets_[head_].count = 0;
framesets_[head_].mask = 0; framesets_[head_].mask = 0;
framesets_[head_].stale = false; framesets_[head_].stale = false;
framesets_[head_].channel1.resize(sources_.size()); //framesets_[head_].channel1.resize(sources_.size());
framesets_[head_].channel2.resize(sources_.size()); //framesets_[head_].channel2.resize(sources_.size());
framesets_[head_].frames.resize(sources_.size());
if (framesets_[head_].sources.size() != sources_.size()) { if (framesets_[head_].sources.size() != sources_.size()) {
framesets_[head_].sources.clear(); framesets_[head_].sources.clear();
......
...@@ -365,7 +365,8 @@ void Streamer::_process(ftl::rgbd::FrameSet &fs) { ...@@ -365,7 +365,8 @@ void Streamer::_process(ftl::rgbd::FrameSet &fs) {
if (!src) continue; if (!src) continue;
if (!fs.sources[j]->isReady()) continue; if (!fs.sources[j]->isReady()) continue;
if (src->clientCount == 0) continue; if (src->clientCount == 0) continue;
if (fs.channel1[j].empty() || (fs.sources[j]->getChannel() != ftl::rgbd::kChanNone && fs.channel2[j].empty())) continue; //if (fs.channel1[j].empty() || (fs.sources[j]->getChannel() != ftl::rgbd::kChanNone && fs.channel2[j].empty())) continue;
if (!fs.frames[j].hasChannel(ftl::rgbd::kChanColour) || !fs.frames[j].hasChannel(fs.sources[j]->getChannel())) continue;
bool hasChan2 = fs.sources[j]->getChannel() != ftl::rgbd::kChanNone; bool hasChan2 = fs.sources[j]->getChannel() != ftl::rgbd::kChanNone;
...@@ -387,14 +388,14 @@ void Streamer::_process(ftl::rgbd::FrameSet &fs) { ...@@ -387,14 +388,14 @@ void Streamer::_process(ftl::rgbd::FrameSet &fs) {
// Receiver only waits for channel 1 by default // Receiver only waits for channel 1 by default
// TODO: Each encode could be done in own thread // TODO: Each encode could be done in own thread
if (hasChan2) { if (hasChan2) {
enc2->encode(fs.channel2[j], src->hq_bitrate, [this,src,hasChan2](const ftl::codecs::Packet &blk){ enc2->encode(fs.frames[j].getChannel<cv::Mat>(fs.sources[j]->getChannel()), src->hq_bitrate, [this,src,hasChan2](const ftl::codecs::Packet &blk){
_transmitPacket(src, blk, 1, hasChan2, true); _transmitPacket(src, blk, 1, hasChan2, true);
}); });
} else { } else {
if (enc2) enc2->reset(); if (enc2) enc2->reset();
} }
enc1->encode(fs.channel1[j], src->hq_bitrate, [this,src,hasChan2](const ftl::codecs::Packet &blk){ enc1->encode(fs.frames[j].getChannel<cv::Mat>(ftl::rgbd::kChanColour), src->hq_bitrate, [this,src,hasChan2](const ftl::codecs::Packet &blk){
_transmitPacket(src, blk, 0, hasChan2, true); _transmitPacket(src, blk, 0, hasChan2, true);
}); });
} }
...@@ -415,14 +416,14 @@ void Streamer::_process(ftl::rgbd::FrameSet &fs) { ...@@ -415,14 +416,14 @@ void Streamer::_process(ftl::rgbd::FrameSet &fs) {
// Important to send channel 2 first if needed... // Important to send channel 2 first if needed...
// Receiver only waits for channel 1 by default // Receiver only waits for channel 1 by default
if (hasChan2) { if (hasChan2) {
enc2->encode(fs.channel2[j], src->lq_bitrate, [this,src,hasChan2](const ftl::codecs::Packet &blk){ enc2->encode(fs.frames[j].getChannel<cv::Mat>(fs.sources[j]->getChannel()), src->lq_bitrate, [this,src,hasChan2](const ftl::codecs::Packet &blk){
_transmitPacket(src, blk, 1, hasChan2, false); _transmitPacket(src, blk, 1, hasChan2, false);
}); });
} else { } else {
if (enc2) enc2->reset(); if (enc2) enc2->reset();
} }
enc1->encode(fs.channel1[j], src->lq_bitrate, [this,src,hasChan2](const ftl::codecs::Packet &blk){ enc1->encode(fs.frames[j].getChannel<cv::Mat>(ftl::rgbd::kChanColour), src->lq_bitrate, [this,src,hasChan2](const ftl::codecs::Packet &blk){
_transmitPacket(src, blk, 0, hasChan2, false); _transmitPacket(src, blk, 0, hasChan2, false);
}); });
} }
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment