From 0d0927f650b3c8d51b2e0964e950e05225211b6a Mon Sep 17 00:00:00 2001
From: Sebastian Hahta <joseha@utu.fi>
Date: Wed, 14 Aug 2019 13:36:34 +0300
Subject: [PATCH] snapshot video capture
---
applications/gui/src/media_panel.cpp | 20 +-
.../include/ftl/rgbd/snapshot.hpp | 59 +++---
components/rgbd-sources/src/snapshot.cpp | 174 ++++++++----------
.../rgbd-sources/src/snapshot_source.cpp | 27 +--
.../rgbd-sources/src/snapshot_source.hpp | 7 +-
components/rgbd-sources/src/source.cpp | 3 +-
components/rgbd-sources/test/source_unit.cpp | 5 +-
7 files changed, 136 insertions(+), 159 deletions(-)
diff --git a/applications/gui/src/media_panel.cpp b/applications/gui/src/media_panel.cpp
index e2a464509..2ee06a69d 100644
--- a/applications/gui/src/media_panel.cpp
+++ b/applications/gui/src/media_panel.cpp
@@ -77,10 +77,9 @@ MediaPanel::MediaPanel(ftl::gui::Screen *screen) : nanogui::Window(screen, ""),
//button = new Button(this, "", ENTYPO_ICON_CONTROLLER_RECORD);
#ifdef HAVE_LIBARCHIVE
-/*
-auto button_snapshot = new Button(this, "", ENTYPO_ICON_IMAGES);
-button_snapshot->setTooltip("Screen capture");
-button_snapshot->setCallback([this] {
+ auto button_snapshot = new Button(this, "", ENTYPO_ICON_IMAGES);
+ button_snapshot->setTooltip("Screen capture");
+ button_snapshot->setCallback([this] {
ftl::gui::Camera *cam = screen_->activeCamera();
if (!cam) return;
@@ -91,18 +90,15 @@ button_snapshot->setCallback([this] {
auto writer = ftl::rgbd::SnapshotWriter(std::string(timestamp) + ".tar.gz");
cv::Mat rgb, depth;
cam->source()->getFrames(rgb, depth);
- if (!writer.addSource("0", cam->source()->getPose(), cam->source()->parameters()) || !writer.addCameraRGBD(
- "0", // TODO
- rgb,
- depth
- )) {
- LOG(ERROR) << "Snapshot failed";
- }
+ writer.addSource( cam->source()->getURI(),
+ cam->source()->parameters(),
+ cam->source()->getPose());
+ writer.addRGBD(0, rgb, depth);
}
catch(std::runtime_error) {
LOG(ERROR) << "Snapshot failed (file error)";
}
- });*/
+ });
#endif
auto popbutton = new PopupButton(this, "", ENTYPO_ICON_LAYERS);
diff --git a/components/rgbd-sources/include/ftl/rgbd/snapshot.hpp b/components/rgbd-sources/include/ftl/rgbd/snapshot.hpp
index 2fc6c98a0..f9bb39756 100644
--- a/components/rgbd-sources/include/ftl/rgbd/snapshot.hpp
+++ b/components/rgbd-sources/include/ftl/rgbd/snapshot.hpp
@@ -69,60 +69,47 @@ private:
void run();
};
-// legacy mode
-struct SnapshotEntry {
- long t;
- cv::Mat rgb;
- cv::Mat depth;
- Eigen::Matrix4d pose;
- ftl::rgbd::Camera params;
- uint status;
- SnapshotEntry() : status(1+2+4+8) {};
+class Snapshot {
+public:
+ size_t getSourcesCount();
+ size_t getFramesCount();
+
+ std::string getSourceURI(size_t camera);
+ ftl::rgbd::Camera getParameters(size_t camera);
+ void getPose(size_t camera, cv::Mat &out);
+ void getPose(size_t camera, Eigen::Matrix4d &out);
+
+ void getLeftRGB(size_t camera, size_t frame, cv::Mat &data);
+ void getLeftDepth(size_t camera, size_t frame, cv::Mat &data);
+
+ size_t n_frames;
+ size_t n_cameras;
+
+ std::vector<std::string> sources;
+ std::vector<ftl::rgbd::Camera> parameters;
+ std::vector<cv::Mat> extrinsic;
+ std::vector<std::vector<cv::Mat>> rgb_left;
+ std::vector<std::vector<cv::Mat>> depth_left;
};
class SnapshotReader {
public:
explicit SnapshotReader(const std::string &filename);
~SnapshotReader();
-
- bool getCameraRGBD(const std::string &id, cv::Mat &rgb, cv::Mat &depth, Eigen::Matrix4d &pose, ftl::rgbd::Camera ¶ms);
- //std::vector<std::string> getIds();
+
+ Snapshot readArchive();
private:
bool readEntry(std::vector<uchar> &data);
- bool readArchive();
bool getDepth(const std::string &name, cv::Mat &data);
bool getRGB(const std::string &name, cv::Mat &data);
- // legacy mode
- SnapshotEntry& getEntry(const std::string &id);
- std::map<std::string, SnapshotEntry> data_;
-
- // new mode
std::map<std::string, std::vector<uchar>> files_;
struct archive *archive_;
struct archive_entry *entry_;
};
-class Snapshot {
-public:
- size_t nSources();
- Eigen::Matrix4f getPose(size_t i);
- int getTime(size_t frame);
- void getLeftRGB(cv::Mat &data);
- void getLeftDepth(cv::Mat &data);
-
-protected:
- std::vector<std::string> sources;
- std::vector<cv::Mat> intrinsic;
- std::vector<cv::Mat> extrinsic;
- std::vector<int> time;
- std::vector<cv::Mat> rgb_left;
- std::vector<cv::Mat> depth_left;
-};
-
-
};
};
diff --git a/components/rgbd-sources/src/snapshot.cpp b/components/rgbd-sources/src/snapshot.cpp
index f3f851836..372c6124d 100644
--- a/components/rgbd-sources/src/snapshot.cpp
+++ b/components/rgbd-sources/src/snapshot.cpp
@@ -249,9 +249,25 @@ void SnapshotStreamWriter::stop() {
if (wasrunning) thread_.join();
}
+size_t Snapshot::getSourcesCount() { return sources.size(); }
+size_t Snapshot::getFramesCount() { return depth_left[0].size(); }
+
+string Snapshot::getSourceURI(size_t camera) { return sources[camera]; }
+ftl::rgbd::Camera Snapshot::getParameters(size_t camera) { return parameters[camera]; }
+void Snapshot::getPose(size_t camera, cv::Mat &out) { out = extrinsic[camera]; }
+void Snapshot::getPose(size_t camera, Eigen::Matrix4d &out) {
+ Mat mat;
+ getPose(camera, mat);
+ cv::cv2eigen(mat, out);
+}
+void Snapshot::getLeftRGB(size_t camera, size_t frame, cv::Mat &data) { data = rgb_left[camera][frame]; }
+void Snapshot::getLeftDepth(size_t camera, size_t frame, cv::Mat &data) { data = depth_left[camera][frame]; }
SnapshotReader::SnapshotReader(const string &filename) {
archive_ = archive_read_new();
+ int retval = ARCHIVE_OK;
+ string msg;
+
if (!archive_) goto error2;
archive_read_support_format_all(archive_);
archive_read_support_filter_all(archive_);
@@ -259,12 +275,22 @@ SnapshotReader::SnapshotReader(const string &filename) {
if (archive_read_open_filename(archive_, filename.c_str(), 4096) != ARCHIVE_OK)
goto error1;
- readArchive();
+ while((retval = archive_read_next_header(archive_, &entry_)) == ARCHIVE_OK) {
+ string path = string(archive_entry_pathname(entry_));
+ vector<uchar> data;
+
+ if (readEntry(data)) { files_[path] = data; }
+ }
+
+ if (retval != ARCHIVE_EOF) { goto error1; }
+
return;
error1:
- LOG(ERROR) << archive_error_string(archive_);
+ msg = archive_error_string(archive_);
archive_read_free(archive_);
+ throw std::runtime_error(msg);
+
error2:
// throw exception; otherwise destructor might be called
throw std::runtime_error("SnapshotReader failed");
@@ -336,7 +362,7 @@ bool SnapshotReader::getRGB(const std::string &name, cv::Mat &data) {
return false;
}
- data = cv::imdecode(data_raw, 0);
+ data = cv::imdecode(data_raw, cv::IMREAD_COLOR);
if (data.empty()) {
LOG(ERROR) << "Error decoding file: " << name;
@@ -346,20 +372,8 @@ bool SnapshotReader::getRGB(const std::string &name, cv::Mat &data) {
return true;
}
-bool SnapshotReader::readArchive() {
- int retval = ARCHIVE_OK;
-
- while((retval = archive_read_next_header(archive_, &entry_)) == ARCHIVE_OK) {
- string path = string(archive_entry_pathname(entry_));
- vector<uchar> data;
-
- if (readEntry(data)) { files_[path] = data; }
- }
-
- if (retval != ARCHIVE_EOF) {
- LOG(ERROR) << archive_error_string(archive_);
- return false;
- }
+Snapshot SnapshotReader::readArchive() {
+ Snapshot result;
if (files_.find("index.yml") != files_.end()) {
LOG(INFO) << "Using new format snapshot archive";
@@ -370,13 +384,14 @@ bool SnapshotReader::readArchive() {
}
FileStorage fs(input, FileStorage::READ | FileStorage::MEMORY);
- vector<string> sources;
- vector<ftl::rgbd::Camera> params;
- vector<Mat> extrinsic;
+ vector<string> &sources = result.sources;
+ vector<ftl::rgbd::Camera> ¶ms = result.parameters;
+ vector<Mat> &extrinsic = result.extrinsic;
+
+ vector<vector<Mat>> &rgb_left = result.rgb_left;
+ vector<vector<Mat>> &depth_left = result.depth_left;
+
vector<string> channels;
- vector<int> times;
- vector<vector<Mat>> rgb_left;
- vector<vector<Mat>> depth_left;
fs["sources"] >> sources;
fs["extrinsic"] >> extrinsic;
@@ -439,83 +454,48 @@ bool SnapshotReader::readArchive() {
}
fs.release();
- return true;
}
+ else {
+ LOG(INFO) << "Using old format snapshot archive";
+
+ result.n_cameras = 1;
+ result.n_frames = 1;
+ Mat &rgb = result.rgb_left.emplace_back().emplace_back();
+ Mat &depth = result.depth_left.emplace_back().emplace_back();
+ Mat &pose = result.extrinsic.emplace_back();
+ Camera ¶ms = result.parameters.emplace_back();
+
+ for (auto const& [path, data] : files_) {
+ if (path.rfind("-") == string::npos) {
+ LOG(WARNING) << "unrecognized file " << path;
+ continue;
+ }
+ string id = path.substr(0, path.find("-"));
- LOG(INFO) << "Using legacy format snapshot archive";
- for (auto const& [path, data] : files_) {
- if (path.rfind("-") == string::npos) {
- LOG(WARNING) << "unrecognized file " << path;
- continue;
- }
- string id = path.substr(0, path.find("-"));
-
- SnapshotEntry &snapshot = getEntry(id);
-
- // TODO: verify that input is valid
- // TODO: check that earlier results are not overwritten (status)
-
- if (path.rfind("-RGB.") != string::npos) {
- snapshot.rgb = cv::imdecode(data, cv::IMREAD_COLOR);
- snapshot.status &= ~1;
- }
- else if (path.rfind("-D.") != string::npos) {
- snapshot.depth = cv::imdecode(data, cv::IMREAD_ANYDEPTH);
- snapshot.depth.convertTo(snapshot.depth, CV_32FC1, 1.0f / 1000.0f);
- snapshot.status &= ~(1 << 1);
- }
- else if (path.rfind("-POSE.pfm") != string::npos) {
- Mat m_ = cv::imdecode(Mat(data), cv::IMREAD_ANYDEPTH);
- if ((m_.rows != 4) || (m_.cols != 4)) continue;
- cv::Matx44d pose_(m_);
- cv::cv2eigen(pose_, snapshot.pose);
- snapshot.status &= ~(1 << 2);
- }
- else if (path.rfind("-PARAMS.json") != string::npos) {
- nlohmann::json j = nlohmann::json::parse(string((const char*) data.data(), data.size()));
- from_json(j, snapshot.params);
- snapshot.status &= ~(1 << 3);
- }
- else {
- LOG(WARNING) << "unknown file " << path;
+ // TODO: verify that input is valid
+ // TODO: check that earlier results are not overwritten (status)
+
+ if (path.rfind("-RGB.") != string::npos) {
+ getRGB(path, rgb);
+ }
+ else if (path.rfind("-D.") != string::npos) {
+ getDepth(path, depth);
+ }
+ else if (path.rfind("-POSE.pfm") != string::npos) {
+ Mat m_ = cv::imdecode(Mat(data), cv::IMREAD_ANYDEPTH);
+ if ((m_.rows != 4) || (m_.cols != 4)) continue;
+ cv::Matx44d pose_(m_);
+ pose = m_;
+ }
+ else if (path.rfind("-PARAMS.json") != string::npos) {
+ nlohmann::json j = nlohmann::json::parse(string((const char*) data.data(), data.size()));
+ from_json(j, params);
+ }
+ else {
+ LOG(WARNING) << "unknown file " << path;
+ }
}
}
-
- return true;
-}
-/*vector<string> SnapshotReader::getIds() {
- vector<string> res;
- res.reserve(data_.size());
- for(auto itr = data_.begin(); itr != data_.end(); ++itr) {
- res.push_back(itr->first);
- }
- return res;
-}*/
-
-SnapshotEntry& SnapshotReader::getEntry(const string &id) {
- /*if (data_.find(id) == data_.end()) {
- data_.emplace(id, SnapshotEntry{});
- }*/
- return data_[id];
+ return result;
}
-
-bool SnapshotReader::getCameraRGBD(const string &id, Mat &rgb, Mat &depth,
- Matrix4d &pose, Camera ¶ms) {
- if (data_.find(id) == data_.end()) {
- LOG(ERROR) << "entry not found: " << id;
- return false;
- }
-
- SnapshotEntry item = getEntry(id);
-
- if (item.status != 0) {
- LOG(ERROR) << "entry incomplete: " << id;
- }
-
- rgb = item.rgb;
- depth = item.depth;
- params = item.params;
- pose = item.pose;
- return true;
-}
\ No newline at end of file
diff --git a/components/rgbd-sources/src/snapshot_source.cpp b/components/rgbd-sources/src/snapshot_source.cpp
index e41417714..7eb5bc674 100644
--- a/components/rgbd-sources/src/snapshot_source.cpp
+++ b/components/rgbd-sources/src/snapshot_source.cpp
@@ -13,22 +13,21 @@ using ftl::rgbd::detail::SnapshotSource;
using std::string;
using std::vector;
-SnapshotSource::SnapshotSource(ftl::rgbd::Source *host, SnapshotReader &reader, const string &id) : detail::Source(host) {
- Eigen::Matrix4d pose;
- reader.getCameraRGBD(id, snap_rgb_, snap_depth_, pose, params_);
-
- rgb_ = snap_rgb_;
- depth_ = snap_depth_;
+SnapshotSource::SnapshotSource(ftl::rgbd::Source *host, Snapshot &snapshot, const string &id) : detail::Source(host) {
+ snapshot_ = snapshot;
+ camera_idx_ = std::atoi(id.c_str());
+ frame_idx_ = 0;
- if (rgb_.empty()) LOG(ERROR) << "Did not load snapshot rgb - " << id;
- if (depth_.empty()) LOG(ERROR) << "Did not load snapshot depth - " << id;
- if (params_.width != rgb_.cols) LOG(ERROR) << "Camera parameters corrupt for " << id;
+ Eigen::Matrix4d pose;
+ snapshot.getPose(camera_idx_, pose);
+ params_ = snapshot.getParameters(camera_idx_);
+ /*
ftl::rgbd::colourCorrection(rgb_, host->value("gamma", 1.0f), host->value("temperature", 6500));
-
host->on("gamma", [this,host](const ftl::config::Event&) {
ftl::rgbd::colourCorrection(rgb_, host->value("gamma", 1.0f), host->value("temperature", 6500));
});
+ */
// Add calibration to config object
host_->getConfig()["focal"] = params_.fx;
@@ -51,11 +50,17 @@ SnapshotSource::SnapshotSource(ftl::rgbd::Source *host, SnapshotReader &reader,
LOG(INFO) << "POSE = " << pose;
- host->setPose(pose);
+ host->setPose(pose);
}
bool SnapshotSource::compute(int n, int b) {
+ snapshot_.getLeftRGB(camera_idx_, frame_idx_, snap_rgb_);
+ snapshot_.getLeftDepth(camera_idx_, frame_idx_, snap_depth_);
+
snap_rgb_.copyTo(rgb_);
snap_depth_.copyTo(depth_);
+
+ frame_idx_ = (frame_idx_ + 1) % snapshot_.getFramesCount();
+
return true;
}
diff --git a/components/rgbd-sources/src/snapshot_source.hpp b/components/rgbd-sources/src/snapshot_source.hpp
index eb4e58594..41bfe690c 100644
--- a/components/rgbd-sources/src/snapshot_source.hpp
+++ b/components/rgbd-sources/src/snapshot_source.hpp
@@ -14,7 +14,7 @@ namespace detail {
class SnapshotSource : public detail::Source {
public:
SnapshotSource(ftl::rgbd::Source *);
- SnapshotSource(ftl::rgbd::Source *, ftl::rgbd::SnapshotReader &reader, const std::string &id);
+ SnapshotSource(ftl::rgbd::Source *, ftl::rgbd::Snapshot &snapshot, const std::string &id);
~SnapshotSource() {};
bool compute(int n, int b);
@@ -22,6 +22,11 @@ class SnapshotSource : public detail::Source {
//void reset();
private:
+ size_t frame_idx_;
+ size_t camera_idx_;
+
+ ftl::rgbd::Snapshot snapshot_;
+
cv::Mat snap_rgb_;
cv::Mat snap_depth_;
};
diff --git a/components/rgbd-sources/src/source.cpp b/components/rgbd-sources/src/source.cpp
index 17e343a00..c52c4100c 100644
--- a/components/rgbd-sources/src/source.cpp
+++ b/components/rgbd-sources/src/source.cpp
@@ -120,7 +120,8 @@ ftl::rgbd::detail::Source *Source::_createFileImpl(const ftl::URI &uri) {
} else if (ext == "tar" || ext == "gz") {
#ifdef HAVE_LIBARCHIVE
ftl::rgbd::SnapshotReader reader(path);
- return new ftl::rgbd::detail::SnapshotSource(this, reader, value("index", std::string("0"))); // TODO: Use URI fragment
+ auto snapshot = reader.readArchive();
+ return new ftl::rgbd::detail::SnapshotSource(this, snapshot, value("index", std::string("0"))); // TODO: Use URI fragment
#else
LOG(ERROR) << "Cannot read snapshots, libarchive not installed";
return nullptr;
diff --git a/components/rgbd-sources/test/source_unit.cpp b/components/rgbd-sources/test/source_unit.cpp
index 1c80e8292..1bbb336aa 100644
--- a/components/rgbd-sources/test/source_unit.cpp
+++ b/components/rgbd-sources/test/source_unit.cpp
@@ -10,9 +10,12 @@ static std::string last_type = "";
namespace ftl {
namespace rgbd {
+class Snapshot {};
+
class SnapshotReader {
public:
SnapshotReader(const std::string &) {}
+ Snapshot readArchive() { return Snapshot(); };
};
namespace detail {
@@ -55,7 +58,7 @@ class NetSource : public ftl::rgbd::detail::Source {
class SnapshotSource : public ftl::rgbd::detail::Source {
public:
- SnapshotSource(ftl::rgbd::Source *host, ftl::rgbd::SnapshotReader &r, const std::string &) : ftl::rgbd::detail::Source(host) {
+ SnapshotSource(ftl::rgbd::Source *host, ftl::rgbd::Snapshot &r, const std::string &) : ftl::rgbd::detail::Source(host) {
last_type = "snapshot";
}
--
GitLab