diff --git a/components/codecs/include/ftl/codecs/channels.hpp b/components/codecs/include/ftl/codecs/channels.hpp
index d1a0e265b0548adc1fc510947f12d408ed3b2840..d1cf07a3aeb55863acb3374fa3c733b12e5ca240 100644
--- a/components/codecs/include/ftl/codecs/channels.hpp
+++ b/components/codecs/include/ftl/codecs/channels.hpp
@@ -36,7 +36,8 @@ enum struct Channel : int {
Configuration = 64, // JSON Data
Calibration = 65, // Camera Parameters Object
Pose = 66, // Eigen::Matrix4d
- Data = 67 // Custom data, any codec.
+ Index = 67,
+ Data = 2048 // Custom data, any codec.
};
inline bool isVideo(Channel c) { return (int)c < 32; };
diff --git a/components/codecs/include/ftl/codecs/packet.hpp b/components/codecs/include/ftl/codecs/packet.hpp
index 14e8fc186d0a55a8142ec5ee9cd5e6ecea02b32b..89df44f9e0128cea76b15adf331796644cb91c6b 100644
--- a/components/codecs/include/ftl/codecs/packet.hpp
+++ b/components/codecs/include/ftl/codecs/packet.hpp
@@ -16,7 +16,14 @@ namespace codecs {
*/
struct Header {
const char magic[4] = {'F','T','L','F'};
- uint8_t version = 1;
+ uint8_t version = 2;
+};
+
+/**
+ * Version 2 header padding for potential indexing use.
+ */
+struct IndexHeader {
+ int64_t reserved[8];
};
/**
diff --git a/components/codecs/src/reader.cpp b/components/codecs/src/reader.cpp
index 41d0697f5c23192a405ac9d13286cb38047a7ec5..53225519a572858a8ceb96e42e9b58ee0c839d43 100644
--- a/components/codecs/src/reader.cpp
+++ b/components/codecs/src/reader.cpp
@@ -22,6 +22,11 @@ bool Reader::begin() {
(*stream_).read((char*)&h, sizeof(h));
if (h.magic[0] != 'F' || h.magic[1] != 'T' || h.magic[2] != 'L' || h.magic[3] != 'F') return false;
+ if (h.version >= 2) {
+ ftl::codecs::IndexHeader ih;
+ (*stream_).read((char*)&ih, sizeof(ih));
+ }
+
// Capture current time to adjust timestamps
timestart_ = (ftl::timer::get_time() / ftl::timer::getInterval()) * ftl::timer::getInterval();
playing_ = true;
diff --git a/components/codecs/src/writer.cpp b/components/codecs/src/writer.cpp
index 6561549f6f2e681b7678b328f4357762eae468a1..d7582f11866b701796ab311016d2a01a5218595a 100644
--- a/components/codecs/src/writer.cpp
+++ b/components/codecs/src/writer.cpp
@@ -14,9 +14,13 @@ Writer::~Writer() {
bool Writer::begin() {
ftl::codecs::Header h;
- h.version = 0;
+ //h.version = 2;
(*stream_).write((const char*)&h, sizeof(h));
+ ftl::codecs::IndexHeader ih;
+ ih.reserved[0] = -1;
+ (*stream_).write((const char*)&ih, sizeof(ih));
+
// Capture current time to adjust timestamps
timestart_ = ftl::timer::get_time();
active_ = true;
diff --git a/components/rgbd-sources/include/ftl/rgbd/source.hpp b/components/rgbd-sources/include/ftl/rgbd/source.hpp
index bb1cda3b251d3ecbf88ca8e799f9430e3f9da71c..586e7a5784f01696eb46c7e87d5f58b6f01b9681 100644
--- a/components/rgbd-sources/include/ftl/rgbd/source.hpp
+++ b/components/rgbd-sources/include/ftl/rgbd/source.hpp
@@ -201,6 +201,12 @@ class Source : public ftl::Configurable {
*/
void notifyRaw(const ftl::codecs::StreamPacket &spkt, const ftl::codecs::Packet &pkt);
+ /**
+ * Notify of a decoded or available pair of frames. This calls the source
+ * callback after having verified the correct resolution of the frames.
+ */
+ void notify(int64_t ts, cv::Mat &c1, cv::Mat &c2);
+
// ==== Inject Data into stream ============================================
/**
diff --git a/components/rgbd-sources/src/source.cpp b/components/rgbd-sources/src/source.cpp
index 7414f43201c5c0e95a539a38e62045629a74eaf3..8e402bf33baf002d2c1c31c92aa0e6a69fbfd3af 100644
--- a/components/rgbd-sources/src/source.cpp
+++ b/components/rgbd-sources/src/source.cpp
@@ -31,6 +31,7 @@ using ftl::rgbd::detail::MiddleburySource;
using ftl::rgbd::capability_t;
using ftl::codecs::Channel;
using ftl::rgbd::detail::FileSource;
+using ftl::rgbd::Camera;
std::map<std::string, ftl::rgbd::Player*> Source::readers__;
@@ -274,3 +275,50 @@ void Source::notifyRaw(const ftl::codecs::StreamPacket &spkt, const ftl::codecs:
i(this, spkt, pkt);
}
}
+
+/*
+ * Scale camera parameters to match resolution.
+ */
+static Camera scaled(Camera &cam, int width, int height) {
+ float scaleX = (float)width / (float)cam.width;
+ float scaleY = (float)height / (float)cam.height;
+
+ CHECK( abs(scaleX - scaleY) < 0.000001f );
+
+ Camera newcam = cam;
+ newcam.width = width;
+ newcam.height = height;
+ newcam.fx *= scaleX;
+ newcam.fy *= scaleY;
+ newcam.cx *= scaleX;
+ newcam.cy *= scaleY;
+
+ return newcam;
+}
+
+void Source::notify(int64_t ts, cv::Mat &c1, cv::Mat &c2) {
+ // Ensure correct scaling of images and parameters.
+ int max_width = max(impl_->params_.width, max(c1.cols, c2.cols));
+ int max_height = max(impl_->params_.height, max(c1.rows, c2.rows));
+
+ // Do we need to scale camera parameters
+ if (impl_->params_.width < max_width || impl_->params_.height < max_height) {
+ impl_->params_ = scaled(impl_->params_, max_width, max_height);
+ }
+
+ // Should channel 1 be scaled?
+ if (c1.cols < max_width || c1.rows < max_height) {
+ cv::resize(c1, c1, cv::Size(max_width, max_height));
+ }
+
+ // Should channel 2 be scaled?
+ if (c2.cols < max_width || c2.rows < max_height) {
+ if (c2.type() == CV_32F) {
+ cv::resize(c2, c2, cv::Size(max_width, max_height), 0.0, 0.0, cv::INTER_NEAREST);
+ } else {
+ cv::resize(c2, c2, cv::Size(max_width, max_height));
+ }
+ }
+
+ if (callback_) callback_(ts, c1, c2);
+}
diff --git a/components/rgbd-sources/src/sources/ftlfile/file_source.cpp b/components/rgbd-sources/src/sources/ftlfile/file_source.cpp
index 323f1ae72179c46ad816ec653ffc940d87cad0f7..d8ebdb399b081da544a34f288e6af928501bc372 100644
--- a/components/rgbd-sources/src/sources/ftlfile/file_source.cpp
+++ b/components/rgbd-sources/src/sources/ftlfile/file_source.cpp
@@ -132,8 +132,9 @@ bool FileSource::compute(int n, int b) {
if (rgb_.empty() || depth_.empty()) return false;
- auto cb = host_->callback();
- if (cb) cb(timestamp_, rgb_, depth_);
+ //auto cb = host_->callback();
+ //if (cb) cb(timestamp_, rgb_, depth_);
+ host_->notify(timestamp_, rgb_, depth_);
return true;
}
diff --git a/components/rgbd-sources/src/sources/net/net.cpp b/components/rgbd-sources/src/sources/net/net.cpp
index 094cbaafb916c799563676922e96bf82f8ceacd6..39227c5e6c36859e3142a8d8518dcd832e48c6e9 100644
--- a/components/rgbd-sources/src/sources/net/net.cpp
+++ b/components/rgbd-sources/src/sources/net/net.cpp
@@ -382,7 +382,8 @@ void NetSource::_recvPacket(short ttimeoff, const ftl::codecs::StreamPacket &spk
adaptive_ = abr_.selectBitrate(frame);
//LOG(INFO) << "Frame finished: " << frame.timestamp;
- auto cb = host_->callback();
+ host_->notify(frame.timestamp, frame.channel1, frame.channel2);
+ /*auto cb = host_->callback();
if (cb) {
try {
cb(frame.timestamp, frame.channel1, frame.channel2);
@@ -391,7 +392,7 @@ void NetSource::_recvPacket(short ttimeoff, const ftl::codecs::StreamPacket &spk
}
} else {
LOG(ERROR) << "NO FRAME CALLBACK";
- }
+ }*/
queue_.freeFrame(frame);