Merge branch 'feature/211/scaling' into 'master'

Implements #211 scaling corrections on recv

Closes #211

See merge request nicolas.pope/ftl!144

components/codecs/include/ftl/codecs/channels.hpp

@@ -36,7 +36,8 @@ enum struct Channel : int {
 	Configuration	= 64,	// JSON Data
 	Calibration		= 65,	// Camera Parameters Object
 	Pose			= 66,	// Eigen::Matrix4d
-	Data			= 67	// Custom data, any codec.
+    Index           = 67,
+	Data			= 2048	// Custom data, any codec.
 };
 
 inline bool isVideo(Channel c) { return (int)c < 32; };

components/codecs/include/ftl/codecs/packet.hpp

@@ -16,7 +16,14 @@ namespace codecs {
  */
 struct Header {
 	const char magic[4] = {'F','T','L','F'};
-	uint8_t version = 1;
+	uint8_t version = 2;
+};
+
+/**
+ * Version 2 header padding for potential indexing use.
+ */
+struct IndexHeader {
+	int64_t reserved[8];
 };
 
 /**

components/codecs/src/reader.cpp

@@ -22,6 +22,11 @@ bool Reader::begin() {
 	(*stream_).read((char*)&h, sizeof(h));
 	if (h.magic[0] != 'F' || h.magic[1] != 'T' || h.magic[2] != 'L' || h.magic[3] != 'F') return false;
 
+	if (h.version >= 2) {
+		ftl::codecs::IndexHeader ih;
+		(*stream_).read((char*)&ih, sizeof(ih));
+	}
+
 	// Capture current time to adjust timestamps
 	timestart_ = (ftl::timer::get_time() / ftl::timer::getInterval()) * ftl::timer::getInterval();
 	playing_ = true;

components/codecs/src/writer.cpp

@@ -14,9 +14,13 @@ Writer::~Writer() {
 
 bool Writer::begin() {
 	ftl::codecs::Header h;
-	h.version = 0;
+	//h.version = 2;
 	(*stream_).write((const char*)&h, sizeof(h));
 
+	ftl::codecs::IndexHeader ih;
+	ih.reserved[0] = -1;
+	(*stream_).write((const char*)&ih, sizeof(ih));
+
 	// Capture current time to adjust timestamps
 	timestart_ = ftl::timer::get_time();
 	active_ = true;

components/rgbd-sources/include/ftl/rgbd/source.hpp

@@ -201,6 +201,12 @@ class Source : public ftl::Configurable {
 	 */
 	void notifyRaw(const ftl::codecs::StreamPacket &spkt, const ftl::codecs::Packet &pkt);
 
+	/**
+	 * Notify of a decoded or available pair of frames. This calls the source
+	 * callback after having verified the correct resolution of the frames.
+	 */
+	void notify(int64_t ts, cv::Mat &c1, cv::Mat &c2);
+
 	// ==== Inject Data into stream ============================================
 
 	/**

components/rgbd-sources/src/source.cpp

@@ -31,6 +31,7 @@ using ftl::rgbd::detail::MiddleburySource;
 using ftl::rgbd::capability_t;
 using ftl::codecs::Channel;
 using ftl::rgbd::detail::FileSource;
+using ftl::rgbd::Camera;
 
 std::map<std::string, ftl::rgbd::Player*> Source::readers__;
 
@@ -274,3 +275,50 @@ void Source::notifyRaw(const ftl::codecs::StreamPacket &spkt, const ftl::codecs:
 		i(this, spkt, pkt);
 	}
 }
+
+/*
+ * Scale camera parameters to match resolution.
+ */
+static Camera scaled(Camera &cam, int width, int height) {
+	float scaleX = (float)width / (float)cam.width;
+	float scaleY = (float)height / (float)cam.height;
+
+	CHECK( abs(scaleX - scaleY) < 0.000001f );
+
+	Camera newcam = cam;
+	newcam.width = width;
+	newcam.height = height;
+	newcam.fx *= scaleX;
+	newcam.fy *= scaleY;
+	newcam.cx *= scaleX;
+	newcam.cy *= scaleY;
+
+	return newcam;
+}
+
+void Source::notify(int64_t ts, cv::Mat &c1, cv::Mat &c2) {
+	// Ensure correct scaling of images and parameters.
+	int max_width = max(impl_->params_.width, max(c1.cols, c2.cols));
+	int max_height = max(impl_->params_.height, max(c1.rows, c2.rows));
+
+	// Do we need to scale camera parameters
+	if (impl_->params_.width < max_width || impl_->params_.height < max_height) {
+		impl_->params_ = scaled(impl_->params_, max_width, max_height);
+	}
+
+	// Should channel 1 be scaled?
+	if (c1.cols < max_width || c1.rows < max_height) {
+		cv::resize(c1, c1, cv::Size(max_width, max_height));
+	}
+
+	// Should channel 2 be scaled?
+	if (c2.cols < max_width || c2.rows < max_height) {
+		if (c2.type() == CV_32F) {
+			cv::resize(c2, c2, cv::Size(max_width, max_height), 0.0, 0.0, cv::INTER_NEAREST);
+		} else {
+			cv::resize(c2, c2, cv::Size(max_width, max_height));
+		}
+	}
+
+	if (callback_) callback_(ts, c1, c2);
+}

components/rgbd-sources/src/sources/ftlfile/file_source.cpp

@@ -132,8 +132,9 @@ bool FileSource::compute(int n, int b) {
 
 	if (rgb_.empty() || depth_.empty()) return false;
 
-	auto cb = host_->callback();
-	if (cb) cb(timestamp_, rgb_, depth_);
+	//auto cb = host_->callback();
+	//if (cb) cb(timestamp_, rgb_, depth_);
+	host_->notify(timestamp_, rgb_, depth_);
     return true;
 }
 

components/rgbd-sources/src/sources/net/net.cpp

@@ -382,7 +382,8 @@ void NetSource::_recvPacket(short ttimeoff, const ftl::codecs::StreamPacket &spk
 
 			adaptive_ = abr_.selectBitrate(frame);
 			//LOG(INFO) << "Frame finished: " << frame.timestamp;
-			auto cb = host_->callback();
+			host_->notify(frame.timestamp, frame.channel1, frame.channel2);
+			/*auto cb = host_->callback();
 			if (cb) {
 				try {
 					cb(frame.timestamp, frame.channel1, frame.channel2);
@@ -391,7 +392,7 @@ void NetSource::_recvPacket(short ttimeoff, const ftl::codecs::StreamPacket &spk
 				}
 			} else {
 				LOG(ERROR) << "NO FRAME CALLBACK";
-			}
+			}*/
 
 			queue_.freeFrame(frame);