Fix H264 decode

components/codecs/include/ftl/codecs/h264.hpp

+#ifndef _FTL_CODECS_H264_HPP_
+#define _FTL_CODECS_H264_HPP_
+
+namespace ftl {
+namespace codecs {
+
+/**
+ * H.264 codec utility functions.
+ */
+namespace h264 {
+
+/**
+ * H264 Network Abstraction Layer Unit types.
+ */
+enum class NALType : int {
+	UNSPECIFIED_0 = 0,
+	CODED_SLICE_NON_IDR = 1,
+	CODED_SLICE_PART_A = 2,
+	CODED_SLICE_PART_B = 3,
+	CODED_SLICE_PART_C = 4,
+	CODED_SLICE_IDR = 5,
+	SEI = 6,
+	SPS = 7,
+	PPS = 8,
+	ACCESS_DELIMITER = 9,
+	EO_SEQ = 10,
+	EO_STREAM = 11,
+	FILTER_DATA = 12,
+	SPS_EXT = 13,
+	PREFIX_NAL_UNIT = 14,
+	SUBSET_SPS = 15,
+	RESERVED_16 = 16,
+	RESERVED_17 = 17,
+	RESERVED_18 = 18,
+	CODED_SLICE_AUX = 19,
+	CODED_SLICE_EXT = 20,
+	CODED_SLICE_DEPTH = 21,
+	RESERVED_22 = 22,
+	RESERVED_23 = 23,
+	UNSPECIFIED_24 = 24,
+	UNSPECIFIED_25,
+	UNSPECIFIED_26,
+	UNSPECIFIED_27,
+	UNSPECIFIED_28,
+	UNSPECIFIED_29,
+	UNSPECIFIED_30,
+	UNSPECIFIED_31
+};
+
+/**
+ * Extract the NAL unit type from the first NAL header.
+ * With NvPipe, the 5th byte contains the NAL Unit header.
+ */
+inline NALType getNALType(const std::vector<uint8_t> &data) {
+	return static_cast<NALType>(data[4] & 0x1F);
+}
+
+/**
+ * Check the H264 bitstream for an I-Frame. With NvPipe, all I-Frames start
+ * with a SPS NAL unit so just check for this.
+ */
+inline bool isIFrame(const std::vector<uint8_t> &data) {
+	return getNALType(data) == NALType::SPS;
+}
+
+}
+}
+}
+
+#endif  // _FTL_CODECS_H264_HPP_

components/codecs/src/decoder.cpp

@@ -10,6 +10,7 @@ Decoder *ftl::codecs::allocateDecoder(const ftl::codecs::Packet &pkt) {
 	switch(pkt.codec) {
 	case codec_t::JPG		:
 	case codec_t::PNG		: return new ftl::codecs::OpenCVDecoder;
+	case codec_t::H264		:
 	case codec_t::HEVC		: return new ftl::codecs::NvPipeDecoder;
 	}
 

components/codecs/src/nvpipe_decoder.cpp

@@ -4,6 +4,7 @@
 
 #include <ftl/cuda_util.hpp>
 #include <ftl/codecs/hevc.hpp>
+#include <ftl/codecs/h264.hpp>
 //#include <cuda_runtime.h>
 
 #include <opencv2/core/cuda/common.hpp>
@@ -87,8 +88,9 @@ bool NvPipeDecoder::decode(const ftl::codecs::Packet &pkt, cv::Mat &out) {
 	if (pkt.codec == ftl::codecs::codec_t::HEVC) {
 		// Obtain NAL unit type
 		if (ftl::codecs::hevc::isIFrame(pkt.data)) seen_iframe_ = true;
+	} else if (pkt.codec == ftl::codecs::codec_t::H264) {
+		if (ftl::codecs::h264::isIFrame(pkt.data)) seen_iframe_ = true;
 	}
-	// TODO: Parse H264 for i-frame check
 
 	if (!seen_iframe_) return false;
 
@@ -127,5 +129,5 @@ bool NvPipeDecoder::decode(const ftl::codecs::Packet &pkt, cv::Mat &out) {
 }
 
 bool NvPipeDecoder::accepts(const ftl::codecs::Packet &pkt) {
-	return pkt.codec == codec_t::HEVC;
+	return pkt.codec == codec_t::HEVC || pkt.codec == codec_t::H264;
 }

components/rgbd-sources/src/sources/ftlfile/file_source.cpp

@@ -34,13 +34,16 @@ FileSource::FileSource(ftl::rgbd::Source *s, ftl::rgbd::Player *r, int sid) : ft
     r->onPacket(sid, [this](const ftl::codecs::StreamPacket &spkt, ftl::codecs::Packet &pkt) {
 		host_->notifyRaw(spkt, pkt);
 
-		// Should config items be parsed here?
+		// Some channels are to be directly handled by the source object and
+		// do not proceed to any subsequent step.
+		// FIXME: Potential problem, these get processed at wrong time
 		if (spkt.channel == Channel::Configuration) {
 			std::tuple<std::string, std::string> cfg;
 			auto unpacked = msgpack::unpack((const char*)pkt.data.data(), pkt.data.size());
 			unpacked.get().convert(cfg);
 
 			LOG(INFO) << "Config Received: " << std::get<1>(cfg);
+			return;
 		}
 		else if (spkt.channel == Channel::Calibration) {
 			_processCalibration(pkt);
@@ -50,7 +53,7 @@ FileSource::FileSource(ftl::rgbd::Source *s, ftl::rgbd::Player *r, int sid) : ft
 			return;
 		}
 
-		// FIXME: For bad and old FTL files
+		// FIXME: For bad and old FTL files where wrong channel is used
 		if (pkt.codec == codec_t::POSE) {
 			_processPose(pkt);
 			return;
@@ -60,6 +63,7 @@ FileSource::FileSource(ftl::rgbd::Source *s, ftl::rgbd::Player *r, int sid) : ft
 		}
 
 
+		// TODO: Check I-Frames for H264
 		if (pkt.codec == codec_t::HEVC) {
 			if (ftl::codecs::hevc::isIFrame(pkt.data)) _removeChannel(spkt.channel);
 		}
@@ -147,9 +151,13 @@ bool FileSource::compute(int n, int b) {
 	int64_t lastts = 0;
 	int lastc = 0;
 
+	// Go through previously read and cached frames in sequence
+	// needs to be done due to P-Frames
 	for (auto i=cache_[cache_read_].begin(); i!=cache_[cache_read_].end(); ++i) {
 		auto &c = *i;
 
+		// Check for verifying that both channels are received, ie. two frames
+		// with the same timestamp.
 		if (c.spkt.timestamp > lastts) {
 			lastts = c.spkt.timestamp;
 			lastc = 1;
@@ -157,11 +165,6 @@ bool FileSource::compute(int n, int b) {
 			lastc++;
 		}
 
-		//LOG(INFO) << "DECODE FRAME: " << c.spkt.timestamp << "," << (int)c.spkt.channel << "," << (int)c.pkt.codec << "," << (int)c.pkt.definition;
-
-		// FIXME: This hack is for old and bad ftl files.
-		//if ((int)c.pkt.codec >= 100) continue;
-
 		if (c.spkt.channel == Channel::Colour) {
 			rgb_.create(cv::Size(ftl::codecs::getWidth(c.pkt.definition),ftl::codecs::getHeight(c.pkt.definition)), CV_8UC3);
 		} else {
@@ -177,6 +180,7 @@ bool FileSource::compute(int n, int b) {
 		}
 	}
 
+	// FIXME: Consider case of Channel::None
 	if (lastc != 2) {
 		LOG(ERROR) << "Channels not in sync (" << sourceid_ << "): " << lastts;
 		return false;
@@ -186,8 +190,7 @@ bool FileSource::compute(int n, int b) {
 
 	if (rgb_.empty() || depth_.empty()) return false;
 
-	//auto cb = host_->callback();
-	//if (cb) cb(timestamp_, rgb_, depth_);
+	// Inform about a decoded frame pair
 	host_->notify(timestamp_, rgb_, depth_);
     return true;
 }