diff --git a/components/codecs/include/ftl/codecs/h264.hpp b/components/codecs/include/ftl/codecs/h264.hpp
new file mode 100644
index 0000000000000000000000000000000000000000..17f649c52220f7ac8b00e6d4ed9b78e18f2847f8
--- /dev/null
+++ b/components/codecs/include/ftl/codecs/h264.hpp
@@ -0,0 +1,70 @@
+#ifndef _FTL_CODECS_H264_HPP_
+#define _FTL_CODECS_H264_HPP_
+
+namespace ftl {
+namespace codecs {
+
+/**
+ * H.264 codec utility functions.
+ */
+namespace h264 {
+
+/**
+ * H264 Network Abstraction Layer Unit types.
+ */
+enum class NALType : int {
+ UNSPECIFIED_0 = 0,
+ CODED_SLICE_NON_IDR = 1,
+ CODED_SLICE_PART_A = 2,
+ CODED_SLICE_PART_B = 3,
+ CODED_SLICE_PART_C = 4,
+ CODED_SLICE_IDR = 5,
+ SEI = 6,
+ SPS = 7,
+ PPS = 8,
+ ACCESS_DELIMITER = 9,
+ EO_SEQ = 10,
+ EO_STREAM = 11,
+ FILTER_DATA = 12,
+ SPS_EXT = 13,
+ PREFIX_NAL_UNIT = 14,
+ SUBSET_SPS = 15,
+ RESERVED_16 = 16,
+ RESERVED_17 = 17,
+ RESERVED_18 = 18,
+ CODED_SLICE_AUX = 19,
+ CODED_SLICE_EXT = 20,
+ CODED_SLICE_DEPTH = 21,
+ RESERVED_22 = 22,
+ RESERVED_23 = 23,
+ UNSPECIFIED_24 = 24,
+ UNSPECIFIED_25,
+ UNSPECIFIED_26,
+ UNSPECIFIED_27,
+ UNSPECIFIED_28,
+ UNSPECIFIED_29,
+ UNSPECIFIED_30,
+ UNSPECIFIED_31
+};
+
+/**
+ * Extract the NAL unit type from the first NAL header.
+ * With NvPipe, the 5th byte contains the NAL Unit header.
+ */
+inline NALType getNALType(const std::vector<uint8_t> &data) {
+ return static_cast<NALType>(data[4] & 0x1F);
+}
+
+/**
+ * Check the H264 bitstream for an I-Frame. With NvPipe, all I-Frames start
+ * with a SPS NAL unit so just check for this.
+ */
+inline bool isIFrame(const std::vector<uint8_t> &data) {
+ return getNALType(data) == NALType::SPS;
+}
+
+}
+}
+}
+
+#endif // _FTL_CODECS_H264_HPP_
diff --git a/components/codecs/src/decoder.cpp b/components/codecs/src/decoder.cpp
index 4cd5437d0cb6ea84251b55bdd139a7c0b799f411..a1809b6157d783782a20e32f46f39f89cd26e8c5 100644
--- a/components/codecs/src/decoder.cpp
+++ b/components/codecs/src/decoder.cpp
@@ -10,6 +10,7 @@ Decoder *ftl::codecs::allocateDecoder(const ftl::codecs::Packet &pkt) {
switch(pkt.codec) {
case codec_t::JPG :
case codec_t::PNG : return new ftl::codecs::OpenCVDecoder;
+ case codec_t::H264 :
case codec_t::HEVC : return new ftl::codecs::NvPipeDecoder;
}
diff --git a/components/codecs/src/nvpipe_decoder.cpp b/components/codecs/src/nvpipe_decoder.cpp
index 97985cd20c4bd2dea7aa86c829b3b151563c3cfb..79e16f3fa77ca9445faab12e33664243f1247293 100644
--- a/components/codecs/src/nvpipe_decoder.cpp
+++ b/components/codecs/src/nvpipe_decoder.cpp
@@ -4,6 +4,7 @@
#include <ftl/cuda_util.hpp>
#include <ftl/codecs/hevc.hpp>
+#include <ftl/codecs/h264.hpp>
//#include <cuda_runtime.h>
#include <opencv2/core/cuda/common.hpp>
@@ -87,8 +88,9 @@ bool NvPipeDecoder::decode(const ftl::codecs::Packet &pkt, cv::Mat &out) {
if (pkt.codec == ftl::codecs::codec_t::HEVC) {
// Obtain NAL unit type
if (ftl::codecs::hevc::isIFrame(pkt.data)) seen_iframe_ = true;
+ } else if (pkt.codec == ftl::codecs::codec_t::H264) {
+ if (ftl::codecs::h264::isIFrame(pkt.data)) seen_iframe_ = true;
}
- // TODO: Parse H264 for i-frame check
if (!seen_iframe_) return false;
@@ -127,5 +129,5 @@ bool NvPipeDecoder::decode(const ftl::codecs::Packet &pkt, cv::Mat &out) {
}
bool NvPipeDecoder::accepts(const ftl::codecs::Packet &pkt) {
- return pkt.codec == codec_t::HEVC;
+ return pkt.codec == codec_t::HEVC || pkt.codec == codec_t::H264;
}
diff --git a/components/rgbd-sources/src/sources/ftlfile/file_source.cpp b/components/rgbd-sources/src/sources/ftlfile/file_source.cpp
index 76cd435e7f6d907db95c6074f2aa9365b10f8652..9597bde796f80ac67f38a5a569190001e13d7065 100644
--- a/components/rgbd-sources/src/sources/ftlfile/file_source.cpp
+++ b/components/rgbd-sources/src/sources/ftlfile/file_source.cpp
@@ -34,13 +34,16 @@ FileSource::FileSource(ftl::rgbd::Source *s, ftl::rgbd::Player *r, int sid) : ft
r->onPacket(sid, [this](const ftl::codecs::StreamPacket &spkt, ftl::codecs::Packet &pkt) {
host_->notifyRaw(spkt, pkt);
- // Should config items be parsed here?
+ // Some channels are to be directly handled by the source object and
+ // do not proceed to any subsequent step.
+ // FIXME: Potential problem, these get processed at wrong time
if (spkt.channel == Channel::Configuration) {
std::tuple<std::string, std::string> cfg;
auto unpacked = msgpack::unpack((const char*)pkt.data.data(), pkt.data.size());
unpacked.get().convert(cfg);
LOG(INFO) << "Config Received: " << std::get<1>(cfg);
+ return;
}
else if (spkt.channel == Channel::Calibration) {
_processCalibration(pkt);
@@ -50,7 +53,7 @@ FileSource::FileSource(ftl::rgbd::Source *s, ftl::rgbd::Player *r, int sid) : ft
return;
}
- // FIXME: For bad and old FTL files
+ // FIXME: For bad and old FTL files where wrong channel is used
if (pkt.codec == codec_t::POSE) {
_processPose(pkt);
return;
@@ -60,6 +63,7 @@ FileSource::FileSource(ftl::rgbd::Source *s, ftl::rgbd::Player *r, int sid) : ft
}
+ // TODO: Check I-Frames for H264
if (pkt.codec == codec_t::HEVC) {
if (ftl::codecs::hevc::isIFrame(pkt.data)) _removeChannel(spkt.channel);
}
@@ -147,9 +151,13 @@ bool FileSource::compute(int n, int b) {
int64_t lastts = 0;
int lastc = 0;
+ // Go through previously read and cached frames in sequence
+ // needs to be done due to P-Frames
for (auto i=cache_[cache_read_].begin(); i!=cache_[cache_read_].end(); ++i) {
auto &c = *i;
+ // Check for verifying that both channels are received, ie. two frames
+ // with the same timestamp.
if (c.spkt.timestamp > lastts) {
lastts = c.spkt.timestamp;
lastc = 1;
@@ -157,11 +165,6 @@ bool FileSource::compute(int n, int b) {
lastc++;
}
- //LOG(INFO) << "DECODE FRAME: " << c.spkt.timestamp << "," << (int)c.spkt.channel << "," << (int)c.pkt.codec << "," << (int)c.pkt.definition;
-
- // FIXME: This hack is for old and bad ftl files.
- //if ((int)c.pkt.codec >= 100) continue;
-
if (c.spkt.channel == Channel::Colour) {
rgb_.create(cv::Size(ftl::codecs::getWidth(c.pkt.definition),ftl::codecs::getHeight(c.pkt.definition)), CV_8UC3);
} else {
@@ -177,6 +180,7 @@ bool FileSource::compute(int n, int b) {
}
}
+ // FIXME: Consider case of Channel::None
if (lastc != 2) {
LOG(ERROR) << "Channels not in sync (" << sourceid_ << "): " << lastts;
return false;
@@ -186,8 +190,7 @@ bool FileSource::compute(int n, int b) {
if (rgb_.empty() || depth_.empty()) return false;
- //auto cb = host_->callback();
- //if (cb) cb(timestamp_, rgb_, depth_);
+ // Inform about a decoded frame pair
host_->notify(timestamp_, rgb_, depth_);
return true;
}