From cda58132ae4aa7b9b22bdde0d7a3444ceaabdd7d Mon Sep 17 00:00:00 2001
From: Nicolas Pope <nicolas.pope@utu.fi>
Date: Tue, 15 Oct 2019 09:27:57 +0300
Subject: [PATCH] Fix stream bugs
---
applications/ftl2mkv/src/main.cpp | 6 +-
applications/player/src/main.cpp | 99 ++++++++--------
applications/reconstruct/src/main.cpp | 6 +-
components/codecs/CMakeLists.txt | 1 +
.../codecs/include/ftl/codecs/channels.hpp | 3 +
components/codecs/include/ftl/codecs/hevc.hpp | 112 ++++++++++++++++++
.../include/ftl/codecs/nvpipe_decoder.hpp | 1 +
.../codecs/include/ftl/codecs/reader.hpp | 2 +-
components/codecs/src/channels.cpp | 93 +++++++++++++++
components/codecs/src/nvpipe_decoder.cpp | 13 +-
components/codecs/src/reader.cpp | 34 ++++--
components/renderers/cpp/src/splat_render.cpp | 2 +
.../include/ftl/rgbd/detail/netframe.hpp | 5 +-
.../rgbd-sources/include/ftl/rgbd/frame.hpp | 7 +-
components/rgbd-sources/src/file_source.cpp | 28 ++++-
components/rgbd-sources/src/file_source.hpp | 1 +
components/rgbd-sources/src/group.cpp | 7 +-
components/rgbd-sources/src/net.cpp | 46 ++++---
components/rgbd-sources/src/streamer.cpp | 1 +
19 files changed, 368 insertions(+), 99 deletions(-)
create mode 100644 components/codecs/include/ftl/codecs/hevc.hpp
create mode 100644 components/codecs/src/channels.cpp
diff --git a/applications/ftl2mkv/src/main.cpp b/applications/ftl2mkv/src/main.cpp
index 4ee909fee..b555dcbf7 100644
--- a/applications/ftl2mkv/src/main.cpp
+++ b/applications/ftl2mkv/src/main.cpp
@@ -3,6 +3,7 @@
#include <ftl/codecs/reader.hpp>
#include <ftl/codecs/packet.hpp>
#include <ftl/rgbd/camera.hpp>
+#include <ftl/codecs/hevc.hpp>
#include <fstream>
@@ -188,10 +189,7 @@ int main(int argc, char **argv) {
bool keyframe = false;
if (pkt.codec == codec_t::HEVC) {
- // Obtain NAL unit type
- int nal_type = (pkt.data[4] >> 1) & 0x3F;
- // A type of 32 = VPS unit (so in this case a key frame)
- if (nal_type == 32) {
+ if (ftl::codecs::hevc::isIFrame(pkt.data)) {
seen_key[spkt.streamID] = true;
keyframe = true;
}
diff --git a/applications/player/src/main.cpp b/applications/player/src/main.cpp
index 751d9969a..cc0f5039a 100644
--- a/applications/player/src/main.cpp
+++ b/applications/player/src/main.cpp
@@ -4,6 +4,7 @@
#include <ftl/codecs/decoder.hpp>
#include <ftl/codecs/packet.hpp>
#include <ftl/rgbd/camera.hpp>
+#include <ftl/timer.hpp>
#include <fstream>
@@ -56,53 +57,57 @@ int main(int argc, char **argv) {
int current_stream = 0;
int current_channel = 0;
- bool res = r.read(90000000000000, [¤t_stream,¤t_channel,&r](const ftl::codecs::StreamPacket &spkt, const ftl::codecs::Packet &pkt) {
- if (spkt.channel != static_cast<ftl::codecs::Channel>(current_channel)) return;
- if (spkt.streamID == current_stream) {
-
- if (pkt.codec == codec_t::POSE) {
- Eigen::Matrix4d p = Eigen::Map<Eigen::Matrix4d>((double*)pkt.data.data());
- LOG(INFO) << "Have pose: " << p;
- return;
- }
-
- if (pkt.codec == codec_t::CALIBRATION) {
- ftl::rgbd::Camera *camera = (ftl::rgbd::Camera*)pkt.data.data();
- LOG(INFO) << "Have calibration: " << camera->fx;
- return;
- }
-
- LOG(INFO) << "Reading packet: (" << (int)spkt.streamID << "," << (int)spkt.channel << ") " << (int)pkt.codec << ", " << (int)pkt.definition;
-
- cv::Mat frame(cv::Size(ftl::codecs::getWidth(pkt.definition),ftl::codecs::getHeight(pkt.definition)), (spkt.channel == Channel::Depth) ? CV_32F : CV_8UC3);
- createDecoder(pkt);
-
- try {
- decoder->decode(pkt, frame);
- } catch (std::exception &e) {
- LOG(INFO) << "Decoder exception: " << e.what();
- }
-
- if (!frame.empty()) {
- if (spkt.channel == Channel::Depth) {
- visualizeDepthMap(frame, frame, 8.0f);
- }
- double time = (double)(spkt.timestamp - r.getStartTime()) / 1000.0;
- cv::putText(frame, std::string("Time: ") + std::to_string(time) + std::string("s"), cv::Point(10,20), cv::FONT_HERSHEY_PLAIN, 1, cv::Scalar(0,0,255));
- cv::imshow("Player", frame);
- }
- int key = cv::waitKey(20);
- if (key >= 48 && key <= 57) {
- current_stream = key - 48;
- } else if (key == 'd') {
- current_channel = (current_channel == 0) ? 1 : 0;
- } else if (key == 27) {
- r.end();
- }
- }
- });
-
- if (!res) LOG(ERROR) << "No frames left";
+ ftl::timer::add(ftl::timer::kTimerMain, [¤t_stream,¤t_channel,&r](int64_t ts) {
+ bool res = r.read(ts, [¤t_stream,¤t_channel,&r](const ftl::codecs::StreamPacket &spkt, const ftl::codecs::Packet &pkt) {
+ if (spkt.channel != static_cast<ftl::codecs::Channel>(current_channel)) return;
+ if (spkt.streamID == current_stream) {
+
+ if (pkt.codec == codec_t::POSE) {
+ Eigen::Matrix4d p = Eigen::Map<Eigen::Matrix4d>((double*)pkt.data.data());
+ LOG(INFO) << "Have pose: " << p;
+ return;
+ }
+
+ if (pkt.codec == codec_t::CALIBRATION) {
+ ftl::rgbd::Camera *camera = (ftl::rgbd::Camera*)pkt.data.data();
+ LOG(INFO) << "Have calibration: " << camera->fx;
+ return;
+ }
+
+ //LOG(INFO) << "Reading packet: (" << (int)spkt.streamID << "," << (int)spkt.channel << ") " << (int)pkt.codec << ", " << (int)pkt.definition;
+
+ cv::Mat frame(cv::Size(ftl::codecs::getWidth(pkt.definition),ftl::codecs::getHeight(pkt.definition)), (spkt.channel == Channel::Depth) ? CV_32F : CV_8UC3);
+ createDecoder(pkt);
+
+ try {
+ decoder->decode(pkt, frame);
+ } catch (std::exception &e) {
+ LOG(INFO) << "Decoder exception: " << e.what();
+ }
+
+ if (!frame.empty()) {
+ if (spkt.channel == Channel::Depth) {
+ visualizeDepthMap(frame, frame, 8.0f);
+ }
+ double time = (double)(spkt.timestamp - r.getStartTime()) / 1000.0;
+ cv::putText(frame, std::string("Time: ") + std::to_string(time) + std::string("s"), cv::Point(10,20), cv::FONT_HERSHEY_PLAIN, 1, cv::Scalar(0,0,255));
+ cv::imshow("Player", frame);
+ }
+ int key = cv::waitKey(1);
+ if (key >= 48 && key <= 57) {
+ current_stream = key - 48;
+ } else if (key == 'd') {
+ current_channel = (current_channel == 0) ? 1 : 0;
+ } else if (key == 27) {
+ ftl::timer::stop(false);
+ }
+ }
+ });
+ if (!res) ftl::timer::stop(false);
+ return res;
+ });
+
+ ftl::timer::start(true);
r.end();
diff --git a/applications/reconstruct/src/main.cpp b/applications/reconstruct/src/main.cpp
index 50d7986f3..d86eb5ea1 100644
--- a/applications/reconstruct/src/main.cpp
+++ b/applications/reconstruct/src/main.cpp
@@ -228,13 +228,13 @@ static void run(ftl::Configurable *root) {
// -------------------------------------------------------------------------
- stream->setLatency(5); // FIXME: This depends on source!?
- stream->add(group);
+ stream->setLatency(6); // FIXME: This depends on source!?
+ //stream->add(group);
stream->run();
bool busy = false;
- group->setLatency(5);
+ group->setLatency(4);
group->setName("ReconGroup");
group->sync([splat,virt,&busy,&slave,&scene_A,&scene_B,&align](ftl::rgbd::FrameSet &fs) -> bool {
//cudaSetDevice(scene->getCUDADevice());
diff --git a/components/codecs/CMakeLists.txt b/components/codecs/CMakeLists.txt
index db41431d4..63e311b24 100644
--- a/components/codecs/CMakeLists.txt
+++ b/components/codecs/CMakeLists.txt
@@ -7,6 +7,7 @@ set(CODECSRC
src/generate.cpp
src/writer.cpp
src/reader.cpp
+ src/channels.cpp
)
if (HAVE_NVPIPE)
diff --git a/components/codecs/include/ftl/codecs/channels.hpp b/components/codecs/include/ftl/codecs/channels.hpp
index 8ef470972..d1a0e265b 100644
--- a/components/codecs/include/ftl/codecs/channels.hpp
+++ b/components/codecs/include/ftl/codecs/channels.hpp
@@ -43,6 +43,9 @@ inline bool isVideo(Channel c) { return (int)c < 32; };
inline bool isAudio(Channel c) { return (int)c >= 32 && (int)c < 64; };
inline bool isData(Channel c) { return (int)c >= 64; };
+std::string name(Channel c);
+int type(Channel c);
+
class Channels {
public:
diff --git a/components/codecs/include/ftl/codecs/hevc.hpp b/components/codecs/include/ftl/codecs/hevc.hpp
new file mode 100644
index 000000000..f658635d6
--- /dev/null
+++ b/components/codecs/include/ftl/codecs/hevc.hpp
@@ -0,0 +1,112 @@
+#ifndef _FTL_CODECS_HEVC_HPP_
+#define _FTL_CODECS_HEVC_HPP_
+
+namespace ftl {
+namespace codecs {
+
+/**
+ * H.265 / HEVC codec utility functions.
+ */
+namespace hevc {
+
+/**
+ * HEVC Network Abstraction Layer Unit types.
+ */
+enum class NALType : int {
+ CODED_SLICE_TRAIL_N = 0,
+ CODED_SLICE_TRAIL_R = 1,
+
+ CODED_SLICE_TSA_N = 2,
+ CODED_SLICE_TSA_R = 3,
+
+ CODED_SLICE_STSA_N = 4,
+ CODED_SLICE_STSA_R = 5,
+
+ CODED_SLICE_RADL_N = 6,
+ CODED_SLICE_RADL_R = 7,
+
+ CODED_SLICE_RASL_N = 8,
+ CODED_SLICE_RASL_R = 9,
+
+ RESERVED_VCL_N10 = 10,
+ RESERVED_VCL_R11 = 11,
+ RESERVED_VCL_N12 = 12,
+ RESERVED_VCL_R13 = 13,
+ RESERVED_VCL_N14 = 14,
+ RESERVED_VCL_R15 = 15,
+
+ CODED_SLICE_BLA_W_LP = 16,
+ CODED_SLICE_BLA_W_RADL = 17,
+ CODED_SLICE_BLA_N_LP = 18,
+ CODED_SLICE_IDR_W_RADL = 19,
+ CODED_SLICE_IDR_N_LP = 20,
+ CODED_SLICE_CRA = 21,
+ RESERVED_IRAP_VCL22 = 22,
+ RESERVED_IRAP_VCL23 = 23,
+
+ RESERVED_VCL24 = 24,
+ RESERVED_VCL25 = 25,
+ RESERVED_VCL26 = 26,
+ RESERVED_VCL27 = 27,
+ RESERVED_VCL28 = 28,
+ RESERVED_VCL29 = 29,
+ RESERVED_VCL30 = 30,
+ RESERVED_VCL31 = 31,
+
+ VPS = 32,
+ SPS = 33,
+ PPS = 34,
+ ACCESS_UNIT_DELIMITER = 35,
+ EOS = 36,
+ EOB = 37,
+ FILLER_DATA = 38,
+ PREFIX_SEI = 39,
+ SUFFIX_SEI = 40,
+
+ RESERVED_NVCL41 = 41,
+ RESERVED_NVCL42 = 42,
+ RESERVED_NVCL43 = 43,
+ RESERVED_NVCL44 = 44,
+ RESERVED_NVCL45 = 45,
+ RESERVED_NVCL46 = 46,
+ RESERVED_NVCL47 = 47,
+ UNSPECIFIED_48 = 48,
+ UNSPECIFIED_49 = 49,
+ UNSPECIFIED_50 = 50,
+ UNSPECIFIED_51 = 51,
+ UNSPECIFIED_52 = 52,
+ UNSPECIFIED_53 = 53,
+ UNSPECIFIED_54 = 54,
+ UNSPECIFIED_55 = 55,
+ UNSPECIFIED_56 = 56,
+ UNSPECIFIED_57 = 57,
+ UNSPECIFIED_58 = 58,
+ UNSPECIFIED_59 = 59,
+ UNSPECIFIED_60 = 60,
+ UNSPECIFIED_61 = 61,
+ UNSPECIFIED_62 = 62,
+ UNSPECIFIED_63 = 63,
+ INVALID = 64
+};
+
+/**
+ * Extract the NAL unit type from the first NAL header.
+ * With NvPipe, the 5th byte contains the NAL Unit header.
+ */
+inline NALType getNALType(const std::vector<uint8_t> &data) {
+ return static_cast<NALType>((data[4] >> 1) & 0x3F);
+}
+
+/**
+ * Check the HEVC bitstream for an I-Frame. With NvPipe, all I-Frames start
+ * with a VPS NAL unit so just check for this.
+ */
+inline bool isIFrame(const std::vector<uint8_t> &data) {
+ return getNALType(data) == NALType::VPS;
+}
+
+}
+}
+}
+
+#endif // _FTL_CODECS_HEVC_HPP_
diff --git a/components/codecs/include/ftl/codecs/nvpipe_decoder.hpp b/components/codecs/include/ftl/codecs/nvpipe_decoder.hpp
index c13b26ec0..75807f05e 100644
--- a/components/codecs/include/ftl/codecs/nvpipe_decoder.hpp
+++ b/components/codecs/include/ftl/codecs/nvpipe_decoder.hpp
@@ -23,6 +23,7 @@ class NvPipeDecoder : public ftl::codecs::Decoder {
bool is_float_channel_;
ftl::codecs::definition_t last_definition_;
MUTEX mutex_;
+ bool seen_iframe_;
};
}
diff --git a/components/codecs/include/ftl/codecs/reader.hpp b/components/codecs/include/ftl/codecs/reader.hpp
index 9e607c77f..cdc50cad3 100644
--- a/components/codecs/include/ftl/codecs/reader.hpp
+++ b/components/codecs/include/ftl/codecs/reader.hpp
@@ -45,7 +45,7 @@ class Reader {
private:
std::istream *stream_;
msgpack::unpacker buffer_;
- std::tuple<StreamPacket,Packet> data_;
+ std::list<std::tuple<StreamPacket,Packet>> data_;
bool has_data_;
int64_t timestart_;
bool playing_;
diff --git a/components/codecs/src/channels.cpp b/components/codecs/src/channels.cpp
new file mode 100644
index 000000000..25b06b533
--- /dev/null
+++ b/components/codecs/src/channels.cpp
@@ -0,0 +1,93 @@
+#include <ftl/codecs/channels.hpp>
+
+#include <opencv2/opencv.hpp>
+
+struct ChannelInfo {
+ const char *name;
+ int type;
+};
+
+static ChannelInfo info[] = {
+ "Colour", CV_8UC3,
+ "Depth", CV_32F,
+ "Right", CV_8UC3,
+ "DepthRight", CV_32F,
+ "Deviation", CV_32F,
+ "Normals", CV_32FC4,
+ "Points", CV_32FC4,
+ "Confidence", CV_32F,
+ "EnergyVector", CV_32FC4,
+ "Flow", CV_32F,
+ "Energy", CV_32F,
+ "Mask", CV_32S,
+ "Density", CV_32F,
+ "LeftGray", CV_8U,
+ "RightGray", CV_8U,
+ "Overlay1", CV_8UC3,
+
+ "NoName", 0,
+ "NoName", 0,
+ "NoName", 0,
+ "NoName", 0,
+ "NoName", 0,
+ "NoName", 0,
+ "NoName", 0,
+ "NoName", 0,
+ "NoName", 0,
+ "NoName", 0,
+ "NoName", 0,
+ "NoName", 0,
+ "NoName", 0,
+ "NoName", 0,
+ "NoName", 0,
+ "NoName", 0,
+
+ "AudioLeft", 0,
+ "AudioRight", 0,
+
+ "NoName", 0,
+ "NoName", 0,
+ "NoName", 0,
+ "NoName", 0,
+ "NoName", 0,
+ "NoName", 0,
+ "NoName", 0,
+ "NoName", 0,
+ "NoName", 0,
+ "NoName", 0,
+ "NoName", 0,
+ "NoName", 0,
+ "NoName", 0,
+ "NoName", 0,
+ "NoName", 0,
+ "NoName", 0,
+ "NoName", 0,
+ "NoName", 0,
+ "NoName", 0,
+ "NoName", 0,
+ "NoName", 0,
+ "NoName", 0,
+ "NoName", 0,
+ "NoName", 0,
+ "NoName", 0,
+ "NoName", 0,
+ "NoName", 0,
+ "NoName", 0,
+ "NoName", 0,
+ "NoName", 0,
+
+ "Configuration", 0,
+ "Calibration", 0,
+ "Pose", 0,
+ "Data", 0
+};
+
+std::string ftl::codecs::name(Channel c) {
+ if (c == Channel::None) return "None";
+ else return info[(int)c].name;
+}
+
+int ftl::codecs::type(Channel c) {
+ if (c == Channel::None) return 0;
+ else return info[(int)c].type;
+}
diff --git a/components/codecs/src/nvpipe_decoder.cpp b/components/codecs/src/nvpipe_decoder.cpp
index fefd5ead5..b5e358388 100644
--- a/components/codecs/src/nvpipe_decoder.cpp
+++ b/components/codecs/src/nvpipe_decoder.cpp
@@ -3,6 +3,7 @@
#include <loguru.hpp>
#include <ftl/cuda_util.hpp>
+#include <ftl/codecs/hevc.hpp>
//#include <cuda_runtime.h>
#include <opencv2/core/cuda/common.hpp>
@@ -11,6 +12,7 @@ using ftl::codecs::NvPipeDecoder;
NvPipeDecoder::NvPipeDecoder() {
nv_decoder_ = nullptr;
+ seen_iframe_ = false;
}
NvPipeDecoder::~NvPipeDecoder() {
@@ -46,13 +48,22 @@ bool NvPipeDecoder::decode(const ftl::codecs::Packet &pkt, cv::Mat &out) {
//LOG(INFO) << "Bitrate=" << (int)bitrate << " width=" << ABRController::getColourWidth(bitrate);
LOG(FATAL) << "Could not create decoder: " << NvPipe_GetError(NULL);
} else {
- LOG(INFO) << "Decoder created";
+ DLOG(INFO) << "Decoder created";
}
+
+ seen_iframe_ = false;
}
// TODO: (Nick) Move to member variable to prevent re-creation
cv::Mat tmp(cv::Size(ftl::codecs::getWidth(pkt.definition),ftl::codecs::getHeight(pkt.definition)), (is_float_frame) ? CV_16U : CV_8UC4);
+ if (pkt.codec == ftl::codecs::codec_t::HEVC) {
+ // Obtain NAL unit type
+ if (ftl::codecs::hevc::isIFrame(pkt.data)) seen_iframe_ = true;
+ }
+
+ if (!seen_iframe_) return false;
+
int rc = NvPipe_Decode(nv_decoder_, pkt.data.data(), pkt.data.size(), tmp.data, tmp.cols, tmp.rows);
if (rc == 0) LOG(ERROR) << "NvPipe decode error: " << NvPipe_GetError(nv_decoder_);
diff --git a/components/codecs/src/reader.cpp b/components/codecs/src/reader.cpp
index b2d146422..41d0697f5 100644
--- a/components/codecs/src/reader.cpp
+++ b/components/codecs/src/reader.cpp
@@ -34,14 +34,18 @@ bool Reader::read(int64_t ts, const std::function<void(const ftl::codecs::Stream
std::unique_lock<std::mutex> lk(mtx_, std::defer_lock);
if (!lk.try_lock()) return true;
- if (has_data_ && get<0>(data_).timestamp <= ts) {
- f(get<0>(data_), get<1>(data_));
- has_data_ = false;
- } else if (has_data_) {
- return false;
+ // Check buffer first for frames already read
+ for (auto i = data_.begin(); i != data_.end();) {
+ if (get<0>(*i).timestamp <= ts) {
+ f(get<0>(*i), get<1>(*i));
+ i = data_.erase(i);
+ } else {
+ ++i;
+ }
}
bool partial = false;
+ int64_t extended_ts = ts + 200; // Buffer 200ms ahead
while (playing_ && stream_->good() || buffer_.nonparsed_size() > 0) {
if (buffer_.nonparsed_size() == 0 || (partial && buffer_.nonparsed_size() < 10000000)) {
@@ -50,7 +54,7 @@ bool Reader::read(int64_t ts, const std::function<void(const ftl::codecs::Stream
//if (stream_->bad()) return false;
int bytes = stream_->gcount();
- if (bytes == 0) return false;
+ if (bytes == 0) break;
buffer_.buffer_consumed(bytes);
partial = false;
}
@@ -61,10 +65,10 @@ bool Reader::read(int64_t ts, const std::function<void(const ftl::codecs::Stream
continue;
}
- std::tuple<StreamPacket,Packet> data;
+ //std::tuple<StreamPacket,Packet> data;
msgpack::object obj = msg.get();
try {
- obj.convert(data);
+ obj.convert(data_.emplace_back());
} catch (std::exception &e) {
LOG(INFO) << "Corrupt message: " << buffer_.nonparsed_size();
//partial = true;
@@ -72,19 +76,25 @@ bool Reader::read(int64_t ts, const std::function<void(const ftl::codecs::Stream
return false;
}
+ auto &data = data_.back();
+
// Adjust timestamp
get<0>(data).timestamp += timestart_;
+ // TODO: Need to read ahead a few frames because there may be a
+ // smaller timestamp after this one... requires a buffer. Ideally this
+ // should be resolved during the write process.
if (get<0>(data).timestamp <= ts) {
f(get<0>(data),get<1>(data));
- } else {
- data_ = data;
- has_data_ = true;
+ data_.pop_back();
+ } else if (get<0>(data).timestamp > extended_ts) {
+ //data_ = data;
+ //has_data_ = true;
return true;
}
}
- return false;
+ return data_.size() > 0;
}
bool Reader::read(int64_t ts) {
diff --git a/components/renderers/cpp/src/splat_render.cpp b/components/renderers/cpp/src/splat_render.cpp
index 9a34d4679..7cba3d7b8 100644
--- a/components/renderers/cpp/src/splat_render.cpp
+++ b/components/renderers/cpp/src/splat_render.cpp
@@ -301,6 +301,8 @@ bool Splatter::render(ftl::rgbd::VirtualSource *src, ftl::rgbd::Frame &out) {
SHARED_LOCK(scene_->mtx, lk);
if (!src->isReady()) return false;
+ scene_->upload(Channel::Colour + Channel::Depth, stream_);
+
const auto &camera = src->parameters();
//cudaSafeCall(cudaSetDevice(scene_->getCUDADevice()));
diff --git a/components/rgbd-sources/include/ftl/rgbd/detail/netframe.hpp b/components/rgbd-sources/include/ftl/rgbd/detail/netframe.hpp
index f01154e12..1abb624c9 100644
--- a/components/rgbd-sources/include/ftl/rgbd/detail/netframe.hpp
+++ b/components/rgbd-sources/include/ftl/rgbd/detail/netframe.hpp
@@ -17,8 +17,9 @@ struct NetFrame {
cv::Mat channel1;
cv::Mat channel2;
volatile int64_t timestamp;
- std::atomic<int> chunk_count;
- int chunk_total;
+ std::atomic<int> chunk_count[2];
+ std::atomic<int> channel_count;
+ int chunk_total[2];
std::atomic<int> tx_size;
int64_t tx_latency;
MUTEX mtx;
diff --git a/components/rgbd-sources/include/ftl/rgbd/frame.hpp b/components/rgbd-sources/include/ftl/rgbd/frame.hpp
index e6025b41a..5e38fe5cc 100644
--- a/components/rgbd-sources/include/ftl/rgbd/frame.hpp
+++ b/components/rgbd-sources/include/ftl/rgbd/frame.hpp
@@ -215,10 +215,10 @@ ftl::cuda::TextureObject<T> &Frame::createTexture(ftl::codecs::Channel c, const
// TODO: Check tex cvType
if (m.tex.devicePtr() == nullptr) {
- LOG(INFO) << "Creating texture object";
+ //LOG(INFO) << "Creating texture object";
m.tex = ftl::cuda::TextureObject<T>(m.gpu);
} else if (m.tex.cvType() != ftl::traits::OpenCVType<T>::value || m.tex.width() != m.gpu.cols || m.tex.height() != m.gpu.rows) {
- LOG(INFO) << "Recreating texture object";
+ LOG(INFO) << "Recreating texture object for '" << ftl::codecs::name(c) << "'";
m.tex.free();
m.tex = ftl::cuda::TextureObject<T>(m.gpu);
}
@@ -248,9 +248,10 @@ ftl::cuda::TextureObject<T> &Frame::createTexture(ftl::codecs::Channel c) {
// TODO: Check tex cvType
if (m.tex.devicePtr() == nullptr) {
- LOG(INFO) << "Creating texture object";
+ //LOG(INFO) << "Creating texture object";
m.tex = ftl::cuda::TextureObject<T>(m.gpu);
} else if (m.tex.cvType() != ftl::traits::OpenCVType<T>::value || m.tex.width() != m.gpu.cols || m.tex.height() != m.gpu.rows || m.tex.devicePtr() != m.gpu.data) {
+ LOG(INFO) << "Recreating texture object for '" << ftl::codecs::name(c) << "'";
m.tex.free();
m.tex = ftl::cuda::TextureObject<T>(m.gpu);
}
diff --git a/components/rgbd-sources/src/file_source.cpp b/components/rgbd-sources/src/file_source.cpp
index bef04d6a4..38acf1a7c 100644
--- a/components/rgbd-sources/src/file_source.cpp
+++ b/components/rgbd-sources/src/file_source.cpp
@@ -1,5 +1,6 @@
#include "file_source.hpp"
+#include <ftl/codecs/hevc.hpp>
#include <ftl/timer.hpp>
using ftl::rgbd::detail::FileSource;
@@ -15,7 +16,7 @@ void FileSource::_createDecoder(int ix, const ftl::codecs::Packet &pkt) {
}
}
- LOG(INFO) << "Create a decoder: " << ix;
+ DLOG(INFO) << "Create a decoder: " << ix;
decoders_[ix] = ftl::codecs::allocateDecoder(pkt);
}
@@ -28,8 +29,10 @@ FileSource::FileSource(ftl::rgbd::Source *s, ftl::codecs::Reader *r, int sid) :
cache_write_ = 0;
realtime_ = host_->value("realtime", true);
timestamp_ = r->getStartTime();
+ sourceid_ = sid;
r->onPacket(sid, [this](const ftl::codecs::StreamPacket &spkt, ftl::codecs::Packet &pkt) {
+ host_->notifyRaw(spkt, pkt);
if (pkt.codec == codec_t::POSE) {
Eigen::Matrix4d p = Eigen::Map<Eigen::Matrix4d>((double*)pkt.data.data());
host_->setPose(p);
@@ -40,10 +43,7 @@ FileSource::FileSource(ftl::rgbd::Source *s, ftl::codecs::Reader *r, int sid) :
has_calibration_ = true;
} else {
if (pkt.codec == codec_t::HEVC) {
- // Obtain NAL unit type
- int nal_type = (pkt.data[4] >> 1) & 0x3F;
- // A type of 32 = VPS unit, hence I-Frame in this case so skip past packets
- if (nal_type == 32) _removeChannel(spkt.channel);
+ if (ftl::codecs::hevc::isIFrame(pkt.data)) _removeChannel(spkt.channel);
}
cache_[cache_write_].emplace_back();
auto &c = cache_[cache_write_].back();
@@ -93,12 +93,23 @@ void FileSource::swap() {
bool FileSource::compute(int n, int b) {
if (cache_read_ < 0) return false;
+ if (cache_[cache_read_].size() == 0) return false;
+
+ int64_t lastts = 0;
+ int lastc = 0;
for (auto i=cache_[cache_read_].begin(); i!=cache_[cache_read_].end(); ++i) {
auto &c = *i;
+ if (c.spkt.timestamp > lastts) {
+ lastts = c.spkt.timestamp;
+ lastc = 1;
+ } else if (c.spkt.timestamp == lastts) {
+ lastc++;
+ }
+
if (c.spkt.channel == Channel::Colour) {
- rgb_.create(cv::Size(ftl::codecs::getWidth(c.pkt.definition),ftl::codecs::getHeight(c.pkt.definition)), CV_8UC4);
+ rgb_.create(cv::Size(ftl::codecs::getWidth(c.pkt.definition),ftl::codecs::getHeight(c.pkt.definition)), CV_8UC3);
} else {
depth_.create(cv::Size(ftl::codecs::getWidth(c.pkt.definition),ftl::codecs::getHeight(c.pkt.definition)), CV_32F);
}
@@ -112,6 +123,11 @@ bool FileSource::compute(int n, int b) {
}
}
+ if (lastc != 2) {
+ LOG(ERROR) << "Channels not in sync (" << sourceid_ << "): " << lastts;
+ return false;
+ }
+
cache_[cache_read_].clear();
if (rgb_.empty() || depth_.empty()) return false;
diff --git a/components/rgbd-sources/src/file_source.hpp b/components/rgbd-sources/src/file_source.hpp
index a9bdc4b36..1f2241ea0 100644
--- a/components/rgbd-sources/src/file_source.hpp
+++ b/components/rgbd-sources/src/file_source.hpp
@@ -38,6 +38,7 @@ class FileSource : public detail::Source {
std::list<PacketPair> cache_[2];
int cache_read_;
int cache_write_;
+ int sourceid_;
ftl::codecs::Decoder *decoders_[2];
diff --git a/components/rgbd-sources/src/group.cpp b/components/rgbd-sources/src/group.cpp
index 278ce8285..4e178c024 100644
--- a/components/rgbd-sources/src/group.cpp
+++ b/components/rgbd-sources/src/group.cpp
@@ -257,8 +257,9 @@ ftl::rgbd::FrameSet *Group::_getFrameset(int f) {
int idx = (head_+kFrameBufferSize-i)%kFrameBufferSize;
if (framesets_[idx].timestamp == lookfor && framesets_[idx].count != sources_.size()) {
- LOG(INFO) << "Required frame not complete (timestamp=" << (framesets_[idx].timestamp) << " buffer=" << i << ")";
+ LOG(WARNING) << "Required frame not complete in '" << name_ << "' (timestamp=" << (framesets_[idx].timestamp) << " buffer=" << i << ")";
//framesets_[idx].stale = true;
+ //return &framesets_[idx];
continue;
}
@@ -302,6 +303,8 @@ void Group::_addFrameset(int64_t timestamp) {
//framesets_[head_].channel2.resize(sources_.size());
framesets_[head_].frames.resize(sources_.size());
+ for (auto &f : framesets_[head_].frames) f.reset();
+
if (framesets_[head_].sources.size() != sources_.size()) {
framesets_[head_].sources.clear();
for (auto s : sources_) framesets_[head_].sources.push_back(s);
@@ -333,6 +336,8 @@ void Group::_addFrameset(int64_t timestamp) {
//framesets_[head_].channel2.resize(sources_.size());
framesets_[head_].frames.resize(sources_.size());
+ for (auto &f : framesets_[head_].frames) f.reset();
+
if (framesets_[head_].sources.size() != sources_.size()) {
framesets_[head_].sources.clear();
for (auto s : sources_) framesets_[head_].sources.push_back(s);
diff --git a/components/rgbd-sources/src/net.cpp b/components/rgbd-sources/src/net.cpp
index 91a990090..dd97e9618 100644
--- a/components/rgbd-sources/src/net.cpp
+++ b/components/rgbd-sources/src/net.cpp
@@ -46,8 +46,11 @@ NetFrame &NetFrameQueue::getFrame(int64_t ts, const cv::Size &s, int c1type, int
for (auto &f : frames_) {
if (f.timestamp == -1) {
f.timestamp = ts;
- f.chunk_count = 0;
- f.chunk_total = 0;
+ f.chunk_count[0] = 0;
+ f.chunk_count[1] = 0;
+ f.chunk_total[0] = 0;
+ f.chunk_total[1] = 0;
+ f.channel_count = 0;
f.tx_size = 0;
f.channel1.create(s, c1type);
f.channel2.create(s, c2type);
@@ -63,8 +66,11 @@ NetFrame &NetFrameQueue::getFrame(int64_t ts, const cv::Size &s, int c1type, int
// Force release of frame!
if (f.timestamp == oldest) {
f.timestamp = ts;
- f.chunk_count = 0;
- f.chunk_total = 0;
+ f.chunk_count[0] = 0;
+ f.chunk_count[1] = 0;
+ f.chunk_total[0] = 0;
+ f.chunk_total[1] = 0;
+ f.channel_count = 0;
f.tx_size = 0;
f.channel1.create(s, c1type);
f.channel2.create(s, c2type);
@@ -246,16 +252,17 @@ void NetSource::_recvPacket(short ttimeoff, const ftl::codecs::StreamPacket &spk
host_->notifyRaw(spkt, pkt);
const ftl::codecs::Channel chan = host_->getChannel();
- const ftl::codecs::Channel rchan = spkt.channel; // & 0x1;
+ const ftl::codecs::Channel rchan = spkt.channel;
+ const int channum = (rchan == Channel::Colour) ? 0 : 1;
- NetFrame &frame = queue_.getFrame(spkt.timestamp, cv::Size(params_.width, params_.height), CV_8UC4, (isFloatChannel(chan) ? CV_32FC1 : CV_8UC4));
+ NetFrame &frame = queue_.getFrame(spkt.timestamp, cv::Size(params_.width, params_.height), CV_8UC3, (isFloatChannel(chan) ? CV_32FC1 : CV_8UC3));
// Update frame statistics
frame.tx_size += pkt.data.size();
// Only decode if this channel is wanted.
if (rchan == Channel::Colour || rchan == chan) {
- _createDecoder((rchan == Channel::Colour) ? 0 : 1, pkt);
+ _createDecoder(channum, pkt);
auto *decoder = (rchan == Channel::Colour) ? decoder_c1_ : decoder_c2_;
if (!decoder) {
LOG(ERROR) << "No frame decoder available";
@@ -282,30 +289,31 @@ void NetSource::_recvPacket(short ttimeoff, const ftl::codecs::StreamPacket &spk
return;
}
- // Calculate how many packets to expect for this frame
- if (frame.chunk_total == 0) {
- // Getting a second channel first means expect double packets
- // FIXME: Assumes each packet has same number of blocks!
- frame.chunk_total = pkt.block_total * spkt.channel_count;
+ // Calculate how many packets to expect for this channel
+ if (frame.chunk_total[channum] == 0) {
+ frame.chunk_total[channum] = pkt.block_total;
}
- ++frame.chunk_count;
+ ++frame.chunk_count[channum];
+ ++frame.channel_count;
- if (frame.chunk_count > frame.chunk_total) LOG(FATAL) << "TOO MANY CHUNKS";
+ if (frame.chunk_count[channum] > frame.chunk_total[channum]) LOG(FATAL) << "TOO MANY CHUNKS";
// Capture tx time of first received chunk
- if (frame.chunk_count == 1) {
+ if (frame.channel_count == 1 && frame.chunk_count[channum] == 1) {
UNIQUE_LOCK(frame.mtx, flk);
- if (frame.chunk_count == 1) {
+ if (frame.chunk_count[channum] == 1) {
frame.tx_latency = int64_t(ttimeoff);
}
}
- // Last chunk now received
- if (frame.chunk_count == frame.chunk_total) {
+ // Last chunk of both channels now received
+ if (frame.channel_count == spkt.channel_count &&
+ frame.chunk_count[0] == frame.chunk_total[0] &&
+ frame.chunk_count[1] == frame.chunk_total[1]) {
UNIQUE_LOCK(frame.mtx, flk);
- if (frame.timestamp >= 0 && frame.chunk_count == frame.chunk_total) {
+ if (frame.timestamp >= 0 && frame.chunk_count[0] == frame.chunk_total[0] && frame.chunk_count[1] == frame.chunk_total[1]) {
timestamp_ = frame.timestamp;
frame.tx_latency = now-(spkt.timestamp+frame.tx_latency);
diff --git a/components/rgbd-sources/src/streamer.cpp b/components/rgbd-sources/src/streamer.cpp
index cab8f99d8..c446445c9 100644
--- a/components/rgbd-sources/src/streamer.cpp
+++ b/components/rgbd-sources/src/streamer.cpp
@@ -48,6 +48,7 @@ Streamer::Streamer(nlohmann::json &config, Universe *net)
//group_.setFPS(value("fps", 20));
group_.setLatency(4);
+ group_.setName("StreamGroup");
compress_level_ = value("compression", 1);
--
GitLab