From f8fe26d620a724fb6760dec5b102d9322411973c Mon Sep 17 00:00:00 2001
From: Nicolas Pope <nicolas.pope@utu.fi>
Date: Sat, 26 Oct 2019 13:12:46 +0300
Subject: [PATCH] Implements #218 red blue swap
---
.../codecs/include/ftl/codecs/bitrates.hpp | 4 +-
.../codecs/include/ftl/codecs/encoder.hpp | 7 ++-
.../include/ftl/codecs/nvpipe_encoder.hpp | 5 ++
.../include/ftl/codecs/opencv_encoder.hpp | 2 +
.../codecs/include/ftl/codecs/packet.hpp | 2 +-
.../codecs/include/ftl/codecs/reader.hpp | 1 +
components/codecs/src/encoder.cpp | 4 +-
components/codecs/src/nvpipe_decoder.cpp | 17 +++++--
components/codecs/src/nvpipe_encoder.cpp | 49 +++++++++++++++++--
components/codecs/src/opencv_encoder.cpp | 8 +++
components/codecs/src/reader.cpp | 7 +++
.../include/ftl/rgbd/streamer.hpp | 1 +
components/rgbd-sources/src/streamer.cpp | 5 +-
13 files changed, 96 insertions(+), 16 deletions(-)
diff --git a/components/codecs/include/ftl/codecs/bitrates.hpp b/components/codecs/include/ftl/codecs/bitrates.hpp
index fd50b9c86..d34ede8ad 100644
--- a/components/codecs/include/ftl/codecs/bitrates.hpp
+++ b/components/codecs/include/ftl/codecs/bitrates.hpp
@@ -24,7 +24,9 @@ enum struct codec_t : uint8_t {
POSE, // 4x4 eigen matrix
MSGPACK,
STRING, // Null terminated string
- RAW // Some unknown binary format
+ RAW, // Some unknown binary format
+
+ Any = 255
};
/**
diff --git a/components/codecs/include/ftl/codecs/encoder.hpp b/components/codecs/include/ftl/codecs/encoder.hpp
index 560810b84..fed8d9575 100644
--- a/components/codecs/include/ftl/codecs/encoder.hpp
+++ b/components/codecs/include/ftl/codecs/encoder.hpp
@@ -33,7 +33,8 @@ enum class device_t {
*/
Encoder *allocateEncoder(
ftl::codecs::definition_t maxdef=ftl::codecs::definition_t::HD1080,
- ftl::codecs::device_t dev=ftl::codecs::device_t::Any);
+ ftl::codecs::device_t dev=ftl::codecs::device_t::Any,
+ ftl::codecs::codec_t codec=ftl::codecs::codec_t::Any);
/**
* Release an encoder to be reused by some other stream.
@@ -47,7 +48,7 @@ void free(Encoder *&e);
class Encoder {
public:
friend Encoder *allocateEncoder(ftl::codecs::definition_t,
- ftl::codecs::device_t);
+ ftl::codecs::device_t, ftl::codecs::codec_t);
friend void free(Encoder *&);
public:
@@ -86,6 +87,8 @@ class Encoder {
virtual void reset() {}
+ virtual bool supports(ftl::codecs::codec_t codec)=0;
+
protected:
bool available;
const ftl::codecs::definition_t max_definition;
diff --git a/components/codecs/include/ftl/codecs/nvpipe_encoder.hpp b/components/codecs/include/ftl/codecs/nvpipe_encoder.hpp
index 3f8de66cf..c2fe379dc 100644
--- a/components/codecs/include/ftl/codecs/nvpipe_encoder.hpp
+++ b/components/codecs/include/ftl/codecs/nvpipe_encoder.hpp
@@ -25,11 +25,16 @@ class NvPipeEncoder : public ftl::codecs::Encoder {
void reset();
+ bool supports(ftl::codecs::codec_t codec) override;
+
+ static constexpr int kFlagRGB = 0x00000001;
+
private:
NvPipe *nvenc_;
definition_t current_definition_;
bool is_float_channel_;
bool was_reset_;
+ ftl::codecs::codec_t preference_;
bool _encoderMatch(const cv::Mat &in, definition_t def);
bool _createEncoder(const cv::Mat &in, definition_t def, bitrate_t rate);
diff --git a/components/codecs/include/ftl/codecs/opencv_encoder.hpp b/components/codecs/include/ftl/codecs/opencv_encoder.hpp
index a34f8811d..aabe9247c 100644
--- a/components/codecs/include/ftl/codecs/opencv_encoder.hpp
+++ b/components/codecs/include/ftl/codecs/opencv_encoder.hpp
@@ -28,6 +28,8 @@ class OpenCVEncoder : public ftl::codecs::Encoder {
bool encode(const cv::Mat &in, ftl::codecs::definition_t definition, ftl::codecs::bitrate_t bitrate,
const std::function<void(const ftl::codecs::Packet&)>&) override;
+ bool supports(ftl::codecs::codec_t codec) override;
+
//bool encode(const cv::cuda::GpuMat &in, std::vector<uint8_t> &out, bitrate_t bix, bool);
private:
diff --git a/components/codecs/include/ftl/codecs/packet.hpp b/components/codecs/include/ftl/codecs/packet.hpp
index 89df44f9e..f2fa53c8f 100644
--- a/components/codecs/include/ftl/codecs/packet.hpp
+++ b/components/codecs/include/ftl/codecs/packet.hpp
@@ -16,7 +16,7 @@ namespace codecs {
*/
struct Header {
const char magic[4] = {'F','T','L','F'};
- uint8_t version = 2;
+ uint8_t version = 3;
};
/**
diff --git a/components/codecs/include/ftl/codecs/reader.hpp b/components/codecs/include/ftl/codecs/reader.hpp
index cdc50cad3..69e19f0a5 100644
--- a/components/codecs/include/ftl/codecs/reader.hpp
+++ b/components/codecs/include/ftl/codecs/reader.hpp
@@ -49,6 +49,7 @@ class Reader {
bool has_data_;
int64_t timestart_;
bool playing_;
+ int version_;
MUTEX mtx_;
diff --git a/components/codecs/src/encoder.cpp b/components/codecs/src/encoder.cpp
index 834c003b7..1863222d6 100644
--- a/components/codecs/src/encoder.cpp
+++ b/components/codecs/src/encoder.cpp
@@ -11,6 +11,7 @@ using ftl::codecs::bitrate_t;
using ftl::codecs::definition_t;
using ftl::codecs::preset_t;
using ftl::codecs::device_t;
+using ftl::codecs::codec_t;
using ftl::codecs::kPresetBest;
using ftl::codecs::kPresetWorst;
using ftl::codecs::kPresetLQThreshold;
@@ -34,7 +35,7 @@ using namespace ftl::codecs::internal;
static MUTEX mutex;
Encoder *ftl::codecs::allocateEncoder(ftl::codecs::definition_t maxdef,
- ftl::codecs::device_t dev) {
+ ftl::codecs::device_t dev, ftl::codecs::codec_t codec) {
UNIQUE_LOCK(mutex, lk);
if (!has_been_init) init_encoders();
@@ -43,6 +44,7 @@ Encoder *ftl::codecs::allocateEncoder(ftl::codecs::definition_t maxdef,
if (!e->available) continue;
if (dev != device_t::Any && dev != e->device) continue;
if (maxdef != definition_t::Any && (maxdef < e->max_definition || maxdef > e->min_definition)) continue;
+ if (codec != codec_t::Any && !e->supports(codec)) continue;
e->available = false;
return e;
diff --git a/components/codecs/src/nvpipe_decoder.cpp b/components/codecs/src/nvpipe_decoder.cpp
index 2ecd69e7c..97985cd20 100644
--- a/components/codecs/src/nvpipe_decoder.cpp
+++ b/components/codecs/src/nvpipe_decoder.cpp
@@ -50,7 +50,7 @@ void cropAndScaleUp(cv::Mat &in, cv::Mat &out) {
bool NvPipeDecoder::decode(const ftl::codecs::Packet &pkt, cv::Mat &out) {
cudaSetDevice(0);
UNIQUE_LOCK(mutex_,lk);
- if (pkt.codec != codec_t::HEVC) return false;
+ if (pkt.codec != codec_t::HEVC && pkt.codec != codec_t::H264) return false;
bool is_float_frame = out.type() == CV_32F;
// Is the previous decoder still valid for current resolution and type?
@@ -68,7 +68,7 @@ bool NvPipeDecoder::decode(const ftl::codecs::Packet &pkt, cv::Mat &out) {
if (nv_decoder_ == nullptr) {
nv_decoder_ = NvPipe_CreateDecoder(
(is_float_frame) ? NVPIPE_UINT16 : NVPIPE_RGBA32,
- NVPIPE_HEVC,
+ (pkt.codec == codec_t::HEVC) ? NVPIPE_HEVC : NVPIPE_H264,
ftl::codecs::getWidth(pkt.definition),
ftl::codecs::getHeight(pkt.definition));
if (!nv_decoder_) {
@@ -88,6 +88,7 @@ bool NvPipeDecoder::decode(const ftl::codecs::Packet &pkt, cv::Mat &out) {
// Obtain NAL unit type
if (ftl::codecs::hevc::isIFrame(pkt.data)) seen_iframe_ = true;
}
+ // TODO: Parse H264 for i-frame check
if (!seen_iframe_) return false;
@@ -106,10 +107,18 @@ bool NvPipeDecoder::decode(const ftl::codecs::Packet &pkt, cv::Mat &out) {
} else {
// Is the received frame the same size as requested output?
if (out.rows == ftl::codecs::getHeight(pkt.definition)) {
- cv::cvtColor(tmp, out, cv::COLOR_BGRA2BGR);
+ if (pkt.flags & 0x1) {
+ cv::cvtColor(tmp, out, cv::COLOR_RGBA2BGR);
+ } else {
+ cv::cvtColor(tmp, out, cv::COLOR_BGRA2BGR);
+ }
} else {
LOG(WARNING) << "Resizing decoded frame from " << tmp.size() << " to " << out.size();
- cv::cvtColor(tmp, tmp, cv::COLOR_BGRA2BGR);
+ if (pkt.flags & 0x1) {
+ cv::cvtColor(tmp, tmp, cv::COLOR_RGBA2BGR);
+ } else {
+ cv::cvtColor(tmp, tmp, cv::COLOR_BGRA2BGR);
+ }
cv::resize(tmp, out, out.size());
}
}
diff --git a/components/codecs/src/nvpipe_encoder.cpp b/components/codecs/src/nvpipe_encoder.cpp
index 80bde1f9f..28407a658 100644
--- a/components/codecs/src/nvpipe_encoder.cpp
+++ b/components/codecs/src/nvpipe_encoder.cpp
@@ -20,6 +20,7 @@ NvPipeEncoder::NvPipeEncoder(definition_t maxdef,
current_definition_ = definition_t::HD1080;
is_float_channel_ = false;
was_reset_ = false;
+ preference_ = codec_t::Any;
}
NvPipeEncoder::~NvPipeEncoder() {
@@ -30,6 +31,14 @@ void NvPipeEncoder::reset() {
was_reset_ = true;
}
+bool NvPipeEncoder::supports(ftl::codecs::codec_t codec) {
+ switch (codec) {
+ case codec_t::H264:
+ case codec_t::HEVC: preference_ = codec; return true;
+ default: return false;
+ }
+}
+
/* Check preset resolution is not better than actual resolution. */
definition_t NvPipeEncoder::_verifiedDefinition(definition_t def, const cv::Mat &in) {
int height = ftl::codecs::getHeight(def);
@@ -103,9 +112,12 @@ bool NvPipeEncoder::encode(const cv::Mat &in, definition_t odefinition, bitrate_
if (tmp.type() == CV_32F) {
tmp.convertTo(tmp, CV_16UC1, 1000);
} else if (tmp.type() == CV_8UC3) {
- cv::cvtColor(tmp, tmp, cv::COLOR_BGR2BGRA);
+ cv::cvtColor(tmp, tmp, cv::COLOR_BGR2RGBA);
+ } else if (tmp.type() == CV_8UC4) {
+ cv::cvtColor(tmp, tmp, cv::COLOR_BGRA2RGBA);
} else {
- //in.copyTo(tmp);
+ LOG(ERROR) << "Unsupported cv::Mat type in Nvidia encoder";
+ return false;
}
// scale/pad to fit output format
@@ -114,10 +126,11 @@ bool NvPipeEncoder::encode(const cv::Mat &in, definition_t odefinition, bitrate_
//std::swap(tmp, tmp2);
Packet pkt;
- pkt.codec = codec_t::HEVC;
+ pkt.codec = (preference_ == codec_t::Any) ? codec_t::HEVC : preference_;
pkt.definition = definition;
pkt.block_total = 1;
pkt.block_number = 0;
+ pkt.flags = NvPipeEncoder::kFlagRGB;
pkt.data.resize(ftl::codecs::kVideoBufferSize);
uint64_t cs = NvPipe_Encode(
@@ -147,9 +160,35 @@ bool NvPipeEncoder::_encoderMatch(const cv::Mat &in, definition_t def) {
((in.type() == CV_8UC3 || in.type() == CV_8UC4) && !is_float_channel_)) && current_definition_ == def;
}
+static uint64_t calculateBitrate(definition_t def, bitrate_t rate) {
+ float scale = 1.0f;
+ switch (rate) {
+ case bitrate_t::High : break;
+ case bitrate_t::Standard : scale = 0.5f; break;
+ case bitrate_t::Low : scale = 0.25f; break;
+ }
+
+ float bitrate = 1.0f; // Megabits
+ switch (def) {
+ case definition_t::UHD4k : bitrate = 24.0f; break;
+ case definition_t::HTC_VIVE : bitrate = 16.0f; break;
+ case definition_t::HD1080 : bitrate = 16.0f; break;
+ case definition_t::HD720 : bitrate = 8.0f; break;
+ case definition_t::SD576 :
+ case definition_t::SD480 : bitrate = 4.0f; break;
+ case definition_t::LD360 : bitrate = 2.0f; break;
+ default : bitrate = 8.0f;
+ }
+
+ return uint64_t(bitrate * 1000.0f * 1000.0f * scale);
+}
+
bool NvPipeEncoder::_createEncoder(const cv::Mat &in, definition_t def, bitrate_t rate) {
if (_encoderMatch(in, def) && nvenc_) return true;
+ uint64_t bitrate = calculateBitrate(def, rate);
+ LOG(INFO) << "Calculated bitrate: " << bitrate;
+
if (in.type() == CV_32F) is_float_channel_ = true;
else is_float_channel_ = false;
current_definition_ = def;
@@ -158,9 +197,9 @@ bool NvPipeEncoder::_createEncoder(const cv::Mat &in, definition_t def, bitrate_
const int fps = 1000/ftl::timer::getInterval();
nvenc_ = NvPipe_CreateEncoder(
(is_float_channel_) ? NVPIPE_UINT16 : NVPIPE_RGBA32,
- NVPIPE_HEVC,
+ (preference_ == codec_t::Any || preference_ == codec_t::HEVC) ? NVPIPE_HEVC : NVPIPE_H264,
(is_float_channel_) ? NVPIPE_LOSSLESS : NVPIPE_LOSSY,
- 16*1000*1000,
+ bitrate,
fps, // FPS
ftl::codecs::getWidth(def), // Output Width
ftl::codecs::getHeight(def) // Output Height
diff --git a/components/codecs/src/opencv_encoder.cpp b/components/codecs/src/opencv_encoder.cpp
index 028395b9e..1ab7a7a31 100644
--- a/components/codecs/src/opencv_encoder.cpp
+++ b/components/codecs/src/opencv_encoder.cpp
@@ -20,6 +20,14 @@ OpenCVEncoder::~OpenCVEncoder() {
}
+bool OpenCVEncoder::supports(ftl::codecs::codec_t codec) {
+ switch (codec) {
+ case codec_t::JPG:
+ case codec_t::PNG: return true;
+ default: return false;
+ }
+}
+
bool OpenCVEncoder::encode(const cv::Mat &in, definition_t definition, bitrate_t bitrate, const std::function<void(const ftl::codecs::Packet&)> &cb) {
cv::Mat tmp;
bool is_colour = in.type() != CV_32F;
diff --git a/components/codecs/src/reader.cpp b/components/codecs/src/reader.cpp
index 53225519a..3727bdd38 100644
--- a/components/codecs/src/reader.cpp
+++ b/components/codecs/src/reader.cpp
@@ -27,6 +27,8 @@ bool Reader::begin() {
(*stream_).read((char*)&ih, sizeof(ih));
}
+ version_ = h.version;
+
// Capture current time to adjust timestamps
timestart_ = (ftl::timer::get_time() / ftl::timer::getInterval()) * ftl::timer::getInterval();
playing_ = true;
@@ -86,6 +88,11 @@ bool Reader::read(int64_t ts, const std::function<void(const ftl::codecs::Stream
// Adjust timestamp
get<0>(data).timestamp += timestart_;
+ // Fix to clear flags for version 2.
+ if (version_ == 2) {
+ get<1>(data).flags = 0;
+ }
+
// TODO: Need to read ahead a few frames because there may be a
// smaller timestamp after this one... requires a buffer. Ideally this
// should be resolved during the write process.
diff --git a/components/rgbd-sources/include/ftl/rgbd/streamer.hpp b/components/rgbd-sources/include/ftl/rgbd/streamer.hpp
index cfa552a86..1fa1cb486 100644
--- a/components/rgbd-sources/include/ftl/rgbd/streamer.hpp
+++ b/components/rgbd-sources/include/ftl/rgbd/streamer.hpp
@@ -158,6 +158,7 @@ class Streamer : public ftl::Configurable {
ftl::timer::TimerHandle timer_job_;
ftl::codecs::device_t hq_devices_;
+ ftl::codecs::codec_t hq_codec_;
enum class Quality {
High,
diff --git a/components/rgbd-sources/src/streamer.cpp b/components/rgbd-sources/src/streamer.cpp
index 6e5a3273d..3eff9e24f 100644
--- a/components/rgbd-sources/src/streamer.cpp
+++ b/components/rgbd-sources/src/streamer.cpp
@@ -45,6 +45,7 @@ Streamer::Streamer(nlohmann::json &config, Universe *net)
encode_mode_ = ftl::rgbd::kEncodeVideo;
hq_devices_ = (value("disable_hardware_encode", false)) ? device_t::Software : device_t::Any;
+ hq_codec_ = value("video_codec", ftl::codecs::codec_t::Any);
//group_.setFPS(value("fps", 20));
group_.setLatency(4);
@@ -446,9 +447,9 @@ void Streamer::_process(ftl::rgbd::FrameSet &fs) {
// Do we need to do high quality encoding?
if (src->hq_count > 0) {
if (!src->hq_encoder_c1) src->hq_encoder_c1 = ftl::codecs::allocateEncoder(
- definition_t::HD1080, hq_devices_);
+ definition_t::HD1080, hq_devices_, hq_codec_);
if (!src->hq_encoder_c2 && hasChan2) src->hq_encoder_c2 = ftl::codecs::allocateEncoder(
- definition_t::HD1080, hq_devices_);
+ definition_t::HD1080, hq_devices_, hq_codec_);
// Do we have the resources to do a HQ encoding?
if (src->hq_encoder_c1 && (!hasChan2 || src->hq_encoder_c2)) {
--
GitLab