From 40079c2c63b7b41a9eb166a2fb6fa76463ac0f7e Mon Sep 17 00:00:00 2001
From: Nicolas Pope <nicolas.pope@utu.fi>
Date: Fri, 31 Jul 2020 08:51:20 +0300
Subject: [PATCH] Client bitrate selection
---
.../codecs/include/ftl/codecs/packet.hpp | 4 +-
components/codecs/src/nvidia_encoder.cpp | 42 ++++++-----
.../streams/include/ftl/streams/netstream.hpp | 1 +
.../streams/include/ftl/streams/sender.hpp | 6 +-
components/streams/src/netstream.cpp | 17 +++--
components/streams/src/sender.cpp | 71 +++++++++++--------
components/streams/test/sender_unit.cpp | 2 +-
7 files changed, 84 insertions(+), 59 deletions(-)
diff --git a/components/codecs/include/ftl/codecs/packet.hpp b/components/codecs/include/ftl/codecs/packet.hpp
index 040f8a1da..65c07580e 100644
--- a/components/codecs/include/ftl/codecs/packet.hpp
+++ b/components/codecs/include/ftl/codecs/packet.hpp
@@ -37,9 +37,9 @@ struct IndexHeader {
struct Packet {
ftl::codecs::codec_t codec;
uint8_t reserved=0;
- uint8_t frame_count; // v4+ Frames included in this packet
+ uint8_t frame_count=1; // v4+ Frames included in this packet
uint8_t bitrate=0; // v4+ For multi-bitrate encoding, 0=highest
- uint8_t flags; // Codec dependent flags (eg. I-Frame or P-Frame)
+ uint8_t flags=0; // Codec dependent flags (eg. I-Frame or P-Frame)
std::vector<uint8_t> data;
MSGPACK_DEFINE(codec, reserved, frame_count, bitrate, flags, data);
diff --git a/components/codecs/src/nvidia_encoder.cpp b/components/codecs/src/nvidia_encoder.cpp
index 62ae6b0f1..10e3d2e0b 100644
--- a/components/codecs/src/nvidia_encoder.cpp
+++ b/components/codecs/src/nvidia_encoder.cpp
@@ -1,5 +1,5 @@
#include <ftl/codecs/nvidia_encoder.hpp>
-//#include <loguru.hpp>
+#include <loguru.hpp>
#include <ftl/timer.hpp>
#include <ftl/codecs/codecs.hpp>
#include <ftl/cuda_util.hpp>
@@ -104,23 +104,27 @@ static ftl::codecs::NvidiaEncoder::Parameters generateParams(const cv::cuda::Gpu
return params;
}
+/*
+ * Using image height and a 0 to 1 rate scale, calculate a Mbps value to be
+ * used.
+ */
static uint64_t calculateBitrate(int64_t pixels, float ratescale) {
- /*float bitrate = 1.0f; // Megabits
- switch (def) {
- case definition_t::UHD4k : bitrate = 40.0f; break;
- case definition_t::HTC_VIVE : bitrate = 32.0f; break;
- case definition_t::HD1080 : bitrate = 12.0f; break;
- case definition_t::HD720 : bitrate = 8.0f; break;
- case definition_t::SD576 :
- case definition_t::SD480 : bitrate = 4.0f; break;
- case definition_t::LD360 : bitrate = 2.0f; break;
- default : bitrate = 16.0f;
- }*/
-
- float bitrate = 16.0f * float(pixels);
-
- //bitrate *= 1000.0f*1000.0f;
- float minrate = 0.05f * bitrate;
+ static constexpr float kTopResolution = 2160.0f;
+ static constexpr float kBottomResolution = 360.0f;
+ static constexpr float kTopBitrate = 40.0f; // Mbps
+ static constexpr float kBottomBitrate = 2.0f; // Mbps
+ static constexpr float kBitrateScale = 1024.0f*1024.0f;
+ static constexpr float kMinBitrateFactor = 0.05f; // 5% of max for resolution
+
+ float resolution = (float(pixels) - kBottomResolution) / (kTopResolution - kBottomResolution);
+ float bitrate = (kTopBitrate - kBottomBitrate) * resolution + kBottomBitrate;
+
+ // Limit to 80Mbps.
+ if (bitrate > 80.0f) bitrate = 80.0f;
+
+ bitrate *= kBitrateScale;
+
+ float minrate = kMinBitrateFactor * bitrate;
return uint64_t((bitrate - minrate)*ratescale + minrate);
}
@@ -210,8 +214,8 @@ bool NvidiaEncoder::_createEncoder(const cv::cuda::GpuMat &in, const ftl::codecs
Parameters params = generateParams(in, pkt);
if (nvenc_ && (params == params_)) return true;
- uint64_t bitrate = calculateBitrate(in.cols*in.rows, float(pkt.bitrate)/255.0f) * pkt.frame_count;
- //LOG(INFO) << "Calculated bitrate " << ((params.is_float) ? "(float)" : "(rgb)") << ": " << bitrate;
+ uint64_t bitrate = calculateBitrate(in.rows, float(pkt.bitrate)/255.0f);
+ LOG(INFO) << "Calculated bitrate " << (float(bitrate) / 1024.0f / 1024.0f) << "Mbps (" << int(pkt.bitrate) << ")";
params_ = params;
diff --git a/components/streams/include/ftl/streams/netstream.hpp b/components/streams/include/ftl/streams/netstream.hpp
index 36ad0d382..f53b37dd7 100644
--- a/components/streams/include/ftl/streams/netstream.hpp
+++ b/components/streams/include/ftl/streams/netstream.hpp
@@ -86,6 +86,7 @@ class Net : public Stream {
int tally_;
std::array<std::atomic<int>,32> reqtally_;
std::unordered_set<ftl::codecs::Channel> last_selected_;
+ uint8_t bitrate_=255;
ftl::Handler<ftl::net::Peer*> connect_cb_;
diff --git a/components/streams/include/ftl/streams/sender.hpp b/components/streams/include/ftl/streams/sender.hpp
index 66b782b0c..0ae404bfb 100644
--- a/components/streams/include/ftl/streams/sender.hpp
+++ b/components/streams/include/ftl/streams/sender.hpp
@@ -87,6 +87,9 @@ class Sender : public ftl::Configurable {
std::unordered_map<int, EncodingState> state_;
std::unordered_map<int, AudioState> audio_state_;
+ std::map<uint8_t, int64_t> bitrate_map_;
+ SHARED_MUTEX bitrate_mtx_;
+ int bitrate_timeout_;
//ftl::codecs::Encoder *_getEncoder(int fsid, int fid, ftl::codecs::Channel c);
void _encodeChannel(ftl::rgbd::FrameSet &fs, ftl::codecs::Channel c, bool reset, bool last_flush);
@@ -96,7 +99,7 @@ class Sender : public ftl::Configurable {
void _encodeDataChannel(ftl::rgbd::FrameSet &fs, ftl::codecs::Channel c, bool reset, bool last_flush);
void _encodeDataChannel(ftl::data::Frame &fs, ftl::codecs::Channel c, bool reset);
- int _generateTiles(const ftl::rgbd::FrameSet &fs, int offset, ftl::codecs::Channel c, cv::cuda::Stream &stream, bool, bool);
+ int _generateTiles(const ftl::rgbd::FrameSet &fs, int offset, ftl::codecs::Channel c, cv::cuda::Stream &stream, bool);
EncodingState &_getTile(int fsid, ftl::codecs::Channel c);
cv::Rect _generateROI(const ftl::rgbd::FrameSet &fs, ftl::codecs::Channel c, int offset, bool stereo);
float _selectFloatMax(ftl::codecs::Channel c);
@@ -105,6 +108,7 @@ class Sender : public ftl::Configurable {
void _sendPersistent(ftl::data::Frame &frame);
bool _checkNeedsIFrame(int64_t ts, bool injecting);
+ uint8_t _getMinBitrate();
};
}
diff --git a/components/streams/src/netstream.cpp b/components/streams/src/netstream.cpp
index 6974066b7..101cdee7d 100644
--- a/components/streams/src/netstream.cpp
+++ b/components/streams/src/netstream.cpp
@@ -63,6 +63,12 @@ Net::Net(nlohmann::json &config, ftl::net::Universe *net) : Stream(config), acti
last_frame_ = 0;
time_peer_ = ftl::UUID(0);
+
+ bitrate_ = static_cast<uint8_t>(std::max(0, std::min(255, value("bitrate", 255))));
+ on("bitrate", [this]() {
+ bitrate_ = static_cast<uint8_t>(std::max(0, std::min(255, value("bitrate", 255))));
+ tally_ = 0;
+ });
}
Net::~Net() {
@@ -195,7 +201,7 @@ bool Net::begin() {
last_selected_ = sel;
for (auto c : changed) {
- _sendRequest(c, kAllFramesets, kAllFrames, 30, 0);
+ _sendRequest(c, kAllFramesets, kAllFrames, 30, 255);
}
}
}
@@ -207,7 +213,7 @@ bool Net::begin() {
const auto &sel = selected(0);
for (auto c : sel) {
- _sendRequest(c, kAllFramesets, kAllFrames, 30, 0);
+ _sendRequest(c, kAllFramesets, kAllFrames, 30, 255);
}
}
tally_ = 30*kTallyScale;
@@ -295,7 +301,7 @@ bool Net::begin() {
active_ = true;
// Initially send a colour request just to create the connection
- _sendRequest(Channel::Colour, kAllFramesets, kAllFrames, 30, 0);
+ _sendRequest(Channel::Colour, kAllFramesets, kAllFrames, 30, 255);
return true;
}
@@ -307,7 +313,7 @@ void Net::reset() {
auto sel = selected(0);
for (auto c : sel) {
- _sendRequest(c, kAllFramesets, kAllFrames, 30, 0);
+ _sendRequest(c, kAllFramesets, kAllFrames, 30, 255);
}
}
tally_ = 30*kTallyScale;
@@ -322,7 +328,8 @@ bool Net::_sendRequest(Channel c, uint8_t frameset, uint8_t frames, uint8_t coun
codec_t::Any, // TODO: Allow specific codec requests
0,
count,
- bitrate,
+ //bitrate,
+ bitrate_,
0
};
diff --git a/components/streams/src/sender.cpp b/components/streams/src/sender.cpp
index ae56595b7..72ab29c71 100644
--- a/components/streams/src/sender.cpp
+++ b/components/streams/src/sender.cpp
@@ -32,6 +32,8 @@ Sender::Sender(nlohmann::json &config) : ftl::Configurable(config), stream_(null
on("iframes", [this]() {
add_iframes_ = value("iframes", 50);
});
+
+ on("bitrate_timeout", bitrate_timeout_, 10000);
}
Sender::~Sender() {
@@ -48,18 +50,35 @@ void Sender::setStream(ftl::stream::Stream*s) {
handle_ = stream_->onPacket([this](const ftl::codecs::StreamPacket &spkt, const ftl::codecs::Packet &pkt) {
if (pkt.data.size() > 0 || !(spkt.flags & ftl::codecs::kFlagRequest)) return true;
- LOG(INFO) << "SENDER REQUEST : " << (int)spkt.channel;
+ if (int(spkt.channel) < 32) {
+ auto now = ftl::timer::get_time();
+
+ // Update the min bitrate selection
+ UNIQUE_LOCK(bitrate_mtx_, lk);
+ if (bitrate_map_.size() > 0) {
+ if (now - bitrate_map_.begin()->second > bitrate_timeout_) {
+ bitrate_map_.erase(bitrate_map_.begin());
+ }
+ }
+ bitrate_map_[pkt.bitrate] = now;
+ }
//if (state_cb_) state_cb_(spkt.channel, spkt.streamID, spkt.frame_number);
if (reqcb_) reqcb_(spkt,pkt);
// Inject state packets
- //do_inject_ = true;
do_inject_.clear();
+
return true;
});
}
+uint8_t Sender::_getMinBitrate() {
+ SHARED_LOCK(bitrate_mtx_, lk);
+ if (bitrate_map_.size() > 0) return bitrate_map_.begin()->first;
+ else return 255;
+}
+
void Sender::onRequest(const ftl::stream::StreamCallback &cb) {
reqcb_ = cb;
}
@@ -84,28 +103,6 @@ static void writeValue(std::vector<unsigned char> &data, T value) {
data.insert(data.end(), pvalue_start, pvalue_start+sizeof(T));
}
-/*static void mergeNALUnits(const std::list<ftl::codecs::Packet> &pkts, ftl::codecs::Packet &pkt) {
- size_t size = 0;
- for (auto i=pkts.begin(); i!=pkts.end(); ++i) size += (*i).data.size();
-
- // TODO: Check Codec, jpg etc can just use last frame.
- // TODO: Also if iframe, can just use that instead
-
- const auto &first = pkts.front();
- pkt.codec = first.codec;
- //pkt.definition = first.definition;
- pkt.frame_count = first.frame_count;
- pkt.bitrate = first.bitrate;
- pkt.flags = first.flags | ftl::codecs::kFlagMultiple; // means merged packets
- pkt.data.reserve(size+pkts.size()*sizeof(int));
-
- for (auto i=pkts.begin(); i!=pkts.end(); ++i) {
- writeValue<int>(pkt.data, (*i).data.size());
- //LOG(INFO) << "NAL Count = " << (*i).data.size();
- pkt.data.insert(pkt.data.end(), (*i).data.begin(), (*i).data.end());
- }
-}*/
-
void Sender::_sendPersistent(ftl::data::Frame &frame) {
auto *session = frame.parent();
if (session) {
@@ -379,7 +376,7 @@ void Sender::resetEncoders(uint32_t fsid) {
void Sender::setActiveEncoders(uint32_t fsid, const std::unordered_set<Channel> &ec) {
for (auto &t : state_) {
- if ((t.first >> 8) == static_cast<int>(fsid)) {
+ if ((t.first >> 16) == static_cast<int>(fsid)) {
if (t.second.encoder[0] && ec.count(static_cast<Channel>(t.first & 0xFF)) == 0) {
// Remove unwanted encoder
ftl::codecs::free(t.second.encoder[0]);
@@ -395,12 +392,24 @@ void Sender::setActiveEncoders(uint32_t fsid, const std::unordered_set<Channel>
}
void Sender::_encodeVideoChannel(ftl::data::FrameSet &fs, Channel c, bool reset, bool last_flush) {
- bool lossless = value("lossless", false);
- int max_bitrate = std::max(0, std::min(255, value("max_bitrate", 128)));
+ bool isfloat = ftl::codecs::type(c) == CV_32F;
+
+ bool lossless = (isfloat) ? value("lossless_float", false) : value("lossless_colour", false);
+ int bitrate = std::max(0, std::min(255, (isfloat) ? value("bitrate_float", 200) : value("bitrate_colour", 64)));
+
+ bitrate = std::min(static_cast<uint8_t>(bitrate), _getMinBitrate());
+
//int min_bitrate = std::max(0, std::min(255, value("min_bitrate", 0))); // TODO: Use this
- codec_t codec = static_cast<codec_t>(value("codec", static_cast<int>(codec_t::Any)));
+ codec_t codec = static_cast<codec_t>(
+ (isfloat) ? value("codec_float", static_cast<int>(codec_t::Any)) :
+ value("codec_colour", static_cast<int>(codec_t::Any)));
+
device_t device = static_cast<device_t>(value("encoder_device", static_cast<int>(device_t::Any)));
+ if (codec == codec_t::Any) {
+ codec = (lossless) ? codec_t::HEVC_LOSSLESS : codec_t::HEVC;
+ }
+
// TODO: Support high res
bool is_stereo = value("stereo", false) && c == Channel::Colour && fs.firstFrame().hasChannel(Channel::Colour2);
@@ -457,7 +466,7 @@ void Sender::_encodeVideoChannel(ftl::data::FrameSet &fs, Channel c, bool reset,
//}
}
- int count = _generateTiles(fs, offset, cc, enc->stream(), lossless, is_stereo);
+ int count = _generateTiles(fs, offset, cc, enc->stream(), is_stereo);
if (count <= 0) {
LOG(ERROR) << "Could not generate tiles.";
break;
@@ -473,7 +482,7 @@ void Sender::_encodeVideoChannel(ftl::data::FrameSet &fs, Channel c, bool reset,
ftl::codecs::Packet pkt;
pkt.frame_count = count;
pkt.codec = codec;
- pkt.bitrate = (!lossless && ftl::codecs::isFloatChannel(cc)) ? max_bitrate : max_bitrate/2;
+ pkt.bitrate = bitrate;
pkt.flags = 0;
// In the event of partial frames, add a flag to indicate that
@@ -668,7 +677,7 @@ float Sender::_selectFloatMax(Channel c) {
}
}
-int Sender::_generateTiles(const ftl::rgbd::FrameSet &fs, int offset, Channel c, cv::cuda::Stream &stream, bool lossless, bool stereo) {
+int Sender::_generateTiles(const ftl::rgbd::FrameSet &fs, int offset, Channel c, cv::cuda::Stream &stream, bool stereo) {
auto &surface = _getTile(fs.id(), c);
const ftl::data::Frame *cframe = nullptr; //&fs.frames[offset];
diff --git a/components/streams/test/sender_unit.cpp b/components/streams/test/sender_unit.cpp
index 59c7f916a..8c8331607 100644
--- a/components/streams/test/sender_unit.cpp
+++ b/components/streams/test/sender_unit.cpp
@@ -209,7 +209,7 @@ TEST_CASE( "Sender::post() video frames" ) {
fs.frames[1].create<cv::cuda::GpuMat>(Channel::Depth).create(cv::Size(1280,720), CV_32F);
fs.frames[1].set<cv::cuda::GpuMat>(Channel::Depth).setTo(cv::Scalar(0.0f));
- sender->set("codec", (int)codec_t::HEVC_LOSSLESS);
+ sender->set("codec_float", (int)codec_t::HEVC_LOSSLESS);
sender->post(fs, Channel::Depth);
REQUIRE( count == 1 );
--
GitLab