From 9512e6854c4e3cc55618b127e4c15e930759dbb8 Mon Sep 17 00:00:00 2001
From: Nicolas Pope <nicolas.pope@utu.fi>
Date: Mon, 20 Jan 2020 09:52:34 +0200
Subject: [PATCH] Implements #273 tile encoding
---
.gitignore | 3 +-
applications/gui/src/camera.cpp | 30 -
applications/gui/src/camera.hpp | 7 -
applications/gui/src/record_window.cpp | 9 +-
applications/gui/src/screen.cpp | 3 +-
applications/gui/src/thumbview.cpp | 5 +-
applications/reconstruct/src/main.cpp | 17 +-
.../codecs/include/ftl/codecs/bitrates.hpp | 144 -----
.../codecs/include/ftl/codecs/codecs.hpp | 98 ++++
.../codecs/include/ftl/codecs/decoder.hpp | 5 +
.../ftl/codecs}/depth_convert_cuda.hpp | 0
.../codecs/include/ftl/codecs/encoder.hpp | 36 +-
.../include/ftl/codecs/nvpipe_decoder.hpp | 1 -
.../include/ftl/codecs/nvpipe_encoder.hpp | 22 +-
.../include/ftl/codecs/opencv_encoder.hpp | 11 +-
.../codecs/include/ftl/codecs/packet.hpp | 2 +-
components/codecs/src/bitrates.cpp | 57 +-
components/codecs/src/depth_convert.cu | 2 +-
components/codecs/src/encoder.cpp | 24 +-
components/codecs/src/nvpipe_decoder.cpp | 55 +-
components/codecs/src/nvpipe_encoder.cpp | 250 +++++----
components/codecs/src/opencv_encoder.cpp | 77 ++-
components/codecs/test/nvpipe_codec_unit.cpp | 523 +++++++++++++++---
components/codecs/test/opencv_codec_unit.cpp | 19 +-
components/common/cpp/src/configuration.cpp | 2 +
.../rgbd-sources/include/ftl/rgbd/format.hpp | 2 +-
.../rgbd-sources/include/ftl/rgbd/frame.hpp | 2 +-
.../rgbd-sources/src/sources/net/net.cpp | 2 +-
.../include/ftl/streams/filestream.hpp | 2 +-
.../streams/include/ftl/streams/netstream.hpp | 2 +
.../streams/include/ftl/streams/receiver.hpp | 3 +-
.../streams/include/ftl/streams/sender.hpp | 23 +-
.../streams/include/ftl/streams/stream.hpp | 15 +-
components/streams/src/filestream.cpp | 14 +-
components/streams/src/netstream.cpp | 44 +-
components/streams/src/receiver.cpp | 164 ++++--
components/streams/src/sender.cpp | 198 ++++++-
components/streams/src/stream.cpp | 25 +-
components/streams/test/CMakeLists.txt | 30 +
components/streams/test/filestream_unit.cpp | 12 +-
components/streams/test/receiver_unit.cpp | 219 ++++++++
components/streams/test/sender_unit.cpp | 299 ++++++++++
python/ftl/ftltype.py | 2 +-
43 files changed, 1797 insertions(+), 663 deletions(-)
delete mode 100644 components/codecs/include/ftl/codecs/bitrates.hpp
create mode 100644 components/codecs/include/ftl/codecs/codecs.hpp
rename components/codecs/{src => include/ftl/codecs}/depth_convert_cuda.hpp (100%)
create mode 100644 components/streams/test/receiver_unit.cpp
create mode 100644 components/streams/test/sender_unit.cpp
diff --git a/.gitignore b/.gitignore
index 23c1a956b..cf767ae36 100644
--- a/.gitignore
+++ b/.gitignore
@@ -10,4 +10,5 @@ lab-designs/viewer
.vscode
doc/
web-service/npm-debug.log
-web-service/server/npm-debug.log
\ No newline at end of file
+web-service/server/npm-debug.log
+gmon.out
\ No newline at end of file
diff --git a/applications/gui/src/camera.cpp b/applications/gui/src/camera.cpp
index 3924410d4..df514b2ff 100644
--- a/applications/gui/src/camera.cpp
+++ b/applications/gui/src/camera.cpp
@@ -140,13 +140,6 @@ ftl::gui::Camera::Camera(ftl::gui::Screen *screen, int fsid, int fid, ftl::codec
sdepth_ = false;
ftime_ = (float)glfwGetTime();
pause_ = false;
- //fileout_ = new std::ofstream();
- /*writer_ = new ftl::codecs::Writer(*fileout_);
- recorder_ = std::function([this](ftl::rgbd::Source *src, const ftl::codecs::StreamPacket &spkt, const ftl::codecs::Packet &pkt) {
- ftl::codecs::StreamPacket s = spkt;
- writer_->write(s, pkt);
- });*/
- recording_ = false;
#ifdef HAVE_OPENVR
vr_mode_ = false;
@@ -425,29 +418,6 @@ void ftl::gui::Camera::setChannel(Channel c) {
#endif
channel_ = c;
- /*switch (c) {
- case Channel::Energy:
- case Channel::Density:
- case Channel::Flow:
- case Channel::Confidence:
- case Channel::Normals:
- case Channel::ColourNormals:
- case Channel::Right:
- src_->setChannel(c);
- break;
-
- case Channel::Deviation:
- if (stats_) { stats_->reset(); }
- src_->setChannel(Channel::Depth);
- break;
-
- case Channel::Depth:
- src_->setChannel(c);
- break;
-
- default: src_->setChannel(Channel::None);
- }*/
- // FIXME: Somehow send channel request...
}
static void visualizeDepthMap( const cv::Mat &depth, cv::Mat &out,
diff --git a/applications/gui/src/camera.hpp b/applications/gui/src/camera.hpp
index 9146d3cce..adc346055 100644
--- a/applications/gui/src/camera.hpp
+++ b/applications/gui/src/camera.hpp
@@ -93,7 +93,6 @@ class Camera {
Screen *screen_;
int fsid_;
int fid_;
- // TODO: Renderer
int width_;
int height_;
@@ -118,12 +117,6 @@ class Camera {
cv::Mat im1_; // first channel (left)
cv::Mat im2_; // second channel ("right")
- // FIXME: Recording will now be broken?
- bool recording_;
- //std::ofstream *fileout_;
- //ftl::codecs::Writer *writer_;
- //ftl::rgbd::RawCallback recorder_;
-
ftl::render::Triangular *renderer_;
ftl::rgbd::Frame frame_;
ftl::rgbd::FrameState state_;
diff --git a/applications/gui/src/record_window.cpp b/applications/gui/src/record_window.cpp
index f74fd61aa..e358206fe 100644
--- a/applications/gui/src/record_window.cpp
+++ b/applications/gui/src/record_window.cpp
@@ -38,16 +38,11 @@ RecordWindow::RecordWindow(nanogui::Widget *parent, ftl::gui::Screen *screen, co
if (s == screen->activeCamera()) {
ix = std::optional<int>(streamNames.size());
}
- // FIXME: Find alternative to source URI
- //streamNames.push_back(s->source()->getURI());
+
streamNames.push_back(std::string("Stream")+std::to_string(i++));
}
auto streamSelect = new ComboBox(this, streamNames);
- // TODO: The function availableChannels() only finds those channels that
- // have been set in camera.cpp. The only channels that are set in
- // camera.cpp currently are Colour and Depth. This means that currently,
- // the list of channels returned by availableChannels() is not accurate
- // and should be fixed.
+
TabWidget *tabWidget = add<TabWidget>();
tabWidget->setFixedWidth(400);
auto snapshot2D = tabWidget->createTab("2D snapshot");
diff --git a/applications/gui/src/screen.cpp b/applications/gui/src/screen.cpp
index 787d20991..bb7ebfa99 100644
--- a/applications/gui/src/screen.cpp
+++ b/applications/gui/src/screen.cpp
@@ -363,7 +363,6 @@ void ftl::gui::Screen::setActiveCamera(ftl::gui::Camera *cam) {
camera_ = cam;
if (cam) {
- // FIXME: Alternative to source URI
status_ = cam->name();
mwindow_->setVisible(true);
mwindow_->cameraChanged();
@@ -397,7 +396,7 @@ bool ftl::gui::Screen::mouseMotionEvent(const Eigen::Vector2i &p, const Eigen::V
}
}
}
- return true; // TODO: return statement was missing; is true correct?
+ return true;
}
bool ftl::gui::Screen::mouseButtonEvent(const nanogui::Vector2i &p, int button, bool down, int modifiers) {
diff --git a/applications/gui/src/thumbview.cpp b/applications/gui/src/thumbview.cpp
index 6c567e516..5fdbd4ec4 100644
--- a/applications/gui/src/thumbview.cpp
+++ b/applications/gui/src/thumbview.cpp
@@ -18,9 +18,10 @@ ThumbView::~ThumbView() {
bool ThumbView::mouseButtonEvent(const nanogui::Vector2i &p, int button, bool down, int modifiers) {
if (button == 0 && !down) {
screen_->setActiveCamera(cam_);
+ return true;
+ } else {
+ return false;
}
-
- return false; // TODO: return statement was missing; is false correct?
}
void ThumbView::draw(NVGcontext *ctx) {
diff --git a/applications/reconstruct/src/main.cpp b/applications/reconstruct/src/main.cpp
index 8108a6b5d..9eabd9e0a 100644
--- a/applications/reconstruct/src/main.cpp
+++ b/applications/reconstruct/src/main.cpp
@@ -11,7 +11,6 @@
#include <ftl/depth_camera.hpp>
#include <ftl/rgbd.hpp>
#include <ftl/rgbd/virtual.hpp>
-#include <ftl/rgbd/streamer.hpp>
#include <ftl/master.hpp>
#include <ftl/rgbd/group.hpp>
#include <ftl/threads.hpp>
@@ -73,16 +72,6 @@ using std::chrono::milliseconds;
using ftl::registration::loadTransformations;
using ftl::registration::saveTransformations;
-static Eigen::Affine3d create_rotation_matrix(float ax, float ay, float az) {
- Eigen::Affine3d rx =
- Eigen::Affine3d(Eigen::AngleAxisd(ax, Eigen::Vector3d(1, 0, 0)));
- Eigen::Affine3d ry =
- Eigen::Affine3d(Eigen::AngleAxisd(ay, Eigen::Vector3d(0, 1, 0)));
- Eigen::Affine3d rz =
- Eigen::Affine3d(Eigen::AngleAxisd(az, Eigen::Vector3d(0, 0, 1)));
- return rz * rx * ry;
-}
-
/* Build a generator using a deprecated list of source objects. */
static ftl::rgbd::Generator *createSourceGenerator(const std::vector<ftl::rgbd::Source*> &srcs) {
// Must find pose for each source...
@@ -217,6 +206,12 @@ static void run(ftl::Configurable *root) {
ftl::stream::Receiver *gen = ftl::create<ftl::stream::Receiver>(root, "receiver");
gen->setStream(stream);
+ sender->onRequest([stream](const ftl::codecs::StreamPacket &spkt, const ftl::codecs::Packet &pkt) {
+ if (spkt.channel == Channel::Colour) {
+ stream->reset();
+ }
+ });
+
int count = 0;
for (auto &s : stream_uris) {
LOG(INFO) << " --- found stream: " << s;
diff --git a/components/codecs/include/ftl/codecs/bitrates.hpp b/components/codecs/include/ftl/codecs/bitrates.hpp
deleted file mode 100644
index c1b9617a3..000000000
--- a/components/codecs/include/ftl/codecs/bitrates.hpp
+++ /dev/null
@@ -1,144 +0,0 @@
-#ifndef _FTL_CODECS_BITRATES_HPP_
-#define _FTL_CODECS_BITRATES_HPP_
-
-#include <cstdint>
-#include <msgpack.hpp>
-
-namespace ftl {
-namespace codecs {
-
-/**
- * Compression format used.
- */
-enum struct codec_t : uint8_t {
- JPG = 0,
- PNG,
- H264,
- HEVC, // H265
- H264_LOSSLESS,
- HEVC_LOSSLESS,
-
- // TODO: Add audio codecs
- WAV,
-
- JSON = 100, // A JSON string
- CALIBRATION, // Camera parameters object
- POSE, // 4x4 eigen matrix
- MSGPACK,
- STRING, // Null terminated string
- RAW, // Some unknown binary format
-
- Any = 255
-};
-
-/**
- * Resolution of encoding.
- */
-enum struct definition_t : uint8_t {
- UHD8k = 0,
- UHD4k = 1,
- HD1080 = 2,
- HD720 = 3,
- SD576 = 4,
- SD480 = 5,
- LD360 = 6,
- Any = 7,
-
- HTC_VIVE = 8,
-
- // TODO: Add audio definitions
-
- Invalid
-};
-
-definition_t findDefinition(int width, int height);
-
-/**
- * Get width in pixels of definition.
- */
-int getWidth(definition_t);
-
-/**
- * Get height in pixels of definition.
- */
-int getHeight(definition_t);
-
-/**
- * General indication of desired quality. Exact bitrate numbers depends also
- * upon chosen definition and codec.
- */
-enum struct bitrate_t {
- High,
- Standard,
- Low
-};
-
-/**
- * Pre-specified definition and quality levels for encoding where kPreset0 is
- * the best quality and kPreset9 is the worst. The use of presets is useful for
- * adaptive bitrate scenarios where the numbers are increased or decreased.
- */
-typedef int8_t preset_t;
-static const preset_t kPreset0 = 0;
-static const preset_t kPreset1 = 1;
-static const preset_t kPreset2 = 2;
-static const preset_t kPreset3 = 3;
-static const preset_t kPreset4 = 4;
-static const preset_t kPreset5 = 5;
-static const preset_t kPreset6 = 6;
-static const preset_t kPreset7 = 7;
-static const preset_t kPreset8 = 8;
-static const preset_t kPreset9 = 9;
-static const preset_t kPresetBest = 0;
-static const preset_t kPresetWorst = 9;
-static const preset_t kPresetLQThreshold = 4;
-
-static const preset_t kPresetHTCVive = -1;
-
-static const preset_t kPresetMinimum = -1;
-
-/**
- * Represents the details of each preset codec configuration.
- */
-struct CodecPreset {
- definition_t res;
- bitrate_t qual;
-};
-
-/**
- * Get preset details structure from preset number.
- */
-const CodecPreset &getPreset(preset_t);
-
-/**
- * Get preset based upon a requested definition. If multiple presets match then
- * the highest quality one is returned.
- */
-const CodecPreset &getPreset(definition_t, definition_t);
-
-/**
- * Get preset based upon a requested definition for colour channel.
- * If multiple presets match then the highest quality one is returned.
- */
-const CodecPreset &getPreset(definition_t);
-
-/**
- * Get the preset id nearest to requested definition for colour and depth.
- */
-preset_t findPreset(definition_t, definition_t);
-
-/**
- * Get the preset id nearest to requested colour definition. If multiple presets
- * match then return highest quality.
- */
-preset_t findPreset(definition_t);
-
-preset_t findPreset(size_t width, size_t height);
-
-}
-}
-
-MSGPACK_ADD_ENUM(ftl::codecs::codec_t);
-MSGPACK_ADD_ENUM(ftl::codecs::definition_t);
-
-#endif // _FTL_CODECS_BITRATES_HPP_
diff --git a/components/codecs/include/ftl/codecs/codecs.hpp b/components/codecs/include/ftl/codecs/codecs.hpp
new file mode 100644
index 000000000..4f643d406
--- /dev/null
+++ b/components/codecs/include/ftl/codecs/codecs.hpp
@@ -0,0 +1,98 @@
+#ifndef _FTL_CODECS_BITRATES_HPP_
+#define _FTL_CODECS_BITRATES_HPP_
+
+#include <cstdint>
+#include <msgpack.hpp>
+
+namespace ftl {
+namespace codecs {
+
+enum struct format_t {
+ BGRA8,
+ RGBA8,
+ VUYA16,
+ F32,
+ U16
+};
+
+static constexpr uint8_t kFlagFlipRGB = 0x01; // Swap blue and red channels [deprecated]
+static constexpr uint8_t kFlagMappedDepth = 0x02; // Use Yuv mapping for float [deprecated]
+static constexpr uint8_t kFlagFloat = 0x04; // Floating point output
+static constexpr uint8_t kFlagMultiple = 0x80; // Multiple video frames in single packet
+
+/**
+ * Compression format used.
+ */
+enum struct codec_t : uint8_t {
+ JPG = 0,
+ PNG,
+ H264,
+ HEVC, // H265
+ H264_LOSSLESS,
+ HEVC_LOSSLESS,
+
+ // TODO: Add audio codecs
+ WAV,
+
+ JSON = 100, // A JSON string
+ CALIBRATION, // Camera parameters object
+ POSE, // 4x4 eigen matrix
+ MSGPACK,
+ STRING, // Null terminated string
+ RAW, // Some unknown binary format
+
+ Invalid = 254,
+ Any = 255
+};
+
+/**
+ * Resolution of encoding.
+ */
+enum struct definition_t : uint8_t {
+ UHD8k = 0,
+ UHD4k = 1,
+ HD1080 = 2,
+ HD720 = 3,
+ SD576 = 4,
+ SD480 = 5,
+ LD360 = 6,
+ Any = 7,
+
+ HTC_VIVE = 8,
+
+ // TODO: Add audio definitions
+
+ Invalid
+};
+
+definition_t findDefinition(int width, int height);
+
+/**
+ * Get width in pixels of definition.
+ */
+int getWidth(definition_t);
+
+/**
+ * Get height in pixels of definition.
+ */
+int getHeight(definition_t);
+
+/**
+ * General indication of desired quality. Exact bitrate numbers depends also
+ * upon chosen definition and codec.
+ */
+enum struct bitrate_t {
+ High,
+ Standard,
+ Low
+};
+
+std::pair<int,int> chooseTileConfig(int size);
+
+}
+}
+
+MSGPACK_ADD_ENUM(ftl::codecs::codec_t);
+MSGPACK_ADD_ENUM(ftl::codecs::definition_t);
+
+#endif // _FTL_CODECS_BITRATES_HPP_
diff --git a/components/codecs/include/ftl/codecs/decoder.hpp b/components/codecs/include/ftl/codecs/decoder.hpp
index 7684990e2..25dcc6a23 100644
--- a/components/codecs/include/ftl/codecs/decoder.hpp
+++ b/components/codecs/include/ftl/codecs/decoder.hpp
@@ -38,6 +38,11 @@ class Decoder {
virtual bool decode(const ftl::codecs::Packet &pkt, cv::cuda::GpuMat &out)=0;
virtual bool accepts(const ftl::codecs::Packet &)=0;
+
+ cv::cuda::Stream &stream() { return stream_; }
+
+ protected:
+ cv::cuda::Stream stream_;
};
}
diff --git a/components/codecs/src/depth_convert_cuda.hpp b/components/codecs/include/ftl/codecs/depth_convert_cuda.hpp
similarity index 100%
rename from components/codecs/src/depth_convert_cuda.hpp
rename to components/codecs/include/ftl/codecs/depth_convert_cuda.hpp
diff --git a/components/codecs/include/ftl/codecs/encoder.hpp b/components/codecs/include/ftl/codecs/encoder.hpp
index ae2d69f9d..54b334a51 100644
--- a/components/codecs/include/ftl/codecs/encoder.hpp
+++ b/components/codecs/include/ftl/codecs/encoder.hpp
@@ -5,14 +5,13 @@
#include <opencv2/opencv.hpp>
#include <opencv2/core/cuda.hpp>
-#include <ftl/codecs/bitrates.hpp>
+#include <ftl/codecs/codecs.hpp>
#include <ftl/codecs/packet.hpp>
namespace ftl {
namespace codecs {
static const unsigned int kVideoBufferSize = 10*1024*1024;
-static constexpr uint8_t kFlagMultiple = 0x80;
class Encoder;
@@ -59,42 +58,31 @@ class Encoder {
virtual ~Encoder();
/**
- * Wrapper encode to allow use of presets.
- */
- virtual bool encode(const cv::cuda::GpuMat &in, ftl::codecs::preset_t preset,
- const std::function<void(const ftl::codecs::Packet&)> &cb);
-
- /**
- * Encode a frame at specified preset and call a callback function for each
- * block as it is encoded. The callback may be called only once with the
- * entire frame or it may be called many times from many threads with
- * partial blocks of a frame. Each packet should be sent over the network
- * to all listening clients.
+ * Encode a frame given as an opencv gpumat. The packet structure should
+ * be filled before calling this function as the codec, definition and
+ * bitrate given in the packet are used as suggestions for the encoder. The
+ * encoder will modify the fields within the packet and will populate the
+ * data element.
*
* @param in An OpenCV image of the frame to compress
- * @param preset Codec preset for resolution and quality
- * @param iframe Full frame if true, else might be a delta frame
- * @param cb Callback containing compressed data
+ * @param fmt Colour format used.
+ * @param pkt Partially filled packet to be encoded into.
* @return True if succeeded with encoding.
*/
- virtual bool encode(
- const cv::cuda::GpuMat &in,
- ftl::codecs::definition_t definition,
- ftl::codecs::bitrate_t bitrate,
- const std::function<void(const ftl::codecs::Packet&)> &cb)=0;
-
- // TODO: Eventually, use GPU memory directly since some encoders can support this
- //virtual bool encode(const cv::cuda::GpuMat &in, std::vector<uint8_t> &out, bitrate_t bix, bool)=0;
+ virtual bool encode(const cv::cuda::GpuMat &in, ftl::codecs::Packet &pkt)=0;
virtual void reset() {}
virtual bool supports(ftl::codecs::codec_t codec)=0;
+ cv::cuda::Stream &stream() { return stream_; }
+
protected:
bool available;
const ftl::codecs::definition_t max_definition;
const ftl::codecs::definition_t min_definition;
const ftl::codecs::device_t device;
+ cv::cuda::Stream stream_;
};
}
diff --git a/components/codecs/include/ftl/codecs/nvpipe_decoder.hpp b/components/codecs/include/ftl/codecs/nvpipe_decoder.hpp
index 82a61feae..990865b60 100644
--- a/components/codecs/include/ftl/codecs/nvpipe_decoder.hpp
+++ b/components/codecs/include/ftl/codecs/nvpipe_decoder.hpp
@@ -26,7 +26,6 @@ class NvPipeDecoder : public ftl::codecs::Decoder {
MUTEX mutex_;
bool seen_iframe_;
cv::cuda::GpuMat tmp_;
- cv::cuda::Stream stream_;
bool _checkIFrame(ftl::codecs::codec_t codec, const unsigned char *data, size_t size);
};
diff --git a/components/codecs/include/ftl/codecs/nvpipe_encoder.hpp b/components/codecs/include/ftl/codecs/nvpipe_encoder.hpp
index ff0f3750b..607d8d40f 100644
--- a/components/codecs/include/ftl/codecs/nvpipe_encoder.hpp
+++ b/components/codecs/include/ftl/codecs/nvpipe_encoder.hpp
@@ -13,13 +13,7 @@ class NvPipeEncoder : public ftl::codecs::Encoder {
ftl::codecs::definition_t mindef);
~NvPipeEncoder();
- bool encode(const cv::cuda::GpuMat &in, ftl::codecs::preset_t preset,
- const std::function<void(const ftl::codecs::Packet&)> &cb) {
- return Encoder::encode(in, preset, cb);
- }
-
- bool encode(const cv::cuda::GpuMat &in, ftl::codecs::definition_t definition, ftl::codecs::bitrate_t bitrate,
- const std::function<void(const ftl::codecs::Packet&)>&) override;
+ bool encode(const cv::cuda::GpuMat &in, ftl::codecs::Packet &pkt) override;
//bool encode(const cv::cuda::GpuMat &in, std::vector<uint8_t> &out, bitrate_t bix, bool);
@@ -32,17 +26,17 @@ class NvPipeEncoder : public ftl::codecs::Encoder {
private:
NvPipe *nvenc_;
- definition_t current_definition_;
- bool is_float_channel_;
+ NvPipe_Codec codec_;
+ NvPipe_Format format_;
+ NvPipe_Compression compression_;
+ uint8_t last_bitrate_;
+
bool was_reset_;
- ftl::codecs::codec_t preference_;
- ftl::codecs::codec_t current_codec_;
cv::cuda::GpuMat tmp_;
cv::cuda::GpuMat tmp2_;
- cv::cuda::Stream stream_;
- bool _encoderMatch(const cv::cuda::GpuMat &in, definition_t def);
- bool _createEncoder(const cv::cuda::GpuMat &in, definition_t def, bitrate_t rate);
+ bool _encoderMatch(const ftl::codecs::Packet &pkt, format_t fmt);
+ bool _createEncoder(const ftl::codecs::Packet &pkt, format_t fmt);
ftl::codecs::definition_t _verifiedDefinition(ftl::codecs::definition_t def, const cv::cuda::GpuMat &in);
};
diff --git a/components/codecs/include/ftl/codecs/opencv_encoder.hpp b/components/codecs/include/ftl/codecs/opencv_encoder.hpp
index 82b0a5ccc..d500e3c5e 100644
--- a/components/codecs/include/ftl/codecs/opencv_encoder.hpp
+++ b/components/codecs/include/ftl/codecs/opencv_encoder.hpp
@@ -20,13 +20,7 @@ class OpenCVEncoder : public ftl::codecs::Encoder {
ftl::codecs::definition_t mindef);
~OpenCVEncoder();
- bool encode(const cv::cuda::GpuMat &in, ftl::codecs::preset_t preset,
- const std::function<void(const ftl::codecs::Packet&)> &cb) {
- return Encoder::encode(in, preset, cb);
- }
-
- bool encode(const cv::cuda::GpuMat &in, ftl::codecs::definition_t definition, ftl::codecs::bitrate_t bitrate,
- const std::function<void(const ftl::codecs::Packet&)>&) override;
+ bool encode(const cv::cuda::GpuMat &in, ftl::codecs::Packet &pkt) override;
bool supports(ftl::codecs::codec_t codec) override;
@@ -35,14 +29,13 @@ class OpenCVEncoder : public ftl::codecs::Encoder {
private:
int chunk_count_;
int chunk_dim_;
- ftl::codecs::preset_t current_preset_;
ftl::codecs::definition_t current_definition_;
std::atomic<int> jobs_;
std::mutex job_mtx_;
std::condition_variable job_cv_;
cv::Mat tmp_;
- bool _encodeBlock(const cv::Mat &in, ftl::codecs::Packet &pkt, ftl::codecs::bitrate_t);
+ bool _encodeBlock(const cv::Mat &in, ftl::codecs::Packet &pkt);
};
}
diff --git a/components/codecs/include/ftl/codecs/packet.hpp b/components/codecs/include/ftl/codecs/packet.hpp
index 0788653c1..f4089188b 100644
--- a/components/codecs/include/ftl/codecs/packet.hpp
+++ b/components/codecs/include/ftl/codecs/packet.hpp
@@ -3,7 +3,7 @@
#include <cstdint>
#include <vector>
-#include <ftl/codecs/bitrates.hpp>
+#include <ftl/codecs/codecs.hpp>
#include <ftl/codecs/channels.hpp>
#include <msgpack.hpp>
diff --git a/components/codecs/src/bitrates.cpp b/components/codecs/src/bitrates.cpp
index fc101f6bb..5781c4f5f 100644
--- a/components/codecs/src/bitrates.cpp
+++ b/components/codecs/src/bitrates.cpp
@@ -1,27 +1,11 @@
-#include <ftl/codecs/bitrates.hpp>
+#include <ftl/codecs/codecs.hpp>
#include <cmath>
-using ftl::codecs::CodecPreset;
-using ftl::codecs::bitrate_t;
-using ftl::codecs::preset_t;
using ftl::codecs::definition_t;
using ftl::codecs::codec_t;
-static const CodecPreset special_presets[] = {
- definition_t::HTC_VIVE, bitrate_t::High
-};
-
-static const CodecPreset presets[] = {
- definition_t::HD1080, bitrate_t::High,
- definition_t::HD720, bitrate_t::High,
- definition_t::SD576, bitrate_t::High,
- definition_t::SD480, bitrate_t::High,
- definition_t::LD360, bitrate_t::Standard,
- definition_t::LD360, bitrate_t::Low
-};
-
static const float kAspectRatio = 1.777778f;
struct Resolution {
@@ -60,43 +44,6 @@ definition_t ftl::codecs::findDefinition(int width, int height) {
best++;
}
- // TODO error!
- return definition_t::Any;
+ return definition_t::Invalid;
}
-/*
-const CodecPreset &ftl::codecs::getPreset(preset_t p) {
- if (p < 0 && p >= -1) return special_presets[std::abs(p+1)];
- if (p > kPresetWorst) return presets[kPresetWorst];
- if (p < kPresetBest) return presets[kPresetBest];
- return presets[p];
-}
-
-preset_t ftl::codecs::findPreset(size_t width, size_t height) {
- int min_error = std::numeric_limits<int>::max();
-
- // Find best definition
- int best_def = (int)definition_t::Invalid;
-
- for (int i=0; i<(int)definition_t::Invalid; ++i) {
- int dw = resolutions[i].width - width;
- int dh = resolutions[i].height - height;
- int error = dw*dw + dh*dh;
- if (error < min_error) {
- min_error = error;
- best_def = i;
- }
- }
-
- // Find preset that matches this best definition
- for (preset_t i=kPresetMinimum; i<=kPresetWorst; ++i) {
- const auto &preset = getPreset(i);
-
- if ((int)preset.res == best_def) {
- return i;
- }
- }
-
- return kPresetWorst;
-}
-*/
diff --git a/components/codecs/src/depth_convert.cu b/components/codecs/src/depth_convert.cu
index aeb814d50..94fa5e4be 100644
--- a/components/codecs/src/depth_convert.cu
+++ b/components/codecs/src/depth_convert.cu
@@ -1,4 +1,4 @@
-#include "depth_convert_cuda.hpp"
+#include <ftl/codecs/depth_convert_cuda.hpp>
#include <opencv2/core/cuda_stream_accessor.hpp>
diff --git a/components/codecs/src/encoder.cpp b/components/codecs/src/encoder.cpp
index 7c7f9a358..4ef9ade5a 100644
--- a/components/codecs/src/encoder.cpp
+++ b/components/codecs/src/encoder.cpp
@@ -7,14 +7,9 @@
#include <loguru.hpp>
using ftl::codecs::Encoder;
-using ftl::codecs::bitrate_t;
using ftl::codecs::definition_t;
-using ftl::codecs::preset_t;
using ftl::codecs::device_t;
using ftl::codecs::codec_t;
-using ftl::codecs::kPresetBest;
-using ftl::codecs::kPresetWorst;
-using ftl::codecs::kPresetLQThreshold;
namespace ftl {
namespace codecs {
@@ -70,10 +65,17 @@ Encoder::~Encoder() {
}
-bool Encoder::encode(const cv::cuda::GpuMat &in, preset_t preset,
- const std::function<void(const ftl::codecs::Packet&)> &cb) {
- const definition_t definition = ftl::codecs::findDefinition(in.size().width, in.size().height);
- const bitrate_t bitrate = bitrate_t::High;
-
- return encode(in, definition, bitrate, cb);
+std::pair<int,int> ftl::codecs::chooseTileConfig(int size) {
+ switch (size) {
+ case 1: return {1,1};
+ case 2: return {2,1};
+ case 3: return {2,2};
+ case 4: return {2,2};
+ case 5: return {3,2};
+ case 6: return {3,2};
+ case 7: return {4,2};
+ case 8: return {4,2};
+ case 9: return {3,3};
+ }
+ return {3,3};
}
diff --git a/components/codecs/src/nvpipe_decoder.cpp b/components/codecs/src/nvpipe_decoder.cpp
index 605586990..339bb665b 100644
--- a/components/codecs/src/nvpipe_decoder.cpp
+++ b/components/codecs/src/nvpipe_decoder.cpp
@@ -10,7 +10,7 @@
#include <opencv2/core/cuda/common.hpp>
-#include "depth_convert_cuda.hpp"
+#include <ftl/codecs/depth_convert_cuda.hpp>
using ftl::codecs::NvPipeDecoder;
@@ -47,7 +47,34 @@ bool NvPipeDecoder::decode(const ftl::codecs::Packet &pkt, cv::cuda::GpuMat &out
cudaSetDevice(0);
UNIQUE_LOCK(mutex_,lk);
if (pkt.codec != codec_t::HEVC && pkt.codec != codec_t::H264 && pkt.codec != codec_t::HEVC_LOSSLESS && pkt.codec != codec_t::H264_LOSSLESS) return false;
- bool is_float_frame = out.type() == CV_32F;
+
+ bool is_float_frame = pkt.flags & ftl::codecs::kFlagFloat;
+ bool islossless = ((pkt.codec == ftl::codecs::codec_t::HEVC || pkt.codec == ftl::codecs::codec_t::H264) && is_float_frame &&
+ !(pkt.flags & 0x2)) || pkt.codec == ftl::codecs::codec_t::HEVC_LOSSLESS || pkt.codec == ftl::codecs::codec_t::H264_LOSSLESS;
+
+ if (is_float_frame && !islossless && out.type() != CV_16UC4) {
+ LOG(ERROR) << "Invalid buffer for lossy float frame";
+ return false;
+ }
+
+ if (is_float_frame && islossless && out.type() != CV_16U) {
+ LOG(ERROR) << "Invalid buffer for lossless float frame";
+ return false;
+ }
+
+ if (!is_float_frame && out.type() != CV_8UC4) {
+ LOG(ERROR) << "Invalid buffer for lossy colour frame: " << out.type();
+ return false;
+ }
+
+ int width = ftl::codecs::getWidth(pkt.definition);
+ int height = ftl::codecs::getHeight(pkt.definition);
+ auto [tx,ty] = ftl::codecs::chooseTileConfig(pkt.frame_count);
+
+ if (tx*width != out.cols || ty*height != out.rows) {
+ LOG(ERROR) << "Received frame too large for output";
+ return false;
+ }
// Is the previous decoder still valid for current resolution and type?
if (nv_decoder_ != nullptr && (last_definition_ != pkt.definition || last_codec_ != pkt.codec || is_float_channel_ != is_float_frame)) {
@@ -59,15 +86,13 @@ bool NvPipeDecoder::decode(const ftl::codecs::Packet &pkt, cv::cuda::GpuMat &out
last_definition_ = pkt.definition;
last_codec_ = pkt.codec;
- bool islossless = ((pkt.codec == ftl::codecs::codec_t::HEVC || pkt.codec == ftl::codecs::codec_t::H264) && is_float_frame && !(pkt.flags & 0x2)) || pkt.codec == ftl::codecs::codec_t::HEVC_LOSSLESS || pkt.codec == ftl::codecs::codec_t::H264_LOSSLESS;
-
// Build a decoder instance of the correct kind
if (nv_decoder_ == nullptr) {
nv_decoder_ = NvPipe_CreateDecoder(
(is_float_frame) ? (islossless) ? NVPIPE_UINT16 : NVPIPE_YUV64 : NVPIPE_RGBA32,
(pkt.codec == codec_t::HEVC || pkt.codec == ftl::codecs::codec_t::HEVC_LOSSLESS) ? NVPIPE_HEVC : NVPIPE_H264,
- ftl::codecs::getWidth(pkt.definition),
- ftl::codecs::getHeight(pkt.definition));
+ out.cols,
+ out.rows);
if (!nv_decoder_) {
//LOG(INFO) << "Bitrate=" << (int)bitrate << " width=" << ABRController::getColourWidth(bitrate);
LOG(FATAL) << "Could not create decoder: " << NvPipe_GetError(NULL);
@@ -76,10 +101,10 @@ bool NvPipeDecoder::decode(const ftl::codecs::Packet &pkt, cv::cuda::GpuMat &out
seen_iframe_ = false;
}
- tmp_.create(cv::Size(ftl::codecs::getWidth(pkt.definition),ftl::codecs::getHeight(pkt.definition)), (!is_float_frame) ? CV_8UC4 : (islossless) ? CV_16U : CV_16UC4);
+ //tmp_.create(cv::Size(ftl::codecs::getWidth(pkt.definition),ftl::codecs::getHeight(pkt.definition)), (!is_float_frame) ? CV_8UC4 : (islossless) ? CV_16U : CV_16UC4);
// Final checks for validity
- if (pkt.data.size() == 0 || tmp_.size() != out.size()) { // || !ftl::codecs::hevc::validNAL(pkt.data)) {
+ if (pkt.data.size() == 0) { // || !ftl::codecs::hevc::validNAL(pkt.data)) {
LOG(ERROR) << "Failed to decode packet";
return false;
}
@@ -100,7 +125,7 @@ bool NvPipeDecoder::decode(const ftl::codecs::Packet &pkt, cv::cuda::GpuMat &out
else return false;
}
- rc = NvPipe_Decode(nv_decoder_, ptr, size, tmp_.data, tmp_.cols, tmp_.rows, tmp_.step);
+ rc = NvPipe_Decode(nv_decoder_, ptr, size, out.data, out.cols, out.rows, out.step);
if (rc == 0) LOG(ERROR) << "NvPipe decode error: " << NvPipe_GetError(nv_decoder_);
ptr += size;
}
@@ -111,17 +136,13 @@ bool NvPipeDecoder::decode(const ftl::codecs::Packet &pkt, cv::cuda::GpuMat &out
LOG(WARNING) << "P-Frame without I-Frame in decoder";
return false;
}
- rc = NvPipe_Decode(nv_decoder_, pkt.data.data(), pkt.data.size(), tmp_.data, tmp_.cols, tmp_.rows, tmp_.step);
+ rc = NvPipe_Decode(nv_decoder_, pkt.data.data(), pkt.data.size(), out.data, out.cols, out.rows, out.step);
if (rc == 0) LOG(ERROR) << "NvPipe decode error: " << NvPipe_GetError(nv_decoder_);
}
- if (is_float_frame) {
+ /*if (is_float_frame) {
if (!islossless) {
//cv::cuda::cvtColor(tmp_, tmp_, cv::COLOR_RGB2YUV, 4, stream_);
- /*cv::Mat tmpHost;
- tmp_.download(tmpHost);
- cv::imshow("DEPTH", tmpHost);
- cv::waitKey(1);*/
ftl::cuda::vuya_to_depth(out, tmp_, 16.0f, stream_);
} else {
@@ -132,9 +153,9 @@ bool NvPipeDecoder::decode(const ftl::codecs::Packet &pkt, cv::cuda::GpuMat &out
if (pkt.flags & 0x1) {
cv::cuda::cvtColor(tmp_, out, cv::COLOR_RGBA2BGRA, 0, stream_);
}
- }
+ }*/
- stream_.waitForCompletion();
+ //stream_.waitForCompletion();
return rc > 0;
}
diff --git a/components/codecs/src/nvpipe_encoder.cpp b/components/codecs/src/nvpipe_encoder.cpp
index 2c3b27636..d34078cc5 100644
--- a/components/codecs/src/nvpipe_encoder.cpp
+++ b/components/codecs/src/nvpipe_encoder.cpp
@@ -1,29 +1,24 @@
#include <ftl/codecs/nvpipe_encoder.hpp>
#include <loguru.hpp>
#include <ftl/timer.hpp>
-#include <ftl/codecs/bitrates.hpp>
+#include <ftl/codecs/codecs.hpp>
#include <ftl/cuda_util.hpp>
#include <opencv2/core/cuda/common.hpp>
-#include "depth_convert_cuda.hpp"
+#include <ftl/codecs/depth_convert_cuda.hpp>
using ftl::codecs::NvPipeEncoder;
using ftl::codecs::bitrate_t;
using ftl::codecs::codec_t;
using ftl::codecs::definition_t;
-using ftl::codecs::preset_t;
-using ftl::codecs::CodecPreset;
+using ftl::codecs::format_t;
using ftl::codecs::Packet;
NvPipeEncoder::NvPipeEncoder(definition_t maxdef,
definition_t mindef) : Encoder(maxdef, mindef, ftl::codecs::device_t::Hardware) {
nvenc_ = nullptr;
- current_definition_ = definition_t::HD1080;
- is_float_channel_ = false;
was_reset_ = false;
- preference_ = codec_t::Any;
- current_codec_ = codec_t::HEVC;
}
NvPipeEncoder::~NvPipeEncoder() {
@@ -39,42 +34,114 @@ bool NvPipeEncoder::supports(ftl::codecs::codec_t codec) {
case codec_t::H264_LOSSLESS:
case codec_t::HEVC_LOSSLESS:
case codec_t::H264:
- case codec_t::HEVC: preference_ = codec; return true;
+ case codec_t::HEVC: return true;
default: return false;
}
}
/* Check preset resolution is not better than actual resolution. */
-definition_t NvPipeEncoder::_verifiedDefinition(definition_t def, const cv::cuda::GpuMat &in) {
+/*definition_t NvPipeEncoder::_verifiedDefinition(definition_t def, const cv::cuda::GpuMat &in) {
int height = ftl::codecs::getHeight(def);
- // FIXME: Make sure this can't go forever
while (height > in.rows) {
def = static_cast<definition_t>(int(def)+1);
height = ftl::codecs::getHeight(def);
}
return def;
-}
+}*/
static bool isLossy(codec_t c) {
return !(c == codec_t::HEVC_LOSSLESS || c == codec_t::H264_LOSSLESS);
}
-bool NvPipeEncoder::encode(const cv::cuda::GpuMat &in, definition_t odefinition, bitrate_t bitrate, const std::function<void(const ftl::codecs::Packet&)> &cb) {
+static bool sanityFormat(int type, ftl::codecs::format_t fmt) {
+ switch(fmt) {
+ case format_t::BGRA8 :
+ case format_t::RGBA8 : return type == CV_8UC4;
+ case format_t::VUYA16 : return type == CV_8UC4;
+ case format_t::F32 : return type == CV_32F;
+ case format_t::U16 : return type == CV_16U;
+ }
+ return false;
+}
+
+static ftl::codecs::format_t formatFromPacket(const ftl::codecs::Packet &pkt) {
+ if (pkt.flags & ftl::codecs::kFlagFloat) {
+ return (pkt.flags & ftl::codecs::kFlagMappedDepth) ? format_t::VUYA16 : format_t::U16;
+ } else {
+ return (pkt.flags & ftl::codecs::kFlagFlipRGB) ? format_t::BGRA8 : format_t::RGBA8;
+ }
+}
+
+static uint64_t calculateBitrate(definition_t def, float ratescale) {
+ float bitrate = 1.0f; // Megabits
+ switch (def) {
+ case definition_t::UHD4k : bitrate = 32.0f; break;
+ case definition_t::HTC_VIVE : bitrate = 16.0f; break;
+ case definition_t::HD1080 : bitrate = 6.0f; break;
+ case definition_t::HD720 : bitrate = 4.0f; break;
+ case definition_t::SD576 :
+ case definition_t::SD480 : bitrate = 2.0f; break;
+ case definition_t::LD360 : bitrate = 1.0f; break;
+ default : bitrate = 8.0f;
+ }
+
+ bitrate *= 1000.0f*1000.0f;
+ float minrate = 0.25f * bitrate;
+ return uint64_t((bitrate - minrate)*ratescale + minrate);
+}
+
+bool NvPipeEncoder::encode(const cv::cuda::GpuMat &in, ftl::codecs::Packet &pkt) {
cudaSetDevice(0);
- auto definition = odefinition; //_verifiedDefinition(odefinition, in);
- auto width = ftl::codecs::getWidth(definition);
- auto height = ftl::codecs::getHeight(definition);
+ if (pkt.codec != codec_t::Any && !supports(pkt.codec)) {
+ pkt.codec = codec_t::Invalid;
+ return false;
+ }
+
+ // Correct for mising flag
+ if (pkt.codec == codec_t::HEVC && (pkt.flags & ftl::codecs::kFlagFloat) && in.type() == CV_8UC4) {
+ pkt.flags |= ftl::codecs::kFlagMappedDepth;
+ }
+
+ ftl::codecs::format_t fmt = formatFromPacket(pkt);
+
+ if (pkt.frame_count == 0) {
+ pkt.definition = definition_t::Invalid;
+ return false;
+ }
+
+ auto [tx,ty] = ftl::codecs::chooseTileConfig(pkt.frame_count);
+ pkt.definition = (pkt.definition == definition_t::Any) ? ftl::codecs::findDefinition(in.cols/tx, in.rows/ty) : pkt.definition;
+ if (pkt.definition == definition_t::Invalid || pkt.definition == definition_t::Any) {
+ LOG(ERROR) << "Could not find appropriate definition";
+ return false;
+ }
+
+ auto width = ftl::codecs::getWidth(pkt.definition);
+ auto height = ftl::codecs::getHeight(pkt.definition);
if (in.empty()) {
LOG(WARNING) << "No data";
return false;
}
+ if (!sanityFormat(in.type(), fmt)) {
+ LOG(ERROR) << "Input type does not match given format";
+ pkt.flags = 0;
+ return false;
+ }
+
+ if (tx*width != in.cols || ty*height != in.rows) {
+ // TODO: Resize if lower definition requested...
+ LOG(ERROR) << "Input size does not match expected: " << in.cols << " != " << tx*width;
+ pkt.definition = definition_t::Invalid;
+ return false;
+ }
+
cv::cuda::GpuMat tmp;
- if (width != in.cols || height != in.rows) {
+ /*if (width != in.cols || height != in.rows) {
LOG(WARNING) << "Mismatch resolution with encoding resolution";
if (in.type() == CV_32F) {
cv::cuda::resize(in, tmp_, cv::Size(width,height), 0.0, 0.0, cv::INTER_NEAREST, stream_);
@@ -82,60 +149,75 @@ bool NvPipeEncoder::encode(const cv::cuda::GpuMat &in, definition_t odefinition,
cv::cuda::resize(in, tmp_, cv::Size(width,height), 0.0, 0.0, cv::INTER_LINEAR, stream_);
}
tmp = tmp_;
- } else {
+ } else {*/
tmp = in;
- }
+ //}
- //LOG(INFO) << "Definition: " << ftl::codecs::getWidth(definition) << "x" << ftl::codecs::getHeight(definition);
+ //LOG(INFO) << "Definition: " << ftl::codecs::getWidth(pkt.definition) << "x" << ftl::codecs::getHeight(pkt.definition);
if (in.empty()) {
LOG(ERROR) << "Missing data for Nvidia encoder";
return false;
}
- if (preference_ == codec_t::Any) preference_ = codec_t::HEVC;
+ if (pkt.codec == codec_t::Any)
+ pkt.codec = ((pkt.flags & ftl::codecs::kFlagFloat) && !(pkt.flags & ftl::codecs::kFlagMappedDepth)) ? codec_t::HEVC_LOSSLESS : codec_t::HEVC;
- if (!_createEncoder(tmp, definition, bitrate)) return false;
+ if (!_createEncoder(pkt, fmt)) return false;
+
+ if (isLossy(pkt.codec) && pkt.bitrate != last_bitrate_) {
+ uint64_t bitrate = calculateBitrate(pkt.definition, float(pkt.bitrate)/255.0f) * pkt.frame_count;
+ const int fps = 1000/ftl::timer::getInterval();
+ LOG(INFO) << "Changing bitrate: " << bitrate;
+ NvPipe_SetBitrate(nvenc_, bitrate, fps);
+ last_bitrate_ = pkt.bitrate;
+ }
//LOG(INFO) << "NvPipe Encode: " << int(definition) << " " << in.cols;
+ //pkt.flags = 0;
+
//cv::Mat tmp;
- if (tmp.type() == CV_32F) {
- if (isLossy(preference_)) {
+ /*if (tmp.type() == CV_32F) {
+ if (isLossy(pkt.codec)) {
// Use special encoding transform
tmp2_.create(tmp.size(), CV_8UC4);
ftl::cuda::depth_to_vuya(tmp, tmp2_, 16.0f, stream_);
+ pkt.flags |= NvPipeEncoder::kFlagMappedDepth;
} else {
tmp.convertTo(tmp2_, CV_16UC1, 1000, stream_);
}
} else if (tmp.type() == CV_8UC3) {
cv::cuda::cvtColor(tmp, tmp2_, cv::COLOR_BGR2RGBA, 0, stream_);
} else if (tmp.type() == CV_8UC4) {
- cv::cuda::cvtColor(tmp, tmp2_, cv::COLOR_BGRA2RGBA, 0, stream_);
+ if (fmt == format_t::BGRA8) {
+ cv::cuda::cvtColor(tmp, tmp2_, cv::COLOR_BGRA2RGBA, 0, stream_);
+ pkt.flags |= NvPipeEncoder::kFlagRGB;
+ } else if (fmt == format_t::VUYA16) {
+ tmp2_ = tmp;
+ }
+ //} else if (tmp.type() == CV_16UC4) {
+
} else {
LOG(ERROR) << "Unsupported cv::Mat type in Nvidia encoder";
return false;
- }
+ }*/
// Make sure conversions complete...
- stream_.waitForCompletion();
+ //stream_.waitForCompletion();
- Packet pkt;
- pkt.codec = preference_;
- pkt.definition = definition;
- pkt.frame_count = 1;
- pkt.bitrate = 0;
- pkt.flags = NvPipeEncoder::kFlagRGB | NvPipeEncoder::kFlagMappedDepth;
+ //pkt.flags = NvPipeEncoder::kFlagRGB | NvPipeEncoder::kFlagMappedDepth;
+ // TODO: Use page locked memory?
pkt.data.resize(ftl::codecs::kVideoBufferSize);
uint64_t cs = NvPipe_Encode(
nvenc_,
- tmp2_.data,
- tmp2_.step,
+ in.data,
+ in.step,
pkt.data.data(),
ftl::codecs::kVideoBufferSize,
- tmp2_.cols,
- tmp2_.rows,
+ in.cols,
+ in.rows,
was_reset_ // Force IFrame!
);
pkt.data.resize(cs);
@@ -145,82 +227,64 @@ bool NvPipeEncoder::encode(const cv::cuda::GpuMat &in, definition_t odefinition,
LOG(ERROR) << "Could not encode video frame: " << NvPipe_GetError(nvenc_);
return false;
} else {
- cb(pkt);
return true;
}
}
-bool NvPipeEncoder::_encoderMatch(const cv::cuda::GpuMat &in, definition_t def) {
- return ((in.type() == CV_32F && is_float_channel_) ||
- ((in.type() == CV_8UC3 || in.type() == CV_8UC4) && !is_float_channel_)) && current_definition_ == def && current_codec_ == preference_;
+static NvPipe_Codec selectCodec(const Packet &pkt) {
+ return (pkt.codec == codec_t::HEVC || pkt.codec == codec_t::HEVC_LOSSLESS) ? NVPIPE_HEVC : NVPIPE_H264;
}
-static uint64_t calculateBitrate(definition_t def, bitrate_t rate) {
- float scale = 1.0f;
- switch (rate) {
- case bitrate_t::High : break;
- case bitrate_t::Standard : scale = 0.5f; break;
- case bitrate_t::Low : scale = 0.25f; break;
+static NvPipe_Compression selectCompression(const Packet &pkt, format_t fmt) {
+ switch (fmt) {
+ case format_t::BGRA8 :
+ case format_t::RGBA8 : return NVPIPE_LOSSY;
+ case format_t::F32 : return (isLossy(pkt.codec)) ? NVPIPE_LOSSY_10BIT_420 : NVPIPE_LOSSLESS;
+ case format_t::VUYA16 : return NVPIPE_LOSSY_10BIT_420; // FIXME: Check codec.
+ case format_t::U16 : return NVPIPE_LOSSLESS;
}
+ return NVPIPE_LOSSY;
+}
- float bitrate = 1.0f; // Megabits
- switch (def) {
- case definition_t::UHD4k : bitrate = 24.0f; break;
- case definition_t::HTC_VIVE : bitrate = 16.0f; break;
- case definition_t::HD1080 : bitrate = 16.0f; break;
- case definition_t::HD720 : bitrate = 8.0f; break;
- case definition_t::SD576 :
- case definition_t::SD480 : bitrate = 4.0f; break;
- case definition_t::LD360 : bitrate = 2.0f; break;
- default : bitrate = 8.0f;
+static NvPipe_Format selectFormat(const Packet &pkt, format_t fmt) {
+ switch (fmt) {
+ case format_t::BGRA8 :
+ case format_t::RGBA8 : return NVPIPE_RGBA32;
+ case format_t::F32 : return (isLossy(pkt.codec)) ? NVPIPE_YUV32 : NVPIPE_UINT16;
+ case format_t::U16 : return NVPIPE_UINT16;
+ case format_t::VUYA16 : return NVPIPE_YUV32;
}
-
- return uint64_t(bitrate * 1000.0f * 1000.0f * scale);
+ return NVPIPE_RGBA32;
}
-bool NvPipeEncoder::_createEncoder(const cv::cuda::GpuMat &in, definition_t def, bitrate_t rate) {
- if (_encoderMatch(in, def) && nvenc_) return true;
+bool NvPipeEncoder::_encoderMatch(const ftl::codecs::Packet &pkt, format_t fmt) {
+ return compression_ == selectCompression(pkt, fmt) &&
+ format_ == selectFormat(pkt, fmt) &&
+ codec_ == selectCodec(pkt);
+}
- if (in.type() == CV_32F) is_float_channel_ = true;
- else is_float_channel_ = false;
- current_definition_ = def;
- current_codec_ = preference_;
+bool NvPipeEncoder::_createEncoder(const ftl::codecs::Packet &pkt, format_t fmt) {
+ if (_encoderMatch(pkt, fmt) && nvenc_) return true;
- uint64_t bitrate = calculateBitrate(def, rate);
- if (is_float_channel_) bitrate *= 2.0f;
+ uint64_t bitrate = calculateBitrate(pkt.definition, float(pkt.bitrate)/255.0f) * pkt.frame_count;
+ //if (is_float_channel_) bitrate *= 2.0f;
//LOG(INFO) << "Calculated bitrate: " << bitrate;
-
- NvPipe_Codec codec;
- NvPipe_Format format;
- NvPipe_Compression compression;
-
- if (is_float_channel_) {
- if (isLossy(preference_)) {
- format = NVPIPE_YUV32;
- compression = NVPIPE_LOSSY_10BIT_420;
- codec = (preference_ == codec_t::HEVC) ? NVPIPE_HEVC : NVPIPE_H264;
- } else {
- format = NVPIPE_UINT16;
- compression = NVPIPE_LOSSLESS;
- codec = (preference_ == codec_t::HEVC_LOSSLESS) ? NVPIPE_HEVC : NVPIPE_H264;
- }
- } else {
- format = NVPIPE_RGBA32;
- compression = NVPIPE_LOSSY;
- codec = (preference_ == codec_t::HEVC || preference_ == codec_t::HEVC_LOSSLESS) ? NVPIPE_HEVC : NVPIPE_H264;
- }
-
+
+ format_ = selectFormat(pkt, fmt);
+ compression_ = selectCompression(pkt, fmt);
+ codec_ = selectCodec(pkt);
+ last_bitrate_ = pkt.bitrate;
if (nvenc_) NvPipe_Destroy(nvenc_);
const int fps = 1000/ftl::timer::getInterval();
nvenc_ = NvPipe_CreateEncoder(
- format,
- codec,
- compression,
+ format_,
+ codec_,
+ compression_,
bitrate,
fps, // FPS
- ftl::codecs::getWidth(def), // Output Width
- ftl::codecs::getHeight(def) // Output Height
+ ftl::codecs::getWidth(pkt.definition), // Output Width
+ ftl::codecs::getHeight(pkt.definition) // Output Height
);
if (!nvenc_) {
diff --git a/components/codecs/src/opencv_encoder.cpp b/components/codecs/src/opencv_encoder.cpp
index 01310c41f..8edc1a616 100644
--- a/components/codecs/src/opencv_encoder.cpp
+++ b/components/codecs/src/opencv_encoder.cpp
@@ -5,10 +5,7 @@
using ftl::codecs::definition_t;
using ftl::codecs::codec_t;
-using ftl::codecs::bitrate_t;
using ftl::codecs::OpenCVEncoder;
-using ftl::codecs::preset_t;
-using ftl::codecs::CodecPreset;
using std::vector;
OpenCVEncoder::OpenCVEncoder(ftl::codecs::definition_t maxdef,
@@ -28,53 +25,53 @@ bool OpenCVEncoder::supports(ftl::codecs::codec_t codec) {
}
}
-bool OpenCVEncoder::encode(const cv::cuda::GpuMat &in, definition_t definition, bitrate_t bitrate, const std::function<void(const ftl::codecs::Packet&)> &cb) {
- bool is_colour = in.type() != CV_32F;
+bool OpenCVEncoder::encode(const cv::cuda::GpuMat &in, ftl::codecs::Packet &pkt) {
+ bool is_colour = !(pkt.flags & ftl::codecs::kFlagFloat);
+
+ if (is_colour && in.type() != CV_8UC4) return false;
+ if (!is_colour && in.type() == CV_8UC4) {
+ LOG(ERROR) << "OpenCV Encoder doesn't support lossy depth";
+ return false;
+ }
+
+ pkt.definition = (pkt.definition == definition_t::Any) ? ftl::codecs::findDefinition(in.cols, in.rows) : pkt.definition;
+
+ if (pkt.definition == definition_t::Invalid || pkt.definition == definition_t::Any) return false;
// Ensure definition does not exceed max
- current_definition_ = ((int)definition < (int)max_definition) ? max_definition : definition;
+ current_definition_ = ((int)pkt.definition < (int)max_definition) ? max_definition : pkt.definition;
in.download(tmp_);
//CHECK(cv::Size(ftl::codecs::getWidth(definition), ftl::codecs::getHeight(definition)) == in.size());
+ int width = ftl::codecs::getWidth(current_definition_);
+ int height = ftl::codecs::getHeight(current_definition_);
+
// Scale down image to match requested definition...
- if (ftl::codecs::getHeight(current_definition_) < in.rows) {
+ /*if (ftl::codecs::getHeight(current_definition_) < in.rows) {
cv::resize(tmp_, tmp_, cv::Size(ftl::codecs::getWidth(current_definition_), ftl::codecs::getHeight(current_definition_)), 0, 0, (is_colour) ? 1 : cv::INTER_NEAREST);
} else {
- }
-
- // Represent float at 16bit int
- if (!is_colour) {
- tmp_.convertTo(tmp_, CV_16UC1, 1000);
- }
+ }*/
+ if (width != in.cols || height != in.rows) return false;
- // FIXME: Chunking is broken so forced to single chunk
- chunk_dim_ = 1; //(definition == definition_t::LD360) ? 1 : 4;
- chunk_count_ = chunk_dim_ * chunk_dim_;
- jobs_ = chunk_count_;
+ if (pkt.codec == codec_t::Any) pkt.codec = (is_colour) ? codec_t::JPG : codec_t::PNG;
//for (int i=0; i<chunk_count_; ++i) {
// Add chunk job to thread pool
//ftl::pool.push([this,i,cb,is_colour,bitrate](int id) {
- ftl::codecs::Packet pkt;
- pkt.bitrate = 0;
- pkt.frame_count = 1;
- pkt.definition = current_definition_;
- pkt.codec = (is_colour) ? codec_t::JPG : codec_t::PNG;
+ //ftl::codecs::Packet pkt;
+ //pkt.bitrate = 0;
+ //pkt.frame_count = 1;
+ //pkt.definition = current_definition_;
+ //pkt.codec = (is_colour) ? codec_t::JPG : codec_t::PNG;
try {
- _encodeBlock(tmp_, pkt, bitrate);
+ _encodeBlock(tmp_, pkt);
} catch(...) {
LOG(ERROR) << "OpenCV encode block exception: ";
}
- try {
- cb(pkt);
- } catch(...) {
- LOG(ERROR) << "OpenCV encoder callback exception";
- }
-
//std::unique_lock<std::mutex> lk(job_mtx_);
//--jobs_;
//if (jobs_ == 0) job_cv_.notify_one();
@@ -90,16 +87,16 @@ bool OpenCVEncoder::encode(const cv::cuda::GpuMat &in, definition_t definition,
return true;
}
-bool OpenCVEncoder::_encodeBlock(const cv::Mat &in, ftl::codecs::Packet &pkt, bitrate_t bitrate) {
- int chunk_width = in.cols / chunk_dim_;
- int chunk_height = in.rows / chunk_dim_;
+bool OpenCVEncoder::_encodeBlock(const cv::Mat &in, ftl::codecs::Packet &pkt) {
+ //int chunk_width = in.cols / 1;
+ //int chunk_height = in.rows / 1;
// Build chunk heads
- int cx = 0; //(pkt.block_number % chunk_dim_) * chunk_width;
- int cy = 0; //(pkt.block_number / chunk_dim_) * chunk_height;
- cv::Rect roi(cx,cy,chunk_width,chunk_height);
+ //int cx = 0; //(pkt.block_number % chunk_dim_) * chunk_width;
+ //int cy = 0; //(pkt.block_number / chunk_dim_) * chunk_height;
+ //cv::Rect roi(cx,cy,chunk_width,chunk_height);
- cv::Mat chunkHead = in(roi);
+ cv::Mat chunkHead = in;
if (pkt.codec == codec_t::PNG) {
vector<int> params = {cv::IMWRITE_PNG_COMPRESSION, 1};
@@ -109,13 +106,7 @@ bool OpenCVEncoder::_encodeBlock(const cv::Mat &in, ftl::codecs::Packet &pkt, bi
}
return true;
} else if (pkt.codec == codec_t::JPG) {
- int q = 95;
-
- switch (bitrate) {
- case bitrate_t::High : q = 95; break;
- case bitrate_t::Standard : q = 75; break;
- case bitrate_t::Low : q = 50; break;
- }
+ int q = int((95.0f - 50.0f) * (float(pkt.bitrate)/255.0f) + 50.0f);
vector<int> params = {cv::IMWRITE_JPEG_QUALITY, q};
cv::imencode(".jpg", chunkHead, pkt.data, params);
diff --git a/components/codecs/test/nvpipe_codec_unit.cpp b/components/codecs/test/nvpipe_codec_unit.cpp
index dccc65f96..2b32e2843 100644
--- a/components/codecs/test/nvpipe_codec_unit.cpp
+++ b/components/codecs/test/nvpipe_codec_unit.cpp
@@ -1,12 +1,12 @@
#include "catch.hpp"
#include <ftl/codecs/nvpipe_encoder.hpp>
#include <ftl/codecs/nvpipe_decoder.hpp>
+#include <ftl/codecs/hevc.hpp>
#include <ftl/threads.hpp>
-using ftl::codecs::CodecPreset;
-using ftl::codecs::preset_t;
using ftl::codecs::definition_t;
using ftl::codecs::codec_t;
+using ftl::codecs::format_t;
ctpl::thread_pool ftl::pool(4);
@@ -22,47 +22,251 @@ namespace ftl {
}
}
-/*
-TEST_CASE( "NvPipeEncoder::encode() - A colour test image at preset 0" ) {
+
+TEST_CASE( "NvPipeEncoder::encode() - A valid colour image" ) {
ftl::codecs::NvPipeEncoder encoder(definition_t::HD1080, definition_t::SD480);
- cv::cuda::GpuMat m(cv::Size(1920,1080), CV_8UC3, cv::Scalar(0,0,0));
+ cv::cuda::GpuMat m(cv::Size(1920,1080), CV_8UC4, cv::Scalar(0,0,0,0));
+
+ ftl::codecs::Packet pkt;
+ pkt.codec = codec_t::Any;
+ pkt.bitrate = 255;
+ pkt.definition = definition_t::Any;
+ pkt.flags = 0;
+ pkt.frame_count = 1;
+
+ SECTION("auto codec and definition, single frame") {
+ bool r = encoder.encode(m, pkt);
- int block_total = 0;
- std::atomic<int> block_count = 0;
- encoder.encode()
- bool r = encoder.encode(m, definition::H, [&block_total, &block_count, preset, m](const ftl::codecs::Packet &pkt) {
+ REQUIRE( r );
REQUIRE( pkt.codec == codec_t::HEVC );
- REQUIRE( pkt.data.size() > 0 );
REQUIRE( pkt.definition == definition_t::HD1080 );
+ REQUIRE( pkt.flags == 0 );
+ REQUIRE( pkt.data.size() > 0 );
+ REQUIRE( ftl::codecs::hevc::validNAL(pkt.data.data(), pkt.data.size()) );
+ }
+
+ SECTION("auto codec and definition, single frame, 1 bitrate") {
+ pkt.bitrate = 1;
+
+ bool r = encoder.encode(m, pkt);
+
+ REQUIRE( r );
+ REQUIRE( pkt.bitrate == 1 );
+ REQUIRE( pkt.data.size() > 0 );
+ REQUIRE( ftl::codecs::hevc::validNAL(pkt.data.data(), pkt.data.size()) );
+ }
+
+ SECTION("invalid frame count of 2") {
+ pkt.frame_count = 2;
+
+ bool r = encoder.encode(m, pkt);
+
+ REQUIRE( !r );
+ REQUIRE( pkt.definition == definition_t::Invalid );
+ REQUIRE( pkt.data.size() == 0 );
+ }
+
+ SECTION("invalid frame count of 0") {
+ pkt.frame_count = 0;
+
+ bool r = encoder.encode(m, pkt);
+
+ REQUIRE( !r );
+ REQUIRE( pkt.definition == definition_t::Invalid );
+ REQUIRE( pkt.data.size() == 0 );
+ }
- block_total = pkt.block_total;
- block_count++;
- });
+ SECTION("invalid float flag") {
+ pkt.flags = ftl::codecs::kFlagFloat;
- REQUIRE( r );
- REQUIRE( block_count == block_total );
+ bool r = encoder.encode(m, pkt);
+
+ REQUIRE( !r );
+ REQUIRE( pkt.flags == 0 );
+ REQUIRE( pkt.data.size() == 0 );
+ }
+
+ SECTION("invalid codec") {
+ pkt.codec = codec_t::JPG;
+
+ bool r = encoder.encode(m, pkt);
+
+ REQUIRE( !r );
+ REQUIRE( pkt.codec == codec_t::Invalid );
+ REQUIRE( pkt.data.size() == 0 );
+ }
+
+ SECTION("invalid definition") {
+ pkt.definition = definition_t::HD720;
+
+ bool r = encoder.encode(m, pkt);
+
+ REQUIRE( !r );
+ REQUIRE( pkt.definition == definition_t::Invalid );
+ REQUIRE( pkt.data.size() == 0 );
+ }
}
-TEST_CASE( "NvPipeEncoder::encode() - A depth test image at preset 0" ) {
+TEST_CASE( "NvPipeEncoder::encode() - A tiled colour image" ) {
ftl::codecs::NvPipeEncoder encoder(definition_t::HD1080, definition_t::SD480);
- cv::cuda::GpuMat m(cv::Size(1920,1080), CV_32F, cv::Scalar(0.0f));
+ cv::cuda::GpuMat m(cv::Size(2560,720), CV_8UC4, cv::Scalar(0,0,0,0));
+
+ SECTION("auto codec and definition, 2x1 frames") {
+ ftl::codecs::Packet pkt;
+ pkt.codec = codec_t::Any;
+ pkt.bitrate = 255;
+ pkt.definition = definition_t::Any;
+ pkt.flags = 0;
+ pkt.frame_count = 2;
- int block_total = 0;
- std::atomic<int> block_count = 0;
+ bool r = encoder.encode(m, pkt);
- bool r = encoder.encode(m, ftl::codecs::kPreset0, [&block_total, &block_count, preset](const ftl::codecs::Packet &pkt) {
+ REQUIRE( r );
REQUIRE( pkt.codec == codec_t::HEVC );
+ REQUIRE( pkt.definition == definition_t::HD720 );
+ REQUIRE( pkt.flags == 0 );
REQUIRE( pkt.data.size() > 0 );
- REQUIRE( pkt.definition == definition_t::HD1080 );
+ REQUIRE( ftl::codecs::hevc::validNAL(pkt.data.data(), pkt.data.size()) );
+ }
+}
+
+TEST_CASE( "NvPipeEncoder::encode() - A valid lossless float image" ) {
+ ftl::codecs::NvPipeEncoder encoder(definition_t::HD1080, definition_t::SD480);
+ cv::cuda::GpuMat m(cv::Size(1280,720), CV_16U, cv::Scalar(0));
+
+ SECTION("auto codec and definition, single frame") {
+ ftl::codecs::Packet pkt;
+ pkt.codec = codec_t::Any;
+ pkt.bitrate = 255;
+ pkt.definition = definition_t::Any;
+ pkt.flags = ftl::codecs::kFlagFloat;
+ pkt.frame_count = 1;
+
+ bool r = encoder.encode(m, pkt);
+
+ REQUIRE( r );
+ REQUIRE( pkt.codec == codec_t::HEVC_LOSSLESS );
+ REQUIRE( pkt.definition == definition_t::HD720 );
+ REQUIRE( pkt.flags == ftl::codecs::kFlagFloat );
+ REQUIRE( pkt.data.size() > 0 );
+ REQUIRE( ftl::codecs::hevc::validNAL(pkt.data.data(), pkt.data.size()) );
+ }
+
+ SECTION("missing float flag") {
+ ftl::codecs::Packet pkt;
+ pkt.codec = codec_t::Any;
+ pkt.bitrate = 255;
+ pkt.definition = definition_t::Any;
+ pkt.flags = 0;
+ pkt.frame_count = 1;
+
+ bool r = encoder.encode(m, pkt);
+
+ REQUIRE( !r );
+ REQUIRE( pkt.data.size() == 0 );
+ }
+
+ SECTION("invalid lossy flag") {
+ ftl::codecs::Packet pkt;
+ pkt.codec = codec_t::Any;
+ pkt.bitrate = 255;
+ pkt.definition = definition_t::Any;
+ pkt.flags = ftl::codecs::kFlagFloat & ftl::codecs::kFlagMappedDepth;
+ pkt.frame_count = 1;
+
+ bool r = encoder.encode(m, pkt);
+
+ REQUIRE( !r );
+ REQUIRE( pkt.data.size() == 0 );
+ }
+}
+
+TEST_CASE( "NvPipeEncoder::encode() - A valid lossy float image" ) {
+ ftl::codecs::NvPipeEncoder encoder(definition_t::HD1080, definition_t::SD480);
+ cv::cuda::GpuMat m(cv::Size(1280,720), CV_8UC4, cv::Scalar(0));
+
+ SECTION("auto codec and definition, single frame") {
+ ftl::codecs::Packet pkt;
+ pkt.codec = codec_t::Any;
+ pkt.bitrate = 255;
+ pkt.definition = definition_t::Any;
+ pkt.flags = ftl::codecs::kFlagFloat | ftl::codecs::kFlagMappedDepth;
+ pkt.frame_count = 1;
+
+ bool r = encoder.encode(m, pkt);
+
+ REQUIRE( r );
+ REQUIRE( pkt.codec == codec_t::HEVC );
+ REQUIRE( pkt.definition == definition_t::HD720 );
+ REQUIRE( pkt.flags == (ftl::codecs::kFlagFloat | ftl::codecs::kFlagMappedDepth) );
+ REQUIRE( pkt.data.size() > 0 );
+ REQUIRE( ftl::codecs::hevc::validNAL(pkt.data.data(), pkt.data.size()) );
+ }
+
+ SECTION("correct codec, missing flag") {
+ ftl::codecs::Packet pkt;
+ pkt.codec = codec_t::HEVC;
+ pkt.bitrate = 255;
+ pkt.definition = definition_t::Any;
+ pkt.flags = ftl::codecs::kFlagFloat;
+ pkt.frame_count = 1;
+
+ bool r = encoder.encode(m, pkt);
+
+ REQUIRE( r );
+ REQUIRE( pkt.codec == codec_t::HEVC );
+ REQUIRE( pkt.definition == definition_t::HD720 );
+ REQUIRE( pkt.flags == (ftl::codecs::kFlagFloat | ftl::codecs::kFlagMappedDepth) );
+ REQUIRE( pkt.data.size() > 0 );
+ REQUIRE( ftl::codecs::hevc::validNAL(pkt.data.data(), pkt.data.size()) );
+ }
+}
+
+TEST_CASE( "NvPipeEncoder::encode() - A tiled lossy float image" ) {
+ ftl::codecs::NvPipeEncoder encoder(definition_t::HD1080, definition_t::SD480);
+ cv::cuda::GpuMat m(cv::Size(2560,720), CV_8UC4, cv::Scalar(0));
+
+ SECTION("auto codec and definition, 2x1 frame") {
+ ftl::codecs::Packet pkt;
+ pkt.codec = codec_t::Any;
+ pkt.bitrate = 255;
+ pkt.definition = definition_t::Any;
+ pkt.flags = ftl::codecs::kFlagFloat & ftl::codecs::kFlagMappedDepth;
+ pkt.frame_count = 2;
+
+ bool r = encoder.encode(m, pkt);
+
+ REQUIRE( r );
+ REQUIRE( pkt.codec == codec_t::HEVC );
+ REQUIRE( pkt.definition == definition_t::HD720 );
+ REQUIRE( pkt.flags == (ftl::codecs::kFlagFloat & ftl::codecs::kFlagMappedDepth) );
+ REQUIRE( pkt.data.size() > 0 );
+ REQUIRE( ftl::codecs::hevc::validNAL(pkt.data.data(), pkt.data.size()) );
+ }
+}
+
+TEST_CASE( "NvPipeEncoder::encode() - A large tiled lossy float image" ) {
+ ftl::codecs::NvPipeEncoder encoder(definition_t::HD1080, definition_t::SD480);
+ cv::cuda::GpuMat m(cv::Size(5120,1440), CV_8UC4, cv::Scalar(0));
+
+ SECTION("auto codec and definition, 4x2 frame") {
+ ftl::codecs::Packet pkt;
+ pkt.codec = codec_t::Any;
+ pkt.bitrate = 255;
+ pkt.definition = definition_t::Any;
+ pkt.flags = ftl::codecs::kFlagFloat & ftl::codecs::kFlagMappedDepth;
+ pkt.frame_count = 7;
- block_total = pkt.block_total;
- block_count++;
- });
+ bool r = encoder.encode(m, pkt);
- REQUIRE( r );
- REQUIRE( block_count == block_total );
+ REQUIRE( r );
+ REQUIRE( pkt.codec == codec_t::HEVC );
+ REQUIRE( pkt.definition == definition_t::HD720 );
+ REQUIRE( pkt.flags == (ftl::codecs::kFlagFloat & ftl::codecs::kFlagMappedDepth) );
+ REQUIRE( pkt.data.size() > 0 );
+ REQUIRE( ftl::codecs::hevc::validNAL(pkt.data.data(), pkt.data.size()) );
+ }
}
-*/
TEST_CASE( "NvPipeDecoder::decode() - A colour test image" ) {
ftl::codecs::NvPipeEncoder encoder(definition_t::HD1080, definition_t::SD480);
@@ -70,53 +274,246 @@ TEST_CASE( "NvPipeDecoder::decode() - A colour test image" ) {
cv::cuda::GpuMat in;
cv::cuda::GpuMat out;
- bool r = false;
- SECTION("FHD in and out, FHD encoding") {
- in = cv::cuda::GpuMat(cv::Size(1920,1080), CV_8UC3, cv::Scalar(255,0,0));
- out = cv::cuda::GpuMat(cv::Size(1920,1080), CV_8UC3, cv::Scalar(0,0,0));
+ //SECTION("FHD in and out, FHD encoding") {
+ in = cv::cuda::GpuMat(cv::Size(1920,1080), CV_8UC4, cv::Scalar(255,0,0,0));
+ out = cv::cuda::GpuMat(cv::Size(1920,1080), CV_8UC4, cv::Scalar(0,0,0,0));
+
+ ftl::codecs::Packet pkt;
+ pkt.codec = codec_t::Any;
+ pkt.bitrate = 255;
+ pkt.definition = definition_t::Any;
+ pkt.frame_count = 1;
+ pkt.flags = 0;
+ bool r = encoder.encode(in, pkt);
+
+ REQUIRE( r );
+ REQUIRE( decoder.decode(pkt, out) );
+ REQUIRE( out.cols == 1920 );
+ REQUIRE( out.type() == CV_8UC4 );
+ //}
+
+ REQUIRE( (cv::cuda::sum(out) != cv::Scalar(0,0,0)) );
+}
+
+TEST_CASE( "NvPipeDecoder::decode() - A tiled colour image" ) {
+ ftl::codecs::NvPipeEncoder encoder(definition_t::HD1080, definition_t::SD480);
+ ftl::codecs::NvPipeDecoder decoder;
+
+ cv::cuda::GpuMat in;
+ cv::cuda::GpuMat out;
+
+ //SECTION("FHD in and out, FHD encoding") {
+ in = cv::cuda::GpuMat(cv::Size(2560,720), CV_8UC4, cv::Scalar(255,0,0,0));
+ out = cv::cuda::GpuMat(cv::Size(2560,720), CV_8UC4, cv::Scalar(0,0,0,0));
+
+ ftl::codecs::Packet pkt;
+ pkt.codec = codec_t::Any;
+ pkt.bitrate = 255;
+ pkt.definition = definition_t::Any;
+ pkt.frame_count = 2;
+ pkt.flags = 0;
+ bool r = encoder.encode(in, pkt);
+
+ REQUIRE( r );
+ REQUIRE( decoder.decode(pkt, out) );
+ REQUIRE( out.cols == 2560 );
+ REQUIRE( out.type() == CV_8UC4 );
+ REQUIRE( pkt.definition == definition_t::HD720 );
+ //}
+
+ REQUIRE( (cv::cuda::sum(out) != cv::Scalar(0,0,0)) );
+}
+
+TEST_CASE( "NvPipeDecoder::decode() - A lossless depth image" ) {
+ ftl::codecs::NvPipeEncoder encoder(definition_t::HD1080, definition_t::SD480);
+ ftl::codecs::NvPipeDecoder decoder;
+
+ cv::cuda::GpuMat in;
+ cv::cuda::GpuMat out;
+
+ //SECTION("FHD in and out, FHD encoding") {
+ in = cv::cuda::GpuMat(cv::Size(1280,720), CV_16U, cv::Scalar(255));
+ out = cv::cuda::GpuMat(cv::Size(1280,720), CV_16U, cv::Scalar(0));
+
+ ftl::codecs::Packet pkt;
+ pkt.codec = codec_t::Any;
+ pkt.bitrate = 255;
+ pkt.definition = definition_t::Any;
+ pkt.frame_count = 1;
+ pkt.flags = ftl::codecs::kFlagFloat;
+ bool r = encoder.encode(in, pkt);
+
+ REQUIRE( r );
+ REQUIRE( decoder.decode(pkt, out) );
+ REQUIRE( out.cols == 1280 );
+ REQUIRE( out.type() == CV_16U );
+ REQUIRE( pkt.definition == definition_t::HD720 );
+ //}
+
+ REQUIRE( (cv::cuda::sum(out) != cv::Scalar(0)) );
+}
+
+TEST_CASE( "NvPipeDecoder::decode() - A lossy depth image" ) {
+ ftl::codecs::NvPipeEncoder encoder(definition_t::HD1080, definition_t::SD480);
+ ftl::codecs::NvPipeDecoder decoder;
+
+ cv::cuda::GpuMat in;
+ cv::cuda::GpuMat out;
+
+ //SECTION("FHD in and out, FHD encoding") {
+ in = cv::cuda::GpuMat(cv::Size(1280,720), CV_8UC4, cv::Scalar(255));
+ out = cv::cuda::GpuMat(cv::Size(1280,720), CV_16UC4, cv::Scalar(0));
+
+ ftl::codecs::Packet pkt;
+ pkt.codec = codec_t::Any;
+ pkt.bitrate = 255;
+ pkt.definition = definition_t::Any;
+ pkt.frame_count = 1;
+ pkt.flags = ftl::codecs::kFlagFloat | ftl::codecs::kFlagMappedDepth;
+ bool r = encoder.encode(in, pkt);
+
+ REQUIRE( r );
+ REQUIRE( decoder.decode(pkt, out) );
+ REQUIRE( out.cols == 1280 );
+ REQUIRE( out.type() == CV_16UC4 );
+ REQUIRE( pkt.definition == definition_t::HD720 );
+ //}
+
+ REQUIRE( (cv::cuda::sum(out) != cv::Scalar(0)) );
+}
+
+TEST_CASE( "NvPipeDecoder::decode() - corrupted packet" ) {
+ ftl::codecs::NvPipeEncoder encoder(definition_t::HD1080, definition_t::SD480);
+ ftl::codecs::NvPipeDecoder decoder;
- r = encoder.encode(in, ftl::codecs::kPreset0, [&out,&decoder](const ftl::codecs::Packet &pkt) {
- REQUIRE( decoder.decode(pkt, out) );
- });
+ cv::cuda::GpuMat in;
+ cv::cuda::GpuMat out;
+
+ SECTION("Corrupted definition") {
+ in = cv::cuda::GpuMat(cv::Size(2560,720), CV_8UC4, cv::Scalar(255,0,0,0));
+ out = cv::cuda::GpuMat(cv::Size(2560,720), CV_8UC4, cv::Scalar(0,0,0,0));
+
+ ftl::codecs::Packet pkt;
+ pkt.codec = codec_t::Any;
+ pkt.bitrate = 255;
+ pkt.definition = definition_t::Any;
+ pkt.frame_count = 2;
+ pkt.flags = 0;
+ bool r = encoder.encode(in, pkt);
+
+ pkt.definition = definition_t::HD1080;
+
+ REQUIRE( r );
+ REQUIRE( !decoder.decode(pkt, out) );
}
- // No longer supported
- /*SECTION("Full HD in, 720 out, FHD encoding") {
- in = cv::cuda::GpuMat(cv::Size(1920,1080), CV_8UC3, cv::Scalar(255,0,0));
- out = cv::cuda::GpuMat(cv::Size(1280,720), CV_8UC3, cv::Scalar(0,0,0));
+ SECTION("Corrupted but supported codec") {
+ in = cv::cuda::GpuMat(cv::Size(2560,720), CV_8UC4, cv::Scalar(255,0,0,0));
+ out = cv::cuda::GpuMat(cv::Size(2560,720), CV_8UC4, cv::Scalar(0,0,0,0));
- r = encoder.encode(in, ftl::codecs::kPreset0, [&out,&decoder](const ftl::codecs::Packet &pkt) {
- REQUIRE( decoder.decode(pkt, out) );
- });
+ ftl::codecs::Packet pkt;
+ pkt.codec = codec_t::Any;
+ pkt.bitrate = 255;
+ pkt.definition = definition_t::Any;
+ pkt.frame_count = 2;
+ pkt.flags = 0;
+ bool r = encoder.encode(in, pkt);
- REQUIRE( (out.rows == 720) );
- }*/
+ pkt.codec = codec_t::H264;
- // No longer supported
- /*SECTION("HHD in, FHD out, FHD encoding") {
- in = cv::cuda::GpuMat(cv::Size(1280,720), CV_8UC3, cv::Scalar(255,0,0));
- out = cv::cuda::GpuMat(cv::Size(1920,1080), CV_8UC3, cv::Scalar(0,0,0));
+ REQUIRE( r );
+ REQUIRE( !decoder.decode(pkt, out) );
+ }
- r = encoder.encode(in, ftl::codecs::kPreset0, [&out,&decoder](const ftl::codecs::Packet &pkt) {
- REQUIRE( decoder.decode(pkt, out) );
- });
+ SECTION("Corrupted and unsupported codec") {
+ in = cv::cuda::GpuMat(cv::Size(2560,720), CV_8UC4, cv::Scalar(255,0,0,0));
+ out = cv::cuda::GpuMat(cv::Size(2560,720), CV_8UC4, cv::Scalar(0,0,0,0));
- REQUIRE( (out.rows == 1080) );
- }*/
+ ftl::codecs::Packet pkt;
+ pkt.codec = codec_t::Any;
+ pkt.bitrate = 255;
+ pkt.definition = definition_t::Any;
+ pkt.frame_count = 2;
+ pkt.flags = 0;
+ bool r = encoder.encode(in, pkt);
- // No longer supported
- /*SECTION("FHD in, HHD out, SD encoding") {
- in = cv::cuda::GpuMat(cv::Size(1920,1080), CV_8UC3, cv::Scalar(255,0,0));
- out = cv::cuda::GpuMat(cv::Size(1280,720), CV_8UC3, cv::Scalar(0,0,0));
+ pkt.codec = codec_t::JPG;
- r = encoder.encode(in, ftl::codecs::kPreset4, [&out,&decoder](const ftl::codecs::Packet &pkt) {
- REQUIRE( decoder.decode(pkt, out) );
- });
+ REQUIRE( r );
+ REQUIRE( !decoder.decode(pkt, out) );
+ }
- REQUIRE( (out.rows == 720) );
- }*/
+ SECTION("Corrupted float flag") {
+ in = cv::cuda::GpuMat(cv::Size(2560,720), CV_8UC4, cv::Scalar(255,0,0,0));
+ out = cv::cuda::GpuMat(cv::Size(2560,720), CV_8UC4, cv::Scalar(0,0,0,0));
- REQUIRE( r );
- REQUIRE( (cv::cuda::sum(out) != cv::Scalar(0,0,0)) );
+ ftl::codecs::Packet pkt;
+ pkt.codec = codec_t::Any;
+ pkt.bitrate = 255;
+ pkt.definition = definition_t::Any;
+ pkt.frame_count = 2;
+ pkt.flags = 0;
+ bool r = encoder.encode(in, pkt);
+
+ pkt.flags = ftl::codecs::kFlagFloat;
+
+ REQUIRE( r );
+ REQUIRE( !decoder.decode(pkt, out) );
+ }
+
+ SECTION("Corrupted float mapped flags") {
+ in = cv::cuda::GpuMat(cv::Size(1280,720), CV_16U, cv::Scalar(255));
+ out = cv::cuda::GpuMat(cv::Size(1280,720), CV_16U, cv::Scalar(0));
+
+ ftl::codecs::Packet pkt;
+ pkt.codec = codec_t::Any;
+ pkt.bitrate = 255;
+ pkt.definition = definition_t::Any;
+ pkt.frame_count = 1;
+ pkt.flags = ftl::codecs::kFlagFloat;
+ bool r = encoder.encode(in, pkt);
+
+ pkt.codec = codec_t::HEVC;
+ pkt.flags = ftl::codecs::kFlagFloat | ftl::codecs::kFlagMappedDepth;
+
+ REQUIRE( r );
+ REQUIRE( !decoder.decode(pkt, out) );
+ }
+
+ SECTION("Missing float flag - lossless") {
+ in = cv::cuda::GpuMat(cv::Size(1280,720), CV_16U, cv::Scalar(255));
+ out = cv::cuda::GpuMat(cv::Size(1280,720), CV_16U, cv::Scalar(0));
+
+ ftl::codecs::Packet pkt;
+ pkt.codec = codec_t::Any;
+ pkt.bitrate = 255;
+ pkt.definition = definition_t::Any;
+ pkt.frame_count = 1;
+ pkt.flags = ftl::codecs::kFlagFloat;
+ bool r = encoder.encode(in, pkt);
+
+ pkt.flags = 0;
+
+ REQUIRE( r );
+ REQUIRE( !decoder.decode(pkt, out) );
+ }
+
+ SECTION("Missing data") {
+ in = cv::cuda::GpuMat(cv::Size(1280,720), CV_16U, cv::Scalar(255));
+ out = cv::cuda::GpuMat(cv::Size(1280,720), CV_16U, cv::Scalar(0));
+
+ ftl::codecs::Packet pkt;
+ pkt.codec = codec_t::Any;
+ pkt.bitrate = 255;
+ pkt.definition = definition_t::Any;
+ pkt.frame_count = 1;
+ pkt.flags = ftl::codecs::kFlagFloat;
+ bool r = encoder.encode(in, pkt);
+
+ pkt.data.resize(0);
+
+ REQUIRE( r );
+ REQUIRE( !decoder.decode(pkt, out) );
+ }
}
diff --git a/components/codecs/test/opencv_codec_unit.cpp b/components/codecs/test/opencv_codec_unit.cpp
index 0982551bd..ec3ec10c0 100644
--- a/components/codecs/test/opencv_codec_unit.cpp
+++ b/components/codecs/test/opencv_codec_unit.cpp
@@ -3,8 +3,7 @@
#include <ftl/codecs/opencv_decoder.hpp>
#include <ftl/threads.hpp>
-using ftl::codecs::CodecPreset;
-using ftl::codecs::preset_t;
+using ftl::codecs::format_t;
using ftl::codecs::definition_t;
using ftl::codecs::codec_t;
@@ -87,12 +86,16 @@ TEST_CASE( "OpenCVDecoder::decode() - A colour test image no resolution change"
cv::cuda::GpuMat in(cv::Size(1024,576), CV_8UC4, cv::Scalar(255,0,0,0));
cv::cuda::GpuMat out(cv::Size(1024,576), CV_8UC4, cv::Scalar(0,0,0,0));
- std::mutex mtx;
+ ftl::codecs::Packet pkt;
+ pkt.codec = codec_t::Any;
+ pkt.definition = definition_t::Any;
+ pkt.bitrate = 255;
+ pkt.flags = 0;
+ pkt.frame_count = 1;
+ bool r = encoder.encode(in, pkt);
- bool r = encoder.encode(in, ftl::codecs::kPreset4, [&mtx, &out,&decoder](const ftl::codecs::Packet &pkt) {
- std::unique_lock<std::mutex> lk(mtx);
- REQUIRE( decoder.decode(pkt, out) );
- });
+ REQUIRE( r );
+ REQUIRE( decoder.decode(pkt, out) );
- REQUIRE( (cv::cuda::sum(out) != cv::Scalar(0,0,0)) );
+ REQUIRE( (cv::cuda::sum(out) != cv::Scalar(0,0,0,0)) );
}
diff --git a/components/common/cpp/src/configuration.cpp b/components/common/cpp/src/configuration.cpp
index c53f50c78..a752a3c1c 100644
--- a/components/common/cpp/src/configuration.cpp
+++ b/components/common/cpp/src/configuration.cpp
@@ -560,11 +560,13 @@ Configurable *ftl::config::configure(ftl::config::json_t &cfg) {
static bool doing_cleanup = false;
void ftl::config::cleanup() {
+ if (doing_cleanup) return;
doing_cleanup = true;
for (auto f : config_instance) {
delete f.second;
}
config_instance.clear();
+ doing_cleanup = false;
}
void ftl::config::removeConfigurable(Configurable *cfg) {
diff --git a/components/rgbd-sources/include/ftl/rgbd/format.hpp b/components/rgbd-sources/include/ftl/rgbd/format.hpp
index 2e86aaf96..e52a6627f 100644
--- a/components/rgbd-sources/include/ftl/rgbd/format.hpp
+++ b/components/rgbd-sources/include/ftl/rgbd/format.hpp
@@ -4,7 +4,7 @@
#include <opencv2/core.hpp>
#include <opencv2/core/cuda.hpp>
#include <ftl/cuda_util.hpp>
-#include <ftl/codecs/bitrates.hpp>
+#include <ftl/codecs/codecs.hpp>
#include <ftl/traits.hpp>
namespace ftl {
diff --git a/components/rgbd-sources/include/ftl/rgbd/frame.hpp b/components/rgbd-sources/include/ftl/rgbd/frame.hpp
index 85b97326b..f4dc76987 100644
--- a/components/rgbd-sources/include/ftl/rgbd/frame.hpp
+++ b/components/rgbd-sources/include/ftl/rgbd/frame.hpp
@@ -11,7 +11,7 @@
#include <ftl/codecs/channels.hpp>
#include <ftl/rgbd/format.hpp>
#include <ftl/rgbd/camera.hpp>
-#include <ftl/codecs/bitrates.hpp>
+#include <ftl/codecs/codecs.hpp>
#include <ftl/codecs/packet.hpp>
#include <ftl/cuda_common.hpp>
diff --git a/components/rgbd-sources/src/sources/net/net.cpp b/components/rgbd-sources/src/sources/net/net.cpp
index 57b494032..7b21594cf 100644
--- a/components/rgbd-sources/src/sources/net/net.cpp
+++ b/components/rgbd-sources/src/sources/net/net.cpp
@@ -8,7 +8,7 @@
#include "colour.hpp"
#include <ftl/rgbd/streamer.hpp>
-#include <ftl/codecs/bitrates.hpp>
+#include <ftl/codecs/codecs.hpp>
using ftl::rgbd::detail::NetFrame;
using ftl::rgbd::detail::NetFrameQueue;
diff --git a/components/streams/include/ftl/streams/filestream.hpp b/components/streams/include/ftl/streams/filestream.hpp
index 7baffd75a..e1677b413 100644
--- a/components/streams/include/ftl/streams/filestream.hpp
+++ b/components/streams/include/ftl/streams/filestream.hpp
@@ -37,7 +37,7 @@ class File : public Stream {
/**
* Manually tick through the frames one per call.
*/
- bool tick();
+ bool tick(int64_t);
enum class Mode {
Read,
diff --git a/components/streams/include/ftl/streams/netstream.hpp b/components/streams/include/ftl/streams/netstream.hpp
index 04cf48afe..3c7aef3c6 100644
--- a/components/streams/include/ftl/streams/netstream.hpp
+++ b/components/streams/include/ftl/streams/netstream.hpp
@@ -56,6 +56,8 @@ class Net : public Stream {
bool end() override;
bool active() override;
+ void reset() override;
+
inline const ftl::UUID &getPeer() const { return peer_; }
private:
diff --git a/components/streams/include/ftl/streams/receiver.hpp b/components/streams/include/ftl/streams/receiver.hpp
index e75c3b598..1d1fe816c 100644
--- a/components/streams/include/ftl/streams/receiver.hpp
+++ b/components/streams/include/ftl/streams/receiver.hpp
@@ -54,6 +54,7 @@ class Receiver : public ftl::Configurable, public ftl::rgbd::Generator {
ftl::rgbd::FrameState state;
ftl::rgbd::Frame frame;
ftl::codecs::Decoder* decoders[32];
+ cv::cuda::GpuMat surface[32];
MUTEX mutex;
ftl::codecs::Channels<0> completed;
};
@@ -62,7 +63,7 @@ class Receiver : public ftl::Configurable, public ftl::rgbd::Generator {
void _processConfig(InternalStates &frame, const ftl::codecs::Packet &pkt);
void _createDecoder(InternalStates &frame, int chan, const ftl::codecs::Packet &pkt);
- InternalStates &_getFrame(const ftl::codecs::StreamPacket &spkt);
+ InternalStates &_getFrame(const ftl::codecs::StreamPacket &spkt, int ix=0);
};
}
diff --git a/components/streams/include/ftl/streams/sender.hpp b/components/streams/include/ftl/streams/sender.hpp
index 2cfc57bb3..31022ee45 100644
--- a/components/streams/include/ftl/streams/sender.hpp
+++ b/components/streams/include/ftl/streams/sender.hpp
@@ -29,16 +29,31 @@ class Sender : public ftl::Configurable {
//void onStateChange(const std::function<void(ftl::codecs::Channel, int, int)>&);
+ void onRequest(const ftl::stream::StreamCallback &);
+
private:
ftl::stream::Stream *stream_;
int64_t timestamp_;
SHARED_MUTEX mutex_;
std::atomic_flag do_inject_;
//std::function<void(ftl::codecs::Channel, int, int)> state_cb_;
-
- std::unordered_map<int, ftl::codecs::Encoder*> encoders_;
-
- ftl::codecs::Encoder *_getEncoder(int fsid, int fid, ftl::codecs::Channel c);
+ ftl::stream::StreamCallback reqcb_;
+
+ struct EncodingState {
+ uint8_t bitrate;
+ ftl::codecs::Encoder *encoder[2];
+ cv::cuda::GpuMat surface;
+ cudaStream_t stream;
+ };
+
+ std::unordered_map<int, EncodingState> state_;
+
+ //ftl::codecs::Encoder *_getEncoder(int fsid, int fid, ftl::codecs::Channel c);
+ void _encodeChannel(const ftl::rgbd::FrameSet &fs, ftl::codecs::Channel c, bool reset);
+ int _generateTiles(const ftl::rgbd::FrameSet &fs, int offset, ftl::codecs::Channel c, cv::cuda::Stream &stream, bool);
+ EncodingState &_getTile(int fsid, ftl::codecs::Channel c);
+ cv::Rect _generateROI(const ftl::rgbd::FrameSet &fs, ftl::codecs::Channel c, int offset);
+ float _selectFloatMax(ftl::codecs::Channel c);
};
}
diff --git a/components/streams/include/ftl/streams/stream.hpp b/components/streams/include/ftl/streams/stream.hpp
index 08064b6c9..2502f8b69 100644
--- a/components/streams/include/ftl/streams/stream.hpp
+++ b/components/streams/include/ftl/streams/stream.hpp
@@ -56,6 +56,13 @@ class Stream : public ftl::Configurable {
*/
virtual bool active()=0;
+ /**
+ * Perform a forced reset of the stream. This means something different
+ * depending on stream type, for example with a network stream it involves
+ * resending all stream requests as if a reconnection had occured.
+ */
+ virtual void reset();
+
/**
* Query available video channels for a frameset.
*/
@@ -70,7 +77,7 @@ class Stream : public ftl::Configurable {
/**
* Change the video channel selection for a frameset.
*/
- void select(int fs, const ftl::codecs::Channels<0> &);
+ void select(int fs, const ftl::codecs::Channels<0> &, bool make=false);
/**
* Number of framesets in stream.
@@ -116,6 +123,8 @@ class Muxer : public Stream {
bool end() override;
bool active() override;
+ void reset() override;
+
private:
struct StreamEntry {
Stream *stream;
@@ -152,6 +161,8 @@ class Broadcast : public Stream {
bool end() override;
bool active() override;
+ void reset() override;
+
private:
std::list<Stream*> streams_;
StreamCallback cb_;
@@ -177,6 +188,8 @@ class Intercept : public Stream {
bool end() override;
bool active() override;
+ void reset() override;
+
private:
Stream *stream_;
StreamCallback cb_;
diff --git a/components/streams/src/filestream.cpp b/components/streams/src/filestream.cpp
index 232a929f6..784b14fb6 100644
--- a/components/streams/src/filestream.cpp
+++ b/components/streams/src/filestream.cpp
@@ -55,7 +55,7 @@ bool File::post(const ftl::codecs::StreamPacket &s, const ftl::codecs::Packet &p
return ostream_->good();
}
-bool File::tick() {
+bool File::tick(int64_t ts) {
if (!active_) return false;
if (mode_ != Mode::Read) {
LOG(ERROR) << "Cannot read from a write only file";
@@ -77,6 +77,7 @@ bool File::tick() {
for (auto i = data_.begin(); i != data_.end(); ++i) {
if (std::get<0>(*i).timestamp <= timestamp_) {
++jobs_;
+ std::get<0>(*i).timestamp = ts;
ftl::pool.push([this,i](int id) {
auto &spkt = std::get<0>(*i);
auto &pkt = std::get<1>(*i);
@@ -141,6 +142,10 @@ bool File::tick() {
if (version_ < 4) {
std::get<0>(data).frame_number = std::get<0>(data).streamID;
std::get<0>(data).streamID = 0;
+ if (isFloatChannel(std::get<0>(data).channel)) std::get<1>(data).flags |= ftl::codecs::kFlagFloat;
+
+ auto codec = std::get<1>(data).codec;
+ if (codec == ftl::codecs::codec_t::HEVC) std::get<1>(data).codec = ftl::codecs::codec_t::HEVC_LOSSLESS;
}
std::get<0>(data).version = 4;
@@ -151,6 +156,7 @@ bool File::tick() {
// the data buffer is already several frames ahead so is processed
// above. Hence, no need to bother parallelising this bit.
if (std::get<0>(data).timestamp <= timestamp_) {
+ std::get<0>(data).timestamp = ts;
if (cb_) {
dlk.lock();
try {
@@ -188,7 +194,7 @@ bool File::tick() {
bool File::run() {
timer_ = ftl::timer::add(ftl::timer::kTimerMain, [this](int64_t ts) {
- tick();
+ tick(ts);
return active_;
});
return true;
@@ -219,10 +225,10 @@ bool File::begin(bool dorun) {
// Capture current time to adjust timestamps
timestart_ = (ftl::timer::get_time() / ftl::timer::getInterval()) * ftl::timer::getInterval();
active_ = true;
- interval_ = ftl::timer::getInterval();
+ interval_ = 50; //ftl::timer::getInterval();
timestamp_ = timestart_;
- tick(); // Do some now!
+ tick(timestart_); // Do some now!
if (dorun) run();
} else if (mode_ == Mode::Write) {
if (!ostream_) ostream_ = new std::ofstream;
diff --git a/components/streams/src/netstream.cpp b/components/streams/src/netstream.cpp
index b5039d71a..b11fdbb7d 100644
--- a/components/streams/src/netstream.cpp
+++ b/components/streams/src/netstream.cpp
@@ -37,6 +37,7 @@ Net::Net(nlohmann::json &config, ftl::net::Universe *net) : Stream(config), net_
}
last_frame_ = 0;
+ last_msg_ = 0;
time_peer_ = ftl::UUID(0);
}
@@ -53,7 +54,7 @@ bool Net::post(const ftl::codecs::StreamPacket &spkt, const ftl::codecs::Packet
if (!active_) return false;
// Check if the channel has been requested recently enough. If not then disable it.
- if (host_ && pkt.data.size() > 0 && static_cast<int>(spkt.channel) >= 0 && static_cast<int>(spkt.channel) < 32) {
+ if (host_ && pkt.data.size() > 0 && spkt.frame_number == 0 && static_cast<int>(spkt.channel) >= 0 && static_cast<int>(spkt.channel) < 32) {
if (reqtally_[static_cast<int>(spkt.channel)] == 0) {
auto sel = selected(0);
sel -= spkt.channel;
@@ -75,9 +76,9 @@ bool Net::post(const ftl::codecs::StreamPacket &spkt, const ftl::codecs::Packet
auto &client = *c;
// Quality filter the packets
- if (client.quality >= 0 && pkt.bitrate != client.quality) {
+ if (pkt.bitrate > 0 && pkt.bitrate != client.quality) {
++c;
- LOG(INFO) << "INCORRECT QUALITY";
+ LOG(INFO) << "Incorrect quality: " << (int)pkt.bitrate << " but requested " << (int)client.quality;
continue;
}
@@ -150,16 +151,18 @@ bool Net::begin() {
if (last_frame_ != spkt.timestamp) {
last_frame_ = spkt.timestamp;
- auto sel = selected(0);
+ if (size() > 0) {
+ auto sel = selected(0);
- // A change in channel selections, so send those requests now
- if (sel != last_selected_) {
- auto changed = sel - last_selected_;
- last_selected_ = sel;
+ // A change in channel selections, so send those requests now
+ if (sel != last_selected_) {
+ auto changed = sel - last_selected_;
+ last_selected_ = sel;
- if (size() > 0) {
- for (auto c : changed) {
- _sendRequest(c, kAllFramesets, kAllFrames, 30, 0);
+ if (size() > 0) {
+ for (auto c : changed) {
+ _sendRequest(c, kAllFramesets, kAllFrames, 30, 0);
+ }
}
}
}
@@ -168,6 +171,8 @@ bool Net::begin() {
if (tally_ <= 5) {
// Yes, so send new requests
if (size() > 0) {
+ auto sel = selected(0);
+
for (auto c : sel) {
_sendRequest(c, kAllFramesets, kAllFrames, 30, 0);
}
@@ -188,7 +193,7 @@ bool Net::begin() {
for (int i=0; i<size(); ++i) {
select(i, selected(i) + spkt.channel);
}
- reqtally_[static_cast<int>(spkt.channel)] = static_cast<int>(pkt.frame_count)*size()*kTallyScale;
+ reqtally_[static_cast<int>(spkt.channel)] = static_cast<int>(pkt.frame_count)*kTallyScale;
} else {
select(spkt.frameSetID(), selected(spkt.frameSetID()) + spkt.channel);
}
@@ -224,6 +229,19 @@ bool Net::begin() {
return true;
}
+void Net::reset() {
+ UNIQUE_LOCK(mutex_, lk);
+
+ if (size() > 0) {
+ auto sel = selected(0);
+
+ for (auto c : sel) {
+ _sendRequest(c, kAllFramesets, kAllFrames, 30, 0);
+ }
+ }
+ tally_ = 30*kTallyScale;
+}
+
bool Net::_sendRequest(Channel c, uint8_t frameset, uint8_t frames, uint8_t count, uint8_t bitrate) {
if (!active_ || host_) return false;
@@ -286,7 +304,7 @@ bool Net::_processRequest(ftl::net::Peer &p, const ftl::codecs::Packet &pkt) {
if (!found) {
auto &client = clients_.emplace_back();
client.peerid = p.id();
- client.quality = 0; // TODO: Use quality given in packet
+ client.quality = 255; // TODO: Use quality given in packet
client.txcount = 0;
client.txmax = static_cast<int>(pkt.frame_count)*kTallyScale;
}
diff --git a/components/streams/src/receiver.cpp b/components/streams/src/receiver.cpp
index 36b454759..ad74db5d7 100644
--- a/components/streams/src/receiver.cpp
+++ b/components/streams/src/receiver.cpp
@@ -1,4 +1,5 @@
#include <ftl/streams/receiver.hpp>
+#include <ftl/codecs/depth_convert_cuda.hpp>
#include "parsers.hpp"
#include "injectors.hpp"
@@ -54,14 +55,16 @@ Receiver::InternalStates::InternalStates() {
for (int i=0; i<32; ++i) decoders[i] = nullptr;
}
-Receiver::InternalStates &Receiver::_getFrame(const StreamPacket &spkt) {
+Receiver::InternalStates &Receiver::_getFrame(const StreamPacket &spkt, int ix) {
+ int fn = spkt.frameNumber()+ix;
+
UNIQUE_LOCK(mutex_, lk);
- while (frames_.size() <= spkt.frameNumber()) {
+ while (frames_.size() <= fn) {
//frames_.resize(spkt.frameNumber()+1);
frames_.push_back(new InternalStates);
- frames_[frames_.size()-1]->state.set("name",std::string("Source ")+std::to_string(spkt.frameNumber()+1));
+ frames_[frames_.size()-1]->state.set("name",std::string("Source ")+std::to_string(fn+1));
}
- auto &f = *frames_[spkt.frameNumber()];
+ auto &f = *frames_[fn];
if (!f.frame.origin()) f.frame.setOrigin(&f.state);
return f;
}
@@ -78,7 +81,7 @@ void Receiver::setStream(ftl::stream::Stream *s) {
const ftl::codecs::Channel rchan = spkt.channel;
const unsigned int channum = (unsigned int)spkt.channel;
- //LOG(INFO) << "PACKET: " << spkt.timestamp << ", " << (int)spkt.channel << ", " << (int)pkt.codec;
+ //LOG(INFO) << "PACKET: " << spkt.timestamp << ", " << (int)spkt.channel << ", " << (int)pkt.codec << ", " << (int)pkt.definition;
// TODO: Allow for multiple framesets
if (spkt.frameSetID() > 0) return;
@@ -89,48 +92,67 @@ void Receiver::setStream(ftl::stream::Stream *s) {
// Dummy no data packet.
if (pkt.data.size() == 0) return;
- InternalStates &frame = _getFrame(spkt);
+ InternalStates &iframe = _getFrame(spkt);
+
+ auto [tx,ty] = ftl::codecs::chooseTileConfig(pkt.frame_count);
+ int width = ftl::codecs::getWidth(pkt.definition);
+ int height = ftl::codecs::getHeight(pkt.definition);
//if (spkt.timestamp > frame.timestamp && !frame.completed.empty()) {
// LOG(WARNING) << "Next frame received";
//return;
//}
- // Deal with the special channels...
- switch (rchan) {
- case Channel::Configuration : frame.state.getConfig() = nlohmann::json::parse(parseConfig(pkt)); return;
- case Channel::Calibration : frame.state.getLeft() = parseCalibration(pkt); return;
- case Channel::Calibration2 : frame.state.getRight() = parseCalibration(pkt); return;
- case Channel::Pose : frame.state.getPose() = parsePose(pkt); return;
- default: break;
+ if (channum >= 64) {
+ for (int i=0; i<pkt.frame_count; ++i) {
+ InternalStates &frame = _getFrame(spkt,i);
+
+ // Deal with the special channels...
+ switch (rchan) {
+ case Channel::Configuration : frame.state.getConfig() = nlohmann::json::parse(parseConfig(pkt)); break;
+ case Channel::Calibration : frame.state.getLeft() = parseCalibration(pkt); break;
+ case Channel::Calibration2 : frame.state.getRight() = parseCalibration(pkt); break;
+ case Channel::Pose : frame.state.getPose() = parsePose(pkt); break;
+ default: break;
+ }
+ }
+ return;
}
- // Packets are for unwanted channel.
- //if (rchan != Channel::Colour && rchan != chan) return;
-
- if (frame.frame.hasChannel(spkt.channel)) {
- // FIXME: Is this a corruption in recording or in playback?
- // Seems to occur in same place in ftl file, one channel is missing
- LOG(ERROR) << "Previous frame not complete: " << frame.timestamp;
- //LOG(ERROR) << " --- " << (string)spkt;
- UNIQUE_LOCK(frame.mutex, lk);
- frame.frame.reset();
- frame.completed.clear();
- }
- frame.timestamp = spkt.timestamp;
+ for (int i=0; i<pkt.frame_count; ++i) {
+ InternalStates &frame = _getFrame(spkt,i);
- // Add channel to frame and allocate memory if required
- const cv::Size size = cv::Size(ftl::codecs::getWidth(pkt.definition), ftl::codecs::getHeight(pkt.definition));
- frame.frame.create<cv::cuda::GpuMat>(spkt.channel).create(size, (isFloatChannel(rchan) ? CV_32FC1 : CV_8UC4));
+ // Packets are for unwanted channel.
+ //if (rchan != Channel::Colour && rchan != chan) return;
+
+ if (frame.frame.hasChannel(spkt.channel)) {
+ // FIXME: Is this a corruption in recording or in playback?
+ // Seems to occur in same place in ftl file, one channel is missing
+ LOG(ERROR) << "Previous frame not complete: " << frame.timestamp;
+ //LOG(ERROR) << " --- " << (string)spkt;
+ UNIQUE_LOCK(frame.mutex, lk);
+ frame.frame.reset();
+ frame.completed.clear();
+ }
+ frame.timestamp = spkt.timestamp;
+
+ // Add channel to frame and allocate memory if required
+ const cv::Size size = cv::Size(width, height);
+ frame.frame.create<cv::cuda::GpuMat>(spkt.channel).create(size, (isFloatChannel(rchan) ? CV_32FC1 : CV_8UC4));
+ }
Packet tmppkt = pkt;
- frame.frame.pushPacket(spkt.channel, tmppkt);
+ iframe.frame.pushPacket(spkt.channel, tmppkt);
//LOG(INFO) << " CODEC = " << (int)pkt.codec << " " << (int)pkt.flags << " " << (int)spkt.channel;
+ //LOG(INFO) << "Decode surface: " << (width*tx) << "x" << (height*ty);
+
+ auto &surface = iframe.surface[static_cast<int>(spkt.channel)];
+ surface.create(height*ty, width*tx, ((isFloatChannel(spkt.channel)) ? ((pkt.flags & 0x2) ? CV_16UC4 : CV_16U) : CV_8UC4));
// Do the actual decode
- _createDecoder(frame, channum, pkt);
- auto *decoder = frame.decoders[channum];
+ _createDecoder(iframe, channum, pkt);
+ auto *decoder = iframe.decoders[channum];
if (!decoder) {
LOG(ERROR) << "No frame decoder available";
return;
@@ -138,40 +160,74 @@ void Receiver::setStream(ftl::stream::Stream *s) {
try {
//LOG(INFO) << "TYPE = " << frame.frame.get<cv::cuda::GpuMat>(spkt.channel).type();
- decoder->decode(pkt, frame.frame.get<cv::cuda::GpuMat>(spkt.channel));
+ if (!decoder->decode(pkt, surface)) {
+ LOG(ERROR) << "Decode failed";
+ //return;
+ }
} catch (std::exception &e) {
LOG(ERROR) << "Decode failed for " << spkt.timestamp << ": " << e.what();
+ //return;
+ }
+
+ /*if (spkt.channel == Channel::Depth && (pkt.flags & 0x2)) {
+ cv::Mat tmp;
+ surface.download(tmp);
+ cv::imshow("Test", tmp);
+ cv::waitKey(1);
+ }*/
+
+ for (int i=0; i<pkt.frame_count; ++i) {
+ InternalStates &frame = _getFrame(spkt,i);
+
+ cv::Rect roi((i % tx)*width, (i / tx)*height, width, height);
+ cv::cuda::GpuMat sroi = surface(roi);
+
+ // Do colour conversion
+ if (isFloatChannel(rchan) && (pkt.flags & 0x2)) {
+ //LOG(INFO) << "VUYA Convert";
+ ftl::cuda::vuya_to_depth(frame.frame.get<cv::cuda::GpuMat>(spkt.channel), sroi, 16.0f, decoder->stream());
+ } else if (isFloatChannel(rchan)) {
+ sroi.convertTo(frame.frame.get<cv::cuda::GpuMat>(spkt.channel), CV_32FC1, 1.0f/1000.0f, decoder->stream());
+ } else {
+ cv::cuda::cvtColor(sroi, frame.frame.get<cv::cuda::GpuMat>(spkt.channel), cv::COLOR_RGBA2BGRA, 0, decoder->stream());
+ }
}
- frame.completed += spkt.channel;
+ decoder->stream().waitForCompletion();
+
+ for (int i=0; i<pkt.frame_count; ++i) {
+ InternalStates &frame = _getFrame(spkt,i);
+
+ frame.completed += spkt.channel;
- // Complete if all requested frames are found
- auto sel = stream_->selected(spkt.frameSetID());
+ // Complete if all requested frames are found
+ auto sel = stream_->selected(spkt.frameSetID());
- if ((frame.completed & sel) == sel) {
- UNIQUE_LOCK(frame.mutex, lk);
if ((frame.completed & sel) == sel) {
- timestamp_ = frame.timestamp;
+ UNIQUE_LOCK(frame.mutex, lk);
+ if ((frame.completed & sel) == sel) {
+ timestamp_ = frame.timestamp;
- //LOG(INFO) << "BUILDER PUSH: " << timestamp_ << ", " << spkt.frameNumber();
+ //LOG(INFO) << "BUILDER PUSH: " << timestamp_ << ", " << spkt.frameNumber();
- if (frame.state.getLeft().width == 0) {
- LOG(WARNING) << "Missing calibration, skipping frame";
- //frame.frame.reset();
- //frame.completed.clear();
- //return;
- }
+ if (frame.state.getLeft().width == 0) {
+ LOG(WARNING) << "Missing calibration, skipping frame";
+ //frame.frame.reset();
+ //frame.completed.clear();
+ //return;
+ }
- // TODO: Have multiple builders for different framesets.
- builder_.push(frame.timestamp, spkt.frameNumber(), frame.frame);
+ // TODO: Have multiple builders for different framesets.
+ builder_.push(frame.timestamp, spkt.frameNumber()+i, frame.frame);
- // Check for any state changes and send them back
- if (frame.state.hasChanged(Channel::Pose)) injectPose(stream_, frame.frame, frame.timestamp, spkt.frameNumber());
- if (frame.state.hasChanged(Channel::Calibration)) injectCalibration(stream_, frame.frame, frame.timestamp, spkt.frameNumber());
- if (frame.state.hasChanged(Channel::Calibration2)) injectCalibration(stream_, frame.frame, frame.timestamp, spkt.frameNumber(), true);
+ // Check for any state changes and send them back
+ if (frame.state.hasChanged(Channel::Pose)) injectPose(stream_, frame.frame, frame.timestamp, spkt.frameNumber()+i);
+ if (frame.state.hasChanged(Channel::Calibration)) injectCalibration(stream_, frame.frame, frame.timestamp, spkt.frameNumber()+i);
+ if (frame.state.hasChanged(Channel::Calibration2)) injectCalibration(stream_, frame.frame, frame.timestamp, spkt.frameNumber()+i, true);
- frame.frame.reset();
- frame.completed.clear();
+ frame.frame.reset();
+ frame.completed.clear();
+ }
}
}
});
diff --git a/components/streams/src/sender.cpp b/components/streams/src/sender.cpp
index 275ad63f0..261c8b2fe 100644
--- a/components/streams/src/sender.cpp
+++ b/components/streams/src/sender.cpp
@@ -1,4 +1,5 @@
#include <ftl/streams/sender.hpp>
+#include <ftl/codecs/depth_convert_cuda.hpp>
#include "injectors.hpp"
@@ -10,16 +11,21 @@ using ftl::codecs::Channel;
using ftl::codecs::definition_t;
using ftl::codecs::device_t;
using ftl::codecs::codec_t;
+using ftl::codecs::format_t;
using ftl::stream::injectCalibration;
using ftl::stream::injectPose;
using ftl::stream::injectConfig;
Sender::Sender(nlohmann::json &config) : ftl::Configurable(config), stream_(nullptr) {
- //do_inject_ = false;
+ do_inject_.test_and_set();
}
Sender::~Sender() {
// Delete all encoders
+ for (auto c : state_) {
+ if (c.second.encoder[0]) ftl::codecs::free(c.second.encoder[0]);
+ if (c.second.encoder[1]) ftl::codecs::free(c.second.encoder[1]);
+ }
}
/*void Sender::onStateChange(const std::function<void(ftl::codecs::Channel,const ftl::rgbd::FrameState&)> &cb) {
@@ -31,9 +37,10 @@ void Sender::setStream(ftl::stream::Stream*s) {
if (stream_) stream_->onPacket(nullptr);
stream_ = s;
stream_->onPacket([this](const ftl::codecs::StreamPacket &spkt, const ftl::codecs::Packet &pkt) {
- LOG(INFO) << "SENDER REQUEST : " << (int)spkt.channel;
+ //LOG(INFO) << "SENDER REQUEST : " << (int)spkt.channel;
//if (state_cb_) state_cb_(spkt.channel, spkt.streamID, spkt.frame_number);
+ if (reqcb_) reqcb_(spkt,pkt);
// Inject state packets
//do_inject_ = true;
@@ -41,6 +48,10 @@ void Sender::setStream(ftl::stream::Stream*s) {
});
}
+void Sender::onRequest(const ftl::stream::StreamCallback &cb) {
+ reqcb_ = cb;
+}
+
template <typename T>
static void writeValue(std::vector<unsigned char> &data, T value) {
unsigned char *pvalue_start = (unsigned char*)&value;
@@ -73,12 +84,12 @@ void Sender::post(const ftl::rgbd::FrameSet &fs) {
if (!stream_) return;
Channels selected;
- if (stream_->size() > 0) selected = stream_->selected(0);
-
Channels available; // but not selected and actually sent.
+ Channels needencoding;
+
+ if (stream_->size() > 0) selected = stream_->selected(0);
bool do_inject = !do_inject_.test_and_set();
- //do_inject_ = false;
for (int i=0; i<fs.frames.size(); ++i) {
const auto &frame = fs.frames[i];
@@ -124,22 +135,7 @@ void Sender::post(const ftl::rgbd::FrameSet &fs) {
//}
}
} else {
- auto *enc = _getEncoder(fs.id, i, cc);
-
- if (enc) {
- // FIXME: Timestamps may not always be aligned to interval.
- //if (do_inject || fs.timestamp % (10*ftl::timer::getInterval()) == 0) enc->reset();
- if (do_inject) enc->reset();
- try {
- enc->encode(frame.get<cv::cuda::GpuMat>(cc), 0, [this,&spkt](const ftl::codecs::Packet &pkt){
- stream_->post(spkt, pkt);
- });
- } catch (std::exception &e) {
- LOG(ERROR) << "Exception in encoder: " << e.what();
- }
- } else {
- LOG(ERROR) << "No encoder";
- }
+ needencoding += c;
}
} else {
available += c;
@@ -164,23 +160,167 @@ void Sender::post(const ftl::rgbd::FrameSet &fs) {
stream_->post(spkt, pkt);
}
+ for (auto c : needencoding) {
+ // TODO: One thread per channel.
+ _encodeChannel(fs, c, do_inject);
+ }
+
//do_inject_ = false;
}
-ftl::codecs::Encoder *Sender::_getEncoder(int fsid, int fid, Channel c) {
- int id = (fsid << 16) | (fid << 8) | (int)c;
+void Sender::_encodeChannel(const ftl::rgbd::FrameSet &fs, Channel c, bool reset) {
+ bool lossless = value("lossless", false);
+ int max_bitrate = std::max(0, std::min(255, value("max_bitrate", 255)));
+ int min_bitrate = std::max(0, std::min(255, value("min_bitrate", 0)));
+ codec_t codec = value("codec", codec_t::Any);
+
+ int offset = 0;
+ while (offset < fs.frames.size()) {
+ StreamPacket spkt;
+ spkt.version = 4;
+ spkt.timestamp = fs.timestamp;
+ spkt.streamID = 0; // FIXME: fs.id;
+ spkt.frame_number = offset;
+ spkt.channel = c;
+
+ auto &tile = _getTile(fs.id, c);
+
+ ftl::codecs::Encoder *enc = tile.encoder[(offset==0)?0:1];
+ if (!enc) {
+ enc = ftl::codecs::allocateEncoder(
+ definition_t::HD1080, device_t::Any, codec);
+ tile.encoder[(offset==0)?0:1] = enc;
+ }
+
+ if (!enc) {
+ LOG(ERROR) << "No encoder";
+ return;
+ }
+
+ int count = _generateTiles(fs, offset, c, enc->stream(), lossless);
+ if (count <= 0) {
+ LOG(ERROR) << "Could not generate tiles.";
+ break;
+ }
+
+ if (enc) {
+ // FIXME: Timestamps may not always be aligned to interval.
+ //if (do_inject || fs.timestamp % (10*ftl::timer::getInterval()) == 0) enc->reset();
+ if (reset) enc->reset();
+
+ try {
+ ftl::codecs::Packet pkt;
+ pkt.frame_count = count;
+ pkt.codec = codec;
+ pkt.definition = definition_t::Any;
+ pkt.bitrate = max_bitrate;
+ pkt.flags = 0;
+
+ if (!lossless && ftl::codecs::isFloatChannel(c)) pkt.flags = ftl::codecs::kFlagFloat | ftl::codecs::kFlagMappedDepth;
+ else if (lossless && ftl::codecs::isFloatChannel(c)) pkt.flags = ftl::codecs::kFlagFloat;
+ else pkt.flags = ftl::codecs::kFlagFlipRGB;
+
+ // Choose correct region of interest into the surface.
+ cv::Rect roi = _generateROI(fs, c, offset);
+ cv::cuda::GpuMat sroi = tile.surface(roi);
+
+ if (enc->encode(sroi, pkt)) {
+ stream_->post(spkt, pkt);
+
+ /*cv::Mat tmp;
+ tile.surface.download(tmp);
+ cv::imshow("Test", tmp);
+ cv::waitKey(1);*/
+ } else {
+ LOG(ERROR) << "Encoding failed";
+ }
+ } catch (std::exception &e) {
+ LOG(ERROR) << "Exception in encoder: " << e.what();
+ }
+ } else {
+ LOG(ERROR) << "No encoder";
+ }
+
+ offset += count;
+ }
+}
+
+cv::Rect Sender::_generateROI(const ftl::rgbd::FrameSet &fs, ftl::codecs::Channel c, int offset) {
+ const ftl::rgbd::Frame *cframe = &fs.frames[offset];
+ int rwidth = cframe->get<cv::cuda::GpuMat>(c).cols;
+ int rheight = cframe->get<cv::cuda::GpuMat>(c).rows;
+ auto [tx,ty] = ftl::codecs::chooseTileConfig(fs.frames.size()-offset);
+ return cv::Rect(0, 0, tx*rwidth, ty*rheight);
+}
+
+Sender::EncodingState &Sender::_getTile(int fsid, Channel c) {
+ int id = (fsid << 8) | (int)c;
{
SHARED_LOCK(mutex_, lk);
- auto i = encoders_.find(id);
- if (i != encoders_.end()) {
+ auto i = state_.find(id);
+ if (i != state_.end()) {
return (*i).second;
}
}
- auto *enc = ftl::codecs::allocateEncoder(
- definition_t::HD1080, device_t::Any, codec_t::Any);
UNIQUE_LOCK(mutex_, lk);
- encoders_[id] = enc;
- return enc;
+ state_[id] = {
+ uint8_t(255),
+ {nullptr,nullptr},
+ cv::cuda::GpuMat(),
+ 0
+ };
+ return state_[id];
+}
+
+float Sender::_selectFloatMax(Channel c) {
+ switch (c) {
+ case Channel::Depth : return 16.0f;
+ default : return 1.0f;
+ }
+}
+
+int Sender::_generateTiles(const ftl::rgbd::FrameSet &fs, int offset, Channel c, cv::cuda::Stream &stream, bool lossless) {
+ auto &surface = _getTile(fs.id, c);
+
+ const ftl::rgbd::Frame *cframe = &fs.frames[offset];
+ auto &m = cframe->get<cv::cuda::GpuMat>(c);
+
+ // Choose tile configuration and allocate memory
+ auto [tx,ty] = ftl::codecs::chooseTileConfig(fs.frames.size());
+ int rwidth = cframe->get<cv::cuda::GpuMat>(c).cols;
+ int rheight = cframe->get<cv::cuda::GpuMat>(c).rows;
+ int width = tx * rwidth;
+ int height = ty * rheight;
+ int tilecount = tx*ty;
+ int count = 0;
+
+ surface.surface.create(height, width, (lossless && m.type() == CV_32F) ? CV_16U : CV_8UC4);
+
+ // Loop over tiles with ROI mats and do colour conversions.
+ while (tilecount > 0 && count+offset < fs.frames.size()) {
+ auto &m = cframe->get<cv::cuda::GpuMat>(c);
+ cv::Rect roi((count % tx)*rwidth, (count / tx)*rheight, rwidth, rheight);
+ cv::cuda::GpuMat sroi = surface.surface(roi);
+
+ if (m.type() == CV_32F) {
+ if (lossless) {
+ m.convertTo(sroi, CV_16UC1, 1000, stream);
+ } else {
+ ftl::cuda::depth_to_vuya(m, sroi, _selectFloatMax(c), stream);
+ }
+ } else if (m.type() == CV_8UC4) {
+ cv::cuda::cvtColor(m, sroi, cv::COLOR_BGRA2RGBA, 0, stream);
+ } else {
+ LOG(ERROR) << "Unsupported colour format";
+ return 0;
+ }
+
+ ++count;
+ --tilecount;
+ cframe = &fs.frames[offset+count];
+ }
+
+ return count;
}
diff --git a/components/streams/src/stream.cpp b/components/streams/src/stream.cpp
index 4a01287f4..2259ba092 100644
--- a/components/streams/src/stream.cpp
+++ b/components/streams/src/stream.cpp
@@ -17,9 +17,10 @@ const ftl::codecs::Channels<0> &Stream::selected(int fs) const {
return state_[fs].selected;
}
-void Stream::select(int fs, const ftl::codecs::Channels<0> &s) {
+void Stream::select(int fs, const ftl::codecs::Channels<0> &s, bool make) {
UNIQUE_LOCK(mtx_, lk);
- if (fs < 0 || fs >= state_.size()) throw ftl::exception("Frameset index out-of-bounds");
+ if (fs < 0 || (!make && fs >= state_.size())) throw ftl::exception("Frameset index out-of-bounds");
+ if (fs >= state_.size()) state_.resize(fs+1);
state_[fs].selected = s;
}
@@ -30,6 +31,10 @@ ftl::codecs::Channels<0> &Stream::available(int fs) {
return state_[fs].available;
}
+void Stream::reset() {
+ // Clear available and selected?
+}
+
// ==== Muxer ==================================================================
Muxer::Muxer(nlohmann::json &config) : Stream(config), nid_(0) {
@@ -108,6 +113,12 @@ bool Muxer::active() {
return r;
}
+void Muxer::reset() {
+ for (auto &s : streams_) {
+ s.stream->reset();
+ }
+}
+
int Muxer::_lookup(int sid, int ssid) {
SHARED_LOCK(mutex_, lk);
auto &se = streams_[sid];
@@ -190,6 +201,12 @@ bool Broadcast::active() {
return r;
}
+void Broadcast::reset() {
+ for (auto &s : streams_) {
+ s->reset();
+ }
+}
+
// ==== Intercept ==============================================================
Intercept::Intercept(nlohmann::json &config) : Stream(config) {
@@ -244,3 +261,7 @@ bool Intercept::end() {
bool Intercept::active() {
return stream_->active();
}
+
+void Intercept::reset() {
+ stream_->reset();
+}
diff --git a/components/streams/test/CMakeLists.txt b/components/streams/test/CMakeLists.txt
index 6fec3245c..859e1bcbd 100644
--- a/components/streams/test/CMakeLists.txt
+++ b/components/streams/test/CMakeLists.txt
@@ -34,3 +34,33 @@ add_test(FileStreamUnitTest filestream_unit)
# ftlcommon ftlcodecs ftlrgbd ftlnet)
#add_test(NetStreamUnitTest netstream_unit)
+
+### Sender Unit ################################################################
+add_executable(sender_unit
+ ./tests.cpp
+ ./sender_unit.cpp
+ ../src/sender.cpp
+ ../src/stream.cpp
+ ../src/injectors.cpp
+ ../src/parsers.cpp
+)
+target_include_directories(sender_unit PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}/../include")
+target_link_libraries(sender_unit
+ ftlcommon ftlcodecs ftlrgbd)
+
+add_test(SenderUnitTest sender_unit)
+
+### Receiver Unit ##############################################################
+add_executable(receiver_unit
+ ./tests.cpp
+ ./receiver_unit.cpp
+ ../src/receiver.cpp
+ ../src/stream.cpp
+ ../src/injectors.cpp
+ ../src/parsers.cpp
+)
+target_include_directories(receiver_unit PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}/../include")
+target_link_libraries(receiver_unit
+ ftlcommon ftlcodecs ftlrgbd)
+
+add_test(ReceiverUnitTest receiver_unit)
diff --git a/components/streams/test/filestream_unit.cpp b/components/streams/test/filestream_unit.cpp
index 54895357e..c58ffaf0e 100644
--- a/components/streams/test/filestream_unit.cpp
+++ b/components/streams/test/filestream_unit.cpp
@@ -72,7 +72,7 @@ TEST_CASE("ftl::stream::File write and read", "[stream]") {
//reader->tick();
REQUIRE( count == 3 );
- REQUIRE( tspkt.timestamp == 0 );
+ REQUIRE( tspkt.timestamp > 0 );
REQUIRE( tspkt.streamID == 2 );
REQUIRE( tspkt.channel == ftl::codecs::Channel::Screen );
}
@@ -104,21 +104,21 @@ TEST_CASE("ftl::stream::File write and read", "[stream]") {
REQUIRE( count == 1 );
//REQUIRE( tspkt.timestamp == 0 );
- auto itime = tspkt.timestamp;
+ //auto itime = tspkt.timestamp;
count = 0;
- reader->tick();
+ reader->tick(0);
std::this_thread::sleep_for(std::chrono::milliseconds(10));
REQUIRE( count == 1 );
- REQUIRE( tspkt.timestamp == itime+ftl::timer::getInterval() );
+ //REQUIRE( tspkt.timestamp == itime+ftl::timer::getInterval() );
count = 0;
- reader->tick();
+ reader->tick(0);
std::this_thread::sleep_for(std::chrono::milliseconds(10));
REQUIRE( count == 1 );
- REQUIRE( tspkt.timestamp == itime+2*ftl::timer::getInterval() );
+ //REQUIRE( tspkt.timestamp == itime+2*ftl::timer::getInterval() );
}
}
diff --git a/components/streams/test/receiver_unit.cpp b/components/streams/test/receiver_unit.cpp
new file mode 100644
index 000000000..86ce60b6c
--- /dev/null
+++ b/components/streams/test/receiver_unit.cpp
@@ -0,0 +1,219 @@
+#include "catch.hpp"
+
+#include <ftl/streams/receiver.hpp>
+#include <ftl/codecs/nvpipe_encoder.hpp>
+#include "../src/injectors.hpp"
+
+using ftl::codecs::definition_t;
+using ftl::codecs::codec_t;
+using ftl::stream::Receiver;
+using ftl::rgbd::Frame;
+using ftl::rgbd::FrameSet;
+using ftl::codecs::Channel;
+using ftl::codecs::Channels;
+using ftl::config::json_t;
+
+class TestStream : public ftl::stream::Stream {
+ public:
+ explicit TestStream(nlohmann::json &config) : ftl::stream::Stream(config) {};
+ ~TestStream() {};
+
+ bool onPacket(const std::function<void(const ftl::codecs::StreamPacket &, const ftl::codecs::Packet &)> &cb) {
+ cb_ = cb;
+ return true;
+ }
+
+ bool post(const ftl::codecs::StreamPacket &spkt, const ftl::codecs::Packet &pkt) {
+ available(spkt.streamID) += spkt.channel;
+ if (pkt.data.size() == 0) {
+ if (spkt.frameSetID() == 255) {
+ for (int i=0; i<size(); ++i) {
+ select(i, selected(i) + spkt.channel);
+ }
+ } else {
+ select(spkt.frameSetID(), selected(spkt.frameSetID()) + spkt.channel);
+ }
+ }
+ if (cb_) cb_(spkt, pkt);
+ return true;
+ }
+
+ bool begin() override { return true; }
+ bool end() override { return true; }
+ bool active() override { return true; }
+
+ private:
+ std::function<void(const ftl::codecs::StreamPacket &, const ftl::codecs::Packet &)> cb_;
+};
+
+
+TEST_CASE( "Receiver generating onFrameSet" ) {
+ json_t global = json_t{{"$id","ftl://test"}};
+ ftl::config::configure(global);
+
+ json_t cfg = json_t{
+ {"$id","ftl://test/1"}
+ };
+ auto *receiver = ftl::create<Receiver>(cfg);
+
+ json_t cfg2 = json_t{
+ {"$id","ftl://test/2"}
+ };
+ TestStream stream(cfg2);
+ receiver->setStream(&stream);
+
+ ftl::codecs::NvPipeEncoder encoder(definition_t::HD1080, definition_t::SD480);
+
+ ftl::codecs::Packet pkt;
+ pkt.codec = codec_t::Any;
+ pkt.bitrate = 255;
+ pkt.definition = definition_t::Any;
+ pkt.flags = 0;
+ pkt.frame_count = 1;
+
+ ftl::codecs::StreamPacket spkt;
+ spkt.version = 4;
+ spkt.timestamp = 10;
+ spkt.frame_number = 0;
+ spkt.channel = Channel::Colour;
+ spkt.streamID = 0;
+
+ ftl::rgbd::Frame dummy;
+ ftl::rgbd::FrameState state;
+ state.getLeft().width = 1280;
+ state.getLeft().height = 720;
+ dummy.setOrigin(&state);
+ ftl::stream::injectCalibration(&stream, dummy, 0, 0);
+
+ ftl::timer::start(false);
+
+ SECTION("a single colour frame") {
+ cv::cuda::GpuMat m(cv::Size(1280,720), CV_8UC4, cv::Scalar(0));
+
+ bool r = encoder.encode(m, pkt);
+ REQUIRE( r );
+
+ stream.post(spkt, pkt);
+
+ int count = 0;
+ receiver->onFrameSet([&count](ftl::rgbd::FrameSet &fs) {
+ ++count;
+
+ REQUIRE( fs.timestamp == 10 );
+ REQUIRE( fs.frames.size() == 1 );
+ REQUIRE( fs.frames[0].hasChannel(Channel::Colour) );
+ REQUIRE( fs.frames[0].get<cv::cuda::GpuMat>(Channel::Colour).rows == 720 );
+ REQUIRE( fs.frames[0].get<cv::cuda::GpuMat>(Channel::Colour).type() == CV_8UC4 );
+
+ return true;
+ });
+
+ int i=10;
+ while (i-- > 0 && count < 1) std::this_thread::sleep_for(std::chrono::milliseconds(10));
+
+ REQUIRE( count == 1 );
+ }
+
+ SECTION("a tiled colour frame") {
+ cv::cuda::GpuMat m(cv::Size(2560,720), CV_8UC4, cv::Scalar(0));
+ ftl::stream::injectCalibration(&stream, dummy, 0, 1);
+
+ pkt.frame_count = 2;
+ bool r = encoder.encode(m, pkt);
+ REQUIRE( r );
+
+ stream.post(spkt, pkt);
+
+ int count = 0;
+ receiver->onFrameSet([&count](ftl::rgbd::FrameSet &fs) {
+ ++count;
+
+ REQUIRE( fs.timestamp == 10 );
+ REQUIRE( fs.frames.size() == 2 );
+ REQUIRE( fs.frames[0].hasChannel(Channel::Colour) );
+ REQUIRE( fs.frames[0].get<cv::cuda::GpuMat>(Channel::Colour).rows == 720 );
+ REQUIRE( fs.frames[0].get<cv::cuda::GpuMat>(Channel::Colour).type() == CV_8UC4 );
+ REQUIRE( fs.frames[1].hasChannel(Channel::Colour) );
+ REQUIRE( fs.frames[1].get<cv::cuda::GpuMat>(Channel::Colour).rows == 720 );
+ REQUIRE( fs.frames[1].get<cv::cuda::GpuMat>(Channel::Colour).type() == CV_8UC4 );
+
+ return true;
+ });
+
+ int i=10;
+ while (i-- > 0 && count < 1) std::this_thread::sleep_for(std::chrono::milliseconds(10));
+
+ REQUIRE( count == 1 );
+ }
+
+ SECTION("a tiled lossy depth frame") {
+ cv::cuda::GpuMat m(cv::Size(2560,720), CV_8UC4, cv::Scalar(0));
+ ftl::stream::injectCalibration(&stream, dummy, 0, 1);
+
+ spkt.channel = Channel::Depth;
+ pkt.frame_count = 2;
+ pkt.flags = ftl::codecs::kFlagFloat | ftl::codecs::kFlagMappedDepth;
+ bool r = encoder.encode(m, pkt);
+ REQUIRE( r );
+
+ stream.post(spkt, pkt);
+
+ int count = 0;
+ receiver->onFrameSet([&count](ftl::rgbd::FrameSet &fs) {
+ ++count;
+
+ REQUIRE( fs.timestamp == 10 );
+ REQUIRE( fs.frames.size() == 2 );
+ REQUIRE( fs.frames[0].hasChannel(Channel::Depth) );
+ REQUIRE( fs.frames[0].get<cv::cuda::GpuMat>(Channel::Depth).rows == 720 );
+ REQUIRE( fs.frames[0].get<cv::cuda::GpuMat>(Channel::Depth).type() == CV_32F );
+ REQUIRE( fs.frames[1].hasChannel(Channel::Depth) );
+ REQUIRE( fs.frames[1].get<cv::cuda::GpuMat>(Channel::Depth).rows == 720 );
+ REQUIRE( fs.frames[1].get<cv::cuda::GpuMat>(Channel::Depth).type() == CV_32F );
+
+ return true;
+ });
+
+ int i=10;
+ while (i-- > 0 && count < 1) std::this_thread::sleep_for(std::chrono::milliseconds(10));
+
+ REQUIRE( count == 1 );
+ }
+
+ SECTION("a tiled lossless depth frame") {
+ cv::cuda::GpuMat m(cv::Size(2560,720), CV_16U, cv::Scalar(0));
+ ftl::stream::injectCalibration(&stream, dummy, 0, 1);
+
+ spkt.channel = Channel::Depth;
+ pkt.frame_count = 2;
+ pkt.flags = ftl::codecs::kFlagFloat;
+ bool r = encoder.encode(m, pkt);
+ REQUIRE( r );
+
+ stream.post(spkt, pkt);
+
+ int count = 0;
+ receiver->onFrameSet([&count](ftl::rgbd::FrameSet &fs) {
+ ++count;
+
+ REQUIRE( fs.timestamp == 10 );
+ REQUIRE( fs.frames.size() == 2 );
+ REQUIRE( fs.frames[0].hasChannel(Channel::Depth) );
+ REQUIRE( fs.frames[0].get<cv::cuda::GpuMat>(Channel::Depth).rows == 720 );
+ REQUIRE( fs.frames[0].get<cv::cuda::GpuMat>(Channel::Depth).type() == CV_32F );
+ REQUIRE( fs.frames[1].hasChannel(Channel::Depth) );
+ REQUIRE( fs.frames[1].get<cv::cuda::GpuMat>(Channel::Depth).rows == 720 );
+ REQUIRE( fs.frames[1].get<cv::cuda::GpuMat>(Channel::Depth).type() == CV_32F );
+
+ return true;
+ });
+
+ int i=10;
+ while (i-- > 0 && count < 1) std::this_thread::sleep_for(std::chrono::milliseconds(10));
+
+ REQUIRE( count == 1 );
+ }
+
+ ftl::timer::stop(true);
+ delete receiver;
+}
diff --git a/components/streams/test/sender_unit.cpp b/components/streams/test/sender_unit.cpp
new file mode 100644
index 000000000..e6d945b51
--- /dev/null
+++ b/components/streams/test/sender_unit.cpp
@@ -0,0 +1,299 @@
+#include "catch.hpp"
+
+#include <ftl/streams/sender.hpp>
+#include <ftl/codecs/hevc.hpp>
+
+using ftl::codecs::definition_t;
+using ftl::codecs::codec_t;
+using ftl::stream::Sender;
+using ftl::rgbd::Frame;
+using ftl::rgbd::FrameSet;
+using ftl::codecs::Channel;
+using ftl::codecs::Channels;
+using ftl::config::json_t;
+
+class TestStream : public ftl::stream::Stream {
+ public:
+ explicit TestStream(nlohmann::json &config) : ftl::stream::Stream(config) {};
+ ~TestStream() {};
+
+ bool onPacket(const std::function<void(const ftl::codecs::StreamPacket &, const ftl::codecs::Packet &)> &cb) {
+ cb_ = cb;
+ return true;
+ }
+
+ bool onIntercept(const std::function<void(const ftl::codecs::StreamPacket &, const ftl::codecs::Packet &)> &cb) {
+ icb_ = cb;
+ return true;
+ }
+
+ bool post(const ftl::codecs::StreamPacket &spkt, const ftl::codecs::Packet &pkt) {
+ available(spkt.streamID) += spkt.channel;
+ if (pkt.data.size() == 0) {
+ if (spkt.frameSetID() == 255) {
+ for (int i=0; i<size(); ++i) {
+ select(i, selected(i) + spkt.channel);
+ }
+ } else {
+ select(spkt.frameSetID(), selected(spkt.frameSetID()) + spkt.channel);
+ }
+ if (cb_) cb_(spkt, pkt);
+ }
+ if (icb_) icb_(spkt, pkt);
+ return true;
+ }
+
+ bool begin() override { return true; }
+ bool end() override { return true; }
+ bool active() override { return true; }
+
+ private:
+ std::function<void(const ftl::codecs::StreamPacket &, const ftl::codecs::Packet &)> cb_;
+ std::function<void(const ftl::codecs::StreamPacket &, const ftl::codecs::Packet &)> icb_;
+};
+
+
+TEST_CASE( "Sender::post() video frames" ) {
+ json_t global = json_t{{"$id","ftl://test"}};
+ ftl::config::configure(global);
+
+ json_t cfg = json_t{
+ {"$id","ftl://test/1"}
+ };
+ auto *sender = ftl::create<Sender>(cfg);
+
+ FrameSet fs;
+ fs.frames.emplace_back();
+ fs.timestamp = 1000;
+
+ json_t cfg2 = json_t{
+ {"$id","ftl://test/2"}
+ };
+ TestStream stream(cfg2);
+ sender->setStream(&stream);
+
+ ftl::codecs::StreamPacket spkt;
+ ftl::codecs::Packet pkt;
+ int count = 0;
+
+ stream.onIntercept([&count,&spkt,&pkt](const ftl::codecs::StreamPacket &pspkt, const ftl::codecs::Packet &ppkt) {
+ spkt = pspkt;
+ pkt = ppkt;
+ ++count;
+ });
+
+ SECTION("a single colour frame") {
+ stream.select(0, Channels(Channel::Colour), true);
+
+ fs.count = 1;
+ fs.frames[0].create<cv::cuda::GpuMat>(Channel::Colour).create(cv::Size(1280,720), CV_8UC4);
+ fs.frames[0].get<cv::cuda::GpuMat>(Channel::Colour).setTo(cv::Scalar(0));
+
+ sender->post(fs);
+
+ REQUIRE( count == 1 );
+ REQUIRE( spkt.version == 4 );
+ REQUIRE( spkt.timestamp == 1000 );
+ REQUIRE( (int)spkt.frame_number == 0 );
+ REQUIRE( spkt.streamID == 0 );
+ REQUIRE( spkt.channel == Channel::Colour );
+ REQUIRE( pkt.codec == codec_t::HEVC );
+ REQUIRE( pkt.definition == definition_t::HD720 );
+ REQUIRE( pkt.data.size() > 0 );
+ REQUIRE( pkt.frame_count == 1 );
+ REQUIRE( ftl::codecs::hevc::validNAL(pkt.data.data(), pkt.data.size()) );
+ }
+
+ SECTION("two colour frames tiled") {
+ stream.select(0, Channels(Channel::Colour), true);
+
+ fs.count = 2;
+ fs.frames[0].create<cv::cuda::GpuMat>(Channel::Colour).create(cv::Size(1280,720), CV_8UC4);
+ fs.frames[0].get<cv::cuda::GpuMat>(Channel::Colour).setTo(cv::Scalar(0));
+ fs.frames.emplace_back();
+ fs.frames[1].create<cv::cuda::GpuMat>(Channel::Colour).create(cv::Size(1280,720), CV_8UC4);
+ fs.frames[1].get<cv::cuda::GpuMat>(Channel::Colour).setTo(cv::Scalar(0));
+
+ sender->post(fs);
+
+ REQUIRE( count == 1 );
+ REQUIRE( spkt.version == 4 );
+ REQUIRE( spkt.timestamp == 1000 );
+ REQUIRE( (int)spkt.frame_number == 0 );
+ REQUIRE( spkt.streamID == 0 );
+ REQUIRE( spkt.channel == Channel::Colour );
+ REQUIRE( pkt.codec == codec_t::HEVC );
+ REQUIRE( pkt.definition == definition_t::HD720 );
+ REQUIRE( pkt.data.size() > 0 );
+ REQUIRE( pkt.frame_count == 2 );
+ REQUIRE( ftl::codecs::hevc::validNAL(pkt.data.data(), pkt.data.size()) );
+ }
+
+ SECTION("two depth frames tiled") {
+ stream.select(0, Channels(Channel::Depth), true);
+
+ fs.count = 2;
+ fs.frames[0].create<cv::cuda::GpuMat>(Channel::Depth).create(cv::Size(1280,720), CV_32F);
+ fs.frames[0].get<cv::cuda::GpuMat>(Channel::Depth).setTo(cv::Scalar(0.0f));
+ fs.frames.emplace_back();
+ fs.frames[1].create<cv::cuda::GpuMat>(Channel::Depth).create(cv::Size(1280,720), CV_32F);
+ fs.frames[1].get<cv::cuda::GpuMat>(Channel::Depth).setTo(cv::Scalar(0.0f));
+
+ sender->post(fs);
+
+ REQUIRE( count == 1 );
+ REQUIRE( spkt.version == 4 );
+ REQUIRE( spkt.timestamp == 1000 );
+ REQUIRE( (int)spkt.frame_number == 0 );
+ REQUIRE( spkt.streamID == 0 );
+ REQUIRE( spkt.channel == Channel::Depth );
+ REQUIRE( pkt.codec == codec_t::HEVC );
+ REQUIRE( pkt.definition == definition_t::HD720 );
+ REQUIRE( pkt.data.size() > 0 );
+ REQUIRE( pkt.flags == (ftl::codecs::kFlagFloat | ftl::codecs::kFlagMappedDepth) );
+ REQUIRE( pkt.frame_count == 2 );
+ REQUIRE( ftl::codecs::hevc::validNAL(pkt.data.data(), pkt.data.size()) );
+ }
+
+ SECTION("10 depth frames tiled") {
+ stream.select(0, Channels(Channel::Depth), true);
+
+ fs.count = 10;
+ fs.frames[0].create<cv::cuda::GpuMat>(Channel::Depth).create(cv::Size(1280,720), CV_32F);
+ fs.frames[0].get<cv::cuda::GpuMat>(Channel::Depth).setTo(cv::Scalar(0.0f));
+
+ for (int i=1; i<10; ++i) {
+ fs.frames.emplace_back();
+ fs.frames[i].create<cv::cuda::GpuMat>(Channel::Depth).create(cv::Size(1280,720), CV_32F);
+ fs.frames[i].get<cv::cuda::GpuMat>(Channel::Depth).setTo(cv::Scalar(0.0f));
+ }
+
+ sender->post(fs);
+
+ REQUIRE( count == 2 );
+ REQUIRE( spkt.version == 4 );
+ REQUIRE( spkt.timestamp == 1000 );
+ REQUIRE( (int)spkt.frame_number == 9 );
+ REQUIRE( spkt.streamID == 0 );
+ REQUIRE( spkt.channel == Channel::Depth );
+ REQUIRE( pkt.codec == codec_t::HEVC );
+ REQUIRE( pkt.definition == definition_t::HD720 );
+ REQUIRE( pkt.data.size() > 0 );
+ REQUIRE( pkt.flags == (ftl::codecs::kFlagFloat | ftl::codecs::kFlagMappedDepth) );
+ REQUIRE( pkt.frame_count == 1 );
+ REQUIRE( ftl::codecs::hevc::validNAL(pkt.data.data(), pkt.data.size()) );
+ }
+
+ SECTION("two lossless depth frames tiled") {
+ stream.select(0, Channels(Channel::Depth), true);
+
+ fs.count = 2;
+ fs.frames[0].create<cv::cuda::GpuMat>(Channel::Depth).create(cv::Size(1280,720), CV_32F);
+ fs.frames[0].get<cv::cuda::GpuMat>(Channel::Depth).setTo(cv::Scalar(0.0f));
+ fs.frames.emplace_back();
+ fs.frames[1].create<cv::cuda::GpuMat>(Channel::Depth).create(cv::Size(1280,720), CV_32F);
+ fs.frames[1].get<cv::cuda::GpuMat>(Channel::Depth).setTo(cv::Scalar(0.0f));
+
+ sender->set("lossless", true);
+ sender->post(fs);
+
+ REQUIRE( count == 1 );
+ REQUIRE( spkt.version == 4 );
+ REQUIRE( spkt.timestamp == 1000 );
+ REQUIRE( (int)spkt.frame_number == 0 );
+ REQUIRE( spkt.streamID == 0 );
+ REQUIRE( spkt.channel == Channel::Depth );
+ REQUIRE( pkt.codec == codec_t::HEVC_LOSSLESS );
+ REQUIRE( pkt.definition == definition_t::HD720 );
+ REQUIRE( pkt.data.size() > 0 );
+ REQUIRE( pkt.flags == (ftl::codecs::kFlagFloat) );
+ REQUIRE( pkt.frame_count == 2 );
+ REQUIRE( ftl::codecs::hevc::validNAL(pkt.data.data(), pkt.data.size()) );
+ }
+
+ SECTION("one frame and two channels") {
+ stream.select(0, Channel::Colour + Channel::Depth, true);
+
+ fs.count = 1;
+ fs.frames[0].create<cv::cuda::GpuMat>(Channel::Colour).create(cv::Size(1280,720), CV_8UC4);
+ fs.frames[0].get<cv::cuda::GpuMat>(Channel::Colour).setTo(cv::Scalar(0));
+ fs.frames[0].create<cv::cuda::GpuMat>(Channel::Depth).create(cv::Size(1280,720), CV_32F);
+ fs.frames[0].get<cv::cuda::GpuMat>(Channel::Depth).setTo(cv::Scalar(0.0f));
+
+ sender->post(fs);
+
+ REQUIRE( count == 2 );
+ REQUIRE( spkt.version == 4 );
+ REQUIRE( spkt.timestamp == 1000 );
+ REQUIRE( (int)spkt.frame_number == 0 );
+ REQUIRE( spkt.streamID == 0 );
+ REQUIRE( spkt.channel == Channel::Depth );
+ REQUIRE( pkt.codec == codec_t::HEVC );
+ REQUIRE( pkt.definition == definition_t::HD720 );
+ REQUIRE( pkt.data.size() > 0 );
+ REQUIRE( pkt.frame_count == 1 );
+ REQUIRE( ftl::codecs::hevc::validNAL(pkt.data.data(), pkt.data.size()) );
+ }
+
+ //ftl::config::cleanup();
+ delete sender;
+}
+
+TEST_CASE( "Sender request to control encoding" ) {
+ json_t global = json_t{{"$id","ftl://test"}};
+ ftl::config::configure(global);
+
+ json_t cfg = json_t{
+ {"$id","ftl://test/1"}
+ };
+ auto *sender = ftl::create<Sender>(cfg);
+
+ FrameSet fs;
+ fs.frames.emplace_back();
+ fs.timestamp = 1000;
+
+ json_t cfg2 = json_t{
+ {"$id","ftl://test/2"}
+ };
+ TestStream stream(cfg2);
+ sender->setStream(&stream);
+
+ ftl::codecs::StreamPacket spkt;
+ ftl::codecs::Packet pkt;
+ int count = 0;
+
+ stream.onIntercept([&count,&spkt,&pkt](const ftl::codecs::StreamPacket &pspkt, const ftl::codecs::Packet &ppkt) {
+ spkt = pspkt;
+ pkt = ppkt;
+ ++count;
+ });
+
+ SECTION("a single colour frame request") {
+ //stream.select(0, Channels(Channel::Colour), true);
+
+ stream.post({
+ 4, 1000, 0, 255, Channel::Colour
+ },{
+ codec_t::Any, definition_t::Any, 255, 255, 0, {}
+ });
+
+ fs.count = 1;
+ fs.frames[0].create<cv::cuda::GpuMat>(Channel::Colour).create(cv::Size(1280,720), CV_8UC4);
+ fs.frames[0].get<cv::cuda::GpuMat>(Channel::Colour).setTo(cv::Scalar(0));
+
+ count = 0;
+ sender->post(fs);
+
+ REQUIRE( count == 5 );
+ REQUIRE( spkt.version == 4 );
+ REQUIRE( spkt.timestamp == 1000 );
+ REQUIRE( (int)spkt.frame_number == 0 );
+ REQUIRE( spkt.streamID == 0 );
+ REQUIRE( spkt.channel == Channel::Colour );
+ REQUIRE( pkt.codec == codec_t::HEVC );
+ REQUIRE( pkt.definition == definition_t::HD720 );
+ REQUIRE( pkt.data.size() > 0 );
+ REQUIRE( pkt.frame_count == 1 );
+ REQUIRE( ftl::codecs::hevc::validNAL(pkt.data.data(), pkt.data.size()) );
+ }
+}
diff --git a/python/ftl/ftltype.py b/python/ftl/ftltype.py
index 4120fdb18..9efad0172 100644
--- a/python/ftl/ftltype.py
+++ b/python/ftl/ftltype.py
@@ -58,7 +58,7 @@ _float_channels = [
def is_float_channel(channel):
return channel in _float_channels
-# components/codecs/include/ftl/codecs/bitrates.hpp
+# components/codecs/include/ftl/codecs/codecs.hpp
class codec_t(IntEnum):
JPG = 0
PNG = 1
--
GitLab