diff --git a/applications/calibration-multi/src/main.cpp b/applications/calibration-multi/src/main.cpp index 1e5a1bf6186afd2656e8d4e193a4e0fc241d27e0..dbea22842d4b0a70546187f48e161def746af207 100644 --- a/applications/calibration-multi/src/main.cpp +++ b/applications/calibration-multi/src/main.cpp @@ -37,7 +37,7 @@ using cv::Vec4d; using ftl::net::Universe; using ftl::rgbd::Source; -using ftl::rgbd::Channel; +using ftl::codecs::Channel; Mat createCameraMatrix(const ftl::rgbd::Camera ¶meters) { Mat m = (cv::Mat_<double>(3,3) << parameters.fx, 0.0, -parameters.cx, 0.0, parameters.fy, -parameters.cy, 0.0, 0.0, 1.0); diff --git a/applications/groupview/src/main.cpp b/applications/groupview/src/main.cpp index 6b32221ef1a13ad05f9e5bb915c1186eca0aa52c..be99d857562bfb48ec120cccccf0737ff2d0b210 100644 --- a/applications/groupview/src/main.cpp +++ b/applications/groupview/src/main.cpp @@ -16,7 +16,7 @@ using std::string; using std::vector; using cv::Size; using cv::Mat; -using ftl::rgbd::Channel; +using ftl::codecs::Channel; // TODO: remove code duplication (function from reconstruction) static void from_json(nlohmann::json &json, map<string, Matrix4d> &transformations) { diff --git a/applications/gui/src/camera.cpp b/applications/gui/src/camera.cpp index d4c85b5d2e5e72b6c6d6c296ebf2921a7d13c636..efd3313224295677153733a9e3c4ca8e3e5eddee 100644 --- a/applications/gui/src/camera.cpp +++ b/applications/gui/src/camera.cpp @@ -7,8 +7,8 @@ using ftl::rgbd::isValidDepth; using ftl::gui::GLTexture; using ftl::gui::PoseWindow; -using ftl::rgbd::Channel; -using ftl::rgbd::Channels; +using ftl::codecs::Channel; +using ftl::codecs::Channels; // TODO(Nick) MOVE class StatisticsImage { diff --git a/applications/gui/src/camera.hpp b/applications/gui/src/camera.hpp index 55042fe0d7b3baf5f24a26e390b1348084bcf320..1c5ebef55dbd9531d56d51659c2104ab614fece2 100644 --- a/applications/gui/src/camera.hpp +++ b/applications/gui/src/camera.hpp @@ -38,11 +38,11 @@ class Camera { void showPoseWindow(); void showSettings(); - void setChannel(ftl::rgbd::Channel c); + void setChannel(ftl::codecs::Channel c); void togglePause(); void isPaused(); - const ftl::rgbd::Channels &availableChannels(); + const ftl::codecs::Channels &availableChannels(); const GLTexture &captureFrame(); const GLTexture &getLeft() const { return texture_; } @@ -71,8 +71,8 @@ class Camera { float lerpSpeed_; bool sdepth_; bool pause_; - ftl::rgbd::Channel channel_; - ftl::rgbd::Channels channels_; + ftl::codecs::Channel channel_; + ftl::codecs::Channels channels_; cv::Mat rgb_; cv::Mat depth_; MUTEX mutex_; diff --git a/applications/gui/src/media_panel.cpp b/applications/gui/src/media_panel.cpp index 9aa81932da0eab2aa814846fccc9f107e176a7ff..826111c9800339eac124f4a59e532e950f7d15e6 100644 --- a/applications/gui/src/media_panel.cpp +++ b/applications/gui/src/media_panel.cpp @@ -12,7 +12,7 @@ #endif using ftl::gui::MediaPanel; -using ftl::rgbd::Channel; +using ftl::codecs::Channel; MediaPanel::MediaPanel(ftl::gui::Screen *screen) : nanogui::Window(screen, ""), screen_(screen) { using namespace nanogui; diff --git a/applications/player/src/main.cpp b/applications/player/src/main.cpp index dc176ed03cff73170f639961e5d328cdf71fac18..751d9969a7ad4844dd7af38d3650322c4981ddb7 100644 --- a/applications/player/src/main.cpp +++ b/applications/player/src/main.cpp @@ -10,6 +10,7 @@ #include <Eigen/Eigen> using ftl::codecs::codec_t; +using ftl::codecs::Channel; static ftl::codecs::Decoder *decoder; @@ -56,7 +57,7 @@ int main(int argc, char **argv) { int current_channel = 0; bool res = r.read(90000000000000, [¤t_stream,¤t_channel,&r](const ftl::codecs::StreamPacket &spkt, const ftl::codecs::Packet &pkt) { - if (spkt.channel != current_channel) return; + if (spkt.channel != static_cast<ftl::codecs::Channel>(current_channel)) return; if (spkt.streamID == current_stream) { if (pkt.codec == codec_t::POSE) { @@ -73,7 +74,7 @@ int main(int argc, char **argv) { LOG(INFO) << "Reading packet: (" << (int)spkt.streamID << "," << (int)spkt.channel << ") " << (int)pkt.codec << ", " << (int)pkt.definition; - cv::Mat frame(cv::Size(ftl::codecs::getWidth(pkt.definition),ftl::codecs::getHeight(pkt.definition)), (spkt.channel == 1) ? CV_32F : CV_8UC3); + cv::Mat frame(cv::Size(ftl::codecs::getWidth(pkt.definition),ftl::codecs::getHeight(pkt.definition)), (spkt.channel == Channel::Depth) ? CV_32F : CV_8UC3); createDecoder(pkt); try { @@ -83,7 +84,7 @@ int main(int argc, char **argv) { } if (!frame.empty()) { - if (spkt.channel == 1) { + if (spkt.channel == Channel::Depth) { visualizeDepthMap(frame, frame, 8.0f); } double time = (double)(spkt.timestamp - r.getStartTime()) / 1000.0; diff --git a/applications/reconstruct/src/ilw.cpp b/applications/reconstruct/src/ilw.cpp index c1bf8499aea122e58723bad6de434ebe4a37cd26..435cd886eba1b83d7530f19f28ccfb09f7e37f3a 100644 --- a/applications/reconstruct/src/ilw.cpp +++ b/applications/reconstruct/src/ilw.cpp @@ -8,8 +8,8 @@ using ftl::ILW; using ftl::detail::ILWData; -using ftl::rgbd::Channel; -using ftl::rgbd::Channels; +using ftl::codecs::Channel; +using ftl::codecs::Channels; using ftl::rgbd::Format; using cv::cuda::GpuMat; diff --git a/applications/reconstruct/src/ilw/ilw.cpp b/applications/reconstruct/src/ilw/ilw.cpp index 1537b556ff4350a18ac6f9ddb14135f249995b9a..c4de69f0a50c8a53afe4daf0557df3408a260cba 100644 --- a/applications/reconstruct/src/ilw/ilw.cpp +++ b/applications/reconstruct/src/ilw/ilw.cpp @@ -8,8 +8,8 @@ using ftl::ILW; using ftl::detail::ILWData; -using ftl::rgbd::Channel; -using ftl::rgbd::Channels; +using ftl::codecs::Channel; +using ftl::codecs::Channels; using ftl::rgbd::Format; using cv::cuda::GpuMat; diff --git a/applications/reconstruct/src/main.cpp b/applications/reconstruct/src/main.cpp index 41359ec65634509df1e00370b4622408fa0e53ec..50d7986f3f13de35c1ed0be28646a8112c054cd7 100644 --- a/applications/reconstruct/src/main.cpp +++ b/applications/reconstruct/src/main.cpp @@ -41,7 +41,7 @@ using std::string; using std::vector; using ftl::rgbd::Source; using ftl::config::json_t; -using ftl::rgbd::Channel; +using ftl::codecs::Channel; using json = nlohmann::json; using std::this_thread::sleep_for; @@ -70,7 +70,7 @@ static void writeSourceProperties(ftl::codecs::Writer &writer, int id, ftl::rgbd spkt.timestamp = 0; spkt.streamID = id; - spkt.channel = 0; + spkt.channel = Channel::Calibration; spkt.channel_count = 1; pkt.codec = ftl::codecs::codec_t::CALIBRATION; pkt.definition = ftl::codecs::definition_t::Any; @@ -81,6 +81,7 @@ static void writeSourceProperties(ftl::codecs::Writer &writer, int id, ftl::rgbd writer.write(spkt, pkt); + spkt.channel = Channel::Pose; pkt.codec = ftl::codecs::codec_t::POSE; pkt.definition = ftl::codecs::definition_t::Any; pkt.block_number = 0; diff --git a/components/rgbd-sources/include/ftl/rgbd/channels.hpp b/components/codecs/include/ftl/codecs/channels.hpp similarity index 69% rename from components/rgbd-sources/include/ftl/rgbd/channels.hpp rename to components/codecs/include/ftl/codecs/channels.hpp index e00647271574584a1305fc86b41491a93e88a88a..8ef470972fdb73ea5fc501e1e4784bd838e3b27d 100644 --- a/components/rgbd-sources/include/ftl/rgbd/channels.hpp +++ b/components/codecs/include/ftl/codecs/channels.hpp @@ -5,32 +5,44 @@ #include <msgpack.hpp> namespace ftl { -namespace rgbd { +namespace codecs { enum struct Channel : int { - None = -1, - Colour = 0, // 8UC3 or 8UC4 - Left = 0, - Depth = 1, // 32S or 32F - Right = 2, // 8UC3 or 8UC4 - Colour2 = 2, - Disparity = 3, - Depth2 = 3, - Deviation = 4, - Normals = 5, // 32FC4 - Points = 6, // 32FC4 - Confidence = 7, // 32F - Contribution = 7, // 32F - EnergyVector, // 32FC4 - Flow, // 32F - Energy, // 32F - Mask, // 32U - Density, // 32F - LeftGray, - RightGray, - Overlay1 + None = -1, + Colour = 0, // 8UC3 or 8UC4 + Left = 0, + Depth = 1, // 32S or 32F + Right = 2, // 8UC3 or 8UC4 + Colour2 = 2, + Disparity = 3, + Depth2 = 3, + Deviation = 4, + Normals = 5, // 32FC4 + Points = 6, // 32FC4 + Confidence = 7, // 32F + Contribution = 7, // 32F + EnergyVector = 8, // 32FC4 + Flow = 9, // 32F + Energy = 10, // 32F + Mask = 11, // 32U + Density = 12, // 32F + LeftGray = 13, // Deprecated + RightGray = 14, // Deprecated + Overlay1 = 15, // Unused + + AudioLeft = 32, + AudioRight = 33, + + Configuration = 64, // JSON Data + Calibration = 65, // Camera Parameters Object + Pose = 66, // Eigen::Matrix4d + Data = 67 // Custom data, any codec. }; +inline bool isVideo(Channel c) { return (int)c < 32; }; +inline bool isAudio(Channel c) { return (int)c >= 32 && (int)c < 64; }; +inline bool isData(Channel c) { return (int)c >= 64; }; + class Channels { public: @@ -39,8 +51,8 @@ class Channels { iterator(const Channels &c, unsigned int ix) : channels_(c), ix_(ix) { } iterator operator++(); iterator operator++(int junk); - inline ftl::rgbd::Channel operator*() { return static_cast<Channel>(static_cast<int>(ix_)); } - //ftl::rgbd::Channel operator->() { return ptr_; } + inline ftl::codecs::Channel operator*() { return static_cast<Channel>(static_cast<int>(ix_)); } + //ftl::codecs::Channel operator->() { return ptr_; } inline bool operator==(const iterator& rhs) { return ix_ == rhs.ix_; } inline bool operator!=(const iterator& rhs) { return ix_ != rhs.ix_; } private: @@ -102,7 +114,7 @@ inline Channels Channels::All() { static const Channels kNoChannels; static const Channels kAllChannels(0xFFFFFFFFu); -inline bool isFloatChannel(ftl::rgbd::Channel chan) { +inline bool isFloatChannel(ftl::codecs::Channel chan) { switch (chan) { case Channel::Depth : //case Channel::Normals : @@ -116,14 +128,14 @@ inline bool isFloatChannel(ftl::rgbd::Channel chan) { } } -MSGPACK_ADD_ENUM(ftl::rgbd::Channel); +MSGPACK_ADD_ENUM(ftl::codecs::Channel); -inline ftl::rgbd::Channels operator|(ftl::rgbd::Channel a, ftl::rgbd::Channel b) { - return ftl::rgbd::Channels(a) | b; +inline ftl::codecs::Channels operator|(ftl::codecs::Channel a, ftl::codecs::Channel b) { + return ftl::codecs::Channels(a) | b; } -inline ftl::rgbd::Channels operator+(ftl::rgbd::Channel a, ftl::rgbd::Channel b) { - return ftl::rgbd::Channels(a) | b; +inline ftl::codecs::Channels operator+(ftl::codecs::Channel a, ftl::codecs::Channel b) { + return ftl::codecs::Channels(a) | b; } #endif // _FTL_RGBD_CHANNELS_HPP_ diff --git a/components/codecs/include/ftl/codecs/packet.hpp b/components/codecs/include/ftl/codecs/packet.hpp index 3b8d17151fb492a89c61686b5f9bcb72193dc08d..988617fc5dd893c1f91a168a185d81cc7f4d341d 100644 --- a/components/codecs/include/ftl/codecs/packet.hpp +++ b/components/codecs/include/ftl/codecs/packet.hpp @@ -4,6 +4,7 @@ #include <cstdint> #include <vector> #include <ftl/codecs/bitrates.hpp> +#include <ftl/codecs/channels.hpp> #include <msgpack.hpp> @@ -43,8 +44,8 @@ struct Packet { struct StreamPacket { int64_t timestamp; uint8_t streamID; // Source number... - uint8_t channel_count; // Number of channels to expect (usually 1 or 2) - uint8_t channel; // Actual channel of this current set of packets + uint8_t channel_count; // Number of channels to expect for this frame to complete (usually 1 or 2) + ftl::codecs::Channel channel; // Actual channel of this current set of packets MSGPACK_DEFINE(timestamp, streamID, channel_count, channel); }; diff --git a/components/codecs/test/CMakeLists.txt b/components/codecs/test/CMakeLists.txt index 89b92059decf25217cb48bbe26a75e1a7a7277a3..74035c2280755ea358e2a53fce471523782cbf0c 100644 --- a/components/codecs/test/CMakeLists.txt +++ b/components/codecs/test/CMakeLists.txt @@ -44,3 +44,14 @@ target_link_libraries(rw_unit add_test(RWUnitTest rw_unit) + +### Channel Unit ############################################################### +add_executable(channel_unit + ./tests.cpp + ./channel_unit.cpp +) +target_include_directories(channel_unit PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}/../include") +target_link_libraries(channel_unit + ftlcommon) + +add_test(ChannelUnitTest channel_unit) diff --git a/components/rgbd-sources/test/channel_unit.cpp b/components/codecs/test/channel_unit.cpp similarity index 95% rename from components/rgbd-sources/test/channel_unit.cpp rename to components/codecs/test/channel_unit.cpp index 25171678540f1970088d84e1e842865a4249455e..bc966c63033f07680f1936e971166ca06acadde9 100644 --- a/components/rgbd-sources/test/channel_unit.cpp +++ b/components/codecs/test/channel_unit.cpp @@ -1,8 +1,8 @@ #include "catch.hpp" -#include <ftl/rgbd/channels.hpp> +#include <ftl/codecs/channels.hpp> -using ftl::rgbd::Channel; -using ftl::rgbd::Channels; +using ftl::codecs::Channel; +using ftl::codecs::Channels; TEST_CASE("channel casting", "") { SECTION("cast channel to channels") { diff --git a/components/codecs/test/readwrite_test.cpp b/components/codecs/test/readwrite_test.cpp index 03a918ebdf9fe9985e3f35a4e9cd6f1448ab46ef..ea8781b1b3b3f9204a26cd2fca527603ee855596 100644 --- a/components/codecs/test/readwrite_test.cpp +++ b/components/codecs/test/readwrite_test.cpp @@ -12,6 +12,7 @@ using ftl::codecs::StreamPacket; using ftl::codecs::Packet; using ftl::codecs::codec_t; using ftl::codecs::definition_t; +using ftl::codecs::Channel; TEST_CASE( "Write and read - Single frame" ) { std::stringstream s; @@ -20,7 +21,7 @@ TEST_CASE( "Write and read - Single frame" ) { StreamPacket spkt; Packet pkt; - spkt.channel = 0; + spkt.channel = Channel::Colour; spkt.timestamp = ftl::timer::get_time(); spkt.streamID = 0; @@ -46,7 +47,7 @@ TEST_CASE( "Write and read - Single frame" ) { REQUIRE(rpkt.codec == codec_t::JSON); REQUIRE(rpkt.data.size() == 3); REQUIRE(rpkt.data[0] == 44); - REQUIRE(rspkt.channel == 0); + REQUIRE(rspkt.channel == Channel::Colour); }); r.end(); @@ -61,7 +62,7 @@ TEST_CASE( "Write and read - Multiple frames" ) { StreamPacket spkt; Packet pkt; - spkt.channel = 0; + spkt.channel = Channel::Colour; spkt.timestamp = ftl::timer::get_time(); spkt.streamID = 0; @@ -93,7 +94,7 @@ TEST_CASE( "Write and read - Multiple frames" ) { REQUIRE(rpkt.codec == codec_t::JSON); REQUIRE(rpkt.data.size() == 3); REQUIRE(rpkt.data[0] == ((n == 1) ? 44 : (n == 2) ? 55 : 66)); - REQUIRE(rspkt.channel == 0); + REQUIRE(rspkt.channel == Channel::Colour); }); r.end(); @@ -108,7 +109,7 @@ TEST_CASE( "Write and read - Multiple streams" ) { StreamPacket spkt; Packet pkt; - spkt.channel = 0; + spkt.channel = Channel::Colour; spkt.timestamp = ftl::timer::get_time(); spkt.streamID = 0; @@ -163,7 +164,7 @@ TEST_CASE( "Write and read - Multiple frames with limit" ) { StreamPacket spkt; Packet pkt; - spkt.channel = 0; + spkt.channel = Channel::Colour; spkt.timestamp = ftl::timer::get_time(); spkt.streamID = 0; @@ -195,7 +196,7 @@ TEST_CASE( "Write and read - Multiple frames with limit" ) { REQUIRE(rpkt.codec == codec_t::JSON); REQUIRE(rpkt.data.size() == 3); REQUIRE(rpkt.data[0] == ((n == 1) ? 44 : (n == 2) ? 55 : 66)); - REQUIRE(rspkt.channel == 0); + REQUIRE(rspkt.channel == Channel::Colour); }); r.end(); @@ -210,7 +211,7 @@ TEST_CASE( "Write and read - Multiple reads" ) { StreamPacket spkt; Packet pkt; - spkt.channel = 0; + spkt.channel = Channel::Colour; spkt.timestamp = ftl::timer::get_time(); spkt.streamID = 0; @@ -242,7 +243,7 @@ TEST_CASE( "Write and read - Multiple reads" ) { REQUIRE(rpkt.codec == codec_t::JSON); REQUIRE(rpkt.data.size() == 3); REQUIRE(rpkt.data[0] == ((n == 1) ? 44 : (n == 2) ? 55 : 66)); - REQUIRE(rspkt.channel == 0); + REQUIRE(rspkt.channel == Channel::Colour); }); REQUIRE( n == 2 ); @@ -254,7 +255,7 @@ TEST_CASE( "Write and read - Multiple reads" ) { REQUIRE(rpkt.codec == codec_t::JSON); REQUIRE(rpkt.data.size() == 3); REQUIRE(rpkt.data[0] == 66 ); - REQUIRE(rspkt.channel == 0); + REQUIRE(rspkt.channel == Channel::Colour); }); r.end(); diff --git a/components/renderers/cpp/include/ftl/render/splat_render.hpp b/components/renderers/cpp/include/ftl/render/splat_render.hpp index 70d326935fe9524cdef2aac040cd82a8fd5ab1d9..bd83d8bd1bda862d71e2ef12f5239a578e2d86e6 100644 --- a/components/renderers/cpp/include/ftl/render/splat_render.hpp +++ b/components/renderers/cpp/include/ftl/render/splat_render.hpp @@ -26,7 +26,7 @@ class Splatter : public ftl::render::Renderer { //void setOutputDevice(int); protected: - void _renderChannel(ftl::rgbd::Frame &out, ftl::rgbd::Channel channel_in, ftl::rgbd::Channel channel_out, cudaStream_t stream); + void _renderChannel(ftl::rgbd::Frame &out, ftl::codecs::Channel channel_in, ftl::codecs::Channel channel_out, cudaStream_t stream); private: int device_; @@ -55,8 +55,8 @@ class Splatter : public ftl::render::Renderer { cudaStream_t stream_; template <typename T> - void __blendChannel(ftl::rgbd::Frame &, ftl::rgbd::Channel in, ftl::rgbd::Channel out, cudaStream_t); - void _blendChannel(ftl::rgbd::Frame &, ftl::rgbd::Channel in, ftl::rgbd::Channel out, cudaStream_t); + void __blendChannel(ftl::rgbd::Frame &, ftl::codecs::Channel in, ftl::codecs::Channel out, cudaStream_t); + void _blendChannel(ftl::rgbd::Frame &, ftl::codecs::Channel in, ftl::codecs::Channel out, cudaStream_t); void _dibr(cudaStream_t); }; diff --git a/components/renderers/cpp/src/splat_render.cpp b/components/renderers/cpp/src/splat_render.cpp index 152f5b3665c9af71a15b7be3915c54bbc3c02b0c..9a34d4679de48c6daa40fd99d3c4fc57bb3eb7dc 100644 --- a/components/renderers/cpp/src/splat_render.cpp +++ b/components/renderers/cpp/src/splat_render.cpp @@ -9,8 +9,8 @@ #include <string> using ftl::render::Splatter; -using ftl::rgbd::Channel; -using ftl::rgbd::Channels; +using ftl::codecs::Channel; +using ftl::codecs::Channels; using ftl::rgbd::Format; using cv::cuda::GpuMat; using std::stoul; @@ -143,7 +143,7 @@ struct AccumSelector<float> { }; template <typename T> -void Splatter::__blendChannel(ftl::rgbd::Frame &output, ftl::rgbd::Channel in, ftl::rgbd::Channel out, cudaStream_t stream) { +void Splatter::__blendChannel(ftl::rgbd::Frame &output, ftl::codecs::Channel in, ftl::codecs::Channel out, cudaStream_t stream) { cv::cuda::Stream cvstream = cv::cuda::StreamAccessor::wrapStream(stream); temp_.create<GpuMat>( AccumSelector<T>::channel, @@ -180,7 +180,7 @@ void Splatter::__blendChannel(ftl::rgbd::Frame &output, ftl::rgbd::Channel in, f ); } -void Splatter::_blendChannel(ftl::rgbd::Frame &output, ftl::rgbd::Channel in, ftl::rgbd::Channel out, cudaStream_t stream) { +void Splatter::_blendChannel(ftl::rgbd::Frame &output, ftl::codecs::Channel in, ftl::codecs::Channel out, cudaStream_t stream) { int type = output.get<GpuMat>(out).type(); // == CV_32F; //ftl::rgbd::isFloatChannel(channel); switch (type) { @@ -424,7 +424,7 @@ bool Splatter::render(ftl::rgbd::VirtualSource *src, ftl::rgbd::Frame &out) { _dibr(stream_); // Need to re-dibr due to pose change _renderChannel(out, Channel::Right, Channel::Right, stream_); } else if (chan != Channel::None) { - if (ftl::rgbd::isFloatChannel(chan)) { + if (ftl::codecs::isFloatChannel(chan)) { out.create<GpuMat>(chan, Format<float>(camera.width, camera.height)); out.get<GpuMat>(chan).setTo(cv::Scalar(0.0f), cvstream); } else { diff --git a/components/rgbd-sources/include/ftl/rgbd/detail/source.hpp b/components/rgbd-sources/include/ftl/rgbd/detail/source.hpp index e98ff38aacd4cf0731ef96b67ecc85732d4c0c7f..0d188c0b163777e842f3bd8c31f91ddd51435269 100644 --- a/components/rgbd-sources/include/ftl/rgbd/detail/source.hpp +++ b/components/rgbd-sources/include/ftl/rgbd/detail/source.hpp @@ -56,7 +56,7 @@ class Source { virtual bool isReady() { return false; }; virtual void setPose(const Eigen::Matrix4d &pose) { }; - virtual Camera parameters(ftl::rgbd::Channel) { return params_; }; + virtual Camera parameters(ftl::codecs::Channel) { return params_; }; protected: capability_t capabilities_; diff --git a/components/rgbd-sources/include/ftl/rgbd/frame.hpp b/components/rgbd-sources/include/ftl/rgbd/frame.hpp index 3bf2b87ce29fb53a1ef44de80556518c2667c410..e6025b41a72f204a9eef24ba119e871fae85d165 100644 --- a/components/rgbd-sources/include/ftl/rgbd/frame.hpp +++ b/components/rgbd-sources/include/ftl/rgbd/frame.hpp @@ -8,7 +8,7 @@ #include <opencv2/core/cuda.hpp> #include <opencv2/core/cuda_stream_accessor.hpp> -#include <ftl/rgbd/channels.hpp> +#include <ftl/codecs/channels.hpp> #include <ftl/rgbd/format.hpp> #include <ftl/codecs/bitrates.hpp> @@ -40,61 +40,61 @@ public: //Frame(const Frame &)=delete; //Frame &operator=(const Frame &)=delete; - void download(ftl::rgbd::Channel c, cv::cuda::Stream stream); - void upload(ftl::rgbd::Channel c, cv::cuda::Stream stream); - void download(ftl::rgbd::Channels c, cv::cuda::Stream stream); - void upload(ftl::rgbd::Channels c, cv::cuda::Stream stream); + void download(ftl::codecs::Channel c, cv::cuda::Stream stream); + void upload(ftl::codecs::Channel c, cv::cuda::Stream stream); + void download(ftl::codecs::Channels c, cv::cuda::Stream stream); + void upload(ftl::codecs::Channels c, cv::cuda::Stream stream); - inline void download(ftl::rgbd::Channel c, cudaStream_t stream=0) { download(c, cv::cuda::StreamAccessor::wrapStream(stream)); }; - inline void upload(ftl::rgbd::Channel c, cudaStream_t stream=0) { upload(c, cv::cuda::StreamAccessor::wrapStream(stream)); }; - inline void download(ftl::rgbd::Channels c, cudaStream_t stream=0) { download(c, cv::cuda::StreamAccessor::wrapStream(stream)); }; - inline void upload(ftl::rgbd::Channels c, cudaStream_t stream=0) { upload(c, cv::cuda::StreamAccessor::wrapStream(stream)); }; + inline void download(ftl::codecs::Channel c, cudaStream_t stream=0) { download(c, cv::cuda::StreamAccessor::wrapStream(stream)); }; + inline void upload(ftl::codecs::Channel c, cudaStream_t stream=0) { upload(c, cv::cuda::StreamAccessor::wrapStream(stream)); }; + inline void download(ftl::codecs::Channels c, cudaStream_t stream=0) { download(c, cv::cuda::StreamAccessor::wrapStream(stream)); }; + inline void upload(ftl::codecs::Channels c, cudaStream_t stream=0) { upload(c, cv::cuda::StreamAccessor::wrapStream(stream)); }; /** * Perform a buffer swap of the selected channels. This is intended to be * a copy from `this` to the passed frame object but by buffer swap * instead of memory copy, meaning `this` may become invalid afterwards. */ - void swapTo(ftl::rgbd::Channels, Frame &); + void swapTo(ftl::codecs::Channels, Frame &); - void swapChannels(ftl::rgbd::Channel, ftl::rgbd::Channel); + void swapChannels(ftl::codecs::Channel, ftl::codecs::Channel); /** * Create a channel with a given format. This will discard any existing * data associated with the channel and ensure all data structures and * memory allocations match the new format. */ - template <typename T> T &create(ftl::rgbd::Channel c, const ftl::rgbd::FormatBase &f); + template <typename T> T &create(ftl::codecs::Channel c, const ftl::rgbd::FormatBase &f); /** * Create a channel but without any format. */ - template <typename T> T &create(ftl::rgbd::Channel c); + template <typename T> T &create(ftl::codecs::Channel c); /** * Create a CUDA texture object for a channel. This version takes a format * argument to also create (or recreate) the associated GpuMat. */ template <typename T> - ftl::cuda::TextureObject<T> &createTexture(ftl::rgbd::Channel c, const ftl::rgbd::Format<T> &f); + ftl::cuda::TextureObject<T> &createTexture(ftl::codecs::Channel c, const ftl::rgbd::Format<T> &f); /** * Create a CUDA texture object for a channel. With this version the GpuMat * must already exist and be of the correct type. */ template <typename T> - ftl::cuda::TextureObject<T> &createTexture(ftl::rgbd::Channel c); + ftl::cuda::TextureObject<T> &createTexture(ftl::codecs::Channel c); - void resetTexture(ftl::rgbd::Channel c); + void resetTexture(ftl::codecs::Channel c); /** * Reset all channels without releasing memory. */ void reset(); - bool empty(ftl::rgbd::Channels c); + bool empty(ftl::codecs::Channels c); - inline bool empty(ftl::rgbd::Channel c) { + inline bool empty(ftl::codecs::Channel c) { auto &m = _get(c); return !hasChannel(c) || (m.host.empty() && m.gpu.empty()); } @@ -102,17 +102,17 @@ public: /** * Is there valid data in channel (either host or gpu). */ - inline bool hasChannel(ftl::rgbd::Channel channel) const { + inline bool hasChannel(ftl::codecs::Channel channel) const { return channels_.has(channel); } - inline ftl::rgbd::Channels getChannels() const { return channels_; } + inline ftl::codecs::Channels getChannels() const { return channels_; } /** * Is the channel data currently located on GPU. This also returns false if * the channel does not exist. */ - inline bool isGPU(ftl::rgbd::Channel channel) const { + inline bool isGPU(ftl::codecs::Channel channel) const { return channels_.has(channel) && gpu_.has(channel); } @@ -120,7 +120,7 @@ public: * Is the channel data currently located on CPU memory. This also returns * false if the channel does not exist. */ - inline bool isCPU(ftl::rgbd::Channel channel) const { + inline bool isCPU(ftl::codecs::Channel channel) const { return channels_.has(channel) && !gpu_.has(channel); } @@ -133,7 +133,7 @@ public: * performed, if necessary, but with a warning since an explicit upload or * download should be used. */ - template <typename T> const T& get(ftl::rgbd::Channel channel) const; + template <typename T> const T& get(ftl::codecs::Channel channel) const; /** * Method to get reference to the channel content. @@ -144,10 +144,10 @@ public: * performed, if necessary, but with a warning since an explicit upload or * download should be used. */ - template <typename T> T& get(ftl::rgbd::Channel channel); + template <typename T> T& get(ftl::codecs::Channel channel); - template <typename T> const ftl::cuda::TextureObject<T> &getTexture(ftl::rgbd::Channel) const; - template <typename T> ftl::cuda::TextureObject<T> &getTexture(ftl::rgbd::Channel); + template <typename T> const ftl::cuda::TextureObject<T> &getTexture(ftl::codecs::Channel) const; + template <typename T> ftl::cuda::TextureObject<T> &getTexture(ftl::codecs::Channel); private: struct ChannelData { @@ -156,31 +156,31 @@ private: cv::cuda::GpuMat gpu; }; - std::array<ChannelData, Channels::kMax> data_; + std::array<ChannelData, ftl::codecs::Channels::kMax> data_; - ftl::rgbd::Channels channels_; // Does it have a channel - ftl::rgbd::Channels gpu_; // Is the channel on a GPU + ftl::codecs::Channels channels_; // Does it have a channel + ftl::codecs::Channels gpu_; // Is the channel on a GPU ftl::rgbd::Source *src_; - inline ChannelData &_get(ftl::rgbd::Channel c) { return data_[static_cast<unsigned int>(c)]; } - inline const ChannelData &_get(ftl::rgbd::Channel c) const { return data_[static_cast<unsigned int>(c)]; } + inline ChannelData &_get(ftl::codecs::Channel c) { return data_[static_cast<unsigned int>(c)]; } + inline const ChannelData &_get(ftl::codecs::Channel c) const { return data_[static_cast<unsigned int>(c)]; } }; // Specialisations -template<> const cv::Mat& Frame::get(ftl::rgbd::Channel channel) const; -template<> const cv::cuda::GpuMat& Frame::get(ftl::rgbd::Channel channel) const; -template<> cv::Mat& Frame::get(ftl::rgbd::Channel channel); -template<> cv::cuda::GpuMat& Frame::get(ftl::rgbd::Channel channel); +template<> const cv::Mat& Frame::get(ftl::codecs::Channel channel) const; +template<> const cv::cuda::GpuMat& Frame::get(ftl::codecs::Channel channel) const; +template<> cv::Mat& Frame::get(ftl::codecs::Channel channel); +template<> cv::cuda::GpuMat& Frame::get(ftl::codecs::Channel channel); -template <> cv::Mat &Frame::create(ftl::rgbd::Channel c, const ftl::rgbd::FormatBase &); -template <> cv::cuda::GpuMat &Frame::create(ftl::rgbd::Channel c, const ftl::rgbd::FormatBase &); -template <> cv::Mat &Frame::create(ftl::rgbd::Channel c); -template <> cv::cuda::GpuMat &Frame::create(ftl::rgbd::Channel c); +template <> cv::Mat &Frame::create(ftl::codecs::Channel c, const ftl::rgbd::FormatBase &); +template <> cv::cuda::GpuMat &Frame::create(ftl::codecs::Channel c, const ftl::rgbd::FormatBase &); +template <> cv::Mat &Frame::create(ftl::codecs::Channel c); +template <> cv::cuda::GpuMat &Frame::create(ftl::codecs::Channel c); template <typename T> -ftl::cuda::TextureObject<T> &Frame::getTexture(ftl::rgbd::Channel c) { +ftl::cuda::TextureObject<T> &Frame::getTexture(ftl::codecs::Channel c) { if (!channels_.has(c)) throw ftl::exception(ftl::Formatter() << "Texture channel does not exist: " << (int)c); if (!gpu_.has(c)) throw ftl::exception("Texture channel is not on GPU"); @@ -195,7 +195,7 @@ ftl::cuda::TextureObject<T> &Frame::getTexture(ftl::rgbd::Channel c) { } template <typename T> -ftl::cuda::TextureObject<T> &Frame::createTexture(ftl::rgbd::Channel c, const ftl::rgbd::Format<T> &f) { +ftl::cuda::TextureObject<T> &Frame::createTexture(ftl::codecs::Channel c, const ftl::rgbd::Format<T> &f) { if (!channels_.has(c)) channels_ += c; if (!gpu_.has(c)) gpu_ += c; @@ -227,7 +227,7 @@ ftl::cuda::TextureObject<T> &Frame::createTexture(ftl::rgbd::Channel c, const ft } template <typename T> -ftl::cuda::TextureObject<T> &Frame::createTexture(ftl::rgbd::Channel c) { +ftl::cuda::TextureObject<T> &Frame::createTexture(ftl::codecs::Channel c) { if (!channels_.has(c)) throw ftl::exception("createTexture needs a format if the channel does not exist"); auto &m = _get(c); diff --git a/components/rgbd-sources/include/ftl/rgbd/frameset.hpp b/components/rgbd-sources/include/ftl/rgbd/frameset.hpp index 2fa39e2eacf19339860e98fa98df44f687ac64c7..f18b52635cde42f3a3e7d2adf047b9fe4048befd 100644 --- a/components/rgbd-sources/include/ftl/rgbd/frameset.hpp +++ b/components/rgbd-sources/include/ftl/rgbd/frameset.hpp @@ -26,8 +26,8 @@ struct FrameSet { bool stale; // True if buffers have been invalidated SHARED_MUTEX mtx; - void upload(ftl::rgbd::Channels, cudaStream_t stream=0); - void download(ftl::rgbd::Channels, cudaStream_t stream=0); + void upload(ftl::codecs::Channels, cudaStream_t stream=0); + void download(ftl::codecs::Channels, cudaStream_t stream=0); void swapTo(ftl::rgbd::FrameSet &); }; diff --git a/components/rgbd-sources/include/ftl/rgbd/source.hpp b/components/rgbd-sources/include/ftl/rgbd/source.hpp index 18413cb8941c56f09f6189895a098a517b276c7a..62286ee55475803e59c0234c7a581103f0686b4c 100644 --- a/components/rgbd-sources/include/ftl/rgbd/source.hpp +++ b/components/rgbd-sources/include/ftl/rgbd/source.hpp @@ -71,9 +71,9 @@ class Source : public ftl::Configurable { /** * Change the second channel source. */ - bool setChannel(ftl::rgbd::Channel c); + bool setChannel(ftl::codecs::Channel c); - ftl::rgbd::Channel getChannel() const { return channel_; } + ftl::codecs::Channel getChannel() const { return channel_; } /** * Perform the hardware or virtual frame grab operation. This should be @@ -146,7 +146,7 @@ class Source : public ftl::Configurable { else return params_; } - const Camera parameters(ftl::rgbd::Channel) const; + const Camera parameters(ftl::codecs::Channel) const; cv::Mat cameraMatrix() const; @@ -236,7 +236,7 @@ class Source : public ftl::Configurable { SHARED_MUTEX mutex_; bool paused_; bool bullet_; - ftl::rgbd::Channel channel_; + ftl::codecs::Channel channel_; cudaStream_t stream_; int64_t timestamp_; std::function<void(int64_t, cv::Mat &, cv::Mat &)> callback_; diff --git a/components/rgbd-sources/include/ftl/rgbd/streamer.hpp b/components/rgbd-sources/include/ftl/rgbd/streamer.hpp index c1a1e4b38a2cdcafea900c2b5728b0c75a9d9b4f..cfa552a862e1d46b9600ce523815eb8fa5efa001 100644 --- a/components/rgbd-sources/include/ftl/rgbd/streamer.hpp +++ b/components/rgbd-sources/include/ftl/rgbd/streamer.hpp @@ -168,14 +168,14 @@ class Streamer : public ftl::Configurable { void _process(ftl::rgbd::FrameSet &); void _cleanUp(); void _addClient(const std::string &source, int N, int rate, const ftl::UUID &peer, const std::string &dest); - void _transmitPacket(detail::StreamSource *src, const ftl::codecs::Packet &pkt, int chan, bool hasChan2, Quality q); + void _transmitPacket(detail::StreamSource *src, const ftl::codecs::Packet &pkt, ftl::codecs::Channel chan, bool hasChan2, Quality q); void _transmitPacket(detail::StreamSource *src, const ftl::codecs::StreamPacket &spkt, const ftl::codecs::Packet &pkt, Quality q); //void _encodeHQAndTransmit(detail::StreamSource *src, const cv::Mat &, const cv::Mat &, int chunk); //void _encodeLQAndTransmit(detail::StreamSource *src, const cv::Mat &, const cv::Mat &, int chunk); //void _encodeAndTransmit(detail::StreamSource *src, ftl::codecs::Encoder *enc1, ftl::codecs::Encoder *enc2, const cv::Mat &, const cv::Mat &); //void _encodeImageChannel1(const cv::Mat &in, std::vector<unsigned char> &out, unsigned int b); - //bool _encodeImageChannel2(const cv::Mat &in, std::vector<unsigned char> &out, ftl::rgbd::channel_t c, unsigned int b); + //bool _encodeImageChannel2(const cv::Mat &in, std::vector<unsigned char> &out, ftl::codecs::Channel_t c, unsigned int b); }; } diff --git a/components/rgbd-sources/src/algorithms/fixstars_sgm.cpp b/components/rgbd-sources/src/algorithms/fixstars_sgm.cpp index 5f8921bda0e562bcc26b454d750a35294ed75537..732c4ffb56a65a2ac8f53841b5dcb45e52ed5280 100644 --- a/components/rgbd-sources/src/algorithms/fixstars_sgm.cpp +++ b/components/rgbd-sources/src/algorithms/fixstars_sgm.cpp @@ -7,7 +7,7 @@ using ftl::algorithms::FixstarsSGM; using cv::Mat; using cv::cuda::GpuMat; -using ftl::rgbd::Channel; +using ftl::codecs::Channel; using ftl::rgbd::Format; //static ftl::Disparity::Register fixstarssgm("libsgm", FixstarsSGM::create); diff --git a/components/rgbd-sources/src/disparity.hpp b/components/rgbd-sources/src/disparity.hpp index 44215871d37b2944c08d072d63afd5bf871082e4..6802869c1c727a60f5ae3b308c86390486404044 100644 --- a/components/rgbd-sources/src/disparity.hpp +++ b/components/rgbd-sources/src/disparity.hpp @@ -50,9 +50,9 @@ class Disparity : public ftl::Configurable { { // FIXME: What were these for? //ftl::rgbd::Frame frame; - //frame.create<cv::cuda::GpuMat>(ftl::rgbd::Channel::Left) = l; - //frame.create<cv::cuda::GpuMat>(ftl::rgbd::Channel::Right) = r; - //frame.create<cv::cuda::GpuMat>(ftl::rgbd::Channel::Disparity) = disp; + //frame.create<cv::cuda::GpuMat>(ftl::codecs::Channel::Left) = l; + //frame.create<cv::cuda::GpuMat>(ftl::codecs::Channel::Right) = r; + //frame.create<cv::cuda::GpuMat>(ftl::codecs::Channel::Disparity) = disp; } /** diff --git a/components/rgbd-sources/src/file_source.cpp b/components/rgbd-sources/src/file_source.cpp index bc0a5fc700ec9b63732761315ec9ccd6fe0e92ad..bef04d6a4ba3a99a2f9a523e1ab870d6c9207d6d 100644 --- a/components/rgbd-sources/src/file_source.cpp +++ b/components/rgbd-sources/src/file_source.cpp @@ -4,6 +4,7 @@ using ftl::rgbd::detail::FileSource; using ftl::codecs::codec_t; +using ftl::codecs::Channel; void FileSource::_createDecoder(int ix, const ftl::codecs::Packet &pkt) { if (decoders_[ix]) { @@ -58,7 +59,7 @@ FileSource::~FileSource() { } -void FileSource::_removeChannel(int channel) { +void FileSource::_removeChannel(ftl::codecs::Channel channel) { int c = 0; for (auto i=cache_[cache_write_].begin(); i != cache_[cache_write_].end(); ++i) { if ((*i).spkt.channel == channel) { @@ -96,16 +97,16 @@ bool FileSource::compute(int n, int b) { for (auto i=cache_[cache_read_].begin(); i!=cache_[cache_read_].end(); ++i) { auto &c = *i; - if (c.spkt.channel == 0) { - rgb_.create(cv::Size(ftl::codecs::getWidth(c.pkt.definition),ftl::codecs::getHeight(c.pkt.definition)), CV_8UC3); + if (c.spkt.channel == Channel::Colour) { + rgb_.create(cv::Size(ftl::codecs::getWidth(c.pkt.definition),ftl::codecs::getHeight(c.pkt.definition)), CV_8UC4); } else { depth_.create(cv::Size(ftl::codecs::getWidth(c.pkt.definition),ftl::codecs::getHeight(c.pkt.definition)), CV_32F); } - _createDecoder(c.spkt.channel, c.pkt); + _createDecoder((c.spkt.channel == Channel::Colour) ? 0 : 1, c.pkt); try { - decoders_[c.spkt.channel]->decode(c.pkt, (c.spkt.channel == 0) ? rgb_ : depth_); + decoders_[(c.spkt.channel == Channel::Colour) ? 0 : 1]->decode(c.pkt, (c.spkt.channel == Channel::Colour) ? rgb_ : depth_); } catch (std::exception &e) { LOG(INFO) << "Decoder exception: " << e.what(); } diff --git a/components/rgbd-sources/src/file_source.hpp b/components/rgbd-sources/src/file_source.hpp index 810b1b56c170400898a81db9963f6c4c1d4cd5d5..a9bdc4b369f9135c741a484f38a432bdf91a2ce3 100644 --- a/components/rgbd-sources/src/file_source.hpp +++ b/components/rgbd-sources/src/file_source.hpp @@ -43,7 +43,7 @@ class FileSource : public detail::Source { bool realtime_; - void _removeChannel(int channel); + void _removeChannel(ftl::codecs::Channel channel); void _createDecoder(int ix, const ftl::codecs::Packet &pkt); }; diff --git a/components/rgbd-sources/src/frame.cpp b/components/rgbd-sources/src/frame.cpp index 0d8dd3282338a77daed06785ee6d74e163a7b12c..851e75af76b96543f1f404303b50a2e3682652ae 100644 --- a/components/rgbd-sources/src/frame.cpp +++ b/components/rgbd-sources/src/frame.cpp @@ -2,8 +2,8 @@ #include <ftl/rgbd/frame.hpp> using ftl::rgbd::Frame; -using ftl::rgbd::Channels; -using ftl::rgbd::Channel; +using ftl::codecs::Channels; +using ftl::codecs::Channel; static cv::Mat none; static cv::cuda::GpuMat noneGPU; @@ -39,14 +39,14 @@ void Frame::upload(Channels c, cv::cuda::Stream stream) { } } -bool Frame::empty(ftl::rgbd::Channels channels) { +bool Frame::empty(ftl::codecs::Channels channels) { for (auto c : channels) { if (empty(c)) return true; } return false; } -void Frame::swapTo(ftl::rgbd::Channels channels, Frame &f) { +void Frame::swapTo(ftl::codecs::Channels channels, Frame &f) { f.reset(); // For all channels in this frame object @@ -74,7 +74,7 @@ void Frame::swapTo(ftl::rgbd::Channels channels, Frame &f) { } } -void Frame::swapChannels(ftl::rgbd::Channel a, ftl::rgbd::Channel b) { +void Frame::swapChannels(ftl::codecs::Channel a, ftl::codecs::Channel b) { auto &m1 = _get(a); auto &m2 = _get(b); cv::swap(m1.host, m2.host); @@ -85,7 +85,7 @@ void Frame::swapChannels(ftl::rgbd::Channel a, ftl::rgbd::Channel b) { m1.tex = std::move(temptex); } -template<> cv::Mat& Frame::get(ftl::rgbd::Channel channel) { +template<> cv::Mat& Frame::get(ftl::codecs::Channel channel) { if (channel == Channel::None) { DLOG(WARNING) << "Cannot get the None channel from a Frame"; none.release(); @@ -105,7 +105,7 @@ template<> cv::Mat& Frame::get(ftl::rgbd::Channel channel) { return _get(channel).host; } -template<> cv::cuda::GpuMat& Frame::get(ftl::rgbd::Channel channel) { +template<> cv::cuda::GpuMat& Frame::get(ftl::codecs::Channel channel) { if (channel == Channel::None) { DLOG(WARNING) << "Cannot get the None channel from a Frame"; noneGPU.release(); @@ -125,7 +125,7 @@ template<> cv::cuda::GpuMat& Frame::get(ftl::rgbd::Channel channel) { return _get(channel).gpu; } -template<> const cv::Mat& Frame::get(ftl::rgbd::Channel channel) const { +template<> const cv::Mat& Frame::get(ftl::codecs::Channel channel) const { if (channel == Channel::None) { LOG(FATAL) << "Cannot get the None channel from a Frame"; } @@ -139,7 +139,7 @@ template<> const cv::Mat& Frame::get(ftl::rgbd::Channel channel) const { return _get(channel).host; } -template<> const cv::cuda::GpuMat& Frame::get(ftl::rgbd::Channel channel) const { +template<> const cv::cuda::GpuMat& Frame::get(ftl::codecs::Channel channel) const { if (channel == Channel::None) { LOG(FATAL) << "Cannot get the None channel from a Frame"; } @@ -156,7 +156,7 @@ template<> const cv::cuda::GpuMat& Frame::get(ftl::rgbd::Channel channel) const return _get(channel).gpu; } -template <> cv::Mat &Frame::create(ftl::rgbd::Channel c, const ftl::rgbd::FormatBase &f) { +template <> cv::Mat &Frame::create(ftl::codecs::Channel c, const ftl::rgbd::FormatBase &f) { if (c == Channel::None) { throw ftl::exception("Cannot create a None channel"); } @@ -172,7 +172,7 @@ template <> cv::Mat &Frame::create(ftl::rgbd::Channel c, const ftl::rgbd::Format return m; } -template <> cv::cuda::GpuMat &Frame::create(ftl::rgbd::Channel c, const ftl::rgbd::FormatBase &f) { +template <> cv::cuda::GpuMat &Frame::create(ftl::codecs::Channel c, const ftl::rgbd::FormatBase &f) { if (c == Channel::None) { throw ftl::exception("Cannot create a None channel"); } @@ -188,7 +188,7 @@ template <> cv::cuda::GpuMat &Frame::create(ftl::rgbd::Channel c, const ftl::rgb return m; } -template <> cv::Mat &Frame::create(ftl::rgbd::Channel c) { +template <> cv::Mat &Frame::create(ftl::codecs::Channel c) { if (c == Channel::None) { throw ftl::exception("Cannot create a None channel"); } @@ -199,7 +199,7 @@ template <> cv::Mat &Frame::create(ftl::rgbd::Channel c) { return m; } -template <> cv::cuda::GpuMat &Frame::create(ftl::rgbd::Channel c) { +template <> cv::cuda::GpuMat &Frame::create(ftl::codecs::Channel c) { if (c == Channel::None) { throw ftl::exception("Cannot create a None channel"); } @@ -210,7 +210,7 @@ template <> cv::cuda::GpuMat &Frame::create(ftl::rgbd::Channel c) { return m; } -void Frame::resetTexture(ftl::rgbd::Channel c) { +void Frame::resetTexture(ftl::codecs::Channel c) { auto &m = _get(c); m.tex.free(); } diff --git a/components/rgbd-sources/src/frameset.cpp b/components/rgbd-sources/src/frameset.cpp index 9b9a807d8599c23141b6c3806546bf8038ef30f9..38232e3f6bc1bbfcb1f3a4963743171d4401fc39 100644 --- a/components/rgbd-sources/src/frameset.cpp +++ b/components/rgbd-sources/src/frameset.cpp @@ -1,16 +1,16 @@ #include <ftl/rgbd/frameset.hpp> using ftl::rgbd::FrameSet; -using ftl::rgbd::Channels; -using ftl::rgbd::Channel; +using ftl::codecs::Channels; +using ftl::codecs::Channel; -void FrameSet::upload(ftl::rgbd::Channels c, cudaStream_t stream) { +void FrameSet::upload(ftl::codecs::Channels c, cudaStream_t stream) { for (auto &f : frames) { f.upload(c, stream); } } -void FrameSet::download(ftl::rgbd::Channels c, cudaStream_t stream) { +void FrameSet::download(ftl::codecs::Channels c, cudaStream_t stream) { for (auto &f : frames) { f.download(c, stream); } diff --git a/components/rgbd-sources/src/group.cpp b/components/rgbd-sources/src/group.cpp index 4a85c3d7cee7a9f897d0f9fdb87ceba1fa86e671..278ce828573b0f822b0328110150cdb7b4697b49 100644 --- a/components/rgbd-sources/src/group.cpp +++ b/components/rgbd-sources/src/group.cpp @@ -10,7 +10,7 @@ using ftl::rgbd::kFrameBufferSize; using std::vector; using std::chrono::milliseconds; using std::this_thread::sleep_for; -using ftl::rgbd::Channel; +using ftl::codecs::Channel; Group::Group() : framesets_(kFrameBufferSize), head_(0) { framesets_[0].timestamp = -1; diff --git a/components/rgbd-sources/src/net.cpp b/components/rgbd-sources/src/net.cpp index 8712f48d84498c0075aea717a0a2582d50ec4a70..91a990090377e9453eb4df8083172c2df148dd54 100644 --- a/components/rgbd-sources/src/net.cpp +++ b/components/rgbd-sources/src/net.cpp @@ -20,7 +20,7 @@ using std::vector; using std::this_thread::sleep_for; using std::chrono::milliseconds; using std::tuple; -using ftl::rgbd::Channel; +using ftl::codecs::Channel; // ===== NetFrameQueue ========================================================= @@ -84,7 +84,7 @@ void NetFrameQueue::freeFrame(NetFrame &f) { // ===== NetSource ============================================================= -bool NetSource::_getCalibration(Universe &net, const UUID &peer, const string &src, ftl::rgbd::Camera &p, ftl::rgbd::Channel chan) { +bool NetSource::_getCalibration(Universe &net, const UUID &peer, const string &src, ftl::rgbd::Camera &p, ftl::codecs::Channel chan) { try { while(true) { auto [cap,buf] = net.call<tuple<unsigned int,vector<unsigned char>>>(peer_, "source_details", src, chan); @@ -245,26 +245,31 @@ void NetSource::_recvPacket(short ttimeoff, const ftl::codecs::StreamPacket &spk // Allow acccess to the raw data elsewhere... host_->notifyRaw(spkt, pkt); - const ftl::rgbd::Channel chan = host_->getChannel(); - int rchan = spkt.channel; // & 0x1; + const ftl::codecs::Channel chan = host_->getChannel(); + const ftl::codecs::Channel rchan = spkt.channel; // & 0x1; - NetFrame &frame = queue_.getFrame(spkt.timestamp, cv::Size(params_.width, params_.height), CV_8UC3, (isFloatChannel(chan) ? CV_32FC1 : CV_8UC3)); + NetFrame &frame = queue_.getFrame(spkt.timestamp, cv::Size(params_.width, params_.height), CV_8UC4, (isFloatChannel(chan) ? CV_32FC1 : CV_8UC4)); // Update frame statistics frame.tx_size += pkt.data.size(); - // Ignore any unwanted second channel - if (!(chan == ftl::rgbd::Channel::None && rchan > 0)) { - _createDecoder(rchan, pkt); - auto *decoder = (rchan == 0) ? decoder_c1_ : decoder_c2_; + // Only decode if this channel is wanted. + if (rchan == Channel::Colour || rchan == chan) { + _createDecoder((rchan == Channel::Colour) ? 0 : 1, pkt); + auto *decoder = (rchan == Channel::Colour) ? decoder_c1_ : decoder_c2_; if (!decoder) { LOG(ERROR) << "No frame decoder available"; return; } - decoder->decode(pkt, (rchan == 0) ? frame.channel1 : frame.channel2); - } else { - //LOG(INFO) << "Unwanted frame"; + decoder->decode(pkt, (rchan == Channel::Colour) ? frame.channel1 : frame.channel2); + } else if (chan != Channel::None && rchan != Channel::Colour) { + // Didn't receive correct second channel so just clear the images + if (isFloatChannel(chan)) { + frame.channel2.setTo(cv::Scalar(0.0f)); + } else { + frame.channel2.setTo(cv::Scalar(0,0,0)); + } } // Apply colour correction to chunk @@ -341,8 +346,8 @@ void NetSource::setPose(const Eigen::Matrix4d &pose) { //Source::setPose(pose); } -ftl::rgbd::Camera NetSource::parameters(ftl::rgbd::Channel chan) { - if (chan == ftl::rgbd::Channel::Right) { +ftl::rgbd::Camera NetSource::parameters(ftl::codecs::Channel chan) { + if (chan == ftl::codecs::Channel::Right) { auto uri = host_->get<string>("uri"); if (!uri) return params_; @@ -357,7 +362,7 @@ ftl::rgbd::Camera NetSource::parameters(ftl::rgbd::Channel chan) { void NetSource::_updateURI() { UNIQUE_LOCK(mutex_,lk); active_ = false; - prev_chan_ = ftl::rgbd::Channel::None; + prev_chan_ = ftl::codecs::Channel::None; auto uri = host_->get<string>("uri"); if (uri_.size() > 0) { @@ -372,7 +377,7 @@ void NetSource::_updateURI() { } peer_ = *p; - has_calibration_ = _getCalibration(*host_->getNet(), peer_, *uri, params_, ftl::rgbd::Channel::Left); + has_calibration_ = _getCalibration(*host_->getNet(), peer_, *uri, params_, ftl::codecs::Channel::Left); host_->getNet()->bind(*uri, [this](short ttimeoff, const ftl::codecs::StreamPacket &spkt, const ftl::codecs::Packet &pkt) { //if (chunk == -1) { @@ -412,7 +417,7 @@ bool NetSource::compute(int n, int b) { // Send k frames before end to prevent unwanted pause // Unless only a single frame is requested if ((N_ <= maxN_/2 && maxN_ > 1) || N_ == 0) { - const ftl::rgbd::Channel chan = host_->getChannel(); + const ftl::codecs::Channel chan = host_->getChannel(); N_ = maxN_; diff --git a/components/rgbd-sources/src/net.hpp b/components/rgbd-sources/src/net.hpp index 51f31861fa3c9c39ea0cb53217e0fa3f764aeef3..0f867a9ab08fee843ba229ba47636938ef04abfa 100644 --- a/components/rgbd-sources/src/net.hpp +++ b/components/rgbd-sources/src/net.hpp @@ -39,7 +39,7 @@ class NetSource : public detail::Source { bool isReady(); void setPose(const Eigen::Matrix4d &pose); - Camera parameters(ftl::rgbd::Channel chan); + Camera parameters(ftl::codecs::Channel chan); void reset(); @@ -57,7 +57,7 @@ class NetSource : public detail::Source { int minB_; int maxN_; int default_quality_; - ftl::rgbd::Channel prev_chan_; + ftl::codecs::Channel prev_chan_; ftl::rgbd::detail::ABRController abr_; int last_bitrate_; @@ -77,7 +77,7 @@ class NetSource : public detail::Source { NetFrameQueue queue_; - bool _getCalibration(ftl::net::Universe &net, const ftl::UUID &peer, const std::string &src, ftl::rgbd::Camera &p, ftl::rgbd::Channel chan); + bool _getCalibration(ftl::net::Universe &net, const ftl::UUID &peer, const std::string &src, ftl::rgbd::Camera &p, ftl::codecs::Channel chan); void _recv(const std::vector<unsigned char> &jpg, const std::vector<unsigned char> &d); void _recvPacket(short ttimeoff, const ftl::codecs::StreamPacket &, const ftl::codecs::Packet &); //void _recvChunk(int64_t frame, short ttimeoff, uint8_t bitrate, int chunk, const std::vector<unsigned char> &jpg, const std::vector<unsigned char> &d); diff --git a/components/rgbd-sources/src/offilter.cpp b/components/rgbd-sources/src/offilter.cpp index 466aa9249517b91ac54726e821401e85272aa082..34c01c1feb73b8171b0890f8d94405ee22a0433b 100644 --- a/components/rgbd-sources/src/offilter.cpp +++ b/components/rgbd-sources/src/offilter.cpp @@ -6,6 +6,7 @@ #include <loguru.hpp> using namespace ftl::rgbd; +using namespace ftl::codecs; using cv::Mat; using cv::Size; diff --git a/components/rgbd-sources/src/source.cpp b/components/rgbd-sources/src/source.cpp index 5539ac612da8bcab109d18dd2a8fcf7afc9c35e9..114ba269394d315a6b88c33e21eb6cef8150e0e2 100644 --- a/components/rgbd-sources/src/source.cpp +++ b/components/rgbd-sources/src/source.cpp @@ -27,7 +27,7 @@ using ftl::rgbd::detail::NetSource; using ftl::rgbd::detail::ImageSource; using ftl::rgbd::detail::MiddleburySource; using ftl::rgbd::capability_t; -using ftl::rgbd::Channel; +using ftl::codecs::Channel; using ftl::rgbd::detail::FileSource; std::map<std::string, ftl::codecs::Reader*> Source::readers__; @@ -321,13 +321,13 @@ bool Source::thumbnail(cv::Mat &t) { return !thumb_.empty(); } -bool Source::setChannel(ftl::rgbd::Channel c) { +bool Source::setChannel(ftl::codecs::Channel c) { channel_ = c; // FIXME:(Nick) Verify channel is supported by this source... return true; } -const ftl::rgbd::Camera Source::parameters(ftl::rgbd::Channel chan) const { +const ftl::rgbd::Camera Source::parameters(ftl::codecs::Channel chan) const { return (impl_) ? impl_->parameters(chan) : parameters(); } diff --git a/components/rgbd-sources/src/stereovideo.cpp b/components/rgbd-sources/src/stereovideo.cpp index 6573f74f4d7cf1f3761f98a66c68c6963e10af31..b84385fcecb3a8c00a94398d998d872c68929951 100644 --- a/components/rgbd-sources/src/stereovideo.cpp +++ b/components/rgbd-sources/src/stereovideo.cpp @@ -12,7 +12,7 @@ using ftl::rgbd::detail::Calibrate; using ftl::rgbd::detail::LocalSource; using ftl::rgbd::detail::StereoVideoSource; -using ftl::rgbd::Channel; +using ftl::codecs::Channel; using std::string; StereoVideoSource::StereoVideoSource(ftl::rgbd::Source *host) @@ -219,7 +219,7 @@ bool StereoVideoSource::compute(int n, int b) { auto &left = frame.get<cv::cuda::GpuMat>(Channel::Left); auto &right = frame.get<cv::cuda::GpuMat>(Channel::Right); - const ftl::rgbd::Channel chan = host_->getChannel(); + const ftl::codecs::Channel chan = host_->getChannel(); if (left.empty() || right.empty()) return false; if (chan == Channel::Depth) { diff --git a/components/rgbd-sources/src/stereovideo.hpp b/components/rgbd-sources/src/stereovideo.hpp index 9d3325e1ac27ec544abeb409149cb89817dc29d2..dfea7937f99777c23f753a7ab2b5bad8c68bb4af 100644 --- a/components/rgbd-sources/src/stereovideo.hpp +++ b/components/rgbd-sources/src/stereovideo.hpp @@ -31,7 +31,7 @@ class StereoVideoSource : public detail::Source { bool retrieve(); bool compute(int n, int b); bool isReady(); - Camera parameters(ftl::rgbd::Channel chan); + Camera parameters(ftl::codecs::Channel chan); //const cv::Mat &getRight() const { return right_; } diff --git a/components/rgbd-sources/src/streamer.cpp b/components/rgbd-sources/src/streamer.cpp index 9d8f68898dc29dfc2d285a63a1c1daea7e26381e..cab8f99d8cdd49b04d8282b040fe09f2e93d7204 100644 --- a/components/rgbd-sources/src/streamer.cpp +++ b/components/rgbd-sources/src/streamer.cpp @@ -17,7 +17,7 @@ using ftl::rgbd::detail::StreamClient; using ftl::rgbd::detail::ABRController; using ftl::codecs::definition_t; using ftl::codecs::device_t; -using ftl::rgbd::Channel; +using ftl::codecs::Channel; using ftl::net::Universe; using std::string; using std::list; @@ -92,7 +92,7 @@ Streamer::Streamer(nlohmann::json &config, Universe *net) }); // Allow remote users to access camera calibration matrix - net->bind("source_details", [this](const std::string &uri, ftl::rgbd::Channel chan) -> tuple<unsigned int,vector<unsigned char>> { + net->bind("source_details", [this](const std::string &uri, ftl::codecs::Channel chan) -> tuple<unsigned int,vector<unsigned char>> { vector<unsigned char> buf; SHARED_LOCK(mutex_,slk); @@ -426,8 +426,11 @@ void Streamer::_process(ftl::rgbd::FrameSet &fs) { if (hasChan2) { // TODO: Stagger the reset between nodes... random phasing if (fs.timestamp % (10*ftl::timer::getInterval()) == 0) enc2->reset(); - enc2->encode(fs.frames[j].get<cv::Mat>(fs.sources[j]->getChannel()), src->hq_bitrate, [this,src,hasChan2](const ftl::codecs::Packet &blk){ - _transmitPacket(src, blk, 1, hasChan2, Quality::High); + + auto chan = fs.sources[j]->getChannel(); + + enc2->encode(fs.frames[j].get<cv::Mat>(chan), src->hq_bitrate, [this,src,hasChan2,chan](const ftl::codecs::Packet &blk){ + _transmitPacket(src, blk, chan, hasChan2, Quality::High); }); } else { if (enc2) enc2->reset(); @@ -436,7 +439,7 @@ void Streamer::_process(ftl::rgbd::FrameSet &fs) { // TODO: Stagger the reset between nodes... random phasing if (fs.timestamp % (10*ftl::timer::getInterval()) == 0) enc1->reset(); enc1->encode(fs.frames[j].get<cv::Mat>(Channel::Colour), src->hq_bitrate, [this,src,hasChan2](const ftl::codecs::Packet &blk){ - _transmitPacket(src, blk, 0, hasChan2, Quality::High); + _transmitPacket(src, blk, Channel::Colour, hasChan2, Quality::High); }); } } @@ -456,15 +459,17 @@ void Streamer::_process(ftl::rgbd::FrameSet &fs) { // Important to send channel 2 first if needed... // Receiver only waits for channel 1 by default if (hasChan2) { - enc2->encode(fs.frames[j].get<cv::Mat>(fs.sources[j]->getChannel()), src->lq_bitrate, [this,src,hasChan2](const ftl::codecs::Packet &blk){ - _transmitPacket(src, blk, 1, hasChan2, Quality::Low); + auto chan = fs.sources[j]->getChannel(); + + enc2->encode(fs.frames[j].get<cv::Mat>(chan), src->lq_bitrate, [this,src,hasChan2,chan](const ftl::codecs::Packet &blk){ + _transmitPacket(src, blk, chan, hasChan2, Quality::Low); }); } else { if (enc2) enc2->reset(); } enc1->encode(fs.frames[j].get<cv::Mat>(Channel::Colour), src->lq_bitrate, [this,src,hasChan2](const ftl::codecs::Packet &blk){ - _transmitPacket(src, blk, 0, hasChan2, Quality::Low); + _transmitPacket(src, blk, Channel::Colour, hasChan2, Quality::Low); }); } } @@ -530,7 +535,7 @@ void Streamer::_process(ftl::rgbd::FrameSet &fs) { } else _cleanUp(); } -void Streamer::_transmitPacket(StreamSource *src, const ftl::codecs::Packet &pkt, int chan, bool hasChan2, Quality q) { +void Streamer::_transmitPacket(StreamSource *src, const ftl::codecs::Packet &pkt, Channel chan, bool hasChan2, Quality q) { ftl::codecs::StreamPacket spkt = { frame_no_, src->id, @@ -567,7 +572,7 @@ void Streamer::_transmitPacket(StreamSource *src, const ftl::codecs::StreamPacke (*c).txcount = (*c).txmax; } else { // Count frame as completed only if last block and channel is 0 - if (pkt.block_number == pkt.block_total - 1 && spkt.channel & 0x1 == 0) ++(*c).txcount; + if (pkt.block_number == pkt.block_total - 1 && spkt.channel == Channel::Colour) ++(*c).txcount; } } catch(...) { (*c).txcount = (*c).txmax; @@ -765,7 +770,7 @@ void Streamer::_encodeImageChannel1(const cv::Mat &in, vector<unsigned char> &ou cv::imencode(".jpg", in, out, jpgparams); } -bool Streamer::_encodeImageChannel2(const cv::Mat &in, vector<unsigned char> &out, ftl::rgbd::channel_t c, unsigned int b) { +bool Streamer::_encodeImageChannel2(const cv::Mat &in, vector<unsigned char> &out, ftl::codecs::Channel_t c, unsigned int b) { if (c == ftl::rgbd::kChanNone) return false; // NOTE: Should not happen if (isFloatChannel(c) && in.type() == CV_16U && in.channels() == 1) { diff --git a/components/rgbd-sources/src/virtual.cpp b/components/rgbd-sources/src/virtual.cpp index 9d9ccc852fd94a6aa701e7da7e4b4f48cad9d0a0..f4bcb4a498973f0a39687f4f0c70a06807dc47ca 100644 --- a/components/rgbd-sources/src/virtual.cpp +++ b/components/rgbd-sources/src/virtual.cpp @@ -2,7 +2,7 @@ using ftl::rgbd::VirtualSource; using ftl::rgbd::Source; -using ftl::rgbd::Channel; +using ftl::codecs::Channel; class VirtualImpl : public ftl::rgbd::detail::Source { public: diff --git a/components/rgbd-sources/test/CMakeLists.txt b/components/rgbd-sources/test/CMakeLists.txt index 78bb6cec7e8c411ffbfe982a1b65320f56439bd5..9611e8081d942a6051e0ea254b3308d406161cd7 100644 --- a/components/rgbd-sources/test/CMakeLists.txt +++ b/components/rgbd-sources/test/CMakeLists.txt @@ -9,17 +9,6 @@ target_link_libraries(source_unit add_test(SourceUnitTest source_unit) -### Channel Unit ############################################################### -add_executable(channel_unit - ./tests.cpp - ./channel_unit.cpp -) -target_include_directories(channel_unit PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}/../include") -target_link_libraries(channel_unit - ftlcommon) - -add_test(ChannelUnitTest channel_unit) - ### Frame Unit ################################################################# add_executable(frame_unit ./tests.cpp diff --git a/components/rgbd-sources/test/frame_unit.cpp b/components/rgbd-sources/test/frame_unit.cpp index 6ad528a28fd677e207b05362c0021f7d302784dc..6d858a62ab7765f1efc6f98ab09c5109f8101603 100644 --- a/components/rgbd-sources/test/frame_unit.cpp +++ b/components/rgbd-sources/test/frame_unit.cpp @@ -2,8 +2,8 @@ #include <ftl/rgbd/frame.hpp> using ftl::rgbd::Frame; -using ftl::rgbd::Channel; -using ftl::rgbd::Channels; +using ftl::codecs::Channel; +using ftl::codecs::Channels; using ftl::rgbd::Format; TEST_CASE("Frame::create() cpu mat", "") {