diff --git a/CMakeLists.txt b/CMakeLists.txt index e9c31bc18ea026b89a95f7ad8f3517bebe18b877..f4fd7641d536f3b0286b76db43ae5014321a3513 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -193,6 +193,8 @@ add_library(beyond-protocol STATIC src/rpc.cpp src/channelUtils.cpp src/service.cpp + src/codecs/golomb.cpp + src/codecs/h264.cpp ) target_include_directories(beyond-protocol PUBLIC diff --git a/examples/CMakeLists.txt b/examples/CMakeLists.txt index 9b28cc491a4e2bcdd252df44267ffa79c52de241..e9dc3c4a5e651b0aaee19c19e4c80fbd8d626d40 100644 --- a/examples/CMakeLists.txt +++ b/examples/CMakeLists.txt @@ -1,6 +1,9 @@ add_executable(read-ftl-file ./read-ftl-file/main.cpp) target_link_libraries(read-ftl-file beyond-protocol Threads::Threads ${OS_LIBS} ${URIPARSER_LIBRARIES} ${UUID_LIBRARIES}) +add_executable(decode-h264 ./decode-h264/main.cpp) +target_link_libraries(decode-h264 beyond-protocol Threads::Threads ${OS_LIBS} ${URIPARSER_LIBRARIES} ${UUID_LIBRARIES}) + add_executable(open-network-stream ./open-network-stream/main.cpp) target_link_libraries(open-network-stream beyond-protocol Threads::Threads ${OS_LIBS} ${URIPARSER_LIBRARIES} ${UUID_LIBRARIES}) diff --git a/examples/decode-h264/main.cpp b/examples/decode-h264/main.cpp new file mode 100644 index 0000000000000000000000000000000000000000..ff04a97453321d90b3e6f6065b636bf033e51d72 --- /dev/null +++ b/examples/decode-h264/main.cpp @@ -0,0 +1,50 @@ +/** + * @file main.cpp + * @copyright Copyright (c) 2022 University of Turku, MIT License + * @author Nicolas Pope + */ + +#include <chrono> +#include <ftl/protocol.hpp> +#include <ftl/protocol/streams.hpp> +#include <ftl/lib/loguru.hpp> +#include <ftl/codec/h264.hpp> + +using ftl::protocol::Codec; +using ftl::protocol::Channel; +using ftl::protocol::StreamPacket; +using ftl::protocol::DataPacket; +using std::this_thread::sleep_for; +using std::chrono::seconds; +using ftl::protocol::StreamProperty; + +int main(int argc, char *argv[]) { + if (argc != 2) return -1; + + auto stream = ftl::getStream(argv[1]); + + const auto parser = std::make_unique<ftl::codec::h264::Parser>(); + + auto h = stream->onPacket([&parser](const StreamPacket &spkt, const DataPacket &pkt) { + if (spkt.channel == Channel::kColour && pkt.codec == Codec::kH264) { + try { + auto slices = parser->parse(pkt.data); + for (const ftl::codec::h264::Slice &s : slices) { + LOG(INFO) << "Slice (" << spkt.timestamp << ")" << std::endl << ftl::codec::h264::prettySlice(s); + } + } catch (const std::exception &e) { + LOG(ERROR) << e.what(); + } + } + return true; + }); + + stream->setProperty(StreamProperty::kLooping, true); + stream->setProperty(StreamProperty::kSpeed, 1); + + if (!stream->begin()) return -1; + sleep_for(seconds(20)); + stream->end(); + + return 0; +} diff --git a/include/ftl/codec/golomb.hpp b/include/ftl/codec/golomb.hpp new file mode 100644 index 0000000000000000000000000000000000000000..9c93248247e000e3c26943e30585a71f65518ae9 --- /dev/null +++ b/include/ftl/codec/golomb.hpp @@ -0,0 +1,95 @@ +#pragma once + +#include <cstddef> +#include <cstdint> + +namespace ftl { +namespace codec { +namespace detail { + +extern const uint8_t golomb_len[512]; +extern const uint8_t golomb_ue_code[512]; +extern const int8_t golomb_se_code[512]; + +struct ParseContext { + const uint8_t *ptr; + size_t index; + size_t length; +}; + +static inline uint32_t bswap_32(uint32_t x) { + x= ((x<<8)&0xFF00FF00) | ((x>>8)&0x00FF00FF); + x= (x>>16) | (x<<16); + return x; +} + +static inline uint32_t read32(const uint8_t *ptr) { + return bswap_32(*reinterpret_cast<const uint32_t*>(ptr)); +} + +static inline unsigned int getBits(ParseContext *ctx, int cnt) { + uint32_t buf = read32(&ctx->ptr[ctx->index >> 3]) << (ctx->index & 0x07); + ctx->index += cnt; + return buf >> (32 - cnt); +} + +static inline unsigned int getBits1(ParseContext *ctx) { + return getBits(ctx, 1); +} + +static inline int log2(unsigned int x) { + #ifdef __GNUC__ + return (31 - __builtin_clz((x)|1)); + #elif _MSC_VER + unsigned long n; + _BitScanReverse(&n, x|1); + return n; + #else + return 0; // TODO(Nick) + #endif +} + +static inline unsigned int golombUnsigned31(ParseContext *ctx) { + uint32_t buf = read32(&ctx->ptr[ctx->index >> 3]) << (ctx->index & 0x07); + buf >>= 32 - 9; + ctx->index += golomb_len[buf]; + return golomb_ue_code[buf]; +} + +static inline unsigned int golombUnsigned(ParseContext *ctx) { + uint32_t buf = read32(&ctx->ptr[ctx->index >> 3]) << (ctx->index & 0x07); + + if (buf >= (1<<27)) { + buf >>= 32 - 9; + ctx->index += golomb_len[buf]; + return golomb_ue_code[buf]; + } else { + int log = 2 * log2(buf) - 31; + buf >>= log; + buf--; + ctx->index += 32 - log; + return buf; + } +} + +static inline int golombSigned(ParseContext *ctx) { + uint32_t buf = read32(&ctx->ptr[ctx->index >> 3]) << (ctx->index & 0x07); + + if (buf >= (1<<27)) { + buf >>= 32 - 9; + ctx->index += golomb_len[buf]; + return golomb_se_code[buf]; + } else { + int log = 2 * log2(buf) - 31; + buf >>= log; + ctx->index += 32 - log; + + if(buf & 1) buf = -(buf>>1); + else buf = (buf>>1); + return buf; + } +} + +} +} +} \ No newline at end of file diff --git a/include/ftl/codec/h264.hpp b/include/ftl/codec/h264.hpp new file mode 100644 index 0000000000000000000000000000000000000000..b1ccd917d8909dfb74ba93d6f786b018647c581f --- /dev/null +++ b/include/ftl/codec/h264.hpp @@ -0,0 +1,283 @@ +/** + * @file h264.hpp + * @copyright Copyright (c) 2020 University of Turku, MIT License + * @author Nicolas Pope + */ + +#pragma once + +#include <vector> +#include <list> +#include <string> +#include <ftl/codec/golomb.hpp> + +namespace ftl { +namespace codec { + +/** + * H.264 codec utility functions. + */ +namespace h264 { + +struct NALHeader { + uint8_t type : 5; + uint8_t ref_idc : 2; + uint8_t forbidden : 1; +}; + +enum class ProfileIDC { + kInvalid = 0, + kBaseline = 66, + kExtended = 88, + kMain = 77, + kHigh = 100, + kHigh10 = 110 +}; + +enum class LevelIDC { + kInvalid = 0, + kLevel1 = 10, + kLevel1_1 = 11, + kLevel1_2 = 12, + kLevel1_3 = 13, + kLevel2 = 20, + kLevel2_1 = 21, + kLevel2_2 = 22, + kLevel3 = 30, + kLevel3_1 = 31, + kLevel3_2 = 32, + kLevel4 = 40, + kLevel4_1 = 41, + kLevel4_2 = 42, + kLevel5 = 50, + kLevel5_1 = 51, + kLevel5_2 = 52, + kLevel6 = 60, + kLevel6_1 = 61, + kLevel6_2 = 62 +}; + +enum class POCType { + kType0 = 0, + kType1 = 1, + kType2 = 2 +}; + +enum class ChromaFormatIDC { + kMonochrome = 0, + k420 = 1, + k422 = 2, + k444 = 3 +}; + +struct PPS { + int id = -1; + int sps_id = 0; + bool cabac = false; + bool pic_order_present = false; + int slice_group_count = 0; + int mb_slice_group_map_type = 0; + unsigned int ref_count[2]; + bool weighted_pred = false; + int weighted_bipred_idc = 0; + int init_qp = 0; + int init_qs = 0; + int chroma_qp_index_offset[2]; + bool deblocking_filter_parameters_present = false; + bool constrained_intra_pred = false; + bool redundant_pic_cnt_present = false; + int transform_8x8_mode = 0; + uint8_t scaling_matrix4[6][16]; + uint8_t scaling_matrix8[2][64]; + uint8_t chroma_qp_table[2][64]; + int chroma_qp_diff = 0; +}; + +struct SPS{ + int id = -1; + ProfileIDC profile_idc = ProfileIDC::kInvalid; + LevelIDC level_idc = LevelIDC::kInvalid; + ChromaFormatIDC chroma_format_idc = ChromaFormatIDC::k420; + int transform_bypass = 0; + int log2_max_frame_num = 0; + int maxFrameNum = 0; + POCType poc_type = POCType::kType0; + int log2_max_poc_lsb = 0; + bool delta_pic_order_always_zero_flag = false; + int offset_for_non_ref_pic = 0; + int offset_for_top_to_bottom_field = 0; + int poc_cycle_length = 0; + int ref_frame_count = 0; + bool gaps_in_frame_num_allowed_flag = false; + int mb_width = 0; + int mb_height = 0; + bool frame_mbs_only_flag = false; + int mb_aff = 0; + bool direct_8x8_inference_flag = false; + int crop = 0; + unsigned int crop_left; + unsigned int crop_right; + unsigned int crop_top; + unsigned int crop_bottom; + bool vui_parameters_present_flag = false; + // AVRational sar; + int video_signal_type_present_flag = 0; + int full_range = 0; + int colour_description_present_flag = 0; + // enum AVColorPrimaries color_primaries; + // enum AVColorTransferCharacteristic color_trc; + // enum AVColorSpace colorspace; + int color_primaries = 0; + int color_trc = 0; + int colorspace = 0; + int timing_info_present_flag = 0; + uint32_t num_units_in_tick = 0; + uint32_t time_scale = 0; + int fixed_frame_rate_flag = 0; + short offset_for_ref_frame[256]; + int bitstream_restriction_flag = 0; + int num_reorder_frames = 0; + int scaling_matrix_present = 0; + uint8_t scaling_matrix4[6][16]; + uint8_t scaling_matrix8[2][64]; + int nal_hrd_parameters_present_flag = 0; + int vcl_hrd_parameters_present_flag = 0; + int pic_struct_present_flag = 0; + int time_offset_length = 0; + int cpb_cnt = 0; + int initial_cpb_removal_delay_length = 0; + int cpb_removal_delay_length = 0; + int dpb_output_delay_length = 0; + int bit_depth_luma = 0; + int bit_depth_chroma = 0; + int residual_color_transform_flag = 0; +}; + +enum class NALSliceType { + kPType, + kBType, + kIType, + kSPType, + kSIType +}; + +/** + * H264 Network Abstraction Layer Unit types. + */ +enum class NALType : int { + UNSPECIFIED_0 = 0, + CODED_SLICE_NON_IDR = 1, + CODED_SLICE_PART_A = 2, + CODED_SLICE_PART_B = 3, + CODED_SLICE_PART_C = 4, + CODED_SLICE_IDR = 5, + SEI = 6, + SPS = 7, + PPS = 8, + ACCESS_DELIMITER = 9, + EO_SEQ = 10, + EO_STREAM = 11, + FILTER_DATA = 12, + SPS_EXT = 13, + PREFIX_NAL_UNIT = 14, + SUBSET_SPS = 15, + RESERVED_16 = 16, + RESERVED_17 = 17, + RESERVED_18 = 18, + CODED_SLICE_AUX = 19, + CODED_SLICE_EXT = 20, + CODED_SLICE_DEPTH = 21, + RESERVED_22 = 22, + RESERVED_23 = 23, + UNSPECIFIED_24 = 24, + UNSPECIFIED_25, + UNSPECIFIED_26, + UNSPECIFIED_27, + UNSPECIFIED_28, + UNSPECIFIED_29, + UNSPECIFIED_30, + UNSPECIFIED_31 +}; + +struct Slice { + NALType type; + int ref_idc = 0; + int frame_number = 9; + bool fieldPicFlag = false; + bool usedForShortTermRef = false; + bool bottomFieldFlag = false; + int idr_pic_id = 0; + int pic_order_cnt_lsb = 0; + int delta_pic_order_cnt_bottom = 0; + int delta_pic_order_cnt[2]; + int redundant_pic_cnt = 0; + bool num_ref_idx_active_override_flag = false; + int num_ref_idx_10_active_minus1 = 0; + bool ref_pic_list_reordering_flag_10 = false; + bool no_output_of_prior_pics_flag = false; + bool long_term_reference_flag = false; + bool adaptive_ref_pic_marking_mode_flag = false; + int prevRefFrameNum = 0; + int picNum = 0; + size_t offset; + size_t size; + bool keyFrame = false; + NALSliceType slice_type; + int repeat_pic; + int pictureStructure; + const PPS *pps; + const SPS *sps; + std::vector<int> refPicList; +}; + +std::string prettySlice(const Slice &s); +std::string prettyPPS(const PPS &pps); +std::string prettySPS(const SPS &sps); + +class Parser { + public: + Parser(); + ~Parser(); + + std::list<Slice> parse(const std::vector<uint8_t> &data); + + private: + PPS pps_; + SPS sps_; + int prevRefFrame_ = 0; + + void _parsePPS(ftl::codec::detail::ParseContext *ctx, size_t length); + void _parseSPS(ftl::codec::detail::ParseContext *ctx, size_t length); + bool _skipToNAL(ftl::codec::detail::ParseContext *ctx); + Slice _createSlice(ftl::codec::detail::ParseContext *ctx, const NALHeader &header, size_t length); +}; + +inline NALType extractNALType(ftl::codec::detail::ParseContext *ctx) { + auto t = static_cast<NALType>(ctx->ptr[ctx->index >> 3] & 0x1F); + ctx->index += 8; + return t; +} + +/** + * Extract the NAL unit type from the first NAL header. + * With NvPipe, the 5th byte contains the NAL Unit header. + */ +inline NALType getNALType(const unsigned char *data, size_t size) { + return (size > 4) ? static_cast<NALType>(data[4] & 0x1F) : NALType::UNSPECIFIED_0; +} + +inline bool validNAL(const unsigned char *data, size_t size) { + return size > 4 && data[0] == 0 && data[1] == 0 && data[2] == 0 && data[3] == 1; +} + +/** + * Check the H264 bitstream for an I-Frame. With NvPipe, all I-Frames start + * with a SPS NAL unit so just check for this. + */ +inline bool isIFrame(const unsigned char *data, size_t size) { + return getNALType(data, size) == NALType::SPS; +} + +} // namespace h264 +} // namespace codec +} // namespace ftl diff --git a/include/ftl/protocol/packet.hpp b/include/ftl/protocol/packet.hpp index 1e375d48d2c17859e9a52bdefab93888cc96e729..485ed85338a3505432daaa4b49d1ff75e6a3f2e0 100644 --- a/include/ftl/protocol/packet.hpp +++ b/include/ftl/protocol/packet.hpp @@ -19,6 +19,7 @@ namespace protocol { static constexpr uint8_t kFlagRequest = 0x01; ///< Used for empty data packets to mark a request for data static constexpr uint8_t kFlagCompleted = 0x02; ///< Last packet for timestamp static constexpr uint8_t kFlagReset = 0x04; ///< Request full data, including key frames. +static constexpr uint8_t kFlagFull = 0x04; ///< If set on EndFrame packet then that frame contained full data static constexpr uint8_t kAllFrames = 255; static constexpr uint8_t kAllFramesets = 255; diff --git a/src/codecs/golomb.cpp b/src/codecs/golomb.cpp new file mode 100644 index 0000000000000000000000000000000000000000..fe796c20b2439f71aa91958dbe39168d41e1dc57 --- /dev/null +++ b/src/codecs/golomb.cpp @@ -0,0 +1,58 @@ +#include <ftl/codec/golomb.hpp> + +const uint8_t ftl::codec::detail::golomb_len[512]={ + 14,13,12,12,11,11,11,11,10,10,10,10,10,10,10,10,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9, + 7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7, + 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5, + 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5, + 3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3, + 3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3, + 3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3, + 3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3, + 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, + 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, + 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, + 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, + 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, + 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, + 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, + 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1 + }; + +const uint8_t ftl::codec::detail::golomb_ue_code[512]={ + 31,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30, + 7, 7, 7, 7, 8, 8, 8, 8, 9, 9, 9, 9,10,10,10,10,11,11,11,11,12,12,12,12,13,13,13,13,14,14,14,14, + 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, + 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 + }; + +const int8_t ftl::codec::detail::golomb_se_code[512]={ + 16, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 8, -8, 9, -9, 10,-10, 11,-11, 12,-12, 13,-13, 14,-14, 15,-15, + 4, 4, 4, 4, -4, -4, -4, -4, 5, 5, 5, 5, -5, -5, -5, -5, 6, 6, 6, 6, -6, -6, -6, -6, 7, 7, 7, 7, -7, -7, -7, -7, + 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, + 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + }; diff --git a/src/codecs/h264.cpp b/src/codecs/h264.cpp new file mode 100644 index 0000000000000000000000000000000000000000..496863e010b0349f79f5cd55ebf1355fcb3aa729 --- /dev/null +++ b/src/codecs/h264.cpp @@ -0,0 +1,420 @@ +/** + * @file h264.cpp + * @copyright Copyright (c) 2022 University of Turku, MIT License + * @author Nicolas Pope + */ + +#include <sstream> +#include <ftl/codec/h264.hpp> +#include <ftl/exception.hpp> +#include <loguru.hpp> + +using ftl::codec::detail::ParseContext; +using ftl::codec::h264::PPS; +using ftl::codec::h264::SPS; +using ftl::codec::h264::Slice; +using ftl::codec::h264::NALType; +using ftl::codec::h264::NALHeader; +using ftl::codec::h264::NALSliceType; +using ftl::codec::h264::ProfileIDC; +using ftl::codec::h264::POCType; +using ftl::codec::h264::LevelIDC; +using ftl::codec::h264::ChromaFormatIDC; +using ftl::codec::detail::golombUnsigned; +using ftl::codec::detail::golombSigned; +using ftl::codec::detail::getBits1; + +static NALHeader extractNALHeader(ParseContext *ctx) { + auto t = *reinterpret_cast<const NALHeader*>(&ctx->ptr[ctx->index >> 3]); + ctx->index += 8; + return t; +} + +bool ftl::codec::h264::Parser::_skipToNAL(ParseContext *ctx) { + uint32_t code = 0xFFFFFFFF; + + while (ctx->index < ctx->length && code != 1) { + code = (code << 8) | ctx->ptr[ctx->index >> 3]; + ctx->index += 8; + } + + return (code == 1); +} + +static void decodeScalingList(ParseContext *ctx, uint8_t *factors, int size) { + if (!getBits1(ctx)) { + // TODO(Nick): Fallback + } else { + int next = 8; + int last = 8; + + for (int i = 0; i < size; i++) { + if (next) { + // TODO(Nick): Actually save the result... + next = ((last + golombSigned(ctx)) & 0xff); + } + if (!i && !next) { + // TODO(Nick): Fallback + break; + } + last = next ? next : last; + } + } +} + +ftl::codec::h264::Parser::Parser() {} + +ftl::codec::h264::Parser::~Parser() {} + +void ftl::codec::h264::Parser::_parseSPS(ParseContext *ctx, size_t length) { + int profile_idc = getBits(ctx, 8); + getBits1(ctx); + getBits1(ctx); + getBits1(ctx); + getBits1(ctx); + getBits(ctx, 4); + + int level_idc = getBits(ctx, 8); + unsigned int sps_id = golombUnsigned31(ctx); + sps_.id = sps_id; + + LOG(INFO) << "Parse SPS " << sps_.id; + + sps_.profile_idc = static_cast<ProfileIDC>(profile_idc); + sps_.level_idc = static_cast<LevelIDC>(level_idc); + + // memset scaling matrix 4 and 8 to 16 + sps_.scaling_matrix_present = 0; + + if (static_cast<int>(sps_.profile_idc) >= 100) { // high profile + sps_.chroma_format_idc = static_cast<ChromaFormatIDC>(golombUnsigned31(ctx)); + if (static_cast<int>(sps_.chroma_format_idc) > 3) { + throw FTL_Error("Invalid chroma format"); + } + if (sps_.chroma_format_idc == ChromaFormatIDC::k444) { + sps_.residual_color_transform_flag = getBits1(ctx); + } + sps_.bit_depth_luma = golombUnsigned(ctx) + 8; + sps_.bit_depth_chroma = golombUnsigned(ctx) + 8; + sps_.transform_bypass = getBits1(ctx); + // scaling matrices? + if (getBits1(ctx)) { + sps_.scaling_matrix_present = 1; + decodeScalingList(ctx, nullptr, 16); + decodeScalingList(ctx, nullptr, 16); + decodeScalingList(ctx, nullptr, 16); + decodeScalingList(ctx, nullptr, 16); + decodeScalingList(ctx, nullptr, 16); + decodeScalingList(ctx, nullptr, 16); + + decodeScalingList(ctx, nullptr, 16); + decodeScalingList(ctx, nullptr, 16); + } + } else { + sps_.chroma_format_idc = ChromaFormatIDC::k420; + sps_.bit_depth_luma = 8; + sps_.bit_depth_chroma = 8; + } + + sps_.log2_max_frame_num = golombUnsigned(ctx) + 4; + sps_.maxFrameNum = 1 << sps_.log2_max_frame_num; + sps_.poc_type = static_cast<POCType>(golombUnsigned31(ctx)); + if (sps_.poc_type == POCType::kType0) { + sps_.log2_max_poc_lsb = golombUnsigned(ctx) + 4; + } else if (sps_.poc_type == POCType::kType1) { + sps_.delta_pic_order_always_zero_flag = getBits1(ctx); + sps_.offset_for_non_ref_pic = golombSigned(ctx); + sps_.offset_for_top_to_bottom_field = golombSigned(ctx); + sps_.poc_cycle_length = golombUnsigned(ctx); + + for (int i = 0; i < sps_.poc_cycle_length; i++) { + sps_.offset_for_ref_frame[i] = golombSigned(ctx); + } + } else { + // fail + } + + sps_.ref_frame_count = golombUnsigned31(ctx); + sps_.gaps_in_frame_num_allowed_flag = getBits1(ctx); + sps_.mb_width = golombUnsigned(ctx) + 1; + sps_.mb_height = golombUnsigned(ctx) + 1; + sps_.frame_mbs_only_flag = getBits1(ctx); + if (!sps_.frame_mbs_only_flag) { + sps_.mb_aff = getBits1(ctx); + } else { + sps_.mb_aff = 0; + } + + sps_.direct_8x8_inference_flag = getBits1(ctx); + sps_.crop = getBits1(ctx); + if (sps_.crop) { + sps_.crop_left = golombUnsigned(ctx); + sps_.crop_right = golombUnsigned(ctx); + sps_.crop_top = golombUnsigned(ctx); + sps_.crop_bottom = golombUnsigned(ctx); + } else { + sps_.crop_left = 0; + sps_.crop_right = 0; + sps_.crop_top = 0; + sps_.crop_bottom = 0; + } + + sps_.vui_parameters_present_flag = getBits1(ctx); + if (sps_.vui_parameters_present_flag) { + // TODO(Nick): decode VUI + } +} + +void ftl::codec::h264::Parser::_parsePPS(ParseContext *ctx, size_t length) { + pps_.id = golombUnsigned(ctx); + pps_.sps_id = golombUnsigned31(ctx); + + LOG(INFO) << "Parse PPS " << pps_.id << ", " << pps_.sps_id; + + pps_.cabac = getBits1(ctx); + pps_.pic_order_present = getBits1(ctx); + pps_.slice_group_count = golombUnsigned(ctx); + if (pps_.slice_group_count > 1) { + pps_.mb_slice_group_map_type = golombUnsigned(ctx); + } + pps_.ref_count[0] = golombUnsigned(ctx) + 1; + pps_.ref_count[1] = golombUnsigned(ctx) + 1; + pps_.weighted_pred = getBits1(ctx); + pps_.weighted_bipred_idc = getBits(ctx, 2); + pps_.init_qp = golombSigned(ctx) + 26; + pps_.init_qs = golombSigned(ctx) + 26; + pps_.chroma_qp_index_offset[0] = golombSigned(ctx); + pps_.deblocking_filter_parameters_present = getBits1(ctx); + pps_.constrained_intra_pred = getBits1(ctx); + pps_.redundant_pic_cnt_present = getBits1(ctx); + pps_.transform_8x8_mode = 0; + + // Copy scaling matrix 4 and 8 from SPS + + if (ctx->index < length) { + // Read some other stuff + } else { + pps_.chroma_qp_index_offset[1] = pps_.chroma_qp_index_offset[0]; + } +} + +Slice ftl::codec::h264::Parser::_createSlice(ParseContext *ctx, const NALHeader &header, size_t length) { + Slice s; + s.type = static_cast<NALType>(header.type); + s.ref_idc = header.ref_idc; + + golombUnsigned(ctx); // skip first_mb_in_slice + s.slice_type = static_cast<NALSliceType>(golombUnsigned31(ctx)); + if (s.type == NALType::CODED_SLICE_IDR) { + s.keyFrame = true; + } else { + s.keyFrame = false; + } + int ppsId = golombUnsigned(ctx); + if (pps_.id != ppsId) { + throw FTL_Error("Unknown PPS"); + } + if (sps_.id != pps_.sps_id) { + throw FTL_Error("Unknown SPS: " << sps_.id << " " << pps_.sps_id); + } + s.pps = &pps_; + s.sps = &sps_; + s.frame_number = getBits(ctx, s.sps->log2_max_frame_num); + + if (!s.sps->frame_mbs_only_flag) { + s.fieldPicFlag = getBits1(ctx); + if (s.fieldPicFlag) { + s.bottomFieldFlag = getBits1(ctx); + } + } + if (s.type == NALType::CODED_SLICE_IDR) { + s.idr_pic_id = golombUnsigned(ctx); + s.prevRefFrameNum = 0; + prevRefFrame_ = s.frame_number; + } else { + s.prevRefFrameNum = prevRefFrame_; + if (s.ref_idc > 0) { + prevRefFrame_ = s.frame_number; + } + } + + if (s.sps->poc_type == POCType::kType0) { + s.pic_order_cnt_lsb = getBits(ctx, s.sps->log2_max_poc_lsb); + if (s.pps->pic_order_present && !s.fieldPicFlag) { + s.delta_pic_order_cnt_bottom = golombSigned(ctx); + } + } + if (s.sps->poc_type == POCType::kType1 && !s.sps->delta_pic_order_always_zero_flag) { + s.delta_pic_order_cnt[0] = golombSigned(ctx); + if (s.pps->pic_order_present && !s.fieldPicFlag) { + s.delta_pic_order_cnt[1] = golombSigned(ctx); + } + } + + if (s.pps->redundant_pic_cnt_present) { + s.redundant_pic_cnt = golombUnsigned(ctx); + } + + if (s.slice_type == NALSliceType::kPType || s.slice_type == NALSliceType::kSPType) { + s.num_ref_idx_active_override_flag = getBits1(ctx); + if (s.num_ref_idx_active_override_flag) { + s.num_ref_idx_10_active_minus1 = golombUnsigned(ctx); + } + } + + if (s.slice_type != NALSliceType::kIType && s.slice_type != NALSliceType::kSIType) { + s.ref_pic_list_reordering_flag_10 = getBits1(ctx); + if (s.ref_pic_list_reordering_flag_10) { + LOG(ERROR) << "Need to parse pic list"; + } + } + + if (s.pps->weighted_pred) { + LOG(ERROR) << "Need to parse weight table"; + } + + if (s.ref_idc != 0) { + if (s.type == NALType::CODED_SLICE_IDR) { + s.no_output_of_prior_pics_flag = getBits1(ctx); + s.long_term_reference_flag = getBits1(ctx); + s.usedForShortTermRef = !s.long_term_reference_flag; + } else { + s.usedForShortTermRef = true; + s.adaptive_ref_pic_marking_mode_flag = getBits1(ctx); + if (s.adaptive_ref_pic_marking_mode_flag) { + LOG(ERROR) << "Parse adaptive ref"; + } + } + } + + s.picNum = s.frame_number % s.sps->maxFrameNum; + + if (s.type != NALType::CODED_SLICE_IDR) { + int numRefFrames = (s.num_ref_idx_active_override_flag) + ? s.num_ref_idx_10_active_minus1 + 1 + : s.sps->ref_frame_count; + s.refPicList.resize(numRefFrames); + int fn = s.frame_number - 1; + for (size_t i = 0; i < s.refPicList.size(); i++) { + s.refPicList[i] = fn--; + } + } + + return s; +} + +std::list<Slice> ftl::codec::h264::Parser::parse(const std::vector<uint8_t> &data) { + std::list<Slice> slices; + Slice slice; + size_t offset = 0; + size_t length = 0; + + ParseContext parseCtx = { + data.data(), 0, 0 + }; + parseCtx.length = data.size() * 8; + _skipToNAL(&parseCtx); + + ParseContext nextCtx = parseCtx; + + while (true) { + bool hasNext = _skipToNAL(&nextCtx); + offset = parseCtx.index; + length = nextCtx.index - parseCtx.index; + // auto type = ftl::codecs::h264::extractNALType(&parseCtx); + auto header = extractNALHeader(&parseCtx); + auto type = static_cast<NALType>(header.type); + + switch (type) { + case NALType::SPS: + _parseSPS(&parseCtx, 0); + if (parseCtx.index > nextCtx.index) { + throw FTL_Error("Bad SPS parse"); + } + break; + case NALType::PPS: + _parsePPS(&parseCtx, 0); + if (parseCtx.index > nextCtx.index) { + throw FTL_Error("Bad PPS parse"); + } + break; + case NALType::CODED_SLICE_IDR: + case NALType::CODED_SLICE_NON_IDR: + slice = _createSlice(&parseCtx, header, 0); + slice.offset = offset / 8; + slice.size = length / 8; + slices.push_back(slice); + break; + default: + LOG(ERROR) << "Unrecognised NAL type: " << int(header.type); + } + + parseCtx = nextCtx; + + if (!hasNext) break; + } + + return slices; +} + +std::string ftl::codec::h264::prettySlice(const Slice &s) { + std::stringstream stream; + stream << " - Type: " << std::to_string(static_cast<int>(s.type)) << std::endl; + stream << " - size: " << std::to_string(s.size) << " bytes" << std::endl; + stream << " - offset: " << std::to_string(s.offset) << " bytes" << std::endl; + stream << " - ref_idc: " << std::to_string(s.ref_idc) << std::endl; + stream << " - frame_num: " << std::to_string(s.frame_number) << std::endl; + stream << " - field_pic_flag: " << std::to_string(s.fieldPicFlag) << std::endl; + stream << " - usedForShortRef: " << std::to_string(s.usedForShortTermRef) << std::endl; + stream << " - slice_type: " << std::to_string(static_cast<int>(s.slice_type)) << std::endl; + stream << " - bottom_field_flag: " << std::to_string(s.bottomFieldFlag) << std::endl; + stream << " - idr_pic_id: " << std::to_string(s.idr_pic_id) << std::endl; + stream << " - redundant_pic_cnt: " << std::to_string(s.redundant_pic_cnt) << std::endl; + stream << " - num_ref_idx_active_override_flag: " + << std::to_string(s.num_ref_idx_active_override_flag) << std::endl; + stream << " - num_ref_idx_10_active_minus1: " + << std::to_string(s.num_ref_idx_10_active_minus1) << std::endl; + stream << " - ref_pic_list_reordering_flag: " << std::to_string(s.ref_pic_list_reordering_flag_10) << std::endl; + stream << " - long_term_reference_flag: " << std::to_string(s.long_term_reference_flag) << std::endl; + stream << " - adaptive_ref_pic_marking_mode_flag: " + << std::to_string(s.adaptive_ref_pic_marking_mode_flag) << std::endl; + stream << " - picNum: " << std::to_string(s.picNum) << std::endl; + stream << " - refPicList (" << std::to_string(s.refPicList.size()) << "): "; + for (int r : s.refPicList) { + stream << std::to_string(r) << ", "; + } + stream << std::endl; + stream << "PPS:" << std::endl << prettyPPS(*s.pps); + stream << "SPS:" << std::endl << prettySPS(*s.sps); + return stream.str(); +} + +std::string ftl::codec::h264::prettyPPS(const PPS &pps) { + std::stringstream stream; + stream << " - id: " << std::to_string(pps.id) << std::endl; + stream << " - sps_id: " << std::to_string(pps.sps_id) << std::endl; + stream << " - pic_order_present: " << std::to_string(pps.pic_order_present) << std::endl; + stream << " - ref_count_0: " << std::to_string(pps.ref_count[0]) << std::endl; + stream << " - ref_count_1: " << std::to_string(pps.ref_count[1]) << std::endl; + stream << " - weighted_pred: " << std::to_string(pps.weighted_pred) << std::endl; + stream << " - init_qp: " << std::to_string(pps.init_qp) << std::endl; + stream << " - init_qs: " << std::to_string(pps.init_qs) << std::endl; + return stream.str(); +} + +std::string ftl::codec::h264::prettySPS(const SPS &sps) { + std::stringstream stream; + stream << " - id: " << std::to_string(sps.id) << std::endl; + stream << " - profile_idc: " << std::to_string(static_cast<int>(sps.profile_idc)) << std::endl; + stream << " - level_idc: " << std::to_string(static_cast<int>(sps.level_idc)) << std::endl; + stream << " - chroma_format_idc: " << std::to_string(static_cast<int>(sps.chroma_format_idc)) << std::endl; + stream << " - transform_bypass: " << std::to_string(sps.transform_bypass) << std::endl; + stream << " - maxFrameNum: " << std::to_string(sps.maxFrameNum) << std::endl; + stream << " - poc_type: " << std::to_string(static_cast<int>(sps.poc_type)) << std::endl; + stream << " - offset_for_non_ref_pic: " << std::to_string(sps.offset_for_non_ref_pic) << std::endl; + stream << " - ref_frame_count: " << std::to_string(sps.ref_frame_count) << std::endl; + stream << " - gaps_in_frame_num_allowed_flag: " << std::to_string(sps.gaps_in_frame_num_allowed_flag) << std::endl; + stream << " - width: " << std::to_string(sps.mb_width * 16) << std::endl; + stream << " - height: " << std::to_string(sps.mb_height * 16) << std::endl; + return stream.str(); +} diff --git a/src/uri.cpp b/src/uri.cpp index 2f43927e7507aafa858ef31ae72e321f2c713133..a0310b8dbeb321101b7846be3d2e612f613d97d2 100644 --- a/src/uri.cpp +++ b/src/uri.cpp @@ -170,8 +170,6 @@ void URI::_parse(uri_t puri) { uriFreeQueryListA(queryList); } - uriFreeUriMembersA(&uri); - auto fraglast = (uri.query.first != NULL) ? uri.query.first : uri.fragment.afterLast; if (uri.fragment.first != NULL && fraglast - uri.fragment.first > 0) { m_frag = std::string(uri.fragment.first, fraglast - uri.fragment.first); @@ -198,6 +196,8 @@ void URI::_parse(uri_t puri) { m_base += std::string(""); } } + + uriFreeUriMembersA(&uri); } }