From 772cf4ac68ce150be36f848039e185dd81f17497 Mon Sep 17 00:00:00 2001
From: Sebastian Hahta <joseha@utu.fi>
Date: Tue, 22 Oct 2019 18:32:29 +0300
Subject: [PATCH] Python module for reading .ftl files
---
python/README.md | 12 ++
python/ftl/__init__.py | 1 +
python/ftl/ftlstream.py | 246 ++++++++++++++++++++++++++++++++++++++++
python/ftl/libde265.py | 203 +++++++++++++++++++++++++++++++++
4 files changed, 462 insertions(+)
create mode 100644 python/README.md
create mode 100644 python/ftl/__init__.py
create mode 100644 python/ftl/ftlstream.py
create mode 100644 python/ftl/libde265.py
diff --git a/python/README.md b/python/README.md
new file mode 100644
index 000000000..d4b687fca
--- /dev/null
+++ b/python/README.md
@@ -0,0 +1,12 @@
+Python support for `.ftl` files. At the moment, only reading RGB channels
+(left/right) supported.
+
+Required **Python** modules:
+
+ * msgpack
+ * numpy
+ * skimage **or** OpenCV
+
+Required libraries
+
+ * libde265 (available on most Linux distributions)
diff --git a/python/ftl/__init__.py b/python/ftl/__init__.py
new file mode 100644
index 000000000..963e374bd
--- /dev/null
+++ b/python/ftl/__init__.py
@@ -0,0 +1 @@
+from . ftlstream import FTLStream
\ No newline at end of file
diff --git a/python/ftl/ftlstream.py b/python/ftl/ftlstream.py
new file mode 100644
index 000000000..7ab436e00
--- /dev/null
+++ b/python/ftl/ftlstream.py
@@ -0,0 +1,246 @@
+import msgpack
+
+import numpy as np
+
+from enum import IntEnum
+from collections import namedtuple
+from . libde265 import Decoder
+
+try:
+ import cv2 as cv
+
+ def _ycrcb2rgb(img):
+ return cv.cvtColor(img, cv.COLOR_YCrCb2RGB)
+
+except ImportError:
+ def _ycrcb2rgb(img):
+ ''' YCrCb to RGB, based on OpenCV documentation definition.
+
+ Note: It seems this implementation is not perfectly equivalent to OpenCV's
+ '''
+
+ rgb = np.zeros(img.shape, np.float)
+
+ Y = img[:,:,0].astype(np.float)
+ Cr = img[:,:,1].astype(np.float)
+ Cb = img[:,:,2].astype(np.float)
+ delta = 128.0
+
+ rgb[:,:,0] = Y + 1.403 * (Cr - delta)
+ rgb[:,:,1] = Y - 0.714 * (Cr - delta) - 0.344 * (Cb - delta)
+ rgb[:,:,2] = Y + 1.773 * (Cb - delta)
+
+ return rgb.round().astype(np.uint8)
+
+# FTL definitions
+
+_packet = namedtuple("Packet", ["codec", "definition", "block_total", "block_number", "flags", "data"])
+_stream_packet = namedtuple("StreamPacket", ["timestamp", "streamID", "chanel_count", "channel"])
+
+_definition_t = {
+ 0 : (),
+ 1 : (),
+ 2 : (1080, 1920),
+ 3 : (720, 1280),
+ 4 : (),
+ 5 : (),
+ 6 : (),
+ 7 : (),
+ 8 : ()
+}
+
+class NALType(IntEnum):
+ CODED_SLICE_TRAIL_N = 0
+ CODED_SLICE_TRAIL_R = 1
+
+ CODED_SLICE_TSA_N = 2
+ CODED_SLICE_TSA_R = 3
+
+ CODED_SLICE_STSA_N = 4
+ CODED_SLICE_STSA_R = 5
+
+ CODED_SLICE_RADL_N = 6
+ CODED_SLICE_RADL_R = 7
+
+ CODED_SLICE_RASL_N = 8
+ CODED_SLICE_RASL_R = 9
+
+ RESERVED_VCL_N10 = 10
+ RESERVED_VCL_R11 = 11
+ RESERVED_VCL_N12 = 12
+ RESERVED_VCL_R13 = 13
+ RESERVED_VCL_N14 = 14
+ RESERVED_VCL_R15 = 15
+
+ CODED_SLICE_BLA_W_LP = 16
+ CODED_SLICE_BLA_W_RADL = 17
+ CODED_SLICE_BLA_N_LP = 18
+ CODED_SLICE_IDR_W_RADL = 19
+ CODED_SLICE_IDR_N_LP = 20
+ CODED_SLICE_CRA = 21
+ RESERVED_IRAP_VCL22 = 22
+ RESERVED_IRAP_VCL23 = 23
+
+ RESERVED_VCL24 = 24
+ RESERVED_VCL25 = 25
+ RESERVED_VCL26 = 26
+ RESERVED_VCL27 = 27
+ RESERVED_VCL28 = 28
+ RESERVED_VCL29 = 29
+ RESERVED_VCL30 = 30
+ RESERVED_VCL31 = 31
+
+ VPS = 32
+ SPS = 33
+ PPS = 34
+ ACCESS_UNIT_DELIMITER = 35
+ EOS = 36
+ EOB = 37
+ FILLER_DATA = 38
+ PREFIX_SEI = 39
+ SUFFIX_SEI = 40
+
+ RESERVED_NVCL41 = 41
+ RESERVED_NVCL42 = 42
+ RESERVED_NVCL43 = 43
+ RESERVED_NVCL44 = 44
+ RESERVED_NVCL45 = 45
+ RESERVED_NVCL46 = 46
+ RESERVED_NVCL47 = 47
+ UNSPECIFIED_48 = 48
+ UNSPECIFIED_49 = 49
+ UNSPECIFIED_50 = 50
+ UNSPECIFIED_51 = 51
+ UNSPECIFIED_52 = 52
+ UNSPECIFIED_53 = 53
+ UNSPECIFIED_54 = 54
+ UNSPECIFIED_55 = 55
+ UNSPECIFIED_56 = 56
+ UNSPECIFIED_57 = 57
+ UNSPECIFIED_58 = 58
+ UNSPECIFIED_59 = 59
+ UNSPECIFIED_60 = 60
+ UNSPECIFIED_61 = 61
+ UNSPECIFIED_62 = 62
+ UNSPECIFIED_63 = 63
+ INVALID = 64
+
+def get_NAL_type(data):
+ if not isinstance(data, bytes):
+ raise ValueError("expected bytes")
+
+ return NALType((data[4] >> 1) & 0x3f)
+
+class FTLStream:
+ def __init__(self, file):
+ self._file = open(file, "br")
+ self._decoders = {}
+ self._frames = {}
+
+ try:
+ magic = self._file.read(5)
+ if magic[:4] != bytearray(ord(c) for c in "FTLF"):
+ raise Exception("wrong magic")
+
+ self._unpacker = msgpack.Unpacker(self._file, raw=True, use_list=False)
+
+ except Exception as ex:
+ self._file.close()
+ raise ex
+
+ self._packets_read = 0
+
+ def __del__(self):
+ self._file.close()
+
+ def _read_next(self):
+ v1, v2 = self._unpacker.unpack()
+ return _stream_packet._make(v1), _packet._make(v2)
+
+ def _update_calib(self, sp, p):
+ ''' Update calibration '''
+ pass
+
+ def _update_pose(self, sp, p):
+ ''' Update pose '''
+ pass
+
+ def _decode_frame_hevc(self, sp, p):
+ ''' Decode HEVC frame '''
+
+ k = (sp.streamID, sp.channel)
+
+ if k not in self._decoders:
+ self._decoders[k] = Decoder(_definition_t[p.definition])
+
+ decoder = self._decoders[k]
+
+ decoder.push_data(p.data)
+ decoder.decode()
+
+ img = decoder.get_next_picture()
+
+ if img is not None:
+ self._frames[k] = _ycrcb2rgb(img)
+
+ def _flush_decoders(self):
+ for decoder in self._decoders.values():
+ decoder.flush_data()
+
+ def read(self):
+ '''
+ Reads data for until the next timestamp. Returns False if there is no
+ more data to read, otherwise returns True.
+ '''
+ if self._packets_read == 0:
+ self._sp, self._p = self._read_next()
+ self._packets_read += 1
+
+ self._frames = {}
+
+ ts = self._sp.timestamp
+ ex = None
+
+ while self._sp.timestamp == ts:
+ try:
+ if self._p.codec == 100: # JSON
+ NotImplementedError("json decoding not implemented")
+
+ elif self._p.codec == 101: # CALIBRATION
+ self._update_calib(self._sp, self._p)
+
+ elif self._p.codec == 102: # POSE
+ self._update_pose(self._sp, self._p)
+
+ elif self._p.codec == 3: # HEVC
+ self._decode_frame_hevc(self._sp, self._p)
+
+ else:
+ raise ValueError("unkowno codec %i" % p.codec)
+
+ except Exception as e:
+ # TODO: Multiple exceptions possible. Re-design read()?
+ ex = e
+
+ try:
+ self._sp, self._p = self._read_next()
+ self._packets_read += 1
+
+ except msgpack.OutOfData:
+ return False
+
+ if ex is not None:
+ raise ex
+
+ return True
+
+ def get_frames(self):
+ ''' Returns all frames '''
+ return self._frames
+
+ def get_frame(self, source, channel):
+ k = (source, channel)
+ if k in self._frames:
+ return self._frames[k]
+ else:
+ return None
diff --git a/python/ftl/libde265.py b/python/ftl/libde265.py
new file mode 100644
index 000000000..74a1dcedb
--- /dev/null
+++ b/python/ftl/libde265.py
@@ -0,0 +1,203 @@
+'''
+Python wrapper for libde265. Only decoding is (partly) implemented.
+
+Requirements:
+ * libde265 library (libde265.so.0)
+ * numpy
+ * opencv or skimage
+
+'''
+
+try:
+ import cv2 as cv
+ def _resize(img, size):
+ return cv.resize(img, dsize=tuple(reversed(size)), interpolation=cv.INTER_CUBIC)
+
+except ImportError:
+ from skimage.transform import resize as resize_skimage
+ def _resize(img, size):
+ # skimage resize() return dtype float64, convert back to uint8
+ # order: 0 nn, 1 bilinear, 3 bicubic
+ return (resize_skimage(img, size, order=3, mode="constant", cval=0) * 255).astype(np.uint8)
+
+import ctypes
+from enum import IntEnum
+
+import numpy as np
+
+# error codes copied from header (de265.h)
+
+class libde265error(IntEnum):
+ DE265_OK = 0
+ DE265_ERROR_NO_SUCH_FILE=1
+ DE265_ERROR_COEFFICIENT_OUT_OF_IMAGE_BOUNDS=4
+ DE265_ERROR_CHECKSUM_MISMATCH=5
+ DE265_ERROR_CTB_OUTSIDE_IMAGE_AREA=6
+ DE265_ERROR_OUT_OF_MEMORY=7
+ DE265_ERROR_CODED_PARAMETER_OUT_OF_RANGE=8
+ DE265_ERROR_IMAGE_BUFFER_FULL=9
+ DE265_ERROR_CANNOT_START_THREADPOOL=10
+ DE265_ERROR_LIBRARY_INITIALIZATION_FAILED=11
+ DE265_ERROR_LIBRARY_NOT_INITIALIZED=12
+ DE265_ERROR_WAITING_FOR_INPUT_DATA=13
+ DE265_ERROR_CANNOT_PROCESS_SEI=14
+ DE265_ERROR_PARAMETER_PARSING=15
+ DE265_ERROR_NO_INITIAL_SLICE_HEADER=16
+ DE265_ERROR_PREMATURE_END_OF_SLICE=17
+ DE265_ERROR_UNSPECIFIED_DECODING_ERROR=18
+ DE265_ERROR_NOT_IMPLEMENTED_YET = 502
+ DE265_WARNING_NO_WPP_CANNOT_USE_MULTITHREADING = 1000
+ DE265_WARNING_WARNING_BUFFER_FULL=1001
+ DE265_WARNING_PREMATURE_END_OF_SLICE_SEGMENT=1002
+ DE265_WARNING_INCORRECT_ENTRY_POINT_OFFSET=1003
+ DE265_WARNING_CTB_OUTSIDE_IMAGE_AREA=1004
+ DE265_WARNING_SPS_HEADER_INVALID=1005
+ DE265_WARNING_PPS_HEADER_INVALID=1006
+ DE265_WARNING_SLICEHEADER_INVALID=1007
+ DE265_WARNING_INCORRECT_MOTION_VECTOR_SCALING=1008
+ DE265_WARNING_NONEXISTING_PPS_REFERENCED=1009
+ DE265_WARNING_NONEXISTING_SPS_REFERENCED=1010
+ DE265_WARNING_BOTH_PREDFLAGS_ZERO=1011
+ DE265_WARNING_NONEXISTING_REFERENCE_PICTURE_ACCESSED=1012
+ DE265_WARNING_NUMMVP_NOT_EQUAL_TO_NUMMVQ=1013
+ DE265_WARNING_NUMBER_OF_SHORT_TERM_REF_PIC_SETS_OUT_OF_RANGE=1014
+ DE265_WARNING_SHORT_TERM_REF_PIC_SET_OUT_OF_RANGE=1015
+ DE265_WARNING_FAULTY_REFERENCE_PICTURE_LIST=1016
+ DE265_WARNING_EOSS_BIT_NOT_SET=1017
+ DE265_WARNING_MAX_NUM_REF_PICS_EXCEEDED=1018
+ DE265_WARNING_INVALID_CHROMA_FORMAT=1019
+ DE265_WARNING_SLICE_SEGMENT_ADDRESS_INVALID=1020
+ DE265_WARNING_DEPENDENT_SLICE_WITH_ADDRESS_ZERO=1021
+ DE265_WARNING_NUMBER_OF_THREADS_LIMITED_TO_MAXIMUM=1022
+ DE265_NON_EXISTING_LT_REFERENCE_CANDIDATE_IN_SLICE_HEADER=1023
+ DE265_WARNING_CANNOT_APPLY_SAO_OUT_OF_MEMORY=1024
+ DE265_WARNING_SPS_MISSING_CANNOT_DECODE_SEI=1025
+ DE265_WARNING_COLLOCATED_MOTION_VECTOR_OUTSIDE_IMAGE_AREA=1026
+
+libde265 = ctypes.cdll.LoadLibrary("libde265.so.0")
+
+libde265.de265_get_error_text.argtypes = [ctypes.c_void_p]
+libde265.de265_get_error_text.restype = ctypes.c_char_p
+libde265.de265_get_version_number_major.restype = ctypes.c_uint32
+libde265.de265_get_version_number_minor.restype = ctypes.c_uint32
+
+libde265.de265_new_decoder.restype = ctypes.c_void_p
+
+libde265.de265_free_decoder.argtypes = [ctypes.c_void_p]
+libde265.de265_free_decoder.restype = ctypes.c_int
+
+libde265.de265_start_worker_threads.argtypes = [ctypes.c_void_p, ctypes.c_int]
+libde265.de265_start_worker_threads.restype = ctypes.c_int
+
+libde265.de265_push_data.argtypes = [ctypes.c_void_p, ctypes.c_void_p, ctypes.c_int, ctypes.c_void_p, ctypes.c_void_p]
+libde265.de265_push_data.restype = ctypes.c_int
+
+libde265.de265_push_NAL.argtypes = [ctypes.c_void_p, ctypes.c_void_p, ctypes.c_int, ctypes.c_void_p, ctypes.c_void_p]
+libde265.de265_push_data.restype = ctypes.c_int
+
+libde265.de265_push_end_of_frame.argtypes = [ctypes.c_void_p]
+
+libde265.de265_flush_data.argtypes = [ctypes.c_void_p]
+libde265.de265_flush_data.restype = ctypes.c_int
+
+libde265.de265_decode.argtypes = [ctypes.c_void_p, ctypes.POINTER(ctypes.c_int)]
+libde265.de265_decode.restype = ctypes.c_int
+
+libde265.de265_get_next_picture.argtypes = [ctypes.c_void_p]
+libde265.de265_get_next_picture.restype = ctypes.c_void_p
+
+libde265.de265_get_image_width.argtypes = [ctypes.c_void_p, ctypes.c_int]
+libde265.de265_get_image_width.restype = ctypes.c_int
+
+libde265.de265_get_image_height.argtypes = [ctypes.c_void_p, ctypes.c_int]
+libde265.de265_get_image_height.restype = ctypes.c_int
+
+libde265.de265_get_bits_per_pixel.argtypes = [ctypes.c_void_p, ctypes.c_int]
+libde265.de265_get_bits_per_pixel.restype = ctypes.c_int
+
+libde265.de265_get_image_plane.argtypes = [ctypes.c_void_p, ctypes.c_int, ctypes.POINTER(ctypes.c_int)]
+libde265.de265_get_image_plane.restype = ctypes.POINTER(ctypes.c_char)
+
+class Decoder:
+ def __init__(self, size, threads=1):
+ self._size = size
+ self._more = ctypes.c_int()
+ self._out_stride = ctypes.c_int()
+ self._ctx = libde265.de265_new_decoder()
+ err = libde265.de265_start_worker_threads(self._ctx, threads)
+ if err:
+ raise Exception(self.get_error_str(err))
+
+ def __del__(self):
+ libde265.de265_free_decoder(self._ctx)
+
+ def get_error_str(self, code):
+ return libde265.de265_get_error_text(code).decode("ascii")
+
+ def push_data(self, data):
+ if not isinstance(data, bytes):
+ raise ValueError("expected bytes")
+
+ err = libde265.de265_push_data(self._ctx, data, len(data), None, None)
+
+ if err:
+ raise Exception(self.get_error_str(err))
+
+ def push_end_of_frame(self):
+ err = libde265.de265_push_end_of_frame(self._ctx)
+
+ if err:
+ raise Exception(self.get_error_str(err))
+
+ def push_NAL(self, data):
+ if not isinstance(data, bytes):
+ raise ValueError("expected bytes")
+
+ err = libde265.de265_push_NAL(self._ctx, data, len(data), None, None)
+
+ if err:
+ raise Exception(self.get_error_str(err))
+
+ def decode(self):
+ err = libde265.de265_decode(self._ctx, self._more)
+
+ if err and err != libde265error.DE265_ERROR_WAITING_FOR_INPUT_DATA:
+ raise Exception(self.get_error_str(err))
+
+ return self._more.value != 0
+
+ def flush_data(self):
+ err = libde265.de265_flush_data(self._ctx)
+
+ if err:
+ raise Exception(self.get_error_str(err))
+
+ def get_next_picture(self):
+ '''
+ Returns next decoded frame. Image in YCbCr format. If no frame available
+ returns None.
+ '''
+ img = libde265.de265_get_next_picture(self._ctx)
+
+ if not img:
+ return None
+
+ res = np.zeros((self._size[0], self._size[1], 3), dtype=np.uint8)
+
+ for c in range(0, 3):
+ size = (libde265.de265_get_image_height(img, c),
+ libde265.de265_get_image_width(img, c))
+
+ bpp = libde265.de265_get_bits_per_pixel(img, c)
+
+ if bpp != 8:
+ raise NotImplementedError("unsupported bits per pixel %i" % bpp)
+
+ img_ptr = libde265.de265_get_image_plane(img, c, self._out_stride)
+
+ ch = np.frombuffer(img_ptr[:size[0] * size[1]], dtype=np.uint8)
+ ch.shape = size
+
+ res[:,:,c] = _resize(ch, self._size)
+
+ return res
\ No newline at end of file
--
GitLab