Merge branch 'feature/python' into 'master'

Python module for reading .ftl files

See merge request nicolas.pope/ftl!141

python/README.md

+Python support for `.ftl` files. At the moment, only reading RGB channels
+(left/right) supported.
+
+Required **Python** modules:
+
+ * msgpack 
+ * numpy
+ * skimage **or** OpenCV 
+
+Required libraries
+
+ * libde265 (available on most Linux distributions)

python/ftl/__init__.py

+from . ftlstream import FTLStream
\ No newline at end of file

python/ftl/ftlstream.py

+import msgpack
+
+import numpy as np
+
+from enum import IntEnum
+from collections import namedtuple
+from . libde265 import Decoder
+
+try:
+    import cv2 as cv
+    
+    def _ycrcb2rgb(img):
+        return cv.cvtColor(img, cv.COLOR_YCrCb2RGB)
+    
+except ImportError:
+    def _ycrcb2rgb(img):
+        ''' YCrCb to RGB, based on OpenCV documentation definition.
+        
+        Note: It seems this implementation is not perfectly equivalent to OpenCV's
+        '''
+        
+        rgb = np.zeros(img.shape, np.float)
+
+        Y = img[:,:,0].astype(np.float)
+        Cr = img[:,:,1].astype(np.float)
+        Cb = img[:,:,2].astype(np.float)
+        delta = 128.0
+
+        rgb[:,:,0] = Y + 1.403 * (Cr - delta)
+        rgb[:,:,1] = Y - 0.714 * (Cr - delta) - 0.344 * (Cb - delta)
+        rgb[:,:,2] = Y + 1.773 * (Cb - delta)
+
+        return rgb.round().astype(np.uint8)
+
+# FTL definitions
+
+_packet = namedtuple("Packet", ["codec", "definition", "block_total", "block_number", "flags", "data"])
+_stream_packet = namedtuple("StreamPacket", ["timestamp", "streamID", "chanel_count", "channel"])
+
+_definition_t = {
+    0 : (),
+    1 : (),
+    2 : (1080, 1920),
+    3 : (720, 1280),
+    4 : (),
+    5 : (),
+    6 : (),
+    7 : (),
+    8 : ()
+}
+
+class NALType(IntEnum):
+    CODED_SLICE_TRAIL_N = 0
+    CODED_SLICE_TRAIL_R = 1
+
+    CODED_SLICE_TSA_N = 2
+    CODED_SLICE_TSA_R = 3
+
+    CODED_SLICE_STSA_N = 4
+    CODED_SLICE_STSA_R = 5
+
+    CODED_SLICE_RADL_N = 6
+    CODED_SLICE_RADL_R = 7
+
+    CODED_SLICE_RASL_N = 8
+    CODED_SLICE_RASL_R = 9
+
+    RESERVED_VCL_N10 = 10
+    RESERVED_VCL_R11 = 11
+    RESERVED_VCL_N12 = 12
+    RESERVED_VCL_R13 = 13
+    RESERVED_VCL_N14 = 14
+    RESERVED_VCL_R15 = 15
+
+    CODED_SLICE_BLA_W_LP = 16
+    CODED_SLICE_BLA_W_RADL = 17
+    CODED_SLICE_BLA_N_LP = 18
+    CODED_SLICE_IDR_W_RADL = 19
+    CODED_SLICE_IDR_N_LP = 20
+    CODED_SLICE_CRA = 21
+    RESERVED_IRAP_VCL22 = 22
+    RESERVED_IRAP_VCL23 = 23
+
+    RESERVED_VCL24 = 24
+    RESERVED_VCL25 = 25
+    RESERVED_VCL26 = 26
+    RESERVED_VCL27 = 27
+    RESERVED_VCL28 = 28
+    RESERVED_VCL29 = 29
+    RESERVED_VCL30 = 30
+    RESERVED_VCL31 = 31
+
+    VPS = 32
+    SPS = 33
+    PPS = 34
+    ACCESS_UNIT_DELIMITER = 35
+    EOS = 36
+    EOB = 37
+    FILLER_DATA = 38
+    PREFIX_SEI = 39
+    SUFFIX_SEI = 40
+
+    RESERVED_NVCL41 = 41
+    RESERVED_NVCL42 = 42
+    RESERVED_NVCL43 = 43
+    RESERVED_NVCL44 = 44
+    RESERVED_NVCL45 = 45
+    RESERVED_NVCL46 = 46
+    RESERVED_NVCL47 = 47
+    UNSPECIFIED_48 = 48
+    UNSPECIFIED_49 = 49
+    UNSPECIFIED_50 = 50
+    UNSPECIFIED_51 = 51
+    UNSPECIFIED_52 = 52
+    UNSPECIFIED_53 = 53
+    UNSPECIFIED_54 = 54
+    UNSPECIFIED_55 = 55
+    UNSPECIFIED_56 = 56
+    UNSPECIFIED_57 = 57
+    UNSPECIFIED_58 = 58
+    UNSPECIFIED_59 = 59
+    UNSPECIFIED_60 = 60
+    UNSPECIFIED_61 = 61
+    UNSPECIFIED_62 = 62
+    UNSPECIFIED_63 = 63
+    INVALID = 64
+
+def get_NAL_type(data):
+    if not isinstance(data, bytes):
+        raise ValueError("expected bytes")
+    
+    return NALType((data[4] >> 1) & 0x3f)
+
+class FTLStream:
+    def __init__(self, file):
+        self._file = open(file, "br")
+        self._decoders = {}
+        self._frames = {}
+        
+        try:
+            magic = self._file.read(5)
+            if magic[:4] != bytearray(ord(c) for c in "FTLF"):
+                raise Exception("wrong magic")
+            
+            self._unpacker = msgpack.Unpacker(self._file, raw=True, use_list=False)
+            
+        except Exception as ex:
+            self._file.close()
+            raise ex
+            
+        self._packets_read = 0
+    
+    def __del__(self):
+        self._file.close()
+    
+    def _read_next(self):
+        v1, v2 = self._unpacker.unpack()
+        return _stream_packet._make(v1), _packet._make(v2)
+    
+    def _update_calib(self, sp, p):
+        ''' Update calibration '''
+        pass
+    
+    def _update_pose(self, sp, p):
+        ''' Update pose '''
+        pass
+    
+    def _decode_frame_hevc(self, sp, p):
+        ''' Decode HEVC frame '''
+        
+        k = (sp.streamID, sp.channel)
+        
+        if k not in self._decoders:
+            self._decoders[k] = Decoder(_definition_t[p.definition])
+        
+        decoder = self._decoders[k]
+        
+        decoder.push_data(p.data)
+        decoder.decode()
+        
+        img = decoder.get_next_picture()
+        
+        if img is not None:
+            self._frames[k] = _ycrcb2rgb(img)
+    
+    def _flush_decoders(self):
+        for decoder in self._decoders.values():
+            decoder.flush_data()
+    
+    def read(self):
+        '''
+        Reads data for until the next timestamp. Returns False if there is no
+        more data to read, otherwise returns True.
+        '''
+        if self._packets_read == 0:
+            self._sp, self._p = self._read_next()
+            self._packets_read += 1
+            
+        self._frames = {}
+        
+        ts = self._sp.timestamp
+        ex = None
+        
+        while self._sp.timestamp == ts:
+            try:
+                if self._p.codec == 100: # JSON
+                    NotImplementedError("json decoding not implemented")
+
+                elif self._p.codec == 101: # CALIBRATION
+                    self._update_calib(self._sp, self._p)
+
+                elif self._p.codec == 102: # POSE
+                    self._update_pose(self._sp, self._p)
+
+                elif self._p.codec == 3: # HEVC
+                    self._decode_frame_hevc(self._sp, self._p)
+
+                else:
+                    raise ValueError("unkowno codec %i" % p.codec)
+            
+            except Exception as e:
+                # TODO: Multiple exceptions possible. Re-design read()?
+                ex = e
+            
+            try:
+                self._sp, self._p = self._read_next()
+                self._packets_read += 1
+            
+            except msgpack.OutOfData:
+                return False
+            
+        if ex is not None:
+            raise ex
+            
+        return True
+    
+    def get_frames(self):
+        ''' Returns all frames '''
+        return self._frames
+    
+    def get_frame(self, source, channel):
+        k = (source, channel)
+        if k in self._frames:
+            return self._frames[k]
+        else:
+            return None

python/ftl/libde265.py

+'''
+Python wrapper for libde265. Only decoding is (partly) implemented.
+
+Requirements:
+ * libde265 library (libde265.so.0)
+ * numpy
+ * opencv or skimage
+
+'''
+
+try:
+    import cv2 as cv
+    def _resize(img, size):
+        return cv.resize(img, dsize=tuple(reversed(size)), interpolation=cv.INTER_CUBIC)
+    
+except ImportError:
+    from skimage.transform import resize as resize_skimage
+    def _resize(img, size):
+        # skimage resize() return dtype float64, convert back to uint8
+        # order: 0 nn, 1 bilinear, 3 bicubic
+        return (resize_skimage(img, size, order=3, mode="constant", cval=0) * 255).astype(np.uint8)
+
+import ctypes
+from enum import IntEnum
+
+import numpy as np
+
+# error codes copied from header (de265.h)
+
+class libde265error(IntEnum):
+    DE265_OK = 0
+    DE265_ERROR_NO_SUCH_FILE=1
+    DE265_ERROR_COEFFICIENT_OUT_OF_IMAGE_BOUNDS=4
+    DE265_ERROR_CHECKSUM_MISMATCH=5
+    DE265_ERROR_CTB_OUTSIDE_IMAGE_AREA=6
+    DE265_ERROR_OUT_OF_MEMORY=7
+    DE265_ERROR_CODED_PARAMETER_OUT_OF_RANGE=8
+    DE265_ERROR_IMAGE_BUFFER_FULL=9
+    DE265_ERROR_CANNOT_START_THREADPOOL=10
+    DE265_ERROR_LIBRARY_INITIALIZATION_FAILED=11
+    DE265_ERROR_LIBRARY_NOT_INITIALIZED=12
+    DE265_ERROR_WAITING_FOR_INPUT_DATA=13
+    DE265_ERROR_CANNOT_PROCESS_SEI=14
+    DE265_ERROR_PARAMETER_PARSING=15
+    DE265_ERROR_NO_INITIAL_SLICE_HEADER=16
+    DE265_ERROR_PREMATURE_END_OF_SLICE=17
+    DE265_ERROR_UNSPECIFIED_DECODING_ERROR=18
+    DE265_ERROR_NOT_IMPLEMENTED_YET = 502
+    DE265_WARNING_NO_WPP_CANNOT_USE_MULTITHREADING = 1000
+    DE265_WARNING_WARNING_BUFFER_FULL=1001
+    DE265_WARNING_PREMATURE_END_OF_SLICE_SEGMENT=1002
+    DE265_WARNING_INCORRECT_ENTRY_POINT_OFFSET=1003
+    DE265_WARNING_CTB_OUTSIDE_IMAGE_AREA=1004
+    DE265_WARNING_SPS_HEADER_INVALID=1005
+    DE265_WARNING_PPS_HEADER_INVALID=1006
+    DE265_WARNING_SLICEHEADER_INVALID=1007
+    DE265_WARNING_INCORRECT_MOTION_VECTOR_SCALING=1008
+    DE265_WARNING_NONEXISTING_PPS_REFERENCED=1009
+    DE265_WARNING_NONEXISTING_SPS_REFERENCED=1010
+    DE265_WARNING_BOTH_PREDFLAGS_ZERO=1011
+    DE265_WARNING_NONEXISTING_REFERENCE_PICTURE_ACCESSED=1012
+    DE265_WARNING_NUMMVP_NOT_EQUAL_TO_NUMMVQ=1013
+    DE265_WARNING_NUMBER_OF_SHORT_TERM_REF_PIC_SETS_OUT_OF_RANGE=1014
+    DE265_WARNING_SHORT_TERM_REF_PIC_SET_OUT_OF_RANGE=1015
+    DE265_WARNING_FAULTY_REFERENCE_PICTURE_LIST=1016
+    DE265_WARNING_EOSS_BIT_NOT_SET=1017
+    DE265_WARNING_MAX_NUM_REF_PICS_EXCEEDED=1018
+    DE265_WARNING_INVALID_CHROMA_FORMAT=1019
+    DE265_WARNING_SLICE_SEGMENT_ADDRESS_INVALID=1020
+    DE265_WARNING_DEPENDENT_SLICE_WITH_ADDRESS_ZERO=1021
+    DE265_WARNING_NUMBER_OF_THREADS_LIMITED_TO_MAXIMUM=1022
+    DE265_NON_EXISTING_LT_REFERENCE_CANDIDATE_IN_SLICE_HEADER=1023
+    DE265_WARNING_CANNOT_APPLY_SAO_OUT_OF_MEMORY=1024
+    DE265_WARNING_SPS_MISSING_CANNOT_DECODE_SEI=1025
+    DE265_WARNING_COLLOCATED_MOTION_VECTOR_OUTSIDE_IMAGE_AREA=1026
+
+libde265 = ctypes.cdll.LoadLibrary("libde265.so.0")
+
+libde265.de265_get_error_text.argtypes = [ctypes.c_void_p]
+libde265.de265_get_error_text.restype = ctypes.c_char_p
+libde265.de265_get_version_number_major.restype = ctypes.c_uint32
+libde265.de265_get_version_number_minor.restype = ctypes.c_uint32
+
+libde265.de265_new_decoder.restype = ctypes.c_void_p
+
+libde265.de265_free_decoder.argtypes = [ctypes.c_void_p]
+libde265.de265_free_decoder.restype = ctypes.c_int
+
+libde265.de265_start_worker_threads.argtypes = [ctypes.c_void_p, ctypes.c_int]
+libde265.de265_start_worker_threads.restype = ctypes.c_int
+
+libde265.de265_push_data.argtypes = [ctypes.c_void_p, ctypes.c_void_p, ctypes.c_int, ctypes.c_void_p, ctypes.c_void_p]
+libde265.de265_push_data.restype = ctypes.c_int
+
+libde265.de265_push_NAL.argtypes = [ctypes.c_void_p, ctypes.c_void_p, ctypes.c_int, ctypes.c_void_p, ctypes.c_void_p]
+libde265.de265_push_data.restype = ctypes.c_int
+
+libde265.de265_push_end_of_frame.argtypes = [ctypes.c_void_p]
+
+libde265.de265_flush_data.argtypes = [ctypes.c_void_p]
+libde265.de265_flush_data.restype = ctypes.c_int
+
+libde265.de265_decode.argtypes = [ctypes.c_void_p, ctypes.POINTER(ctypes.c_int)]
+libde265.de265_decode.restype = ctypes.c_int
+
+libde265.de265_get_next_picture.argtypes = [ctypes.c_void_p]
+libde265.de265_get_next_picture.restype = ctypes.c_void_p
+
+libde265.de265_get_image_width.argtypes = [ctypes.c_void_p, ctypes.c_int]
+libde265.de265_get_image_width.restype = ctypes.c_int
+
+libde265.de265_get_image_height.argtypes = [ctypes.c_void_p, ctypes.c_int]
+libde265.de265_get_image_height.restype = ctypes.c_int
+
+libde265.de265_get_bits_per_pixel.argtypes = [ctypes.c_void_p, ctypes.c_int]
+libde265.de265_get_bits_per_pixel.restype = ctypes.c_int
+
+libde265.de265_get_image_plane.argtypes = [ctypes.c_void_p, ctypes.c_int, ctypes.POINTER(ctypes.c_int)]
+libde265.de265_get_image_plane.restype = ctypes.POINTER(ctypes.c_char)
+
+class Decoder:
+    def __init__(self, size, threads=1):
+        self._size = size
+        self._more = ctypes.c_int()
+        self._out_stride = ctypes.c_int()
+        self._ctx = libde265.de265_new_decoder()
+        err = libde265.de265_start_worker_threads(self._ctx, threads)
+        if err:
+            raise Exception(self.get_error_str(err))
+        
+    def __del__(self):
+        libde265.de265_free_decoder(self._ctx)
+    
+    def get_error_str(self, code):
+        return libde265.de265_get_error_text(code).decode("ascii")
+    
+    def push_data(self, data):
+        if not isinstance(data, bytes):
+            raise ValueError("expected bytes")
+        
+        err = libde265.de265_push_data(self._ctx, data, len(data), None, None)
+        
+        if err:
+            raise Exception(self.get_error_str(err))
+            
+    def push_end_of_frame(self):
+        err = libde265.de265_push_end_of_frame(self._ctx)
+        
+        if err:
+            raise Exception(self.get_error_str(err))
+            
+    def push_NAL(self, data):
+        if not isinstance(data, bytes):
+            raise ValueError("expected bytes")
+        
+        err = libde265.de265_push_NAL(self._ctx, data, len(data), None, None)
+        
+        if err:
+            raise Exception(self.get_error_str(err))
+            
+    def decode(self):
+        err = libde265.de265_decode(self._ctx, self._more)
+        
+        if err and err != libde265error.DE265_ERROR_WAITING_FOR_INPUT_DATA:
+            raise Exception(self.get_error_str(err))
+        
+        return self._more.value != 0
+    
+    def flush_data(self):
+        err = libde265.de265_flush_data(self._ctx)
+        
+        if err:
+            raise Exception(self.get_error_str(err))
+        
+    def get_next_picture(self):
+        '''
+        Returns next decoded frame. Image in YCbCr format. If no frame available
+        returns None.
+        '''
+        img = libde265.de265_get_next_picture(self._ctx)
+        
+        if not img:
+            return None
+        
+        res = np.zeros((self._size[0], self._size[1], 3), dtype=np.uint8)
+        
+        for c in range(0, 3):
+            size = (libde265.de265_get_image_height(img, c),
+                    libde265.de265_get_image_width(img, c))
+            
+            bpp = libde265.de265_get_bits_per_pixel(img, c)
+
+            if bpp != 8:
+                raise NotImplementedError("unsupported bits per pixel %i" % bpp)
+            
+            img_ptr = libde265.de265_get_image_plane(img, c, self._out_stride)
+            
+            ch = np.frombuffer(img_ptr[:size[0] * size[1]], dtype=np.uint8)
+            ch.shape = size
+            
+            res[:,:,c] = _resize(ch, self._size)
+
+        return res
\ No newline at end of file