Skip to content
Snippets Groups Projects

Compare revisions

Changes are shown as if the source revision was being merged into the target revision. Learn more about comparing revisions.

Source

Select target project
No results found

Target

Select target project
  • nicolaspope/ftl
1 result
Show changes
Hide whitespace changes
Inline Side-by-side
python/ftl/ftlstream.py 0 → 100644
View file @ c9629aed
import msgpack
import numpy as np
from enum import IntEnum
from collections import namedtuple
from . libde265 import Decoder
try:
import cv2 as cv
def _ycrcb2rgb(img):
return cv.cvtColor(img, cv.COLOR_YCrCb2RGB)
except ImportError:
def _ycrcb2rgb(img):
''' YCrCb to RGB, based on OpenCV documentation definition.
Note: It seems this implementation is not perfectly equivalent to OpenCV's
'''
rgb = np.zeros(img.shape, np.float)
Y = img[:,:,0].astype(np.float)
Cr = img[:,:,1].astype(np.float)
Cb = img[:,:,2].astype(np.float)
delta = 128.0
rgb[:,:,0] = Y + 1.403 * (Cr - delta)
rgb[:,:,1] = Y - 0.714 * (Cr - delta) - 0.344 * (Cb - delta)
rgb[:,:,2] = Y + 1.773 * (Cb - delta)
return rgb.round().astype(np.uint8)
# FTL definitions
_packet = namedtuple("Packet", ["codec", "definition", "block_total", "block_number", "flags", "data"])
_stream_packet = namedtuple("StreamPacket", ["timestamp", "streamID", "chanel_count", "channel"])
_definition_t = {
0 : (),
1 : (),
2 : (1080, 1920),
3 : (720, 1280),
4 : (),
5 : (),
6 : (),
7 : (),
8 : ()
}
class NALType(IntEnum):
CODED_SLICE_TRAIL_N = 0
CODED_SLICE_TRAIL_R = 1
CODED_SLICE_TSA_N = 2
CODED_SLICE_TSA_R = 3
CODED_SLICE_STSA_N = 4
CODED_SLICE_STSA_R = 5
CODED_SLICE_RADL_N = 6
CODED_SLICE_RADL_R = 7
CODED_SLICE_RASL_N = 8
CODED_SLICE_RASL_R = 9
RESERVED_VCL_N10 = 10
RESERVED_VCL_R11 = 11
RESERVED_VCL_N12 = 12
RESERVED_VCL_R13 = 13
RESERVED_VCL_N14 = 14
RESERVED_VCL_R15 = 15
CODED_SLICE_BLA_W_LP = 16
CODED_SLICE_BLA_W_RADL = 17
CODED_SLICE_BLA_N_LP = 18
CODED_SLICE_IDR_W_RADL = 19
CODED_SLICE_IDR_N_LP = 20
CODED_SLICE_CRA = 21
RESERVED_IRAP_VCL22 = 22
RESERVED_IRAP_VCL23 = 23
RESERVED_VCL24 = 24
RESERVED_VCL25 = 25
RESERVED_VCL26 = 26
RESERVED_VCL27 = 27
RESERVED_VCL28 = 28
RESERVED_VCL29 = 29
RESERVED_VCL30 = 30
RESERVED_VCL31 = 31
VPS = 32
SPS = 33
PPS = 34
ACCESS_UNIT_DELIMITER = 35
EOS = 36
EOB = 37
FILLER_DATA = 38
PREFIX_SEI = 39
SUFFIX_SEI = 40
RESERVED_NVCL41 = 41
RESERVED_NVCL42 = 42
RESERVED_NVCL43 = 43
RESERVED_NVCL44 = 44
RESERVED_NVCL45 = 45
RESERVED_NVCL46 = 46
RESERVED_NVCL47 = 47
UNSPECIFIED_48 = 48
UNSPECIFIED_49 = 49
UNSPECIFIED_50 = 50
UNSPECIFIED_51 = 51
UNSPECIFIED_52 = 52
UNSPECIFIED_53 = 53
UNSPECIFIED_54 = 54
UNSPECIFIED_55 = 55
UNSPECIFIED_56 = 56
UNSPECIFIED_57 = 57
UNSPECIFIED_58 = 58
UNSPECIFIED_59 = 59
UNSPECIFIED_60 = 60
UNSPECIFIED_61 = 61
UNSPECIFIED_62 = 62
UNSPECIFIED_63 = 63
INVALID = 64
def get_NAL_type(data):
if not isinstance(data, bytes):
raise ValueError("expected bytes")
return NALType((data[4] >> 1) & 0x3f)
class FTLStream:
def __init__(self, file):
self._file = open(file, "br")
self._decoders = {}
self._frames = {}
try:
magic = self._file.read(5)
if magic[:4] != bytearray(ord(c) for c in "FTLF"):
raise Exception("wrong magic")
self._unpacker = msgpack.Unpacker(self._file, raw=True, use_list=False)
except Exception as ex:
self._file.close()
raise ex
self._packets_read = 0
def __del__(self):
self._file.close()
def _read_next(self):
v1, v2 = self._unpacker.unpack()
return _stream_packet._make(v1), _packet._make(v2)
def _update_calib(self, sp, p):
''' Update calibration '''
pass
def _update_pose(self, sp, p):
''' Update pose '''
pass
def _decode_frame_hevc(self, sp, p):
''' Decode HEVC frame '''
k = (sp.streamID, sp.channel)
if k not in self._decoders:
self._decoders[k] = Decoder(_definition_t[p.definition])
decoder = self._decoders[k]
decoder.push_data(p.data)
decoder.decode()
img = decoder.get_next_picture()
if img is not None:
self._frames[k] = _ycrcb2rgb(img)
def _flush_decoders(self):
for decoder in self._decoders.values():
decoder.flush_data()
def read(self):
'''
Reads data for until the next timestamp. Returns False if there is no
more data to read, otherwise returns True.
'''
if self._packets_read == 0:
self._sp, self._p = self._read_next()
self._packets_read += 1
self._frames = {}
ts = self._sp.timestamp
ex = None
while self._sp.timestamp == ts:
try:
if self._p.codec == 100: # JSON
NotImplementedError("json decoding not implemented")
elif self._p.codec == 101: # CALIBRATION
self._update_calib(self._sp, self._p)
elif self._p.codec == 102: # POSE
self._update_pose(self._sp, self._p)
elif self._p.codec == 3: # HEVC
self._decode_frame_hevc(self._sp, self._p)
else:
raise ValueError("unkowno codec %i" % p.codec)
except Exception as e:
# TODO: Multiple exceptions possible. Re-design read()?
ex = e
try:
self._sp, self._p = self._read_next()
self._packets_read += 1
except msgpack.OutOfData:
return False
if ex is not None:
raise ex
return True
def get_frames(self):
''' Returns all frames '''
return self._frames
def get_frame(self, source, channel):
k = (source, channel)
if k in self._frames:
return self._frames[k]
else:
return None
python/ftl/libde265.py 0 → 100644
View file @ c9629aed
'''
Python wrapper for libde265. Only decoding is (partly) implemented.
Requirements:
* libde265 library (libde265.so.0)
* numpy
* opencv or skimage
'''
try:
import cv2 as cv
def _resize(img, size):
return cv.resize(img, dsize=tuple(reversed(size)), interpolation=cv.INTER_CUBIC)
except ImportError:
from skimage.transform import resize as resize_skimage
def _resize(img, size):
# skimage resize() return dtype float64, convert back to uint8
# order: 0 nn, 1 bilinear, 3 bicubic
return (resize_skimage(img, size, order=3, mode="constant", cval=0) * 255).astype(np.uint8)
import ctypes
from enum import IntEnum
import numpy as np
# error codes copied from header (de265.h)
class libde265error(IntEnum):
DE265_OK = 0
DE265_ERROR_NO_SUCH_FILE=1
DE265_ERROR_COEFFICIENT_OUT_OF_IMAGE_BOUNDS=4
DE265_ERROR_CHECKSUM_MISMATCH=5
DE265_ERROR_CTB_OUTSIDE_IMAGE_AREA=6
DE265_ERROR_OUT_OF_MEMORY=7
DE265_ERROR_CODED_PARAMETER_OUT_OF_RANGE=8
DE265_ERROR_IMAGE_BUFFER_FULL=9
DE265_ERROR_CANNOT_START_THREADPOOL=10
DE265_ERROR_LIBRARY_INITIALIZATION_FAILED=11
DE265_ERROR_LIBRARY_NOT_INITIALIZED=12
DE265_ERROR_WAITING_FOR_INPUT_DATA=13
DE265_ERROR_CANNOT_PROCESS_SEI=14
DE265_ERROR_PARAMETER_PARSING=15
DE265_ERROR_NO_INITIAL_SLICE_HEADER=16
DE265_ERROR_PREMATURE_END_OF_SLICE=17
DE265_ERROR_UNSPECIFIED_DECODING_ERROR=18
DE265_ERROR_NOT_IMPLEMENTED_YET = 502
DE265_WARNING_NO_WPP_CANNOT_USE_MULTITHREADING = 1000
DE265_WARNING_WARNING_BUFFER_FULL=1001
DE265_WARNING_PREMATURE_END_OF_SLICE_SEGMENT=1002
DE265_WARNING_INCORRECT_ENTRY_POINT_OFFSET=1003
DE265_WARNING_CTB_OUTSIDE_IMAGE_AREA=1004
DE265_WARNING_SPS_HEADER_INVALID=1005
DE265_WARNING_PPS_HEADER_INVALID=1006
DE265_WARNING_SLICEHEADER_INVALID=1007
DE265_WARNING_INCORRECT_MOTION_VECTOR_SCALING=1008
DE265_WARNING_NONEXISTING_PPS_REFERENCED=1009
DE265_WARNING_NONEXISTING_SPS_REFERENCED=1010
DE265_WARNING_BOTH_PREDFLAGS_ZERO=1011
DE265_WARNING_NONEXISTING_REFERENCE_PICTURE_ACCESSED=1012
DE265_WARNING_NUMMVP_NOT_EQUAL_TO_NUMMVQ=1013
DE265_WARNING_NUMBER_OF_SHORT_TERM_REF_PIC_SETS_OUT_OF_RANGE=1014
DE265_WARNING_SHORT_TERM_REF_PIC_SET_OUT_OF_RANGE=1015
DE265_WARNING_FAULTY_REFERENCE_PICTURE_LIST=1016
DE265_WARNING_EOSS_BIT_NOT_SET=1017
DE265_WARNING_MAX_NUM_REF_PICS_EXCEEDED=1018
DE265_WARNING_INVALID_CHROMA_FORMAT=1019
DE265_WARNING_SLICE_SEGMENT_ADDRESS_INVALID=1020
DE265_WARNING_DEPENDENT_SLICE_WITH_ADDRESS_ZERO=1021
DE265_WARNING_NUMBER_OF_THREADS_LIMITED_TO_MAXIMUM=1022
DE265_NON_EXISTING_LT_REFERENCE_CANDIDATE_IN_SLICE_HEADER=1023
DE265_WARNING_CANNOT_APPLY_SAO_OUT_OF_MEMORY=1024
DE265_WARNING_SPS_MISSING_CANNOT_DECODE_SEI=1025
DE265_WARNING_COLLOCATED_MOTION_VECTOR_OUTSIDE_IMAGE_AREA=1026
libde265 = ctypes.cdll.LoadLibrary("libde265.so.0")
libde265.de265_get_error_text.argtypes = [ctypes.c_void_p]
libde265.de265_get_error_text.restype = ctypes.c_char_p
libde265.de265_get_version_number_major.restype = ctypes.c_uint32
libde265.de265_get_version_number_minor.restype = ctypes.c_uint32
libde265.de265_new_decoder.restype = ctypes.c_void_p
libde265.de265_free_decoder.argtypes = [ctypes.c_void_p]
libde265.de265_free_decoder.restype = ctypes.c_int
libde265.de265_start_worker_threads.argtypes = [ctypes.c_void_p, ctypes.c_int]
libde265.de265_start_worker_threads.restype = ctypes.c_int
libde265.de265_push_data.argtypes = [ctypes.c_void_p, ctypes.c_void_p, ctypes.c_int, ctypes.c_void_p, ctypes.c_void_p]
libde265.de265_push_data.restype = ctypes.c_int
libde265.de265_push_NAL.argtypes = [ctypes.c_void_p, ctypes.c_void_p, ctypes.c_int, ctypes.c_void_p, ctypes.c_void_p]
libde265.de265_push_data.restype = ctypes.c_int
libde265.de265_push_end_of_frame.argtypes = [ctypes.c_void_p]
libde265.de265_flush_data.argtypes = [ctypes.c_void_p]
libde265.de265_flush_data.restype = ctypes.c_int
libde265.de265_decode.argtypes = [ctypes.c_void_p, ctypes.POINTER(ctypes.c_int)]
libde265.de265_decode.restype = ctypes.c_int
libde265.de265_get_next_picture.argtypes = [ctypes.c_void_p]
libde265.de265_get_next_picture.restype = ctypes.c_void_p
libde265.de265_get_image_width.argtypes = [ctypes.c_void_p, ctypes.c_int]
libde265.de265_get_image_width.restype = ctypes.c_int
libde265.de265_get_image_height.argtypes = [ctypes.c_void_p, ctypes.c_int]
libde265.de265_get_image_height.restype = ctypes.c_int
libde265.de265_get_bits_per_pixel.argtypes = [ctypes.c_void_p, ctypes.c_int]
libde265.de265_get_bits_per_pixel.restype = ctypes.c_int
libde265.de265_get_image_plane.argtypes = [ctypes.c_void_p, ctypes.c_int, ctypes.POINTER(ctypes.c_int)]
libde265.de265_get_image_plane.restype = ctypes.POINTER(ctypes.c_char)
class Decoder:
def __init__(self, size, threads=1):
self._size = size
self._more = ctypes.c_int()
self._out_stride = ctypes.c_int()
self._ctx = libde265.de265_new_decoder()
err = libde265.de265_start_worker_threads(self._ctx, threads)
if err:
raise Exception(self.get_error_str(err))
def __del__(self):
libde265.de265_free_decoder(self._ctx)
def get_error_str(self, code):
return libde265.de265_get_error_text(code).decode("ascii")
def push_data(self, data):
if not isinstance(data, bytes):
raise ValueError("expected bytes")
err = libde265.de265_push_data(self._ctx, data, len(data), None, None)
if err:
raise Exception(self.get_error_str(err))
def push_end_of_frame(self):
err = libde265.de265_push_end_of_frame(self._ctx)
if err:
raise Exception(self.get_error_str(err))
def push_NAL(self, data):
if not isinstance(data, bytes):
raise ValueError("expected bytes")
err = libde265.de265_push_NAL(self._ctx, data, len(data), None, None)
if err:
raise Exception(self.get_error_str(err))
def decode(self):
err = libde265.de265_decode(self._ctx, self._more)
if err and err != libde265error.DE265_ERROR_WAITING_FOR_INPUT_DATA:
raise Exception(self.get_error_str(err))
return self._more.value != 0
def flush_data(self):
err = libde265.de265_flush_data(self._ctx)
if err:
raise Exception(self.get_error_str(err))
def get_next_picture(self):
'''
Returns next decoded frame. Image in YCbCr format. If no frame available
returns None.
'''
img = libde265.de265_get_next_picture(self._ctx)
if not img:
return None
res = np.zeros((self._size[0], self._size[1], 3), dtype=np.uint8)
for c in range(0, 3):
size = (libde265.de265_get_image_height(img, c),
libde265.de265_get_image_width(img, c))
bpp = libde265.de265_get_bits_per_pixel(img, c)
if bpp != 8:
raise NotImplementedError("unsupported bits per pixel %i" % bpp)
img_ptr = libde265.de265_get_image_plane(img, c, self._out_stride)
ch = np.frombuffer(img_ptr[:size[0] * size[1]], dtype=np.uint8)
ch.shape = size
res[:,:,c] = _resize(ch, self._size)
return res
\ No newline at end of file