From 96961a1ad98a5bd42021701528c421ed03009113 Mon Sep 17 00:00:00 2001
From: Nicolas Pope <nwpope@utu.fi>
Date: Sat, 25 Jul 2020 13:49:31 +0300
Subject: [PATCH] WIP Add some thumbnail code
---
components/codecs/include/ftl/codecs/channels.hpp | 1 +
.../rgbd-sources/src/sources/stereovideo/device.hpp | 4 +++-
.../rgbd-sources/src/sources/stereovideo/opencv.cpp | 12 +++++++++++-
.../rgbd-sources/src/sources/stereovideo/opencv.hpp | 2 +-
.../rgbd-sources/src/sources/stereovideo/pylon.cpp | 3 ++-
.../rgbd-sources/src/sources/stereovideo/pylon.hpp | 2 +-
.../src/sources/stereovideo/stereovideo.cpp | 4 ++--
components/streams/src/feed.cpp | 7 +++++++
8 files changed, 28 insertions(+), 7 deletions(-)
diff --git a/components/codecs/include/ftl/codecs/channels.hpp b/components/codecs/include/ftl/codecs/channels.hpp
index b6aba67dd..17197c31b 100644
--- a/components/codecs/include/ftl/codecs/channels.hpp
+++ b/components/codecs/include/ftl/codecs/channels.hpp
@@ -56,6 +56,7 @@ enum struct Channel : int {
MetaData = 71, // Map of string pairs (key, value)
Capabilities = 72, // Unordered set of int capabilities
CalibrationData = 73, // Just for stereo intrinsics/extrinsics etc
+ Thumbnail = 74, // Small JPG thumbnail, sometimes updated
Data = 2048, // Custom data, any codec.
Faces = 2049, // Data about detected faces
diff --git a/components/rgbd-sources/src/sources/stereovideo/device.hpp b/components/rgbd-sources/src/sources/stereovideo/device.hpp
index 40f526c0a..f575ea3ae 100644
--- a/components/rgbd-sources/src/sources/stereovideo/device.hpp
+++ b/components/rgbd-sources/src/sources/stereovideo/device.hpp
@@ -7,6 +7,8 @@
namespace ftl {
namespace rgbd {
+class Frame;
+
namespace detail {
class StereoRectification;
@@ -33,7 +35,7 @@ class Device : public Configurable {
//virtual const std::vector<DeviceDetails> &listDevices()=0;
virtual bool grab()=0;
- virtual bool get(cv::cuda::GpuMat &l, cv::cuda::GpuMat &r, cv::cuda::GpuMat &h_l, cv::Mat &h_r, StereoRectification *c, cv::cuda::Stream &stream)=0;
+ virtual bool get(ftl::rgbd::Frame &frame, cv::cuda::GpuMat &l, cv::cuda::GpuMat &r, cv::cuda::GpuMat &h_l, cv::Mat &h_r, StereoRectification *c, cv::cuda::Stream &stream)=0;
virtual unsigned int width() const =0;
virtual unsigned int height() const =0;
diff --git a/components/rgbd-sources/src/sources/stereovideo/opencv.cpp b/components/rgbd-sources/src/sources/stereovideo/opencv.cpp
index aefae673e..ba48e7b26 100644
--- a/components/rgbd-sources/src/sources/stereovideo/opencv.cpp
+++ b/components/rgbd-sources/src/sources/stereovideo/opencv.cpp
@@ -8,12 +8,14 @@
#include <chrono>
#include <ftl/threads.hpp>
#include <ftl/profiler.hpp>
+#include <ftl/rgbd/frame.hpp>
#include "opencv.hpp"
#include "rectification.hpp"
#include <opencv2/core.hpp>
#include <opencv2/opencv.hpp>
#include <opencv2/xphoto.hpp>
+#include <opencv2/imgcodecs.hpp>
#include <ftl/timer.hpp>
@@ -323,7 +325,7 @@ bool OpenCVDevice::grab() {
return true;
}
-bool OpenCVDevice::get(cv::cuda::GpuMat &l_out, cv::cuda::GpuMat &r_out,
+bool OpenCVDevice::get(ftl::rgbd::Frame &frame, cv::cuda::GpuMat &l_out, cv::cuda::GpuMat &r_out,
cv::cuda::GpuMat &l_hres_out, cv::Mat &r_hres_out, StereoRectification *c, cv::cuda::Stream &stream) {
Mat l, r ,hres;
@@ -413,6 +415,14 @@ bool OpenCVDevice::get(cv::cuda::GpuMat &l_out, cv::cuda::GpuMat &r_out,
}
//r_out.upload(r, stream);
+ if (!frame.hasChannel(Channel::Thumbnail)) {
+ cv::Mat thumb;
+ cv::resize(l, thumb, cv::Size(320,240));
+ auto &thumbdata = frame.create<std::vector<uint8_t>>(Channel::Thumbnail);
+ std::vector<int> params = {cv::IMWRITE_JPEG_QUALITY, 70};
+ cv::imencode(".jpg", thumb, thumbdata, params);
+ }
+
if (camera_b_) {
//FTL_Profile("WaitCamB", 0.05);
future_b.wait();
diff --git a/components/rgbd-sources/src/sources/stereovideo/opencv.hpp b/components/rgbd-sources/src/sources/stereovideo/opencv.hpp
index 1ea18d8f3..e0f10ca1e 100644
--- a/components/rgbd-sources/src/sources/stereovideo/opencv.hpp
+++ b/components/rgbd-sources/src/sources/stereovideo/opencv.hpp
@@ -20,7 +20,7 @@ class OpenCVDevice : public ftl::rgbd::detail::Device {
static std::vector<DeviceDetails> listDevices();
bool grab() override;
- bool get(cv::cuda::GpuMat &l, cv::cuda::GpuMat &r, cv::cuda::GpuMat &h_l, cv::Mat &h_r, StereoRectification *c, cv::cuda::Stream &stream) override;
+ bool get(ftl::rgbd::Frame &frame, cv::cuda::GpuMat &l, cv::cuda::GpuMat &r, cv::cuda::GpuMat &h_l, cv::Mat &h_r, StereoRectification *c, cv::cuda::Stream &stream) override;
unsigned int width() const override { return dwidth_; }
unsigned int height() const override { return dheight_; }
diff --git a/components/rgbd-sources/src/sources/stereovideo/pylon.cpp b/components/rgbd-sources/src/sources/stereovideo/pylon.cpp
index f47901447..160c389e1 100644
--- a/components/rgbd-sources/src/sources/stereovideo/pylon.cpp
+++ b/components/rgbd-sources/src/sources/stereovideo/pylon.cpp
@@ -6,6 +6,7 @@
#include <ftl/threads.hpp>
#include <ftl/rgbd/source.hpp>
#include <ftl/profiler.hpp>
+#include <ftl/rgbd/frame.hpp>
#include <pylon/PylonIncludes.h>
#include <pylon/BaslerUniversalInstantCamera.h>
@@ -215,7 +216,7 @@ bool PylonDevice::_retrieveFrames(Pylon::CGrabResultPtr &result, Pylon::CBaslerU
return true;
}
-bool PylonDevice::get(cv::cuda::GpuMat &l_out, cv::cuda::GpuMat &r_out, cv::cuda::GpuMat &h_l, cv::Mat &h_r, StereoRectification *c, cv::cuda::Stream &stream) {
+bool PylonDevice::get(ftl::rgbd::Frame &frame, cv::cuda::GpuMat &l_out, cv::cuda::GpuMat &r_out, cv::cuda::GpuMat &h_l, cv::Mat &h_r, StereoRectification *c, cv::cuda::Stream &stream) {
if (!isReady()) return false;
Mat l, r ,hres;
diff --git a/components/rgbd-sources/src/sources/stereovideo/pylon.hpp b/components/rgbd-sources/src/sources/stereovideo/pylon.hpp
index 6331b418d..70c82a3da 100644
--- a/components/rgbd-sources/src/sources/stereovideo/pylon.hpp
+++ b/components/rgbd-sources/src/sources/stereovideo/pylon.hpp
@@ -22,7 +22,7 @@ class PylonDevice : public ftl::rgbd::detail::Device {
static std::vector<DeviceDetails> listDevices();
bool grab() override;
- bool get(cv::cuda::GpuMat &l, cv::cuda::GpuMat &r, cv::cuda::GpuMat &h_l, cv::Mat &h_r, StereoRectification *c, cv::cuda::Stream &stream) override;
+ bool get(ftl::rgbd::Frame &frame, cv::cuda::GpuMat &l, cv::cuda::GpuMat &r, cv::cuda::GpuMat &h_l, cv::Mat &h_r, StereoRectification *c, cv::cuda::Stream &stream) override;
unsigned int width() const override { return width_; }
unsigned int height() const override { return height_; };
diff --git a/components/rgbd-sources/src/sources/stereovideo/stereovideo.cpp b/components/rgbd-sources/src/sources/stereovideo/stereovideo.cpp
index 93883e5aa..547e84382 100644
--- a/components/rgbd-sources/src/sources/stereovideo/stereovideo.cpp
+++ b/components/rgbd-sources/src/sources/stereovideo/stereovideo.cpp
@@ -341,7 +341,7 @@ bool StereoVideoSource::retrieve(ftl::rgbd::Frame &frame) {
if (lsrc_->isStereo()) {
cv::cuda::GpuMat &left = frame.create<cv::cuda::GpuMat>(Channel::Left);
cv::cuda::GpuMat &right = frame.create<cv::cuda::GpuMat>(Channel::Right);
- if (!lsrc_->get(left, right, hres, hres_r, rectification_.get(), stream2_)) {
+ if (!lsrc_->get(frame, left, right, hres, hres_r, rectification_.get(), stream2_)) {
frame.remove(Channel::Left);
frame.remove(Channel::Right);
}
@@ -349,7 +349,7 @@ bool StereoVideoSource::retrieve(ftl::rgbd::Frame &frame) {
else {
cv::cuda::GpuMat &left = frame.create<cv::cuda::GpuMat>(Channel::Left);
cv::cuda::GpuMat right;
- if (!lsrc_->get(left, right, hres, hres_r, rectification_.get(), stream2_)) {
+ if (!lsrc_->get(frame, left, right, hres, hres_r, rectification_.get(), stream2_)) {
frame.remove(Channel::Left);
}
}
diff --git a/components/streams/src/feed.cpp b/components/streams/src/feed.cpp
index a4bea0efe..9a5974093 100644
--- a/components/streams/src/feed.cpp
+++ b/components/streams/src/feed.cpp
@@ -182,6 +182,10 @@ Feed::Feed(nlohmann::json &config, ftl::net::Universe*net) :
std::atomic_store(&latest_.at(fs->frameset()), fs);
+ if (fs->hasAnyChanged(Channel::Thumbnail)) {
+ _saveThumbnail(fs);
+ }
+
for (auto* filter : filters_) {
// TODO: smarter update (update only when changed) instead of
// filter->channels_available_ = fs->channels();
@@ -253,6 +257,9 @@ Feed::~Feed() {
}
}
+void Feed::_saveThumbnail(const ftl::data::FrameSetPtr& fs) {
+ // TODO: Put thumb somewhere here...
+}
uint32_t Feed::allocateFrameSetId(const std::string &group) {
if (group.size() == 0) {
--
GitLab