diff --git a/components/codecs/include/ftl/codecs/channels.hpp b/components/codecs/include/ftl/codecs/channels.hpp
index b6aba67dd9e142c90ed3bb45fce31a2775c1be62..17197c31b126e5e8bf373b139c8efd7e56e5d9b6 100644
--- a/components/codecs/include/ftl/codecs/channels.hpp
+++ b/components/codecs/include/ftl/codecs/channels.hpp
@@ -56,6 +56,7 @@ enum struct Channel : int {
MetaData = 71, // Map of string pairs (key, value)
Capabilities = 72, // Unordered set of int capabilities
CalibrationData = 73, // Just for stereo intrinsics/extrinsics etc
+ Thumbnail = 74, // Small JPG thumbnail, sometimes updated
Data = 2048, // Custom data, any codec.
Faces = 2049, // Data about detected faces
diff --git a/components/rgbd-sources/src/sources/stereovideo/device.hpp b/components/rgbd-sources/src/sources/stereovideo/device.hpp
index 40f526c0adfe53f83f2191ee400bf53c8a16d3e9..f575ea3aec6b23ccbfe755690e3e6bc4a94e7b6d 100644
--- a/components/rgbd-sources/src/sources/stereovideo/device.hpp
+++ b/components/rgbd-sources/src/sources/stereovideo/device.hpp
@@ -7,6 +7,8 @@
namespace ftl {
namespace rgbd {
+class Frame;
+
namespace detail {
class StereoRectification;
@@ -33,7 +35,7 @@ class Device : public Configurable {
//virtual const std::vector<DeviceDetails> &listDevices()=0;
virtual bool grab()=0;
- virtual bool get(cv::cuda::GpuMat &l, cv::cuda::GpuMat &r, cv::cuda::GpuMat &h_l, cv::Mat &h_r, StereoRectification *c, cv::cuda::Stream &stream)=0;
+ virtual bool get(ftl::rgbd::Frame &frame, cv::cuda::GpuMat &l, cv::cuda::GpuMat &r, cv::cuda::GpuMat &h_l, cv::Mat &h_r, StereoRectification *c, cv::cuda::Stream &stream)=0;
virtual unsigned int width() const =0;
virtual unsigned int height() const =0;
diff --git a/components/rgbd-sources/src/sources/stereovideo/opencv.cpp b/components/rgbd-sources/src/sources/stereovideo/opencv.cpp
index aefae673e1cc320308e47da6cb65b08faae0e070..ba48e7b262dfc45c646ef66baeb9a62750531b84 100644
--- a/components/rgbd-sources/src/sources/stereovideo/opencv.cpp
+++ b/components/rgbd-sources/src/sources/stereovideo/opencv.cpp
@@ -8,12 +8,14 @@
#include <chrono>
#include <ftl/threads.hpp>
#include <ftl/profiler.hpp>
+#include <ftl/rgbd/frame.hpp>
#include "opencv.hpp"
#include "rectification.hpp"
#include <opencv2/core.hpp>
#include <opencv2/opencv.hpp>
#include <opencv2/xphoto.hpp>
+#include <opencv2/imgcodecs.hpp>
#include <ftl/timer.hpp>
@@ -323,7 +325,7 @@ bool OpenCVDevice::grab() {
return true;
}
-bool OpenCVDevice::get(cv::cuda::GpuMat &l_out, cv::cuda::GpuMat &r_out,
+bool OpenCVDevice::get(ftl::rgbd::Frame &frame, cv::cuda::GpuMat &l_out, cv::cuda::GpuMat &r_out,
cv::cuda::GpuMat &l_hres_out, cv::Mat &r_hres_out, StereoRectification *c, cv::cuda::Stream &stream) {
Mat l, r ,hres;
@@ -413,6 +415,14 @@ bool OpenCVDevice::get(cv::cuda::GpuMat &l_out, cv::cuda::GpuMat &r_out,
}
//r_out.upload(r, stream);
+ if (!frame.hasChannel(Channel::Thumbnail)) {
+ cv::Mat thumb;
+ cv::resize(l, thumb, cv::Size(320,240));
+ auto &thumbdata = frame.create<std::vector<uint8_t>>(Channel::Thumbnail);
+ std::vector<int> params = {cv::IMWRITE_JPEG_QUALITY, 70};
+ cv::imencode(".jpg", thumb, thumbdata, params);
+ }
+
if (camera_b_) {
//FTL_Profile("WaitCamB", 0.05);
future_b.wait();
diff --git a/components/rgbd-sources/src/sources/stereovideo/opencv.hpp b/components/rgbd-sources/src/sources/stereovideo/opencv.hpp
index 1ea18d8f3a3492290c68b89bbe963769567b34d1..e0f10ca1e5329cff94545d00a97e20e638ae1b78 100644
--- a/components/rgbd-sources/src/sources/stereovideo/opencv.hpp
+++ b/components/rgbd-sources/src/sources/stereovideo/opencv.hpp
@@ -20,7 +20,7 @@ class OpenCVDevice : public ftl::rgbd::detail::Device {
static std::vector<DeviceDetails> listDevices();
bool grab() override;
- bool get(cv::cuda::GpuMat &l, cv::cuda::GpuMat &r, cv::cuda::GpuMat &h_l, cv::Mat &h_r, StereoRectification *c, cv::cuda::Stream &stream) override;
+ bool get(ftl::rgbd::Frame &frame, cv::cuda::GpuMat &l, cv::cuda::GpuMat &r, cv::cuda::GpuMat &h_l, cv::Mat &h_r, StereoRectification *c, cv::cuda::Stream &stream) override;
unsigned int width() const override { return dwidth_; }
unsigned int height() const override { return dheight_; }
diff --git a/components/rgbd-sources/src/sources/stereovideo/pylon.cpp b/components/rgbd-sources/src/sources/stereovideo/pylon.cpp
index f479014475a0366efcfe6cd822fa52a54191e2e2..160c389e10a8ae59a529a3dc7a4ac2b9fee8ee28 100644
--- a/components/rgbd-sources/src/sources/stereovideo/pylon.cpp
+++ b/components/rgbd-sources/src/sources/stereovideo/pylon.cpp
@@ -6,6 +6,7 @@
#include <ftl/threads.hpp>
#include <ftl/rgbd/source.hpp>
#include <ftl/profiler.hpp>
+#include <ftl/rgbd/frame.hpp>
#include <pylon/PylonIncludes.h>
#include <pylon/BaslerUniversalInstantCamera.h>
@@ -215,7 +216,7 @@ bool PylonDevice::_retrieveFrames(Pylon::CGrabResultPtr &result, Pylon::CBaslerU
return true;
}
-bool PylonDevice::get(cv::cuda::GpuMat &l_out, cv::cuda::GpuMat &r_out, cv::cuda::GpuMat &h_l, cv::Mat &h_r, StereoRectification *c, cv::cuda::Stream &stream) {
+bool PylonDevice::get(ftl::rgbd::Frame &frame, cv::cuda::GpuMat &l_out, cv::cuda::GpuMat &r_out, cv::cuda::GpuMat &h_l, cv::Mat &h_r, StereoRectification *c, cv::cuda::Stream &stream) {
if (!isReady()) return false;
Mat l, r ,hres;
diff --git a/components/rgbd-sources/src/sources/stereovideo/pylon.hpp b/components/rgbd-sources/src/sources/stereovideo/pylon.hpp
index 6331b418deb50297f3106c55543ecce076c63fa4..70c82a3da316fa7b7f21bab45830b8880f363768 100644
--- a/components/rgbd-sources/src/sources/stereovideo/pylon.hpp
+++ b/components/rgbd-sources/src/sources/stereovideo/pylon.hpp
@@ -22,7 +22,7 @@ class PylonDevice : public ftl::rgbd::detail::Device {
static std::vector<DeviceDetails> listDevices();
bool grab() override;
- bool get(cv::cuda::GpuMat &l, cv::cuda::GpuMat &r, cv::cuda::GpuMat &h_l, cv::Mat &h_r, StereoRectification *c, cv::cuda::Stream &stream) override;
+ bool get(ftl::rgbd::Frame &frame, cv::cuda::GpuMat &l, cv::cuda::GpuMat &r, cv::cuda::GpuMat &h_l, cv::Mat &h_r, StereoRectification *c, cv::cuda::Stream &stream) override;
unsigned int width() const override { return width_; }
unsigned int height() const override { return height_; };
diff --git a/components/rgbd-sources/src/sources/stereovideo/stereovideo.cpp b/components/rgbd-sources/src/sources/stereovideo/stereovideo.cpp
index 93883e5aafed354f24b311fa54842e2a43c59262..547e84382260ba3f2eb27c8937b95511427d6d6c 100644
--- a/components/rgbd-sources/src/sources/stereovideo/stereovideo.cpp
+++ b/components/rgbd-sources/src/sources/stereovideo/stereovideo.cpp
@@ -341,7 +341,7 @@ bool StereoVideoSource::retrieve(ftl::rgbd::Frame &frame) {
if (lsrc_->isStereo()) {
cv::cuda::GpuMat &left = frame.create<cv::cuda::GpuMat>(Channel::Left);
cv::cuda::GpuMat &right = frame.create<cv::cuda::GpuMat>(Channel::Right);
- if (!lsrc_->get(left, right, hres, hres_r, rectification_.get(), stream2_)) {
+ if (!lsrc_->get(frame, left, right, hres, hres_r, rectification_.get(), stream2_)) {
frame.remove(Channel::Left);
frame.remove(Channel::Right);
}
@@ -349,7 +349,7 @@ bool StereoVideoSource::retrieve(ftl::rgbd::Frame &frame) {
else {
cv::cuda::GpuMat &left = frame.create<cv::cuda::GpuMat>(Channel::Left);
cv::cuda::GpuMat right;
- if (!lsrc_->get(left, right, hres, hres_r, rectification_.get(), stream2_)) {
+ if (!lsrc_->get(frame, left, right, hres, hres_r, rectification_.get(), stream2_)) {
frame.remove(Channel::Left);
}
}
diff --git a/components/streams/src/feed.cpp b/components/streams/src/feed.cpp
index a4bea0efeff283f61d59647417caa35bd9b33eb6..9a5974093b496d43897da9e57f7d9fcc077652dd 100644
--- a/components/streams/src/feed.cpp
+++ b/components/streams/src/feed.cpp
@@ -182,6 +182,10 @@ Feed::Feed(nlohmann::json &config, ftl::net::Universe*net) :
std::atomic_store(&latest_.at(fs->frameset()), fs);
+ if (fs->hasAnyChanged(Channel::Thumbnail)) {
+ _saveThumbnail(fs);
+ }
+
for (auto* filter : filters_) {
// TODO: smarter update (update only when changed) instead of
// filter->channels_available_ = fs->channels();
@@ -253,6 +257,9 @@ Feed::~Feed() {
}
}
+void Feed::_saveThumbnail(const ftl::data::FrameSetPtr& fs) {
+ // TODO: Put thumb somewhere here...
+}
uint32_t Feed::allocateFrameSetId(const std::string &group) {
if (group.size() == 0) {