From d05d242ffcbad82ca367d2184337961a68220a7c Mon Sep 17 00:00:00 2001
From: Sebastian Hahta <joseha@utu.fi>
Date: Fri, 31 Jul 2020 12:00:54 +0300
Subject: [PATCH] dual/stereo view for camera
---
applications/gui2/src/modules/camera.cpp | 50 +++++++++++++---------
applications/gui2/src/modules/camera.hpp | 13 +++---
applications/gui2/src/views/camera.cpp | 54 ++++++++++++++++++++----
applications/gui2/src/views/camera.hpp | 5 +++
4 files changed, 90 insertions(+), 32 deletions(-)
diff --git a/applications/gui2/src/modules/camera.cpp b/applications/gui2/src/modules/camera.cpp
index c362d6a48..363639d53 100644
--- a/applications/gui2/src/modules/camera.cpp
+++ b/applications/gui2/src/modules/camera.cpp
@@ -179,8 +179,8 @@ void Camera::initiate_(ftl::data::Frame &frame) {
if (!view) return;
view->onClose([this](){
- filter->remove();
- filter = nullptr;
+ filter_->remove();
+ filter_ = nullptr;
nframes_ = -1;
auto *mod = this->screen->getModule<ftl::gui2::Statistics>();
@@ -191,7 +191,7 @@ void Camera::initiate_(ftl::data::Frame &frame) {
mod->getJSON(StatisticsPanel::CAMERA_DETAILS).clear();
});
- setChannel(channel);
+ setChannel(channel_);
screen->setView(view);
view->refresh();
@@ -239,10 +239,10 @@ void Camera::activate(ftl::data::FrameID id) {
//std::mutex m;
//std::condition_variable cv;
- filter = io->feed()->filter(std::unordered_set<unsigned int>{id.frameset()}, {Channel::Left});
- filter->on(
+ filter_ = io->feed()->filter(std::unordered_set<unsigned int>{id.frameset()}, {Channel::Left});
+ filter_->on(
[this, speaker = io->speaker()](ftl::data::FrameSetPtr fs){
- if (paused) return true;
+ if (paused_) return true;
std::atomic_store(¤t_fs_, fs);
std::atomic_store(&latest_, fs);
@@ -294,7 +294,7 @@ void Camera::activate(ftl::data::FrameID id) {
}
);
- auto sets = filter->getLatestFrameSets();
+ auto sets = filter_->getLatestFrameSets();
if (sets.size() > 0) {
std::atomic_store(¤t_fs_, sets.front());
std::atomic_store(&latest_, sets.front());
@@ -311,8 +311,8 @@ void Camera::activate(ftl::data::FrameID id) {
}
void Camera::setChannel(Channel c) {
- channel = c;
- filter->select({c});
+ channel_ = c;
+ filter_->select({Channel::Colour, c});
}
std::string Camera::getActiveSourceURI() {
@@ -349,25 +349,25 @@ bool Camera::isRecording() {
void Camera::stopRecording() {
io->feed()->stopRecording();
- filter->select({channel});
+ filter_->select({channel_});
}
void Camera::startRecording(const std::string &filename, const std::unordered_set<ftl::codecs::Channel> &channels) {
- filter->select(channels);
- io->feed()->startRecording(filter, filename);
+ filter_->select(channels);
+ io->feed()->startRecording(filter_, filename);
}
void Camera::startStreaming(const std::unordered_set<ftl::codecs::Channel> &channels) {
- filter->select(channels);
- io->feed()->startStreaming(filter);
+ filter_->select(channels);
+ io->feed()->startStreaming(filter_);
}
void Camera::snapshot(const std::string &filename) {
auto ptr = std::atomic_load(&latest_);
if (ptr) {
auto &frame = ptr->frames[frame_idx];
- if (frame.hasChannel(channel)) {
- const auto &snap = frame.get<cv::Mat>(channel);
+ if (frame.hasChannel(channel_)) {
+ const auto &snap = frame.get<cv::Mat>(channel_);
cv::Mat output;
cv::cvtColor(snap, output, cv::COLOR_BGRA2BGR);
cv::imwrite(filename, output);
@@ -376,8 +376,15 @@ void Camera::snapshot(const std::string &filename) {
}
ftl::cuda::TextureObject<uchar4>& Camera::getFrame() {
+ return getFrame(channel_);
+}
+
+ftl::cuda::TextureObject<uchar4>& Camera::getFrame(ftl::codecs::Channel channel) {
if (std::atomic_load(¤t_fs_)) {
auto& frame = current_fs_->frames[frame_idx].cast<ftl::rgbd::Frame>();
+
+ current_frame_colour_ = frame.getTexture<uchar4>(Channel::Left);
+
if (frame.hasChannel(channel)) {
current_frame_ = colouriser_->colourise(frame, channel, 0);
} else {
@@ -385,10 +392,11 @@ ftl::cuda::TextureObject<uchar4>& Camera::getFrame() {
}
std::atomic_store(¤t_fs_, {});
}
- return current_frame_;
+ if (channel == Channel::Left) { return current_frame_colour_; }
+ else { return current_frame_; }
}
-bool Camera::getFrame(ftl::cuda::TextureObject<uchar4>& frame) {
+bool Camera::getFrame(ftl::cuda::TextureObject<uchar4>& frame, ftl::codecs::Channel channel) {
if (std::atomic_load(¤t_fs_).get() != nullptr) {
frame = getFrame();
return true;
@@ -396,10 +404,14 @@ bool Camera::getFrame(ftl::cuda::TextureObject<uchar4>& frame) {
return false;
}
+bool Camera::getFrame(ftl::cuda::TextureObject<uchar4>& frame) {
+ return getFrame(frame, channel_);
+}
+
bool Camera::hasFrame() {
auto ptr = std::atomic_load(¤t_fs_);
if (ptr && ptr->frames.size() > (unsigned int)(frame_idx)) {
- return ptr->frames[frame_idx].hasChannel(channel);
+ return ptr->frames[frame_idx].hasChannel(channel_);
}
return false;
}
diff --git a/applications/gui2/src/modules/camera.hpp b/applications/gui2/src/modules/camera.hpp
index bdcb6dcaa..5ad2c3557 100644
--- a/applications/gui2/src/modules/camera.hpp
+++ b/applications/gui2/src/modules/camera.hpp
@@ -21,8 +21,8 @@ public:
virtual void activate(ftl::data::FrameID id);
void setChannel(ftl::codecs::Channel c);
- void setPaused(bool set) { paused = set; };
- bool isPaused() { return paused; }
+ void setPaused(bool set) { paused_ = set; };
+ bool isPaused() { return paused_; }
float volume();
void setVolume(float v);
@@ -31,7 +31,9 @@ public:
* will stay valid until getFrame() is called again. Always returns a
* reference to internal buffer. */
ftl::cuda::TextureObject<uchar4>& getFrame();
+ ftl::cuda::TextureObject<uchar4>& getFrame(ftl::codecs::Channel channel);
bool getFrame(ftl::cuda::TextureObject<uchar4>&);
+ bool getFrame(ftl::cuda::TextureObject<uchar4>&, ftl::codecs::Channel channel);
std::unordered_set<ftl::codecs::Channel> availableChannels();
@@ -68,9 +70,9 @@ public:
private:
int frame_idx = -1;
ftl::data::FrameID frame_id_;
- ftl::codecs::Channel channel = ftl::codecs::Channel::Colour;
- ftl::stream::Feed::Filter *filter = nullptr;
- std::atomic_bool paused = false; // TODO: implement in InputOutput
+ ftl::codecs::Channel channel_ = ftl::codecs::Channel::Colour;
+ ftl::stream::Feed::Filter *filter_ = nullptr;
+ std::atomic_bool paused_ = false; // TODO: implement in InputOutput
bool has_seen_frame_ = false;
ftl::codecs::Touch point_;
bool live_=false;
@@ -85,6 +87,7 @@ private:
ftl::data::FrameSetPtr current_fs_;
ftl::data::FrameSetPtr latest_;
ftl::cuda::TextureObject<uchar4> current_frame_;
+ ftl::cuda::TextureObject<uchar4> current_frame_colour_;
std::unique_ptr<ftl::render::Colouriser> colouriser_;
std::unique_ptr<ftl::overlay::Overlay> overlay_;
diff --git a/applications/gui2/src/views/camera.cpp b/applications/gui2/src/views/camera.cpp
index 76ff8efab..2794a96ec 100644
--- a/applications/gui2/src/views/camera.cpp
+++ b/applications/gui2/src/views/camera.cpp
@@ -166,7 +166,7 @@ void RecordOptions::show(const std::function<void(bool)> &cb) {
class MediaPanel : public FixedWindow {
public:
- MediaPanel(nanogui::Widget *parent, Camera* ctrl);
+ MediaPanel(nanogui::Widget *parent, Camera* ctrl, CameraView* view);
virtual ~MediaPanel();
void setAvailableChannels(const std::unordered_set<ftl::codecs::Channel> &channels);
@@ -183,14 +183,15 @@ public:
private:
std::vector<nanogui::Widget*> buttons(); // channel buttons
Camera* ctrl_;
+ CameraView* view_;
RecordOptions *record_opts_=nullptr;
public:
EIGEN_MAKE_ALIGNED_OPERATOR_NEW
};
-MediaPanel::MediaPanel(nanogui::Widget *parent, ftl::gui2::Camera* ctrl) :
- ftl::gui2::FixedWindow(parent, ""), ctrl_(ctrl) {
+MediaPanel::MediaPanel(nanogui::Widget *parent, ftl::gui2::Camera* ctrl, CameraView* view) :
+ ftl::gui2::FixedWindow(parent, ""), ctrl_(ctrl), view_(view) {
LOG(INFO) << __func__ << " (" << this << ")";
using namespace nanogui;
@@ -268,6 +269,12 @@ MediaPanel::MediaPanel(nanogui::Widget *parent, ftl::gui2::Camera* ctrl) :
}
});
+ auto button_stereo = new nanogui::Button(this, "", ENTYPO_ICON_GRID);
+ button_stereo->setFlags(nanogui::Button::Flags::ToggleButton);
+ button_stereo->setChangeCallback([view = view_](bool v){
+ view->setStereo(v);
+ });
+
// Channel select. Creates buttons for 32 channels and sets available ones
// visible (a bit of a hack, only used here and setAvailableChannels())
@@ -364,10 +371,11 @@ nanogui::Button* MediaPanel::addButton(int pos) {
// ==== CameraView =============================================================
CameraView::CameraView(ftl::gui2::Screen* parent, ftl::gui2::Camera* ctrl) :
- View(parent), enable_zoom_(false), enable_pan_(false), ctrl_(ctrl) {
+ View(parent), enable_zoom_(false), enable_pan_(false), ctrl_(ctrl),
+ stereoim_(nullptr) {
imview_ = new ftl::gui2::FTLImageView(this);
- panel_ = new ftl::gui2::MediaPanel(screen(), ctrl);
+ panel_ = new ftl::gui2::MediaPanel(screen(), ctrl, this);
auto *mod = ctrl_->screen->getModule<ftl::gui2::Statistics>();
if (ctrl_->isMovable()) {
@@ -415,6 +423,25 @@ CameraView::~CameraView() {
}
}
+void CameraView::setStereo(bool v) {
+ if (v) {
+ if (!stereoim_) {
+ removeChild(imview_);
+ stereoim_ = new StereoImageView(this);
+ imview_ = stereoim_->right();
+ performLayout(screen()->nvgContext());
+ }
+ }
+ else {
+ if (stereoim_) {
+ removeChild(stereoim_);
+ imview_ = new FTLImageView(this);
+ stereoim_ = nullptr;
+ performLayout(screen()->nvgContext());
+ }
+ }
+}
+
void CameraView::refresh() {
bool was_valid = imview_->texture().isValid();
@@ -481,6 +508,9 @@ void CameraView::draw(NVGcontext*ctx) {
try {
// TODO: Select shader to flip if VR capability found...
imview_->copyFrom(ctrl_->getFrame());
+ if (stereoim_) {
+ stereoim_->left()->copyFrom(ctrl_->getFrame(Channel::Left));
+ }
}
catch (std::exception& e) {
gui()->showError("Exception", e.what());
@@ -500,9 +530,17 @@ void CameraView::draw(NVGcontext*ctx) {
}
void CameraView::performLayout(NVGcontext* ctx) {
- imview_->setSize(size());
- if (!(enable_zoom_ && enable_pan_)) {
- imview_->fit();
+ if (stereoim_) {
+ stereoim_->setFixedSize(size());
+ if (!(enable_zoom_ && enable_pan_)) {
+ stereoim_->fit();
+ }
+ }
+ else {
+ imview_->setSize(size());
+ if (!(enable_zoom_ && enable_pan_)) {
+ imview_->fit();
+ }
}
View::performLayout(ctx);
}
diff --git a/applications/gui2/src/views/camera.hpp b/applications/gui2/src/views/camera.hpp
index efeb4221b..3c1c27a81 100644
--- a/applications/gui2/src/views/camera.hpp
+++ b/applications/gui2/src/views/camera.hpp
@@ -28,6 +28,8 @@ public:
void setZoom(bool enable);
void setPan(bool enable);
+ void setStereo(bool v);
+
protected:
bool enable_zoom_;
bool enable_pan_;
@@ -36,6 +38,9 @@ protected:
FTLImageView* imview_;
nanogui::Window *context_menu_;
+private:
+ StereoImageView* stereoim_;
+
public:
EIGEN_MAKE_ALIGNED_OPERATOR_NEW
};
--
GitLab