Skip to content
Snippets Groups Projects
Commit de3d0aad authored by Sebastian Hahta's avatar Sebastian Hahta
Browse files

WIP: (major) gui refactor

parent 653c52ea
No related branches found
No related tags found
1 merge request!316Resolves #343 GUI and Frame Refactor
Showing
with 880 additions and 143 deletions
...@@ -394,8 +394,8 @@ if (WIN32) # TODO(nick) Should do based upon compiler (VS) ...@@ -394,8 +394,8 @@ if (WIN32) # TODO(nick) Should do based upon compiler (VS)
set(OS_LIBS "") set(OS_LIBS "")
else() else()
add_definitions(-DUNIX) add_definitions(-DUNIX)
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++17 -fPIC -msse3 -Wall") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++17 -fPIC -msse3 -Wall -Werror=return-type")
set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -D_DEBUG -pg") set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -O0 -D_DEBUG -pg")
set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} -O3 -mfpmath=sse") set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} -O3 -mfpmath=sse")
set(OS_LIBS "dl") set(OS_LIBS "dl")
endif() endif()
......
...@@ -2,17 +2,34 @@ ...@@ -2,17 +2,34 @@
#include_directories(${PROJECT_SOURCE_DIR}/reconstruct/include) #include_directories(${PROJECT_SOURCE_DIR}/reconstruct/include)
#include_directories(${PROJECT_BINARY_DIR}) #include_directories(${PROJECT_BINARY_DIR})
function(add_gui_module NAME)
list(APPEND GUI2SRC "src/modules/${NAME}/control.cpp")
get_filename_component(FULLPATH "src/modules/${NAME}/view.cpp" ABSOLUTE)
if (EXISTS ${FULLPATH})
list(APPEND GUI2SRC "src/modules/${NAME}/view.cpp")
endif()
set(GUI2SRC ${GUI2SRC} PARENT_SCOPE)
endfunction()
set(GUI2SRC set(GUI2SRC
src/main.cpp src/main.cpp
src/inputoutput.cpp src/inputoutput.cpp
src/screen.cpp src/screen.cpp
src/view.cpp src/view.cpp
src/gltexture.cpp src/gltexture.cpp
src/modules/home.cpp src/frameview.cpp
) )
add_gui_module("config")
add_gui_module("camera")
add_gui_module("thumbnails")
add_gui_module("renderer")
add_gui_module("record")
if (HAVE_OPENVR) if (HAVE_OPENVR)
list(APPEND GUI2SRC "src/vr/vr.cpp") #list(APPEND GUI2SRC "src/vr/vr.cpp")
endif() endif()
# Various preprocessor definitions have been generated by NanoGUI # Various preprocessor definitions have been generated by NanoGUI
......
GUI
Nanogui based graphical user interface.
General:
* Do not modify gui outside gui thread (main). Modifications must be done in
GUI callbacks or draw().
* Expensive processing should be moved out of gui thread (draw() and callbacks)
* Module is only required to implement Module. Each module is expected to be
loaded only once (this design decision could be modified).
Classes
Screen
* Implements main screen: toolbar and view
* Interface for registering new modules.
* Interface for adding/removing buttons
* Interface for setting active View. Inactive view is removed and destroyed if
no other references are remaining.
* Note: toolbar could be a module, but other modules likely assume it is
always available anyways.
Module (controller)
* GUI module class wraps pointers for io, config and net. Initialization should
add necessary buttons to Screen
* Build necessary callbacks to process data from InputOutput to view.
Note: If callback passes data to view, callback handle should be owned by
the view or Module has to keep a nanogui reference to the View.
View
* active view will be the main window; only one view can be active at time
* button callbacks (eg. those registered by module init) may change active view
* Destroyed when view is changed. Object lifetime can be used to remove
callbacks from InputOutput (TODO: only one active callback supported at the
moment)
* Implementations do not have to inherit from View. Popup/Window/Widget... can
be used to implement UI components available from any mode (config, record).
InputOutput
* Contains all relevant datastructures and objects for FTL system.
* Network
* Pipelines (incl. rendering) TODO
* Audio player TODO
* Recording TODO
NanoGUI notes:
* Window instances can only be deleted with dispose().
* If sub-windows are created, they have to hold to a reference to parent
object if they share resources (eg. in lambda callbacks
[ref = std::move(ref)] could be used).
#include <loguru.hpp>
#include <nanogui/screen.h>
#include <ftl/cuda_common.hpp>
#include <ftl/cuda_texture.hpp>
#include <ftl/exception.hpp>
#include "gltexture.hpp"
#include "frameview.hpp"
using ftl::gui2::FrameView;
namespace {
// modify gl_Position to flip y axis (flip sign)
constexpr char const *const GLFrameViewVertexShader =
R"(#version 330
uniform vec2 scaleFactor;
uniform vec2 position;
in vec2 vertex;
out vec2 uv;
void main() {
uv = vertex;
vec2 scaledVertex = (vertex * scaleFactor) + position;
gl_Position = vec4(2.0*scaledVertex.x - 1.0,
1.0 - 2.0*scaledVertex.y,
0.0, 1.0);
})";
// set color.w = 1.0f ; possibly bug: incorrect value from getTexture()?
constexpr char const *const GLFrameViewFragmentShader =
R"(#version 330
uniform sampler2D image;
out vec4 color;
in vec2 uv;
void main() {
color = texture(image, uv);
color.w = 1.0f;
})";
}
void buildFrameViewShader(nanogui::GLShader &shader) {
shader.init("GLFrameViewShader", GLFrameViewVertexShader,
GLFrameViewFragmentShader);
nanogui::MatrixXu indices(3, 2);
indices.col(0) << 0, 1, 2;
indices.col(1) << 2, 3, 1;
nanogui::MatrixXf vertices(2, 4);
vertices.col(0) << 0, 0;
vertices.col(1) << 1, 0;
vertices.col(2) << 0, 1;
vertices.col(3) << 1, 1;
shader.bind();
shader.uploadIndices(indices);
shader.uploadAttrib("vertex", vertices);
}
FrameView::FrameView(nanogui::Widget *parent) :
Widget(parent), texture(ftl::gui2::GLTexture::Type::BGRA) {
if (glfwGetCurrentContext() == nullptr) {
throw FTL_Error("No current OpenGL context");
}
buildFrameViewShader(mShader);
}
void FrameView::draw(NVGcontext *ctx) {
Widget::draw(ctx);
if (flush_) {
nvgEndFrame(ctx); // Flush the NanoVG draw stack, not necessary to call nvgBeginFrame afterwards.
}
if (copy) {
auto &buffer = frame.createTexture<uchar4>(channel, true);
texture.make(buffer.width(), buffer.height());
auto dst = texture.map(stream);
cudaMemcpy2D(dst.data, dst.step1(), buffer.devicePtr(), buffer.pitch(), buffer.width()*4, buffer.height(), cudaMemcpyDeviceToDevice);
texture.unmap(stream);
copy = false;
}
if (!texture.isValid()) {
return;
}
const nanogui::Vector2f imageSize = nanogui::Vector2f(texture.width(), texture.height());
const nanogui::Vector2f widgetSize = size().cast<float>();
const nanogui::Vector2f screenSize = screen()->size().cast<float>();
// scale to fill screen
float scale = widgetSize.cwiseQuotient(imageSize).minCoeff();
const nanogui::Vector2f scaleFactor = imageSize.cwiseQuotient(screenSize)* scale;
const nanogui::Vector2f scaledImageSize = scale * imageSize;
// center
const nanogui::Vector2f offset = (widgetSize - scaledImageSize) / 2;
const nanogui::Vector2f positionInScreen = absolutePosition().cast<float>() + offset;
const nanogui::Vector2f imagePosition = positionInScreen.cwiseQuotient(screenSize);
mShader.bind();
glEnable(GL_SCISSOR_TEST);
float r = screen()->pixelRatio();
glScissor(positionInScreen.x() * r,
(screenSize.y() - positionInScreen.y() - size().y()) * r,
size().x() * r, size().y() * r);
glActiveTexture(GL_TEXTURE0);
glBindTexture(GL_TEXTURE_2D, texture.texture());
mShader.setUniform("image", 0);
mShader.setUniform("scaleFactor", scaleFactor);
mShader.setUniform("position", imagePosition);
mShader.drawIndexed(GL_TRIANGLES, 0, 2);
glDisable(GL_SCISSOR_TEST);
}
void FrameView::set(ftl::rgbd::Frame &f, ftl::codecs::Channel c, cudaStream_t s, bool cp) {
f.swapTo(frame);
channel = c;
stream = s;
if (cp) {
copy = true;
}
}
#include <ftl/rgbd/frame.hpp>
#include <nanogui/widget.h>
#include <nanogui/glutil.h>
namespace ftl {
namespace gui2 {
/// Widget for ftl::cuda::TextureObject<uchar4>
class FrameView : public nanogui::Widget {
public:
FrameView(nanogui::Widget* parent);
virtual void draw(NVGcontext *ctx) override;
/** Set frame. If copy == true, buffer will be copied to OpenGL framebuffer
* at next draw() call. */
void set(ftl::rgbd::Frame &frame, ftl::codecs::Channel channel, cudaStream_t stream=0, bool copy=false);
/** should NanoVG draw stack be flushed before drawing the texture? */
void setFlush(bool v) { flush_ = v; }
private:
ftl::rgbd::Frame frame;
ftl::codecs::Channel channel;
cudaStream_t stream;
GLTexture texture;
nanogui::GLShader mShader;
std::atomic<bool> copy = false;
bool flush_ = false;
};
}
}
...@@ -16,7 +16,6 @@ GLTexture::GLTexture(GLTexture::Type type) { ...@@ -16,7 +16,6 @@ GLTexture::GLTexture(GLTexture::Type type) {
cuda_res_ = nullptr; cuda_res_ = nullptr;
width_ = 0; width_ = 0;
height_ = 0; height_ = 0;
changed_ = true;
type_ = type; type_ = type;
} }
...@@ -24,31 +23,6 @@ GLTexture::~GLTexture() { ...@@ -24,31 +23,6 @@ GLTexture::~GLTexture() {
//glDeleteTextures(1, &glid_); //glDeleteTextures(1, &glid_);
} }
void GLTexture::update(cv::Mat &m) {
LOG(INFO) << "DEPRECATED";
if (m.rows == 0) return;
if (glid_ == std::numeric_limits<unsigned int>::max()) {
glGenTextures(1, &glid_);
glBindTexture(GL_TEXTURE_2D, glid_);
//cv::Mat m(cv::Size(100,100), CV_8UC3);
if (type_ == Type::BGRA) {
glTexImage2D(GL_TEXTURE_2D, 0, GL_RGBA8, m.cols, m.rows, 0, GL_BGRA, GL_UNSIGNED_BYTE, m.data);
} else if (type_ == Type::Float) {
glTexImage2D(GL_TEXTURE_2D, 0, GL_R32F, m.cols, m.rows, 0, GL_RED, GL_FLOAT, m.data);
}
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_NEAREST);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR);
} else {
//glBindTexture(GL_TEXTURE_2D, glid_);
// TODO Allow for other formats
//glTexImage2D(GL_TEXTURE_2D, 0, GL_RGBA8, m.cols, m.rows, 0, GL_BGRA, GL_UNSIGNED_BYTE, m.data);
}
auto err = glGetError();
if (err != 0) LOG(ERROR) << "OpenGL Texture error: " << err;
}
void GLTexture::make(int width, int height) { void GLTexture::make(int width, int height) {
if (width != width_ || height != height_) { if (width != width_ || height != height_) {
free(); free();
...@@ -114,6 +88,7 @@ void GLTexture::free() { ...@@ -114,6 +88,7 @@ void GLTexture::free() {
} }
cv::cuda::GpuMat GLTexture::map(cudaStream_t stream) { cv::cuda::GpuMat GLTexture::map(cudaStream_t stream) {
mtx_.lock();
void *devptr; void *devptr;
size_t size; size_t size;
cudaSafeCall(cudaGraphicsMapResources(1, &cuda_res_, stream)); cudaSafeCall(cudaGraphicsMapResources(1, &cuda_res_, stream));
...@@ -122,8 +97,9 @@ cv::cuda::GpuMat GLTexture::map(cudaStream_t stream) { ...@@ -122,8 +97,9 @@ cv::cuda::GpuMat GLTexture::map(cudaStream_t stream) {
} }
void GLTexture::unmap(cudaStream_t stream) { void GLTexture::unmap(cudaStream_t stream) {
// note: code must not throw, otherwise mtx_.unlock() does not happen
cudaSafeCall(cudaGraphicsUnmapResources(1, &cuda_res_, stream)); cudaSafeCall(cudaGraphicsUnmapResources(1, &cuda_res_, stream));
changed_ = true;
//glActiveTexture(GL_TEXTURE0); //glActiveTexture(GL_TEXTURE0);
glBindBuffer( GL_PIXEL_UNPACK_BUFFER, glbuf_); glBindBuffer( GL_PIXEL_UNPACK_BUFFER, glbuf_);
...@@ -139,6 +115,8 @@ void GLTexture::unmap(cudaStream_t stream) { ...@@ -139,6 +115,8 @@ void GLTexture::unmap(cudaStream_t stream) {
glPixelStorei(GL_UNPACK_ROW_LENGTH, 0); glPixelStorei(GL_UNPACK_ROW_LENGTH, 0);
glBindTexture(GL_TEXTURE_2D, 0); glBindTexture(GL_TEXTURE_2D, 0);
glBindBuffer( GL_PIXEL_UNPACK_BUFFER, 0); glBindBuffer( GL_PIXEL_UNPACK_BUFFER, 0);
mtx_.unlock();
} }
unsigned int GLTexture::texture() const { unsigned int GLTexture::texture() const {
...@@ -153,6 +131,6 @@ unsigned int GLTexture::texture() const { ...@@ -153,6 +131,6 @@ unsigned int GLTexture::texture() const {
return glid_; return glid_;
} else { } else {
return glid_; throw FTL_Error("No OpenGL texture; use make() first");
} }
} }
#ifndef _FTL_GUI_GLTEXTURE_HPP_ #pragma once
#define _FTL_GUI_GLTEXTURE_HPP_
#include <opencv2/core/mat.hpp> #include <ftl/cuda_common.hpp>
#include <cuda_runtime.h>
struct cudaGraphicsResource; struct cudaGraphicsResource;
...@@ -21,32 +18,34 @@ class GLTexture { ...@@ -21,32 +18,34 @@ class GLTexture {
explicit GLTexture(Type); explicit GLTexture(Type);
~GLTexture(); ~GLTexture();
void update(cv::Mat &m);
void make(int width, int height);
unsigned int texture() const;
bool isValid() const { return glid_ != std::numeric_limits<unsigned int>::max(); } bool isValid() const { return glid_ != std::numeric_limits<unsigned int>::max(); }
int width() const { return width_; }
int height() const { return height_; }
cv::cuda::GpuMat map(cudaStream_t stream); std::mutex& mutex() { return mtx_; }
void unmap(cudaStream_t stream);
// acquire mutex before make() or free()
void make(int width, int height);
void free(); void free();
unsigned int texture() const;
int width() const { return width_; } cv::cuda::GpuMat map(cudaStream_t stream);
int height() const { return height_; } void unmap(cudaStream_t stream);
private: private:
unsigned int glid_; unsigned int glid_;
unsigned int glbuf_; unsigned int glbuf_;
int width_; int width_;
int height_; int height_;
int stride_; int stride_;
bool changed_;
Type type_; Type type_;
std::mutex mtx_; // for locking while in use (opengl thread calls lock() or cuda mapped)
cudaGraphicsResource *cuda_res_; cudaGraphicsResource *cuda_res_;
}; };
} }
} }
#endif // _FTL_GUI_GLTEXTURE_HPP_
...@@ -8,11 +8,13 @@ using ftl::gui2::InputOutput; ...@@ -8,11 +8,13 @@ using ftl::gui2::InputOutput;
using ftl::codecs::Channel; using ftl::codecs::Channel;
InputOutput::InputOutput(ftl::Configurable *root, ftl::net::Universe *net) : net_(net) { InputOutput::InputOutput(ftl::Configurable *root, ftl::net::Universe *net) :
audio_callbacks_(), video_callbacks_(), net_(net) {
UNIQUE_LOCK(mtx_, lk); UNIQUE_LOCK(mtx_, lk);
controller_ = std::unique_ptr<ftl::ctrl::Master>(new ftl::ctrl::Master(root, net)); master_ = std::unique_ptr<ftl::ctrl::Master>(new ftl::ctrl::Master(root, net));
controller_->onLog([](const ftl::ctrl::LogEvent &e){ master_->onLog([](const ftl::ctrl::LogEvent &e){
const int v = e.verbosity; const int v = e.verbosity;
switch (v) { switch (v) {
case -2: LOG(ERROR) << "Remote log: " << e.message; break; case -2: LOG(ERROR) << "Remote log: " << e.message; break;
...@@ -78,11 +80,16 @@ InputOutput::InputOutput(ftl::Configurable *root, ftl::net::Universe *net) : net ...@@ -78,11 +80,16 @@ InputOutput::InputOutput(ftl::Configurable *root, ftl::net::Universe *net) : net
} }
}); });
receiver_->onFrameSet([this](ftl::rgbd::FrameSet &fs){ return process_callbacks_video(fs); }); receiver_->onFrameSet([this](ftl::rgbd::FrameSet &fs){
//processFrameSet_(fs);
video_callbacks_.trigger(fs);
return true;
});
speaker_ = std::unique_ptr<ftl::audio::Speaker>(ftl::create<ftl::audio::Speaker>(root, "speaker_test")); speaker_ = std::unique_ptr<ftl::audio::Speaker>(ftl::create<ftl::audio::Speaker>(root, "speaker_test"));
receiver_->onAudio([this](ftl::audio::FrameSet &fs) { receiver_->onAudio([this](ftl::audio::FrameSet &fs) {
audio_callbacks_.trigger(fs);
/* /*
if (framesets_.size() == 0) return true; if (framesets_.size() == 0) return true;
auto *c = screen_->activeCamera(); auto *c = screen_->activeCamera();
...@@ -91,7 +98,8 @@ InputOutput::InputOutput(ftl::Configurable *root, ftl::net::Universe *net) : net ...@@ -91,7 +98,8 @@ InputOutput::InputOutput(ftl::Configurable *root, ftl::net::Universe *net) : net
speaker_->queue(fs.timestamp, fs.frames[0]); speaker_->queue(fs.timestamp, fs.frames[0]);
//LOG(INFO) << "Audio delay = " << (fs.timestamp - framesets_[0]->timestamp + renddelay); //LOG(INFO) << "Audio delay = " << (fs.timestamp - framesets_[0]->timestamp + renddelay);
*/return true; */
return true;
}); });
/*ftl::timer::add(ftl::timer::kTimerMain, [this](int64_t ts) { /*ftl::timer::add(ftl::timer::kTimerMain, [this](int64_t ts) {
...@@ -155,28 +163,71 @@ InputOutput::InputOutput(ftl::Configurable *root, ftl::net::Universe *net) : net ...@@ -155,28 +163,71 @@ InputOutput::InputOutput(ftl::Configurable *root, ftl::net::Universe *net) : net
stream_->begin(); stream_->begin();
} }
bool InputOutput::process_callbacks_video(ftl::rgbd::FrameSet &fs) { void InputOutput::processFrameSet_(ftl::rgbd::FrameSet &fs) {
UNIQUE_LOCK(mtx_, lk); // Request the channels required by current camera configuration
for (auto &f : cb_video_) { /*if (fromstream) {
f(fs); auto cs = _aggregateChannels(fs.id);
auto avail = static_cast<const ftl::stream::Stream*>(interceptor_)->available(fs.id);
if (cs.has(Channel::Depth) && !avail.has(Channel::Depth) && avail.has(Channel::Right)) {
cs -= Channel::Depth;
cs += Channel::Right;
} }
return true; interceptor_->select(fs.id, cs);
}*/
// Make sure there are enough framesets allocated
/*{
UNIQUE_LOCK(mutex_, lk);
_checkFrameSets(fs.id);
}*/
// !config()->value("drop_partial_framesets", false)
if (!fs.test(ftl::data::FSFlag::PARTIAL)) {
// Enforce interpolated colour and GPU upload
for (size_t i=0; i<fs.frames.size(); ++i) {
if (!fs.hasFrame(i)) {
continue;
} }
fs.frames[i].createTexture<uchar4>(Channel::Colour, true);
bool InputOutput::process_callbacks_audio(ftl::audio::FrameSet &fs) { // TODO: Do all channels. This is a fix for screen capture sources.
UNIQUE_LOCK(mtx_, lk); // pre_pipelines_[fs.id]->getStream()
for (auto &f : cb_audio_) { if (!fs.frames[i].isGPU(Channel::Colour)) {
f(fs); fs.frames[i].upload(ftl::codecs::Channels<0>(Channel::Colour), 0);
} }
return true;
} }
void InputOutput::subscribe(const std::function<bool(ftl::rgbd::FrameSet&)> &f) { //fs.mask &= pre_pipelines_[fs.id]->value("frame_mask", 0xFFFF);
UNIQUE_LOCK(mtx_, lk);
cb_video_.push_back(f); /*{
FTL_Profile("Prepipe",0.020);
pre_pipelines_[fs.id]->apply(fs, fs, 0);
}*/
//fs.swapTo(*framesets_[fs.id]);
} else {
LOG(WARNING) << "Dropping frameset: " << fs.timestamp;
} }
void InputOutput::subscribe(const std::function<bool(ftl::audio::FrameSet&)> &f) { /*
UNIQUE_LOCK(mtx_, lk); size_t i=0;
cb_audio_.push_back(f); for (auto cam : cameras_) {
// Only update the camera periodically unless the active camera
if (screen_->activeCamera() == cam.second.camera ||
(screen_->activeCamera() == nullptr && cycle_ % cameras_.size() == i++)) cam.second.camera->update(framesets_);
ftl::codecs::Channels<0> channels;
if (fromstream) channels = cstream->available(fs.id);
//if ((*framesets_[fs.id]).frames.size() > 0) channels += (*framesets_[fs.id]).frames[0].getChannels();
cam.second.camera->update(fs.id, channels);
}*/
}
ftl::Handle InputOutput::addCallback(const std::function<bool(ftl::rgbd::FrameSet&)> f) {
return video_callbacks_.on(f);
}
ftl::Handle InputOutput::addCallback(const std::function<bool(ftl::audio::FrameSet&)> f) {
return audio_callbacks_.on(f);
} }
...@@ -3,6 +3,7 @@ ...@@ -3,6 +3,7 @@
#include <memory> #include <memory>
#include <mutex> #include <mutex>
#include <ftl/handle.hpp>
#include <ftl/configuration.hpp> #include <ftl/configuration.hpp>
#include <ftl/net/universe.hpp> #include <ftl/net/universe.hpp>
#include <ftl/master.hpp> #include <ftl/master.hpp>
...@@ -21,33 +22,27 @@ public: ...@@ -21,33 +22,27 @@ public:
InputOutput(const InputOutput&) = delete; InputOutput(const InputOutput&) = delete;
void operator=(const InputOutput&) = delete; void operator=(const InputOutput&) = delete;
void subscribe(const std::function<bool(ftl::rgbd::FrameSet&)>&); ftl::Handle addCallback(std::function<bool(ftl::rgbd::FrameSet&)>);
void subscribe(const std::function<bool(ftl::audio::FrameSet&)>&); ftl::Handle addCallback(std::function<bool(ftl::audio::FrameSet&)>);
void unsubscribe();
void unsubscribe_audio();
void unsubscribe_video();
ftl::net::Universe* net() const; ftl::net::Universe* net() const;
ftl::ctrl::Master* master() const { return master_.get(); };
private: private:
bool process_callbacks_video(ftl::rgbd::FrameSet &fs); ftl::Handler<ftl::audio::FrameSet&> audio_callbacks_;
bool process_callbacks_audio(ftl::audio::FrameSet &fs); ftl::Handler<ftl::rgbd::FrameSet&> video_callbacks_;
void processFrameSet_(ftl::rgbd::FrameSet &fs);
std::mutex mtx_; std::mutex mtx_;
ftl::net::Universe* net_; ftl::net::Universe* net_;
std::unique_ptr<ftl::ctrl::Master> controller_; std::unique_ptr<ftl::ctrl::Master> master_;
std::unique_ptr<ftl::stream::Muxer> stream_; std::unique_ptr<ftl::stream::Muxer> stream_;
std::unique_ptr<ftl::stream::Intercept> interceptor_; std::unique_ptr<ftl::stream::Intercept> interceptor_;
std::unique_ptr<ftl::stream::File> recorder_; std::unique_ptr<ftl::stream::File> recorder_;
std::unique_ptr<ftl::stream::Receiver> receiver_; std::unique_ptr<ftl::stream::Receiver> receiver_;
std::unique_ptr<ftl::audio::Speaker> speaker_; std::unique_ptr<ftl::audio::Speaker> speaker_;
//std::unordered_map<std::string, ftl::stream::Stream*> available_;
std::vector<uintptr_t> cb_video_id_;
std::vector<uintptr_t> cb_audio_id_;
std::vector<std::function<bool(ftl::rgbd::FrameSet&)>> cb_video_;
std::vector<std::function<bool(ftl::audio::FrameSet&)>> cb_audio_;
int frameset_counter_ = 0; int frameset_counter_ = 0;
}; };
......
...@@ -12,77 +12,120 @@ ...@@ -12,77 +12,120 @@
#include <cuda_gl_interop.h> #include <cuda_gl_interop.h>
#include "inputoutput.hpp" #include "inputoutput.hpp"
#include "modules.hpp" #include "module.hpp"
#include "screen.hpp" #include "screen.hpp"
#include "modules.hpp"
using std::unique_ptr; using std::unique_ptr;
using std::make_unique;
/**
* FTL Graphical User Interface
* Single screen, loads configuration and sets up networking and input/output.
* Loads required modules to gui.
*/
class FTLGui {
public:
FTLGui(int argc, char **argv);
~FTLGui();
template<typename T>
T* loadModule(const std::string &name);
void mainloop();
private:
std::unique_ptr<ftl::Configurable> root_;
std::unique_ptr<ftl::net::Universe> net_;
std::unique_ptr<ftl::gui2::InputOutput> io_;
nanogui::ref<ftl::gui2::Screen> screen_;
};
template<typename T>
T* FTLGui::loadModule(const std::string &name) {
return screen_->addModule<T>(name, root_.get(), screen_.get(), io_.get());
}
int main(int argc, char **argv) { FTLGui::FTLGui(int argc, char **argv) {
using namespace ftl::gui2;
screen_ = new Screen();
int cuda_device; int cuda_device;
cudaSafeCall(cudaGetDevice(&cuda_device)); cudaSafeCall(cudaGetDevice(&cuda_device));
//cudaSafeCall(cudaGLSetGLDevice(cuda_device)); //cudaSafeCall(cudaGLSetGLDevice(cuda_device));
auto root = unique_ptr<ftl::Configurable>(ftl::configure(argc, argv, "gui_default")); root_ = unique_ptr<ftl::Configurable>(ftl::configure(argc, argv, "gui_default"));
auto net = unique_ptr<ftl::net::Universe>(ftl::create<ftl::net::Universe>(root.get(), "net")); net_ = unique_ptr<ftl::net::Universe>(ftl::create<ftl::net::Universe>(root_.get(), "net"));
io_ = make_unique<ftl::gui2::InputOutput>(root_.get(), net_.get());
net_->start();
net_->waitConnections();
loadModule<ThumbnailsController>("home")->activate();
loadModule<Camera>("camera");
loadModule<ConfigWindowController>("configwindow");
loadModule<Renderer>("renderer");
//loadModule<RecordController>("record");
}
auto io = ftl::gui2::InputOutput(root.get(), net.get()); FTLGui::~FTLGui() {
net_->shutdown();
}
net->start(); void FTLGui::mainloop() {
net->waitConnections(); // implements similar main loop as nanogui::mainloop()
ftl::timer::start(); ftl::timer::start();
try { screen_->setVisible(true);
nanogui::init(); screen_->drawAll();
{
nanogui::ref<ftl::gui2::Screen> gui =
new ftl::gui2::Screen();
nanogui::ref<ftl::gui2::HomeView> home =
new ftl::gui2::HomeView(root.get(), &io);
gui->setView(home);
gui->drawAll();
gui->setVisible(true);
float last_draw_time = 0.0f; float last_draw_time = 0.0f;
while (ftl::running) { while (ftl::running) {
if (!gui->visible()) { if (!screen_->visible()) {
ftl::running = false; ftl::running = false;
} }
else if (glfwWindowShouldClose(gui->glfwWindow())) { else if (glfwWindowShouldClose(screen_->glfwWindow())) {
gui->setVisible(false); screen_->setVisible(false);
ftl::running = false; ftl::running = false;
} }
else { else {
float now = float(glfwGetTime()); float now = float(glfwGetTime());
float delta = now - last_draw_time; //float delta = now - last_draw_time;
// Generate poses and render and virtual frame here // Generate poses and render and virtual frame here
// at full FPS (25 without VR and 90 with VR currently) // at full FPS (25 without VR and 90 with VR currently)
//gui->drawFast(); //screen_->render();
// Only draw the GUI at 25fps // Only draw the GUI at 25fps
if (delta >= 0.04f) { //if (delta >= 0.04f) {
last_draw_time = now; last_draw_time = now;
gui->drawAll(); screen_->drawAll();
} //}
} }
// Wait for mouse/keyboard or empty refresh events // Wait for mouse/keyboard or empty refresh events
glfwPollEvents(); glfwWaitEvents(); // VR headest issues
//glfwPollEvents();
} }
// Process events once more // Process events once more
glfwPollEvents(); glfwPollEvents();
LOG(INFO) << "Stopping...";
ftl::timer::stop(false);
ftl::pool.stop(true);
LOG(INFO) << "All threads stopped.";
} }
nanogui::shutdown(); ////////////////////////////////////////////////////////////////////////////////
int main(int argc, char **argv) {
nanogui::init();
FTLGui gui(argc, argv);
try {
gui.mainloop();
} }
catch (const ftl::exception &e) { catch (const ftl::exception &e) {
...@@ -99,6 +142,12 @@ int main(int argc, char **argv) { ...@@ -99,6 +142,12 @@ int main(int argc, char **argv) {
return -1; return -1;
} }
net->shutdown(); nanogui::shutdown();
LOG(INFO) << "Stopping...";
ftl::timer::stop(false);
ftl::pool.stop(true);
LOG(INFO) << "All threads stopped.";
return 0; return 0;
} }
#pragma once
#include "view.hpp"
#include "inputoutput.hpp"
#include <ftl/configurable.hpp>
#include <nanogui/entypo.h>
#include <nanogui/button.h>
namespace ftl {
namespace gui2 {
class Screen;
class Module : public ftl::Configurable {
public:
Module(nlohmann::json &config, Screen *screen, InputOutput *io) :
Configurable(config), screen(screen), io(io) {}
/// Cerform any initialization
virtual void init() {};
virtual ~Module() {};
protected:
ftl::gui2::Screen* const screen;
ftl::gui2::InputOutput* const io;
};
}
}
#pragma once #pragma once
#include "modules/home.hpp"
#include "modules/thumbnails/control.hpp"
#include "modules/record/control.hpp"
#include "modules/camera/control.hpp"
#include "modules/config/control.hpp"
#include "modules/renderer/control.hpp"
#include "control.hpp"
#include "view.hpp"
using ftl::gui2::Camera;
void Camera::activate() {
auto view = new ftl::gui2::CameraView(screen);
view->setHandle(
io->addCallback([this, view](ftl::rgbd::FrameSet& fs){
view->update(fs, source_idx);
screen->redraw();
return true;
}));
screen->setView(view);
}
/*void Camera::deactivate() {
io->removeCallbackVideo();
io->removeCallbackAudio();
}*/
void Camera::setSource(int idx) {
source_idx = idx;
}
#pragma once
#include "../../module.hpp"
#include "../../screen.hpp"
namespace ftl {
namespace gui2 {
class Camera : public Module {
public:
using Module::Module;
virtual void activate();
void setSource(int);
private:
int source_idx = -1;
};
}
}
#include <nanogui/screen.h>
#include "view.hpp"
using ftl::gui2::CameraView;
CameraView::CameraView(nanogui::Widget* parent) : View(parent) {
fview = new ftl::gui2::FrameView(this);
fview->setFlush(false); // buffer will be rendered as "background"
}
void CameraView::draw(NVGcontext *ctx) {
fview->setFixedSize(size());
performLayout(ctx);
View::draw(ctx);
}
void CameraView::update(ftl::rgbd::FrameSet &fs, int idx) {
auto channel = ftl::codecs::Channel::Colour;
if (!fs.hasFrame(idx)) {
return;
}
auto &frame = fs.frames[idx];
if (!frame.hasChannel(channel)) {
return;
}
fview->set(frame, channel, 0, true);
}
#pragma once
#include "../../view.hpp"
#include "../../gltexture.hpp"
#include "../../frameview.hpp"
#include <nanogui/imageview.h>
namespace ftl {
namespace gui2 {
class CameraView : public View {
public:
CameraView(nanogui::Widget* parent);
virtual void draw(NVGcontext *ctx) override;
void update(ftl::rgbd::FrameSet&fs, int id);
void setHandle(ftl::Handle&& handle) { handle_ = std::move(handle); }
private:
ftl::gui2::FrameView *fview = nullptr;
ftl::Handle handle_;
};
}
}
#include "control.hpp"
using ftl::gui2::ConfigWindowController;
void ConfigWindowController::init() {
button = screen->addButton();
button->setIcon(ENTYPO_ICON_COG);
button->setTooltip("Config");
button->setCallback([this](){
button->setPushed(false);
show();
});
button->setVisible(true);
}
void ConfigWindowController::show() {
if (screen->childIndex(window) == -1) {
window = new ftl::gui2::ConfigWindow(screen, io->master());
}
window->requestFocus();
window->setVisible(true);
screen->performLayout();
}
ConfigWindowController::~ConfigWindowController() {
// remove window?
}
#pragma once
#include "../../module.hpp"
#include "../../screen.hpp"
#include "view.hpp"
namespace ftl {
namespace gui2 {
/**
* Controller for thumbnail view.
*/
class ConfigWindowController : public Module {
public:
using Module::Module;
virtual ~ConfigWindowController();
virtual void init() override;
virtual void show();
private:
nanogui::ToolButton *button;
ftl::gui2::ConfigWindow *window = nullptr;
};
}
}
#include <loguru.hpp>
#include <nanogui/layout.h>
#include <nanogui/label.h>
#include <nanogui/button.h>
#include <nanogui/entypo.h>
#include <nanogui/formhelper.h>
#include <nanogui/vscrollpanel.h>
#include <nanogui/opengl.h>
#include <nlohmann/json.hpp>
#include <vector>
#include <string>
#include "view.hpp"
using ftl::gui2::ConfigWindow;
using std::string;
using std::vector;
using ftl::config::json_t;
class SearchBox : public nanogui::TextBox {
private:
std::vector<std::string> configurables_;
Widget *buttons_;
std::string previous;
void _setVisible(const std::string &str) {
// Check whether the search string has changed to prevent
// unnecessary searching.
if (str != previous) {
for (int i = configurables_.size()-1; i >= 0; --i) {
if (configurables_[i].find(mValueTemp) != std::string::npos) {
buttons_->childAt(i)->setVisible(true);
} else {
buttons_->childAt(i)->setVisible(false);
}
}
previous = str;
}
}
public:
SearchBox(Widget *parent, std::vector<std::string> &configurables) : nanogui::TextBox(parent, ""), configurables_(configurables) {
setAlignment(TextBox::Alignment::Left);
setEditable(true);
setPlaceholder("Search");
}
~SearchBox() {
}
bool keyboardEvent(int key, int scancode, int action, int modifier) {
TextBox::keyboardEvent(key, scancode, action, modifier);
_setVisible(mValueTemp);
return true;
}
void setButtons(Widget *buttons) {
buttons_ = buttons;
}
};
static std::string titleForURI(const ftl::URI &uri) {
auto *cfg = ftl::config::find(uri.getBaseURI());
if (cfg && cfg->get<std::string>("title")) {
return *cfg->get<std::string>("title");
} else if (uri.getPath().size() > 0) {
return uri.getPathSegment(-1);
} else {
return uri.getHost();
}
}
ConfigWindow::ConfigWindow(nanogui::Widget *parent, ftl::ctrl::Master *ctrl)
: nanogui::Window(parent, "Settings"), ctrl_(ctrl) {
using namespace nanogui;
setLayout(new GroupLayout());
setPosition(Vector2i(parent->width()/2.0f - 100.0f, parent->height()/2.0f - 100.0f));
auto close = new nanogui::Button(buttonPanel(), "", ENTYPO_ICON_CROSS);
close->setCallback([this](){ dispose();});
auto configurables = ftl::config::list();
const auto size = configurables.size();
new Label(this, "Select Configurable","sans-bold");
auto searchBox = new SearchBox(this, configurables);
auto vscroll = new VScrollPanel(this);
vscroll->setFixedHeight(300);
auto buttons = new Widget(vscroll);
buttons->setLayout(new BoxLayout(Orientation::Vertical, Alignment::Fill));
searchBox->setButtons(buttons);
std::vector<std::string> configurable_titles(size);
for (size_t i = 0; i < size; ++i) {
ftl::URI uri(configurables[i]);
std::string label = uri.getFragment();
size_t pos = label.find_last_of("/");
if (pos != std::string::npos) label = label.substr(pos+1);
std::string parentName = configurables[i];
size_t pos2 = parentName.find_last_of("/");
if (pos2 != std::string::npos) parentName = parentName.substr(0,pos2);
// FIXME: Does not indicated parent indentation ... needs sorting?
if (i > 0 && parentName == configurables[i-1]) {
ftl::URI uri(configurables[i-1]);
configurable_titles[i-1] = std::string("[") + titleForURI(uri) + std::string("] ") + uri.getFragment();
auto *prev = dynamic_cast<Button*>(buttons->childAt(buttons->childCount()-1));
prev->setCaption(configurable_titles[i-1]);
prev->setBackgroundColor(nanogui::Color(0.3f,0.3f,0.3f,1.0f));
prev->setTextColor(nanogui::Color(1.0f,1.0f,1.0f,1.0f));
prev->setIconPosition(Button::IconPosition::Left);
prev->setIcon(ENTYPO_ICON_FOLDER);
}
configurable_titles[i] = label;
auto itembutton = new nanogui::Button(buttons, configurable_titles[i]);
std::string c = configurables[i];
itembutton->setTooltip(c);
itembutton->setBackgroundColor(nanogui::Color(0.9f,0.9f,0.9f,0.9f));
itembutton->setCallback([this,c]() {
_buildForm(c);
setVisible(false);
dispose();
});
}
}
ConfigWindow::~ConfigWindow() {
LOG(INFO) << "ConfigWindow::~ConfigWindow()";
}
void ConfigWindow::_addElements(nanogui::FormHelper *form, const std::string &suri) {
using namespace nanogui;
Configurable *configurable = ftl::config::find(suri);
ftl::config::json_t data;
if (configurable) {
configurable->refresh();
data = configurable->getConfig();
}
for (auto i=data.begin(); i!=data.end(); ++i) {
if (i.key() == "$id") continue;
if (i.key() == "$ref" && i.value().is_string()) {
const std::string suri = std::string(i.value().get<string>());
_addElements(form, suri);
continue;
}
if (i.value().is_boolean()) {
string key = i.key();
form->addVariable<bool>(i.key(), [this,data,key,suri](const bool &b){
ftl::config::update(suri+"/"+key, b);
}, [data,key]() -> bool {
return data[key].get<bool>();
});
} else if (i.value().is_number_integer()) {
string key = i.key();
form->addVariable<int>(i.key(), [this,data,key,suri](const int &f){
ftl::config::update(suri+"/"+key, f);
}, [data,key]() -> int {
return data[key].get<int>();
});
} else if (i.value().is_number_float()) {
string key = i.key();
form->addVariable<float>(i.key(), [this,data,key,suri](const float &f){
ftl::config::update(suri+"/"+key, f);
}, [data,key]() -> float {
return data[key].get<float>();
});
} else if (i.value().is_string()) {
string key = i.key();
form->addVariable<string>(i.key(), [this,data,key,suri](const string &f){
ftl::config::update(suri+"/"+key, f);
}, [data,key]() -> string {
return data[key].get<string>();
});
} else if (i.value().is_object()) {
string key = i.key();
// Checking the URI with exists() prevents unloaded local configurations from being shown.
if (suri.find('#') != string::npos && exists(suri+string("/")+key)) {
form->addButton(key, [this,suri,key]() {
_buildForm(suri+string("/")+key);
})->setIcon(ENTYPO_ICON_FOLDER);
} else if (exists(suri+string("#")+key)) {
form->addButton(key, [this,suri,key]() {
_buildForm(suri+string("#")+key);
})->setIcon(ENTYPO_ICON_FOLDER);
}
}
}
}
void ConfigWindow::_buildForm(const std::string &suri) {
using namespace nanogui;
ftl::URI uri(suri);
FormHelper *form = new FormHelper(this->screen());
form->addWindow(Vector2i(100,50), uri.getFragment());
form->window()->setTheme(theme());
_addElements(form, suri);
// prevent parent window from being destroyed too early
incRef();
auto close = new nanogui::Button(
form->window()->buttonPanel(),
"",
ENTYPO_ICON_CROSS);
close->setCallback([this, form](){
form->window()->dispose();
decRef();
});
form->window()->screen()->performLayout();
}
bool ConfigWindow::exists(const std::string &uri) {
return ftl::config::find(uri) != nullptr;
}
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment