Skip to content
Snippets Groups Projects
Commit a766c238 authored by Nicolas Pope's avatar Nicolas Pope
Browse files

Merge branch 'feature/guishader' into 'master' 

Feature/guishader

See merge request nicolas.pope/ftl!26
parents f7c8434f d13c3649
No related branches found
No related tags found
1 merge request!26Feature/guishader
Pipeline #11296 passed
Stage: all
Hide whitespace changes
Inline Side-by-side
applications/gui/src/main.cpp
+ 216
− 2
View file @ a766c238
......@@ -29,6 +29,88 @@ using ftl::rgbd::Source;
};*/
namespace {
constexpr char const *const defaultImageViewVertexShader =
R"(#version 330
uniform vec2 scaleFactor;
uniform vec2 position;
in vec2 vertex;
out vec2 uv;
void main() {
uv = vertex;
vec2 scaledVertex = (vertex * scaleFactor) + position;
gl_Position = vec4(2.0*scaledVertex.x - 1.0,
1.0 - 2.0*scaledVertex.y,
0.0, 1.0);
})";
constexpr char const *const defaultImageViewFragmentShader =
R"(#version 330
uniform sampler2D image;
out vec4 color;
in vec2 uv;
void main() {
color = texture(image, uv);
})";
class GLTexture {
public:
GLTexture() {
glGenTextures(1, &glid_);
glBindTexture(GL_TEXTURE_2D, glid_);
cv::Mat m(cv::Size(100,100), CV_8UC3);
glTexImage2D(GL_TEXTURE_2D, 0, GL_RGB8, m.cols, m.rows, 0, GL_RGB, GL_UNSIGNED_BYTE, m.data);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_NEAREST);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR);
}
~GLTexture() {
glDeleteTextures(1, &glid_);
}
void update(cv::Mat &m) {
if (m.rows == 0) return;
glBindTexture(GL_TEXTURE_2D, glid_);
// TODO Allow for other formats
glTexImage2D(GL_TEXTURE_2D, 0, GL_RGB8, m.cols, m.rows, 0, GL_BGR, GL_UNSIGNED_BYTE, m.data);
auto err = glGetError();
if (err != 0) LOG(ERROR) << "OpenGL Texture error: " << err;
}
unsigned int texture() const { return glid_; }
private:
unsigned int glid_;
};
template<class T>
Eigen::Matrix<T,4,4> lookAt
(
Eigen::Matrix<T,3,1> const & eye,
Eigen::Matrix<T,3,1> const & center,
Eigen::Matrix<T,3,1> const & up
)
{
typedef Eigen::Matrix<T,4,4> Matrix4;
typedef Eigen::Matrix<T,3,1> Vector3;
Vector3 f = (center - eye).normalized();
Vector3 u = up.normalized();
Vector3 s = f.cross(u).normalized();
u = s.cross(f);
Matrix4 res;
res << s.x(),s.y(),s.z(),-s.dot(eye),
u.x(),u.y(),u.z(),-u.dot(eye),
-f.x(),-f.y(),-f.z(),f.dot(eye),
0,0,0,1;
return res;
}
}
class FTLApplication : public nanogui::Screen {
public:
......@@ -37,22 +119,154 @@ class FTLApplication : public nanogui::Screen {
net_ = net;
auto cwindow = new ftl::gui::ControlWindow(this, controller);
auto swindow = new ftl::gui::SourceWindow(this, controller);
swindow_ = new ftl::gui::SourceWindow(this, controller);
//src_ = nullptr;
eye_ = Eigen::Vector3f(0.0f, 0.0f, 0.0f);
centre_ = Eigen::Vector3f(0.0f, 0.0f, -4.0f);
up_ = Eigen::Vector3f(0,1.0f,0);
lookPoint_ = Eigen::Vector3f(0.0f,0.0f,-4.0f);
lerpSpeed_ = 0.4f;
depth_ = false;
mShader.init("RGBDShader", defaultImageViewVertexShader,
defaultImageViewFragmentShader);
MatrixXu indices(3, 2);
indices.col(0) << 0, 1, 2;
indices.col(1) << 2, 3, 1;
MatrixXf vertices(2, 4);
vertices.col(0) << 0, 0;
vertices.col(1) << 1, 0;
vertices.col(2) << 0, 1;
vertices.col(3) << 1, 1;
mShader.bind();
mShader.uploadIndices(indices);
mShader.uploadAttrib("vertex", vertices);
setVisible(true);
performLayout();
}
~FTLApplication() {
mShader.free();
}
bool mouseButtonEvent(const nanogui::Vector2i &p, int button, bool down, int modifiers) {
if (Screen::mouseButtonEvent(p, button, down, modifiers)) {
return true;
} else {
auto src_ = swindow_->getSource();
if (src_ && src_->isReady() && down) {
Eigen::Vector4f camPos = src_->point(p[0],p[1]);
camPos *= -1.0f;
Eigen::Vector4f worldPos = src_->getPose() * camPos;
lookPoint_ = Eigen::Vector3f(worldPos[0],worldPos[1],worldPos[2]);
LOG(INFO) << "Depth at click = " << -camPos[2];
return true;
}
return false;
}
}
bool keyboardEvent(int key, int scancode, int action, int modifiers) {
if (Screen::keyboardEvent(key, scancode, action, modifiers)) {
return true;
} else {
LOG(INFO) << "Key press" << key << " - " << action;
if (key == 81 || key == 83) {
// TODO Should rotate around lookAt object, but requires correct depth
Eigen::Quaternion<float> q; q = Eigen::AngleAxis<float>((key == 81) ? 0.01f : -0.01f, up_);
eye_ = (q * (eye_ - centre_)) + centre_;
return true;
} else if (key == 84 || key == 82) {
float scalar = (key == 84) ? 0.99f : 1.01f;
eye_ = ((eye_ - centre_) * scalar) + centre_;
return true;
}
return false;
}
}
virtual void draw(NVGcontext *ctx) {
nvgText(ctx, 10, 10, "FT-Lab Remote Presence System", NULL);
using namespace Eigen;
auto src_ = swindow_->getSource();
Vector2f imageSize(0, 0);
if (src_) {
cv::Mat rgb, depth;
centre_ += (lookPoint_ - centre_) * (lerpSpeed_ * 0.1f);
Eigen::Matrix4f viewPose = lookAt<float>(eye_,centre_,up_).inverse();
src_->setPose(viewPose);
src_->grab();
src_->getFrames(rgb, depth);
if (swindow_->getDepth()) {
if (depth.rows > 0) {
imageSize = Vector2f(depth.cols,depth.rows);
cv::Mat idepth;
depth.convertTo(idepth, CV_8U, 255.0f / 10.0f); // TODO(nick)
applyColorMap(idepth, idepth, cv::COLORMAP_JET);
texture_.update(idepth);
mImageID = texture_.texture();
}
} else {
if (rgb.rows > 0) {
imageSize = Vector2f(rgb.cols,rgb.rows);
texture_.update(rgb);
mImageID = texture_.texture();
}
}
}
if (imageSize[0] > 0) {
Vector2f screenSize = size().cast<float>();
auto mScale = (screenSize.cwiseQuotient(imageSize).minCoeff());
Vector2f scaleFactor = mScale * imageSize.cwiseQuotient(screenSize);
Vector2f positionInScreen(0.0f, 0.0f);
auto mOffset = (screenSize - (screenSize.cwiseProduct(scaleFactor))) / 2;
Vector2f positionAfterOffset = positionInScreen + mOffset;
Vector2f imagePosition = positionAfterOffset.cwiseQuotient(screenSize);
//glEnable(GL_SCISSOR_TEST);
//float r = screen->pixelRatio();
/* glScissor(positionInScreen.x() * r,
(screenSize.y() - positionInScreen.y() - size().y()) * r,
size().x() * r, size().y() * r);*/
mShader.bind();
glActiveTexture(GL_TEXTURE0);
glBindTexture(GL_TEXTURE_2D, mImageID);
mShader.setUniform("image", 0);
mShader.setUniform("scaleFactor", scaleFactor);
mShader.setUniform("position", imagePosition);
mShader.drawIndexed(GL_TRIANGLES, 0, 2);
//glDisable(GL_SCISSOR_TEST);
}
nvgText(ctx, 10, 20, "FT-Lab Remote Presence System", NULL);
/* Draw the user interface */
screen()->performLayout(ctx);
Screen::draw(ctx);
}
private:
ftl::gui::SourceWindow *swindow_;
//std::vector<SourceViews> sources_;
ftl::net::Universe *net_;
nanogui::GLShader mShader;
GLuint mImageID;
//Source *src_;
GLTexture texture_;
Eigen::Vector3f eye_;
Eigen::Vector3f centre_;
Eigen::Vector3f up_;
Eigen::Vector3f lookPoint_;
float lerpSpeed_;
bool depth_;
};
int main(int argc, char **argv) {
......
applications/gui/src/src_window.cpp
+ 13
− 11
View file @ a766c238
......@@ -159,15 +159,16 @@ SourceWindow::SourceWindow(nanogui::Widget *parent, ftl::ctrl::Master *ctrl)
using namespace nanogui;
depth_ = false;
src_ = ftl::create<Source>(ctrl->getRoot(), "source", ctrl->getNet());
Widget *tools = new Widget(this);
tools->setLayout(new BoxLayout(Orientation::Horizontal,
Alignment::Middle, 0, 6));
//Widget *tools = new Widget(this);
// tools->setLayout(new BoxLayout(Orientation::Horizontal,
// Alignment::Middle, 0, 6));
new Label(tools, "Select source","sans-bold");
new Label(this, "Select source","sans-bold");
available_ = ctrl->getNet()->findAll<string>("list_streams");
auto select = new ComboBox(tools, available_);
auto select = new ComboBox(this, available_);
select->setCallback([this,select](int ix) {
LOG(INFO) << "Change source: " << ix;
src_->set("uri", available_[ix]);
......@@ -178,10 +179,11 @@ SourceWindow::SourceWindow(nanogui::Widget *parent, ftl::ctrl::Master *ctrl)
select->setItems(available_);
});
auto depth = new Button(tools, "Depth");
auto depth = new Button(this, "Depth");
depth->setFlags(Button::ToggleButton);
depth->setChangeCallback([this](bool state) {
image_->setDepth(state);
//image_->setDepth(state);
depth_ = state;
});
#ifdef HAVE_LIBARCHIVE
......@@ -210,11 +212,11 @@ SourceWindow::SourceWindow(nanogui::Widget *parent, ftl::ctrl::Master *ctrl)
});
#endif
auto imageView = new VirtualCameraView(this);
//auto imageView = new VirtualCameraView(this);
//cam.view = imageView;
imageView->setGridThreshold(20);
imageView->setSource(src_);
image_ = imageView;
//imageView->setGridThreshold(20);
//imageView->setSource(src_);
//image_ = imageView;
}
SourceWindow::~SourceWindow() {
......
applications/gui/src/src_window.hpp
+ 4
− 0
View file @ a766c238
......@@ -21,9 +21,13 @@ class SourceWindow : public nanogui::Window {
SourceWindow(nanogui::Widget *parent, ftl::ctrl::Master *ctrl);
~SourceWindow();
ftl::rgbd::Source *getSource() const { return src_; }
bool getDepth() const { return depth_; }
private:
ftl::ctrl::Master *ctrl_;
ftl::rgbd::Source *src_;
bool depth_;
VirtualCameraView *image_;
std::vector<std::string> available_;
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment