diff --git a/applications/gui/src/main.cpp b/applications/gui/src/main.cpp
index 220dd20ce55e33e8f409435baedaab3aca338f30..47c1aa7c5b5025f2824ce4822c6e1d8aa16858fb 100644
--- a/applications/gui/src/main.cpp
+++ b/applications/gui/src/main.cpp
@@ -29,6 +29,88 @@ using ftl::rgbd::Source;
};*/
+namespace {
+ constexpr char const *const defaultImageViewVertexShader =
+ R"(#version 330
+ uniform vec2 scaleFactor;
+ uniform vec2 position;
+ in vec2 vertex;
+ out vec2 uv;
+ void main() {
+ uv = vertex;
+ vec2 scaledVertex = (vertex * scaleFactor) + position;
+ gl_Position = vec4(2.0*scaledVertex.x - 1.0,
+ 1.0 - 2.0*scaledVertex.y,
+ 0.0, 1.0);
+ })";
+
+ constexpr char const *const defaultImageViewFragmentShader =
+ R"(#version 330
+ uniform sampler2D image;
+ out vec4 color;
+ in vec2 uv;
+ void main() {
+ color = texture(image, uv);
+ })";
+
+
+ class GLTexture {
+ public:
+ GLTexture() {
+ glGenTextures(1, &glid_);
+ glBindTexture(GL_TEXTURE_2D, glid_);
+ cv::Mat m(cv::Size(100,100), CV_8UC3);
+ glTexImage2D(GL_TEXTURE_2D, 0, GL_RGB8, m.cols, m.rows, 0, GL_RGB, GL_UNSIGNED_BYTE, m.data);
+ glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
+ glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
+ glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_NEAREST);
+ glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR);
+ }
+ ~GLTexture() {
+ glDeleteTextures(1, &glid_);
+ }
+
+ void update(cv::Mat &m) {
+ if (m.rows == 0) return;
+ glBindTexture(GL_TEXTURE_2D, glid_);
+ // TODO Allow for other formats
+ glTexImage2D(GL_TEXTURE_2D, 0, GL_RGB8, m.cols, m.rows, 0, GL_BGR, GL_UNSIGNED_BYTE, m.data);
+ auto err = glGetError();
+ if (err != 0) LOG(ERROR) << "OpenGL Texture error: " << err;
+ }
+
+ unsigned int texture() const { return glid_; }
+
+ private:
+ unsigned int glid_;
+ };
+
+ template<class T>
+ Eigen::Matrix<T,4,4> lookAt
+ (
+ Eigen::Matrix<T,3,1> const & eye,
+ Eigen::Matrix<T,3,1> const & center,
+ Eigen::Matrix<T,3,1> const & up
+ )
+ {
+ typedef Eigen::Matrix<T,4,4> Matrix4;
+ typedef Eigen::Matrix<T,3,1> Vector3;
+
+ Vector3 f = (center - eye).normalized();
+ Vector3 u = up.normalized();
+ Vector3 s = f.cross(u).normalized();
+ u = s.cross(f);
+
+ Matrix4 res;
+ res << s.x(),s.y(),s.z(),-s.dot(eye),
+ u.x(),u.y(),u.z(),-u.dot(eye),
+ -f.x(),-f.y(),-f.z(),f.dot(eye),
+ 0,0,0,1;
+
+ return res;
+ }
+}
+
class FTLApplication : public nanogui::Screen {
public:
@@ -37,22 +119,154 @@ class FTLApplication : public nanogui::Screen {
net_ = net;
auto cwindow = new ftl::gui::ControlWindow(this, controller);
- auto swindow = new ftl::gui::SourceWindow(this, controller);
+ swindow_ = new ftl::gui::SourceWindow(this, controller);
+
+ //src_ = nullptr;
+ eye_ = Eigen::Vector3f(0.0f, 0.0f, 0.0f);
+ centre_ = Eigen::Vector3f(0.0f, 0.0f, -4.0f);
+ up_ = Eigen::Vector3f(0,1.0f,0);
+ lookPoint_ = Eigen::Vector3f(0.0f,0.0f,-4.0f);
+ lerpSpeed_ = 0.4f;
+ depth_ = false;
+
+ mShader.init("RGBDShader", defaultImageViewVertexShader,
+ defaultImageViewFragmentShader);
+
+ MatrixXu indices(3, 2);
+ indices.col(0) << 0, 1, 2;
+ indices.col(1) << 2, 3, 1;
+
+ MatrixXf vertices(2, 4);
+ vertices.col(0) << 0, 0;
+ vertices.col(1) << 1, 0;
+ vertices.col(2) << 0, 1;
+ vertices.col(3) << 1, 1;
+
+ mShader.bind();
+ mShader.uploadIndices(indices);
+ mShader.uploadAttrib("vertex", vertices);
setVisible(true);
performLayout();
}
+ ~FTLApplication() {
+ mShader.free();
+ }
+
+ bool mouseButtonEvent(const nanogui::Vector2i &p, int button, bool down, int modifiers) {
+ if (Screen::mouseButtonEvent(p, button, down, modifiers)) {
+ return true;
+ } else {
+ auto src_ = swindow_->getSource();
+ if (src_ && src_->isReady() && down) {
+ Eigen::Vector4f camPos = src_->point(p[0],p[1]);
+ camPos *= -1.0f;
+ Eigen::Vector4f worldPos = src_->getPose() * camPos;
+ lookPoint_ = Eigen::Vector3f(worldPos[0],worldPos[1],worldPos[2]);
+ LOG(INFO) << "Depth at click = " << -camPos[2];
+ return true;
+ }
+ return false;
+ }
+ }
+
+ bool keyboardEvent(int key, int scancode, int action, int modifiers) {
+ if (Screen::keyboardEvent(key, scancode, action, modifiers)) {
+ return true;
+ } else {
+ LOG(INFO) << "Key press" << key << " - " << action;
+ if (key == 81 || key == 83) {
+ // TODO Should rotate around lookAt object, but requires correct depth
+ Eigen::Quaternion<float> q; q = Eigen::AngleAxis<float>((key == 81) ? 0.01f : -0.01f, up_);
+ eye_ = (q * (eye_ - centre_)) + centre_;
+ return true;
+ } else if (key == 84 || key == 82) {
+ float scalar = (key == 84) ? 0.99f : 1.01f;
+ eye_ = ((eye_ - centre_) * scalar) + centre_;
+ return true;
+ }
+ return false;
+ }
+ }
+
virtual void draw(NVGcontext *ctx) {
- nvgText(ctx, 10, 10, "FT-Lab Remote Presence System", NULL);
+ using namespace Eigen;
+
+ auto src_ = swindow_->getSource();
+ Vector2f imageSize(0, 0);
+
+ if (src_) {
+ cv::Mat rgb, depth;
+ centre_ += (lookPoint_ - centre_) * (lerpSpeed_ * 0.1f);
+ Eigen::Matrix4f viewPose = lookAt<float>(eye_,centre_,up_).inverse();
+
+ src_->setPose(viewPose);
+ src_->grab();
+ src_->getFrames(rgb, depth);
+
+ if (swindow_->getDepth()) {
+ if (depth.rows > 0) {
+ imageSize = Vector2f(depth.cols,depth.rows);
+ cv::Mat idepth;
+ depth.convertTo(idepth, CV_8U, 255.0f / 10.0f); // TODO(nick)
+ applyColorMap(idepth, idepth, cv::COLORMAP_JET);
+ texture_.update(idepth);
+ mImageID = texture_.texture();
+ }
+ } else {
+ if (rgb.rows > 0) {
+ imageSize = Vector2f(rgb.cols,rgb.rows);
+ texture_.update(rgb);
+ mImageID = texture_.texture();
+ }
+ }
+ }
+
+ if (imageSize[0] > 0) {
+ Vector2f screenSize = size().cast<float>();
+ auto mScale = (screenSize.cwiseQuotient(imageSize).minCoeff());
+ Vector2f scaleFactor = mScale * imageSize.cwiseQuotient(screenSize);
+ Vector2f positionInScreen(0.0f, 0.0f);
+ auto mOffset = (screenSize - (screenSize.cwiseProduct(scaleFactor))) / 2;
+ Vector2f positionAfterOffset = positionInScreen + mOffset;
+ Vector2f imagePosition = positionAfterOffset.cwiseQuotient(screenSize);
+ //glEnable(GL_SCISSOR_TEST);
+ //float r = screen->pixelRatio();
+ /* glScissor(positionInScreen.x() * r,
+ (screenSize.y() - positionInScreen.y() - size().y()) * r,
+ size().x() * r, size().y() * r);*/
+ mShader.bind();
+ glActiveTexture(GL_TEXTURE0);
+ glBindTexture(GL_TEXTURE_2D, mImageID);
+ mShader.setUniform("image", 0);
+ mShader.setUniform("scaleFactor", scaleFactor);
+ mShader.setUniform("position", imagePosition);
+ mShader.drawIndexed(GL_TRIANGLES, 0, 2);
+ //glDisable(GL_SCISSOR_TEST);
+ }
+
+ nvgText(ctx, 10, 20, "FT-Lab Remote Presence System", NULL);
/* Draw the user interface */
+ screen()->performLayout(ctx);
Screen::draw(ctx);
}
private:
+ ftl::gui::SourceWindow *swindow_;
//std::vector<SourceViews> sources_;
ftl::net::Universe *net_;
+ nanogui::GLShader mShader;
+ GLuint mImageID;
+ //Source *src_;
+ GLTexture texture_;
+ Eigen::Vector3f eye_;
+ Eigen::Vector3f centre_;
+ Eigen::Vector3f up_;
+ Eigen::Vector3f lookPoint_;
+ float lerpSpeed_;
+ bool depth_;
};
int main(int argc, char **argv) {
diff --git a/applications/gui/src/src_window.cpp b/applications/gui/src/src_window.cpp
index 15b8c5439234b87ba6e414209f644718ed3626a9..98b760676a06099b7994ccbb0dc2db5d67b48783 100644
--- a/applications/gui/src/src_window.cpp
+++ b/applications/gui/src/src_window.cpp
@@ -159,15 +159,16 @@ SourceWindow::SourceWindow(nanogui::Widget *parent, ftl::ctrl::Master *ctrl)
using namespace nanogui;
+ depth_ = false;
src_ = ftl::create<Source>(ctrl->getRoot(), "source", ctrl->getNet());
- Widget *tools = new Widget(this);
- tools->setLayout(new BoxLayout(Orientation::Horizontal,
- Alignment::Middle, 0, 6));
+ //Widget *tools = new Widget(this);
+ // tools->setLayout(new BoxLayout(Orientation::Horizontal,
+ // Alignment::Middle, 0, 6));
- new Label(tools, "Select source","sans-bold");
+ new Label(this, "Select source","sans-bold");
available_ = ctrl->getNet()->findAll<string>("list_streams");
- auto select = new ComboBox(tools, available_);
+ auto select = new ComboBox(this, available_);
select->setCallback([this,select](int ix) {
LOG(INFO) << "Change source: " << ix;
src_->set("uri", available_[ix]);
@@ -178,10 +179,11 @@ SourceWindow::SourceWindow(nanogui::Widget *parent, ftl::ctrl::Master *ctrl)
select->setItems(available_);
});
- auto depth = new Button(tools, "Depth");
+ auto depth = new Button(this, "Depth");
depth->setFlags(Button::ToggleButton);
depth->setChangeCallback([this](bool state) {
- image_->setDepth(state);
+ //image_->setDepth(state);
+ depth_ = state;
});
#ifdef HAVE_LIBARCHIVE
@@ -210,11 +212,11 @@ SourceWindow::SourceWindow(nanogui::Widget *parent, ftl::ctrl::Master *ctrl)
});
#endif
- auto imageView = new VirtualCameraView(this);
+ //auto imageView = new VirtualCameraView(this);
//cam.view = imageView;
- imageView->setGridThreshold(20);
- imageView->setSource(src_);
- image_ = imageView;
+ //imageView->setGridThreshold(20);
+ //imageView->setSource(src_);
+ //image_ = imageView;
}
SourceWindow::~SourceWindow() {
diff --git a/applications/gui/src/src_window.hpp b/applications/gui/src/src_window.hpp
index 4b92f2e1f66b10046207063d3b3745d3b7dd9b28..6258267c62fcec81c688a89a9361c4c58d166938 100644
--- a/applications/gui/src/src_window.hpp
+++ b/applications/gui/src/src_window.hpp
@@ -21,9 +21,13 @@ class SourceWindow : public nanogui::Window {
SourceWindow(nanogui::Widget *parent, ftl::ctrl::Master *ctrl);
~SourceWindow();
+ ftl::rgbd::Source *getSource() const { return src_; }
+ bool getDepth() const { return depth_; }
+
private:
ftl::ctrl::Master *ctrl_;
ftl::rgbd::Source *src_;
+ bool depth_;
VirtualCameraView *image_;
std::vector<std::string> available_;