Skip to content
GitLab
Explore
Sign in
Primary navigation
Search or go to…
Project
F
ftl
Manage
Activity
Members
Labels
Plan
Issues
Issue boards
Milestones
Wiki
Requirements
Code
Merge requests
Repository
Branches
Commits
Tags
Repository graph
Compare revisions
Snippets
Locked files
Build
Pipelines
Jobs
Pipeline schedules
Test cases
Artifacts
Deploy
Releases
Package Registry
Container Registry
Model registry
Operate
Environments
Terraform modules
Monitor
Incidents
Service Desk
Analyze
Value stream analytics
Contributor analytics
CI/CD analytics
Repository analytics
Code review analytics
Issue analytics
Insights
Model experiments
Help
Help
Support
GitLab documentation
Compare GitLab plans
Community forum
Contribute to GitLab
Provide feedback
Keyboard shortcuts
?
Snippets
Groups
Projects
Show more breadcrumbs
Nicolas Pope
ftl
Commits
a766c238
Commit
a766c238
authored
5 years ago
by
Nicolas Pope
Browse files
Options
Downloads
Plain Diff
Merge branch 'feature/guishader' into 'master'
Feature/guishader See merge request nicolas.pope/ftl!26
parents
f7c8434f
d13c3649
No related branches found
No related tags found
1 merge request
!26
Feature/guishader
Pipeline
#11296
passed
5 years ago
Stage: all
Changes
3
Pipelines
1
Hide whitespace changes
Inline
Side-by-side
Showing
3 changed files
applications/gui/src/main.cpp
+216
-2
216 additions, 2 deletions
applications/gui/src/main.cpp
applications/gui/src/src_window.cpp
+13
-11
13 additions, 11 deletions
applications/gui/src/src_window.cpp
applications/gui/src/src_window.hpp
+4
-0
4 additions, 0 deletions
applications/gui/src/src_window.hpp
with
233 additions
and
13 deletions
applications/gui/src/main.cpp
+
216
−
2
View file @
a766c238
...
...
@@ -29,6 +29,88 @@ using ftl::rgbd::Source;
};*/
namespace
{
constexpr
char
const
*
const
defaultImageViewVertexShader
=
R"(#version 330
uniform vec2 scaleFactor;
uniform vec2 position;
in vec2 vertex;
out vec2 uv;
void main() {
uv = vertex;
vec2 scaledVertex = (vertex * scaleFactor) + position;
gl_Position = vec4(2.0*scaledVertex.x - 1.0,
1.0 - 2.0*scaledVertex.y,
0.0, 1.0);
})"
;
constexpr
char
const
*
const
defaultImageViewFragmentShader
=
R"(#version 330
uniform sampler2D image;
out vec4 color;
in vec2 uv;
void main() {
color = texture(image, uv);
})"
;
class
GLTexture
{
public:
GLTexture
()
{
glGenTextures
(
1
,
&
glid_
);
glBindTexture
(
GL_TEXTURE_2D
,
glid_
);
cv
::
Mat
m
(
cv
::
Size
(
100
,
100
),
CV_8UC3
);
glTexImage2D
(
GL_TEXTURE_2D
,
0
,
GL_RGB8
,
m
.
cols
,
m
.
rows
,
0
,
GL_RGB
,
GL_UNSIGNED_BYTE
,
m
.
data
);
glTexParameteri
(
GL_TEXTURE_2D
,
GL_TEXTURE_WRAP_S
,
GL_CLAMP_TO_EDGE
);
glTexParameteri
(
GL_TEXTURE_2D
,
GL_TEXTURE_WRAP_T
,
GL_CLAMP_TO_EDGE
);
glTexParameteri
(
GL_TEXTURE_2D
,
GL_TEXTURE_MAG_FILTER
,
GL_NEAREST
);
glTexParameteri
(
GL_TEXTURE_2D
,
GL_TEXTURE_MIN_FILTER
,
GL_LINEAR
);
}
~
GLTexture
()
{
glDeleteTextures
(
1
,
&
glid_
);
}
void
update
(
cv
::
Mat
&
m
)
{
if
(
m
.
rows
==
0
)
return
;
glBindTexture
(
GL_TEXTURE_2D
,
glid_
);
// TODO Allow for other formats
glTexImage2D
(
GL_TEXTURE_2D
,
0
,
GL_RGB8
,
m
.
cols
,
m
.
rows
,
0
,
GL_BGR
,
GL_UNSIGNED_BYTE
,
m
.
data
);
auto
err
=
glGetError
();
if
(
err
!=
0
)
LOG
(
ERROR
)
<<
"OpenGL Texture error: "
<<
err
;
}
unsigned
int
texture
()
const
{
return
glid_
;
}
private
:
unsigned
int
glid_
;
};
template
<
class
T
>
Eigen
::
Matrix
<
T
,
4
,
4
>
lookAt
(
Eigen
::
Matrix
<
T
,
3
,
1
>
const
&
eye
,
Eigen
::
Matrix
<
T
,
3
,
1
>
const
&
center
,
Eigen
::
Matrix
<
T
,
3
,
1
>
const
&
up
)
{
typedef
Eigen
::
Matrix
<
T
,
4
,
4
>
Matrix4
;
typedef
Eigen
::
Matrix
<
T
,
3
,
1
>
Vector3
;
Vector3
f
=
(
center
-
eye
).
normalized
();
Vector3
u
=
up
.
normalized
();
Vector3
s
=
f
.
cross
(
u
).
normalized
();
u
=
s
.
cross
(
f
);
Matrix4
res
;
res
<<
s
.
x
(),
s
.
y
(),
s
.
z
(),
-
s
.
dot
(
eye
),
u
.
x
(),
u
.
y
(),
u
.
z
(),
-
u
.
dot
(
eye
),
-
f
.
x
(),
-
f
.
y
(),
-
f
.
z
(),
f
.
dot
(
eye
),
0
,
0
,
0
,
1
;
return
res
;
}
}
class
FTLApplication
:
public
nanogui
::
Screen
{
public:
...
...
@@ -37,22 +119,154 @@ class FTLApplication : public nanogui::Screen {
net_
=
net
;
auto
cwindow
=
new
ftl
::
gui
::
ControlWindow
(
this
,
controller
);
auto
swindow
=
new
ftl
::
gui
::
SourceWindow
(
this
,
controller
);
swindow_
=
new
ftl
::
gui
::
SourceWindow
(
this
,
controller
);
//src_ = nullptr;
eye_
=
Eigen
::
Vector3f
(
0.0
f
,
0.0
f
,
0.0
f
);
centre_
=
Eigen
::
Vector3f
(
0.0
f
,
0.0
f
,
-
4.0
f
);
up_
=
Eigen
::
Vector3f
(
0
,
1.0
f
,
0
);
lookPoint_
=
Eigen
::
Vector3f
(
0.0
f
,
0.0
f
,
-
4.0
f
);
lerpSpeed_
=
0.4
f
;
depth_
=
false
;
mShader
.
init
(
"RGBDShader"
,
defaultImageViewVertexShader
,
defaultImageViewFragmentShader
);
MatrixXu
indices
(
3
,
2
);
indices
.
col
(
0
)
<<
0
,
1
,
2
;
indices
.
col
(
1
)
<<
2
,
3
,
1
;
MatrixXf
vertices
(
2
,
4
);
vertices
.
col
(
0
)
<<
0
,
0
;
vertices
.
col
(
1
)
<<
1
,
0
;
vertices
.
col
(
2
)
<<
0
,
1
;
vertices
.
col
(
3
)
<<
1
,
1
;
mShader
.
bind
();
mShader
.
uploadIndices
(
indices
);
mShader
.
uploadAttrib
(
"vertex"
,
vertices
);
setVisible
(
true
);
performLayout
();
}
~
FTLApplication
()
{
mShader
.
free
();
}
bool
mouseButtonEvent
(
const
nanogui
::
Vector2i
&
p
,
int
button
,
bool
down
,
int
modifiers
)
{
if
(
Screen
::
mouseButtonEvent
(
p
,
button
,
down
,
modifiers
))
{
return
true
;
}
else
{
auto
src_
=
swindow_
->
getSource
();
if
(
src_
&&
src_
->
isReady
()
&&
down
)
{
Eigen
::
Vector4f
camPos
=
src_
->
point
(
p
[
0
],
p
[
1
]);
camPos
*=
-
1.0
f
;
Eigen
::
Vector4f
worldPos
=
src_
->
getPose
()
*
camPos
;
lookPoint_
=
Eigen
::
Vector3f
(
worldPos
[
0
],
worldPos
[
1
],
worldPos
[
2
]);
LOG
(
INFO
)
<<
"Depth at click = "
<<
-
camPos
[
2
];
return
true
;
}
return
false
;
}
}
bool
keyboardEvent
(
int
key
,
int
scancode
,
int
action
,
int
modifiers
)
{
if
(
Screen
::
keyboardEvent
(
key
,
scancode
,
action
,
modifiers
))
{
return
true
;
}
else
{
LOG
(
INFO
)
<<
"Key press"
<<
key
<<
" - "
<<
action
;
if
(
key
==
81
||
key
==
83
)
{
// TODO Should rotate around lookAt object, but requires correct depth
Eigen
::
Quaternion
<
float
>
q
;
q
=
Eigen
::
AngleAxis
<
float
>
((
key
==
81
)
?
0.01
f
:
-
0.01
f
,
up_
);
eye_
=
(
q
*
(
eye_
-
centre_
))
+
centre_
;
return
true
;
}
else
if
(
key
==
84
||
key
==
82
)
{
float
scalar
=
(
key
==
84
)
?
0.99
f
:
1.01
f
;
eye_
=
((
eye_
-
centre_
)
*
scalar
)
+
centre_
;
return
true
;
}
return
false
;
}
}
virtual
void
draw
(
NVGcontext
*
ctx
)
{
nvgText
(
ctx
,
10
,
10
,
"FT-Lab Remote Presence System"
,
NULL
);
using
namespace
Eigen
;
auto
src_
=
swindow_
->
getSource
();
Vector2f
imageSize
(
0
,
0
);
if
(
src_
)
{
cv
::
Mat
rgb
,
depth
;
centre_
+=
(
lookPoint_
-
centre_
)
*
(
lerpSpeed_
*
0.1
f
);
Eigen
::
Matrix4f
viewPose
=
lookAt
<
float
>
(
eye_
,
centre_
,
up_
).
inverse
();
src_
->
setPose
(
viewPose
);
src_
->
grab
();
src_
->
getFrames
(
rgb
,
depth
);
if
(
swindow_
->
getDepth
())
{
if
(
depth
.
rows
>
0
)
{
imageSize
=
Vector2f
(
depth
.
cols
,
depth
.
rows
);
cv
::
Mat
idepth
;
depth
.
convertTo
(
idepth
,
CV_8U
,
255.0
f
/
10.0
f
);
// TODO(nick)
applyColorMap
(
idepth
,
idepth
,
cv
::
COLORMAP_JET
);
texture_
.
update
(
idepth
);
mImageID
=
texture_
.
texture
();
}
}
else
{
if
(
rgb
.
rows
>
0
)
{
imageSize
=
Vector2f
(
rgb
.
cols
,
rgb
.
rows
);
texture_
.
update
(
rgb
);
mImageID
=
texture_
.
texture
();
}
}
}
if
(
imageSize
[
0
]
>
0
)
{
Vector2f
screenSize
=
size
().
cast
<
float
>
();
auto
mScale
=
(
screenSize
.
cwiseQuotient
(
imageSize
).
minCoeff
());
Vector2f
scaleFactor
=
mScale
*
imageSize
.
cwiseQuotient
(
screenSize
);
Vector2f
positionInScreen
(
0.0
f
,
0.0
f
);
auto
mOffset
=
(
screenSize
-
(
screenSize
.
cwiseProduct
(
scaleFactor
)))
/
2
;
Vector2f
positionAfterOffset
=
positionInScreen
+
mOffset
;
Vector2f
imagePosition
=
positionAfterOffset
.
cwiseQuotient
(
screenSize
);
//glEnable(GL_SCISSOR_TEST);
//float r = screen->pixelRatio();
/* glScissor(positionInScreen.x() * r,
(screenSize.y() - positionInScreen.y() - size().y()) * r,
size().x() * r, size().y() * r);*/
mShader
.
bind
();
glActiveTexture
(
GL_TEXTURE0
);
glBindTexture
(
GL_TEXTURE_2D
,
mImageID
);
mShader
.
setUniform
(
"image"
,
0
);
mShader
.
setUniform
(
"scaleFactor"
,
scaleFactor
);
mShader
.
setUniform
(
"position"
,
imagePosition
);
mShader
.
drawIndexed
(
GL_TRIANGLES
,
0
,
2
);
//glDisable(GL_SCISSOR_TEST);
}
nvgText
(
ctx
,
10
,
20
,
"FT-Lab Remote Presence System"
,
NULL
);
/* Draw the user interface */
screen
()
->
performLayout
(
ctx
);
Screen
::
draw
(
ctx
);
}
private
:
ftl
::
gui
::
SourceWindow
*
swindow_
;
//std::vector<SourceViews> sources_;
ftl
::
net
::
Universe
*
net_
;
nanogui
::
GLShader
mShader
;
GLuint
mImageID
;
//Source *src_;
GLTexture
texture_
;
Eigen
::
Vector3f
eye_
;
Eigen
::
Vector3f
centre_
;
Eigen
::
Vector3f
up_
;
Eigen
::
Vector3f
lookPoint_
;
float
lerpSpeed_
;
bool
depth_
;
};
int
main
(
int
argc
,
char
**
argv
)
{
...
...
This diff is collapsed.
Click to expand it.
applications/gui/src/src_window.cpp
+
13
−
11
View file @
a766c238
...
...
@@ -159,15 +159,16 @@ SourceWindow::SourceWindow(nanogui::Widget *parent, ftl::ctrl::Master *ctrl)
using
namespace
nanogui
;
depth_
=
false
;
src_
=
ftl
::
create
<
Source
>
(
ctrl
->
getRoot
(),
"source"
,
ctrl
->
getNet
());
Widget
*
tools
=
new
Widget
(
this
);
tools
->
setLayout
(
new
BoxLayout
(
Orientation
::
Horizontal
,
Alignment
::
Middle
,
0
,
6
));
//
Widget *tools = new Widget(this);
//
tools->setLayout(new BoxLayout(Orientation::Horizontal,
//
Alignment::Middle, 0, 6));
new
Label
(
t
ool
s
,
"Select source"
,
"sans-bold"
);
new
Label
(
t
hi
s
,
"Select source"
,
"sans-bold"
);
available_
=
ctrl
->
getNet
()
->
findAll
<
string
>
(
"list_streams"
);
auto
select
=
new
ComboBox
(
t
ool
s
,
available_
);
auto
select
=
new
ComboBox
(
t
hi
s
,
available_
);
select
->
setCallback
([
this
,
select
](
int
ix
)
{
LOG
(
INFO
)
<<
"Change source: "
<<
ix
;
src_
->
set
(
"uri"
,
available_
[
ix
]);
...
...
@@ -178,10 +179,11 @@ SourceWindow::SourceWindow(nanogui::Widget *parent, ftl::ctrl::Master *ctrl)
select
->
setItems
(
available_
);
});
auto
depth
=
new
Button
(
t
ool
s
,
"Depth"
);
auto
depth
=
new
Button
(
t
hi
s
,
"Depth"
);
depth
->
setFlags
(
Button
::
ToggleButton
);
depth
->
setChangeCallback
([
this
](
bool
state
)
{
image_
->
setDepth
(
state
);
//image_->setDepth(state);
depth_
=
state
;
});
#ifdef HAVE_LIBARCHIVE
...
...
@@ -210,11 +212,11 @@ SourceWindow::SourceWindow(nanogui::Widget *parent, ftl::ctrl::Master *ctrl)
});
#endif
auto
imageView
=
new
VirtualCameraView
(
this
);
//
auto imageView = new VirtualCameraView(this);
//cam.view = imageView;
imageView
->
setGridThreshold
(
20
);
imageView
->
setSource
(
src_
);
image_
=
imageView
;
//
imageView->setGridThreshold(20);
//
imageView->setSource(src_);
//
image_ = imageView;
}
SourceWindow
::~
SourceWindow
()
{
...
...
This diff is collapsed.
Click to expand it.
applications/gui/src/src_window.hpp
+
4
−
0
View file @
a766c238
...
...
@@ -21,9 +21,13 @@ class SourceWindow : public nanogui::Window {
SourceWindow
(
nanogui
::
Widget
*
parent
,
ftl
::
ctrl
::
Master
*
ctrl
);
~
SourceWindow
();
ftl
::
rgbd
::
Source
*
getSource
()
const
{
return
src_
;
}
bool
getDepth
()
const
{
return
depth_
;
}
private
:
ftl
::
ctrl
::
Master
*
ctrl_
;
ftl
::
rgbd
::
Source
*
src_
;
bool
depth_
;
VirtualCameraView
*
image_
;
std
::
vector
<
std
::
string
>
available_
;
...
...
This diff is collapsed.
Click to expand it.
Preview
0%
Loading
Try again
or
attach a new file
.
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Save comment
Cancel
Please
register
or
sign in
to comment