diff --git a/applications/gui/src/camera.cpp b/applications/gui/src/camera.cpp
index 0407bd45c56417199195ab104bcbd45e8d407b29..53a0d0593a457921cc82dfe2759484a1967f18dc 100644
--- a/applications/gui/src/camera.cpp
+++ b/applications/gui/src/camera.cpp
@@ -129,7 +129,29 @@ void ftl::gui::Camera::drawUpdated(std::vector<ftl::rgbd::FrameSet*> &fss) {
}
void ftl::gui::Camera::draw(std::vector<ftl::rgbd::FrameSet*> &fss) {
- if (fid_ != 255) return;
+ if (fid_ != 255) {
+ for (auto *fs : fss) {
+ if (!usesFrameset(fs->id)) continue;
+
+ ftl::rgbd::Frame *frame = nullptr;
+
+ if ((size_t)fid_ >= fs->frames.size()) return;
+ frame = &fs->frames[fid_];
+
+ auto &buf = colouriser_->colourise(*frame, channel_, 0);
+
+ // For non-virtual cameras, copy the CUDA texture into the opengl
+ // texture device-to-device.
+ texture1_.make(buf.width(), buf.height());
+ auto dst1 = texture1_.map(0);
+ cudaMemcpy2D(dst1.data, dst1.step1(), buf.devicePtr(), buf.pitch(), buf.width()*4, buf.height(), cudaMemcpyDeviceToDevice);
+ texture1_.unmap(0);
+
+ width_ = texture1_.width();
+ height_ = texture1_.height();
+ return;
+ }
+ }
//if (fsid_ >= fss.size()) return;
//auto &fs = *fss[fsid_];
@@ -270,6 +292,14 @@ void ftl::gui::Camera::_draw(std::vector<ftl::rgbd::FrameSet*> &fss) {
frame_.reset();
frame_.setOrigin(&state_);
+ // Make sure an OpenGL pixel buffer exists
+ texture1_.make(state_.getLeft().width, state_.getLeft().height);
+ if (isStereo()) texture2_.make(state_.getRight().width, state_.getRight().height);
+
+ // Map the GL pixel buffer to a GpuMat
+ frame_.create<cv::cuda::GpuMat>(Channel::Colour) = texture1_.map(renderer_->getCUDAStream());
+ if (isStereo()) frame_.create<cv::cuda::GpuMat>(Channel::Colour2) = texture2_.map((renderer2_) ? renderer2_->getCUDAStream() : 0);
+
{
FTL_Profile("Render",0.034);
renderer_->begin(frame_, Channel::Colour);
@@ -317,10 +347,17 @@ void ftl::gui::Camera::_draw(std::vector<ftl::rgbd::FrameSet*> &fss) {
channels_ = frame_.getChannels();
+ // Unmap GL buffer from CUDA and finish updating GL texture
+ texture1_.unmap(renderer_->getCUDAStream());
+ if (isStereo()) texture2_.unmap(renderer2_->getCUDAStream());
+
+ width_ = texture1_.width();
+ height_ = texture1_.height();
+
if (isStereo()) {
- _downloadFrames(frame_.getTexture<uchar4>(Channel::Colour), frame_.getTexture<uchar4>(Channel::Colour2));
+ //_downloadFrames(frame_.getTexture<uchar4>(Channel::Colour), frame_.getTexture<uchar4>(Channel::Colour2));
} else {
- _downloadFrame(frame_.getTexture<uchar4>(Channel::Colour));
+ //_downloadFrame(frame_.getTexture<uchar4>(Channel::Colour));
}
if (screen_->root()->value("show_poses", false)) {
@@ -358,38 +395,6 @@ void ftl::gui::Camera::_draw(std::vector<ftl::rgbd::FrameSet*> &fss) {
}
}
-void ftl::gui::Camera::_downloadFrames(ftl::cuda::TextureObject<uchar4> &a, ftl::cuda::TextureObject<uchar4> &b) {
- im1_.create(cv::Size(a.width(), a.height()), CV_8UC4);
- a.to_gpumat().download(im1_);
-
- // OpenGL (0,0) bottom left
- cv::flip(im1_, im1_, 0);
-
- width_ = im1_.cols;
- height_ = im1_.rows;
-
- im2_.create(cv::Size(b.width(), b.height()), CV_8UC4);
- b.to_gpumat().download(im2_);
- cv::flip(im2_, im2_, 0);
-
- if (im2_.cols != im1_.cols || im2_.rows != im1_.rows) {
- throw FTL_Error("Left and right images are different sizes");
- }
-}
-
-void ftl::gui::Camera::_downloadFrame(ftl::cuda::TextureObject<uchar4> &a) {
- im1_.create(cv::Size(a.width(), a.height()), CV_8UC4);
- a.to_gpumat().download(im1_);
-
- // OpenGL (0,0) bottom left
- cv::flip(im1_, im1_, 0);
-
- width_ = im1_.cols;
- height_ = im1_.rows;
-
- im2_ = cv::Mat();
-}
-
void ftl::gui::Camera::update(int fsid, const ftl::codecs::Channels<0> &c) {
if (!isVirtual() && ((1 << fsid) & fsmask_)) {
channels_ = c;
@@ -408,11 +413,6 @@ void ftl::gui::Camera::update(std::vector<ftl::rgbd::FrameSet*> &fss) {
//if (fss.size() <= fsid_) return;
if (fid_ == 255) {
name_ = "Virtual Camera";
- // Do a draw if not active. If active the draw function will be called
- // directly.
- if (screen_->activeCamera() != this) {
- //_draw(fss);
- }
} else {
for (auto *fs : fss) {
if (!usesFrameset(fs->id)) continue;
@@ -422,13 +422,6 @@ void ftl::gui::Camera::update(std::vector<ftl::rgbd::FrameSet*> &fss) {
if ((size_t)fid_ >= fs->frames.size()) return;
frame = &fs->frames[fid_];
- auto &buf = colouriser_->colourise(*frame, channel_, 0);
- if (isStereo() && frame->hasChannel(Channel::Right)) {
- _downloadFrames(buf, frame->createTexture<uchar4>(Channel::Right));
- } else {
- _downloadFrame(buf);
- }
-
auto n = frame->get<std::string>("name");
if (n) {
name_ = *n;
@@ -715,10 +708,10 @@ const void ftl::gui::Camera::captureFrame() {
{
UNIQUE_LOCK(mutex_, lk);
if (im1_.rows != 0) {
- texture1_.update(im1_);
+ //texture1_.update(im1_);
}
if (isStereo() && im2_.rows != 0) {
- texture2_.update(im2_);
+ //texture2_.update(im2_);
}
}
}
diff --git a/applications/gui/src/camera.hpp b/applications/gui/src/camera.hpp
index 9e034c83a9509aa7663648d3f12ecc8390b39bd0..8a62198d966686631a9fe299efc531bd50572b90 100644
--- a/applications/gui/src/camera.hpp
+++ b/applications/gui/src/camera.hpp
@@ -166,8 +166,6 @@ class Camera {
float baseline_;
#endif
- void _downloadFrames(ftl::cuda::TextureObject<uchar4> &, ftl::cuda::TextureObject<uchar4> &);
- void _downloadFrame(ftl::cuda::TextureObject<uchar4> &);
void _draw(std::vector<ftl::rgbd::FrameSet*> &fss);
};
diff --git a/applications/gui/src/gltexture.cpp b/applications/gui/src/gltexture.cpp
index d12c193796ededa636c096de35f9809cad7857e3..ec83997a3ba72ad744e4fd429ccc89151f24247a 100644
--- a/applications/gui/src/gltexture.cpp
+++ b/applications/gui/src/gltexture.cpp
@@ -3,10 +3,20 @@
#include <nanogui/opengl.h>
#include <loguru.hpp>
+#include <ftl/cuda_common.hpp>
+#include <cuda_gl_interop.h>
+
+#include <ftl/exception.hpp>
+
using ftl::gui::GLTexture;
GLTexture::GLTexture() {
glid_ = std::numeric_limits<unsigned int>::max();
+ glbuf_ = std::numeric_limits<unsigned int>::max();
+ cuda_res_ = nullptr;
+ width_ = 0;
+ height_ = 0;
+ changed_ = true;
}
GLTexture::~GLTexture() {
@@ -14,6 +24,7 @@ GLTexture::~GLTexture() {
}
void GLTexture::update(cv::Mat &m) {
+ LOG(INFO) << "DEPRECATED";
if (m.rows == 0) return;
if (glid_ == std::numeric_limits<unsigned int>::max()) {
glGenTextures(1, &glid_);
@@ -25,10 +36,98 @@ void GLTexture::update(cv::Mat &m) {
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_NEAREST);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR);
} else {
- glBindTexture(GL_TEXTURE_2D, glid_);
+ //glBindTexture(GL_TEXTURE_2D, glid_);
// TODO Allow for other formats
- glTexImage2D(GL_TEXTURE_2D, 0, GL_RGBA8, m.cols, m.rows, 0, GL_BGRA, GL_UNSIGNED_BYTE, m.data);
+ //glTexImage2D(GL_TEXTURE_2D, 0, GL_RGBA8, m.cols, m.rows, 0, GL_BGRA, GL_UNSIGNED_BYTE, m.data);
}
auto err = glGetError();
if (err != 0) LOG(ERROR) << "OpenGL Texture error: " << err;
}
+
+void GLTexture::make(int width, int height) {
+ if (width != width_ || height != height_) {
+ free();
+ }
+
+ width_ = width;
+ height_ = height;
+
+ if (width == 0 || height == 0) {
+ throw FTL_Error("Invalid texture size");
+ }
+
+ if (glid_ == std::numeric_limits<unsigned int>::max()) {
+ glGenTextures(1, &glid_);
+ glBindTexture(GL_TEXTURE_2D, glid_);
+ //cv::Mat m(cv::Size(100,100), CV_8UC3);
+ glTexImage2D(GL_TEXTURE_2D, 0, GL_RGBA8, width, height, 0, GL_BGRA, GL_UNSIGNED_BYTE, nullptr);
+ glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
+ glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
+ glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_NEAREST);
+ glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR);
+ //auto err = glGetError();
+ //if (err != 0) LOG(ERROR) << "OpenGL Texture error: " << err;
+
+ glBindTexture(GL_TEXTURE_2D, 0);
+
+ glGenBuffers(1, &glbuf_);
+ // Make this the current UNPACK buffer (OpenGL is state-based)
+ glBindBuffer(GL_PIXEL_UNPACK_BUFFER, glbuf_);
+ // Allocate data for the buffer. 4-channel 8-bit image
+ glBufferData(GL_PIXEL_UNPACK_BUFFER, width * height * 4, NULL, GL_DYNAMIC_COPY);
+
+ cudaSafeCall(cudaGraphicsGLRegisterBuffer(&cuda_res_, glbuf_, cudaGraphicsRegisterFlagsWriteDiscard));
+ glBindBuffer(GL_PIXEL_UNPACK_BUFFER, 0);
+ }
+}
+
+void GLTexture::free() {
+ if (glid_ != std::numeric_limits<unsigned int>::max()) {
+ glDeleteTextures(1, &glid_);
+ glid_ = std::numeric_limits<unsigned int>::max();
+ }
+
+ if (glbuf_ != std::numeric_limits<unsigned int>::max()) {
+ cudaSafeCall(cudaGraphicsUnregisterResource( cuda_res_ ));
+ cuda_res_ = nullptr;
+ glDeleteBuffers(1, &glbuf_);
+ glbuf_ = std::numeric_limits<unsigned int>::max();
+ }
+}
+
+cv::cuda::GpuMat GLTexture::map(cudaStream_t stream) {
+ void *devptr;
+ size_t size;
+ cudaSafeCall(cudaGraphicsMapResources(1, &cuda_res_, stream));
+ cudaSafeCall(cudaGraphicsResourceGetMappedPointer(&devptr, &size, cuda_res_));
+ return cv::cuda::GpuMat(height_, width_, CV_8UC4, devptr);
+}
+
+void GLTexture::unmap(cudaStream_t stream) {
+ cudaSafeCall(cudaGraphicsUnmapResources(1, &cuda_res_, stream));
+ changed_ = true;
+
+ //glActiveTexture(GL_TEXTURE0);
+ glBindBuffer( GL_PIXEL_UNPACK_BUFFER, glbuf_);
+ // Select the appropriate texture
+ glBindTexture( GL_TEXTURE_2D, glid_);
+ // Make a texture from the buffer
+ glTexSubImage2D( GL_TEXTURE_2D, 0, 0, 0, width_, height_, GL_BGRA, GL_UNSIGNED_BYTE, NULL);
+ glBindBuffer( GL_PIXEL_UNPACK_BUFFER, 0);
+}
+
+unsigned int GLTexture::texture() const {
+ if (glbuf_ < std::numeric_limits<unsigned int>::max()) {
+ /*//glActiveTexture(GL_TEXTURE0);
+ glBindBuffer( GL_PIXEL_UNPACK_BUFFER, glbuf_);
+ // Select the appropriate texture
+ glBindTexture( GL_TEXTURE_2D, glid_);
+ // Make a texture from the buffer
+ glTexSubImage2D( GL_TEXTURE_2D, 0, 0, 0, width_, height_, GL_BGRA, GL_UNSIGNED_BYTE, NULL);
+ glBindBuffer( GL_PIXEL_UNPACK_BUFFER, 0);*/
+
+ return glid_;
+ } else {
+ return glid_;
+ }
+}
diff --git a/applications/gui/src/gltexture.hpp b/applications/gui/src/gltexture.hpp
index 88bf1a18bff29634f09c902c9a56707d74a9d61d..c9e5146bbc26ebfd3175bd37ad1113d0f3360415 100644
--- a/applications/gui/src/gltexture.hpp
+++ b/applications/gui/src/gltexture.hpp
@@ -3,6 +3,10 @@
#include <opencv2/core/mat.hpp>
+#include <cuda_runtime.h>
+
+struct cudaGraphicsResource;
+
namespace ftl {
namespace gui {
@@ -12,11 +16,26 @@ class GLTexture {
~GLTexture();
void update(cv::Mat &m);
- unsigned int texture() const { return glid_; }
+ void make(int width, int height);
+ unsigned int texture() const;
bool isValid() const { return glid_ != std::numeric_limits<unsigned int>::max(); }
+ cv::cuda::GpuMat map(cudaStream_t stream);
+ void unmap(cudaStream_t stream);
+
+ void free();
+
+ int width() const { return width_; }
+ int height() const { return height_; }
+
private:
unsigned int glid_;
+ unsigned int glbuf_;
+ int width_;
+ int height_;
+ bool changed_;
+
+ cudaGraphicsResource *cuda_res_;
};
}
diff --git a/applications/gui/src/main.cpp b/applications/gui/src/main.cpp
index 71afa31edd64936fb7ef534a2aa5e53208bbfd1a..2677d67358af7a7bb9906e2d1ee93b99a372df5b 100644
--- a/applications/gui/src/main.cpp
+++ b/applications/gui/src/main.cpp
@@ -8,11 +8,17 @@
#include "screen.hpp"
+#include <cuda_gl_interop.h>
+
int main(int argc, char **argv) {
auto root = ftl::configure(argc, argv, "gui_default");
ftl::net::Universe *net = ftl::create<ftl::net::Universe>(root, "net");
+ int cuda_device;
+ cudaSafeCall(cudaGetDevice(&cuda_device));
+ //cudaSafeCall(cudaGLSetGLDevice(cuda_device));
+
ftl::ctrl::Master *controller = new ftl::ctrl::Master(root, net);
controller->onLog([](const ftl::ctrl::LogEvent &e){
const int v = e.verbosity;
diff --git a/applications/gui/src/screen.cpp b/applications/gui/src/screen.cpp
index efd853eaf9fdc7596871284bc84a9c0b8624171e..7d144d5cbe727d110133281e5adec09b726ee3ab 100644
--- a/applications/gui/src/screen.cpp
+++ b/applications/gui/src/screen.cpp
@@ -38,7 +38,7 @@ namespace {
in vec2 vertex;
out vec2 uv;
void main() {
- uv = vertex;
+ uv = vec2(vertex.x, 1.0 - vertex.y);
vec2 scaledVertex = (vertex * scaleFactor) + position;
gl_Position = vec4(2.0*scaledVertex.x - 1.0,
2.0*scaledVertex.y - 1.0,
@@ -520,13 +520,14 @@ void ftl::gui::Screen::draw(NVGcontext *ctx) {
if (camera_) {
imageSize = {camera_->width(), camera_->height()};
+ glActiveTexture(GL_TEXTURE0);
mImageID = camera_->getLeft().texture();
leftEye_ = mImageID;
rightEye_ = camera_->getRight().texture();
//if (camera_->getChannel() != ftl::codecs::Channel::Left) { mImageID = rightEye_; }
- if (mImageID < std::numeric_limits<unsigned int>::max() && imageSize[0] > 0) {
+ if (camera_->getLeft().isValid() && imageSize[0] > 0) {
auto mScale = (screenSize.cwiseQuotient(imageSize).minCoeff()) * zoom_;
Vector2f scaleFactor = mScale * imageSize.cwiseQuotient(screenSize);
Vector2f positionInScreen(pos_x_, pos_y_);
@@ -541,8 +542,10 @@ void ftl::gui::Screen::draw(NVGcontext *ctx) {
mShader.bind();
glActiveTexture(GL_TEXTURE0);
glBindTexture(GL_TEXTURE_2D, leftEye_);
+ //camera_->getLeft().texture();
glActiveTexture(GL_TEXTURE1);
glBindTexture(GL_TEXTURE_2D, (camera_->isStereo() && camera_->getRight().isValid()) ? rightEye_ : leftEye_);
+ //(camera_->isStereo() && camera_->getRight().isValid()) ? camera_->getRight().texture() : camera_->getLeft().texture();
mShader.setUniform("image1", 0);
mShader.setUniform("image2", 1);
mShader.setUniform("blendAmount", (camera_->isStereo()) ? root_->value("blending", 0.5f) : 1.0f);
@@ -550,6 +553,8 @@ void ftl::gui::Screen::draw(NVGcontext *ctx) {
mShader.setUniform("position", imagePosition);
mShader.drawIndexed(GL_TRIANGLES, 0, 2);
//glDisable(GL_SCISSOR_TEST);
+
+ glBindBuffer(GL_PIXEL_UNPACK_BUFFER, 0);
}
} else {
// Must periodically render the cameras here to update any thumbnails.
diff --git a/applications/gui/src/src_window.cpp b/applications/gui/src/src_window.cpp
index 5620e1928632dae4a9f34390fee702f5827bb526..07fcb3ce3b2191151eda2518ac10b1a02c7f62b4 100644
--- a/applications/gui/src/src_window.cpp
+++ b/applications/gui/src/src_window.cpp
@@ -368,26 +368,28 @@ void SourceWindow::draw(NVGcontext *ctx) {
UNIQUE_LOCK(mutex_, lk);
//refresh_thumbs_ = false;
- if (thumbs_.size() < cameras_.size()) thumbs_.resize(cameras_.size());
+ //if (thumbs_.size() < cameras_.size()) thumbs_.resize(cameras_.size());
//for (size_t i=0; i<thumbs_.size(); ++i) {
int i = 0;
for (auto &camera : cameras_) {
cv::Mat t;
auto *cam = camera.second.camera;
- if (cam) {
+ //if (cam) {
//cam->draw(framesets_);
- if (cam->thumbnail(t)) {
- thumbs_[i].update(t);
- }
- }
+ // if (cam->thumbnail(t)) {
+ // thumbs_[i].update(t);
+ // }
+ //}
if (!camera.second.thumbview) camera.second.thumbview = new ftl::gui::ThumbView(ipanel_, screen_, cam);
+ camera.second.thumbview->setFixedSize(nanogui::Vector2i(320,180));
/*if ((size_t)ipanel_->childCount() < i+1) {
new ftl::gui::ThumbView(ipanel_, screen_, cam);
}*/
- if (thumbs_[i].isValid()) dynamic_cast<nanogui::ImageView*>(camera.second.thumbview)->bindImage(thumbs_[i].texture());
+ //if (thumbs_[i].isValid()) dynamic_cast<nanogui::ImageView*>(camera.second.thumbview)->bindImage(thumbs_[i].texture());
+ if (cam->getLeft().isValid()) dynamic_cast<nanogui::ImageView*>(camera.second.thumbview)->bindImage(cam->getLeft().texture());
++i;
}
diff --git a/components/renderers/cpp/include/ftl/render/CUDARender.hpp b/components/renderers/cpp/include/ftl/render/CUDARender.hpp
index 4b74bcc490b9367eed7568a9a7d2de6b37db8e62..7a9d46e5b32befa9a1f3338ecf4212ed80562cac 100644
--- a/components/renderers/cpp/include/ftl/render/CUDARender.hpp
+++ b/components/renderers/cpp/include/ftl/render/CUDARender.hpp
@@ -35,6 +35,8 @@ class CUDARender : public ftl::render::Renderer {
params_.viewPortMode = mode;
}
+ cudaStream_t getCUDAStream() const { return stream_; }
+
protected:
void _renderChannel(ftl::rgbd::Frame &out, ftl::codecs::Channel channel_in, const Eigen::Matrix4d &t, cudaStream_t stream);