From 7fc400f21bd399f6508e0e46ffe0befc6321716e Mon Sep 17 00:00:00 2001
From: Nicolas Pope <nwpope@utu.fi>
Date: Wed, 11 Nov 2020 17:44:45 +0200
Subject: [PATCH] Add initial gpu utility code
---
SDK/CPP/private/frame_impl.cpp | 4 +--
SDK/CPP/private/frame_impl.hpp | 2 +-
SDK/CPP/public/CMakeLists.txt | 4 +++
SDK/CPP/public/include/voltu/cuda.hpp | 27 +++++++++++++++
SDK/CPP/public/include/voltu/opencv.hpp | 14 ++++++--
SDK/CPP/public/include/voltu/system.hpp | 2 +-
SDK/CPP/public/include/voltu/types/frame.hpp | 2 +-
SDK/CPP/public/samples/basic_file/main.cpp | 2 +-
.../public/samples/basic_virtual_cam/main.cpp | 2 +-
.../public/samples/fusion_evaluator/main.cpp | 7 +++-
SDK/CPP/public/voltu.cpp | 20 +++++++++++
SDK/CPP/public/voltu_cv.cpp | 33 +++++++++++++++----
12 files changed, 103 insertions(+), 16 deletions(-)
create mode 100644 SDK/CPP/public/include/voltu/cuda.hpp
diff --git a/SDK/CPP/private/frame_impl.cpp b/SDK/CPP/private/frame_impl.cpp
index a91ad3e6a..059b508ff 100644
--- a/SDK/CPP/private/frame_impl.cpp
+++ b/SDK/CPP/private/frame_impl.cpp
@@ -15,9 +15,9 @@ FrameImpl::~FrameImpl()
}
-std::list<voltu::ImagePtr> FrameImpl::getImageSet(voltu::Channel c)
+std::vector<voltu::ImagePtr> FrameImpl::getImageSet(voltu::Channel c)
{
- std::list<voltu::ImagePtr> result;
+ std::vector<voltu::ImagePtr> result;
ftl::codecs::Channel channel = ftl::codecs::Channel::Colour;
switch (c)
diff --git a/SDK/CPP/private/frame_impl.hpp b/SDK/CPP/private/frame_impl.hpp
index eb5caa5a6..38750ce42 100644
--- a/SDK/CPP/private/frame_impl.hpp
+++ b/SDK/CPP/private/frame_impl.hpp
@@ -16,7 +16,7 @@ public:
FrameImpl();
~FrameImpl() override;
- std::list<voltu::ImagePtr> getImageSet(voltu::Channel) override;
+ std::vector<voltu::ImagePtr> getImageSet(voltu::Channel) override;
voltu::PointCloudPtr getPointCloud(voltu::PointCloudFormat cloudfmt, voltu::PointFormat pointfmt) override;
diff --git a/SDK/CPP/public/CMakeLists.txt b/SDK/CPP/public/CMakeLists.txt
index f1ea62a4e..e3d1c339a 100644
--- a/SDK/CPP/public/CMakeLists.txt
+++ b/SDK/CPP/public/CMakeLists.txt
@@ -43,6 +43,10 @@ endif()
add_library(voltu_sdk STATIC ${VOLTU_SRCS})
+if (WITH_OPENCV)
+ target_compile_definitions(voltu_sdk PUBLIC WITH_OPENCV)
+endif()
+
target_include_directories(voltu_sdk
PUBLIC include)
target_link_libraries(voltu_sdk ${OS_LIBS} Threads::Threads ${OPTIONAL_DEPENDENCIES} Eigen3::Eigen)
diff --git a/SDK/CPP/public/include/voltu/cuda.hpp b/SDK/CPP/public/include/voltu/cuda.hpp
new file mode 100644
index 000000000..972f4964b
--- /dev/null
+++ b/SDK/CPP/public/include/voltu/cuda.hpp
@@ -0,0 +1,27 @@
+#pragma once
+
+#include <voltu/types/image.hpp>
+#include
+#include <memory>
+
+namespace voltu
+{
+
+/**
+ * @brief CUDA Processing Stream.
+ *
+ * An instance of this class is mapped to a single CUDA stream, so any of the
+ * available operations will occur within that stream. It is therefore
+ * necessary to call `waitCompletion` after all steps have been finished.
+ */
+class CUDAProc
+{
+public:
+ virtual bool waitCompletion(int timeout, bool except=false) = 0;
+
+ virtual void* getInternalStream() = 0;
+
+ virtual void visualiseDepthEnhancement(const voltu::ImagePtr >, const voltu::ImagePtr &depth_old, const voltu::ImagePtr &depth_new, const voltu::ImagePtr &colour) = 0;
+};
+
+}
diff --git a/SDK/CPP/public/include/voltu/opencv.hpp b/SDK/CPP/public/include/voltu/opencv.hpp
index 7119cbc87..1e8e858ab 100644
--- a/SDK/CPP/public/include/voltu/opencv.hpp
+++ b/SDK/CPP/public/include/voltu/opencv.hpp
@@ -7,19 +7,29 @@
#pragma once
#include <opencv2/core/mat.hpp>
-#include <opencv2/core/cuda_types.hpp>
+#include <opencv2/core/cuda.hpp>
#include <voltu/types/image.hpp>
namespace voltu
{
-namespace cv
+namespace opencv
{
void convert(voltu::ImagePtr img, ::cv::Mat &mat);
void convert(voltu::ImagePtr img, ::cv::cuda::GpuMat &mat);
+::cv::cuda::GpuMat toGpuMat(voltu::ImagePtr img);
+
void visualise(voltu::ImagePtr img, ::cv::Mat &mat);
}
+
+struct GpuUtilities
+{
+ void (*visualiseDepthEnhancement)(const voltu::ImagePtr >, const voltu::ImagePtr &depth_old, const voltu::ImagePtr &depth_new, const voltu::ImagePtr &colour) = nullptr;
+};
+
+extern GpuUtilities gpu;
+
}
\ No newline at end of file
diff --git a/SDK/CPP/public/include/voltu/system.hpp b/SDK/CPP/public/include/voltu/system.hpp
index 8fb9c744c..df7b4938e 100644
--- a/SDK/CPP/public/include/voltu/system.hpp
+++ b/SDK/CPP/public/include/voltu/system.hpp
@@ -78,7 +78,7 @@ public:
* Identifiers (URIs), with some non-standard additions. A few examples
* are:
* * `file:///home/user/file.ftl`
- * * `tcp://localhost:9001/*`
+ * * `tcp://localhost:9001/`
* * `ftl://my.stream.name/room1`
* * `ws://ftlab.utu.fi/lab/`
* * `./file.ftl`
diff --git a/SDK/CPP/public/include/voltu/types/frame.hpp b/SDK/CPP/public/include/voltu/types/frame.hpp
index 30b7c4ddc..7f03fc604 100644
--- a/SDK/CPP/public/include/voltu/types/frame.hpp
+++ b/SDK/CPP/public/include/voltu/types/frame.hpp
@@ -22,7 +22,7 @@ class Frame
public:
virtual ~Frame() = default;
- PY_API PY_RV_LIFETIME_PARENT virtual std::list<voltu::ImagePtr> getImageSet(voltu::Channel channel) = 0;
+ PY_API PY_RV_LIFETIME_PARENT virtual std::vector<voltu::ImagePtr> getImageSet(voltu::Channel channel) = 0;
PY_API PY_RV_LIFETIME_PARENT virtual voltu::PointCloudPtr getPointCloud(voltu::PointCloudFormat cloudfmt, voltu::PointFormat pointfmt) = 0;
diff --git a/SDK/CPP/public/samples/basic_file/main.cpp b/SDK/CPP/public/samples/basic_file/main.cpp
index c98386427..9af26c1c0 100644
--- a/SDK/CPP/public/samples/basic_file/main.cpp
+++ b/SDK/CPP/public/samples/basic_file/main.cpp
@@ -45,7 +45,7 @@ int main(int argc, char **argv)
for (auto img : imgset)
{
cv::Mat m;
- voltu::cv::visualise(img, m);
+ voltu::opencv::visualise(img, m);
cv::imshow(string("Image-") + img->getName(), m);
}
diff --git a/SDK/CPP/public/samples/basic_virtual_cam/main.cpp b/SDK/CPP/public/samples/basic_virtual_cam/main.cpp
index f1b2634b7..47dea0583 100644
--- a/SDK/CPP/public/samples/basic_virtual_cam/main.cpp
+++ b/SDK/CPP/public/samples/basic_virtual_cam/main.cpp
@@ -69,7 +69,7 @@ int main(int argc, char **argv)
for (auto img : imgset)
{
cv::Mat m;
- voltu::cv::convert(img, m);
+ voltu::opencv::convert(img, m);
cv::imshow(string("Camera-") + img->getName(), m);
}
diff --git a/SDK/CPP/public/samples/fusion_evaluator/main.cpp b/SDK/CPP/public/samples/fusion_evaluator/main.cpp
index c4073baf4..f9b96468c 100644
--- a/SDK/CPP/public/samples/fusion_evaluator/main.cpp
+++ b/SDK/CPP/public/samples/fusion_evaluator/main.cpp
@@ -107,6 +107,10 @@ int main(int argc, char **argv)
op1->property("visibility_carving")->setBool(do_carving);
op1->property("mls_iterations")->setInt(iters);
+ cv::Mat old_depth;
+ auto oldimgset = frame->getImageSet(voltu::Channel::kDepth);
+ voltu::opencv::toGpuMat(oldimgset[sourceno]).download(old_depth);
+
pipe->submit(frame);
pipe->waitCompletion(3000, true);
@@ -123,7 +127,8 @@ int main(int argc, char **argv)
{
if (srccount++ < sourceno) continue;
cv::Mat m;
- voltu::cv::visualise(img, m);
+ voltu::opencv::toGpuMat(img).download(m);
+ voltu::opencv::visualise(img, m);
cv::imshow(string("Image-") + img->getName(), m);
break;
}
diff --git a/SDK/CPP/public/voltu.cpp b/SDK/CPP/public/voltu.cpp
index 2c4d19712..d558ff7b0 100644
--- a/SDK/CPP/public/voltu.cpp
+++ b/SDK/CPP/public/voltu.cpp
@@ -8,6 +8,10 @@
#include <voltu/types/errors.hpp>
#include <voltu/voltu.hpp>
+#ifdef WITH_OPENCV
+#include <voltu/opencv.hpp>
+#endif
+
#if defined(WIN32)
#include <windows.h>
#pragma comment(lib, "User32.lib")
@@ -23,6 +27,10 @@
static bool g_init = false;
+#ifdef WITH_OPENCV
+voltu::GpuUtilities voltu::gpu;
+#endif
+
typedef void* Library;
static Library loadLibrary(const char *file)
@@ -132,6 +140,18 @@ std::shared_ptr<voltu::System> voltu::instance()
throw voltu::exceptions::RuntimeVersionMismatch();
}
+#ifdef WITH_OPENCV
+ auto gpuinit = (voltu::GpuUtilities* (*)())getFunction(handle, "voltu_utilities_gpu");
+ if (gpuinit)
+ {
+ gpu = *gpuinit();
+ }
+ else
+ {
+ //throw voltu::exceptions::LibraryLoadFailed();
+ }
+#endif
+
return instance;
}
else
diff --git a/SDK/CPP/public/voltu_cv.cpp b/SDK/CPP/public/voltu_cv.cpp
index e9951dc37..5e55e7908 100644
--- a/SDK/CPP/public/voltu_cv.cpp
+++ b/SDK/CPP/public/voltu_cv.cpp
@@ -9,7 +9,7 @@
#include <opencv2/imgproc.hpp>
-void voltu::cv::convert(voltu::ImagePtr img, ::cv::Mat &mat)
+void voltu::opencv::convert(voltu::ImagePtr img, ::cv::Mat &mat)
{
voltu::ImageData data = img->getHost();
@@ -31,23 +31,44 @@ void voltu::cv::convert(voltu::ImagePtr img, ::cv::Mat &mat)
}
}
-void voltu::cv::convert(voltu::ImagePtr img, ::cv::cuda::GpuMat &mat)
+void voltu::opencv::convert(voltu::ImagePtr img, ::cv::cuda::GpuMat &mat)
{
+ mat = voltu::opencv::toGpuMat(img);
+}
+
+cv::cuda::GpuMat voltu::opencv::toGpuMat(voltu::ImagePtr img)
+{
+ voltu::ImageData data = img->getDevice();
+
+ if (data.format == voltu::ImageFormat::kBGRA8)
+ {
+
+ }
+ else if (data.format == voltu::ImageFormat::kFloat32)
+ {
+ return ::cv::cuda::GpuMat(
+ data.height,
+ data.width,
+ CV_32F,
+ data.data
+ );
+ }
+
throw voltu::exceptions::NotImplemented();
}
-void voltu::cv::visualise(voltu::ImagePtr img, ::cv::Mat &mat)
+void voltu::opencv::visualise(voltu::ImagePtr img, ::cv::Mat &mat)
{
voltu::ImageData data = img->getHost();
if (data.format == voltu::ImageFormat::kBGRA8)
{
- voltu::cv::convert(img, mat);
+ voltu::opencv::convert(img, mat);
}
else if (data.format == voltu::ImageFormat::kFloat32)
{
::cv::Mat tmp;
- voltu::cv::convert(img, tmp);
+ voltu::opencv::convert(img, tmp);
float maxdepth = 8.0f; // TODO: Get from intrinsics
@@ -65,7 +86,7 @@ void voltu::cv::visualise(voltu::ImagePtr img, ::cv::Mat &mat)
else if (data.format == voltu::ImageFormat::kFloat16_4)
{
::cv::Mat tmp;
- voltu::cv::convert(img, tmp);
+ voltu::opencv::convert(img, tmp);
tmp.convertTo(tmp, CV_32FC4);
tmp += 1.0f;
tmp *= 127.0f;
--
GitLab