From 02e566a278e7d758eb2e9e09651923163c911dd0 Mon Sep 17 00:00:00 2001
From: Nicolas Pope <nwpope@utu.fi>
Date: Sat, 6 Jun 2020 15:55:09 +0300
Subject: [PATCH] WIP does grab frames but not fast enough
---
CMakeLists.txt | 5 +-
applications/gui/src/main.cpp | 8 +
applications/vision/src/main.cpp | 8 +
cmake/FindPylon.cmake | 24 +--
components/rgbd-sources/CMakeLists.txt | 6 +-
components/rgbd-sources/src/source.cpp | 11 ++
.../rgbd-sources/src/sources/pylon/pylon.cpp | 161 ++++++++++++++++++
.../rgbd-sources/src/sources/pylon/pylon.hpp | 39 +++++
components/rgbd-sources/test/source_unit.cpp | 13 ++
9 files changed, 250 insertions(+), 25 deletions(-)
create mode 100644 components/rgbd-sources/src/sources/pylon/pylon.cpp
create mode 100644 components/rgbd-sources/src/sources/pylon/pylon.hpp
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 446407f04..3f2c96fb1 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -10,7 +10,6 @@ include(GNUInstallDirs)
include(CTest)
enable_testing()
-option(WITH_PYLON "Use Pylon for Basler cameras" ON)
option(WITH_OPTFLOW "Use NVIDIA Optical Flow if available" OFF)
option(WITH_OPENVR "Build with OpenVR support" OFF)
option(WITH_OPUS "Use Opus audio compression" ON)
@@ -49,9 +48,7 @@ find_package( URIParser REQUIRED )
find_package( MsgPack REQUIRED )
find_package( Eigen3 REQUIRED )
-if (WITH_PYLON)
- find_package( Pylon )
-endif()
+find_package( Pylon )
VERSION_STR_TO_INTS(OPENCV_MAJOR OPENCV_MINOR OPENCV_PATCH ${OpenCV_VERSION})
math(EXPR OPENCV_NUMBER "(${OPENCV_MAJOR} * 10000) + (${OPENCV_MINOR} * 100) + ${OPENCV_PATCH}")
diff --git a/applications/gui/src/main.cpp b/applications/gui/src/main.cpp
index 03146915e..182faf06e 100644
--- a/applications/gui/src/main.cpp
+++ b/applications/gui/src/main.cpp
@@ -10,8 +10,16 @@
#include <cuda_gl_interop.h>
+#ifdef HAVE_PYLON
+#include <pylon/PylonIncludes.h>
+#endif
+
int main(int argc, char **argv) {
+#ifdef HAVE_PYLON
+ Pylon::PylonAutoInitTerm autoInitTerm;
+#endif
+
auto root = ftl::configure(argc, argv, "gui_default");
ftl::net::Universe *net = ftl::create<ftl::net::Universe>(root, "net");
diff --git a/applications/vision/src/main.cpp b/applications/vision/src/main.cpp
index 489a384b2..d855dd935 100644
--- a/applications/vision/src/main.cpp
+++ b/applications/vision/src/main.cpp
@@ -34,6 +34,10 @@
#include "opencv2/highgui.hpp"
#include "opencv2/core/utility.hpp"
+#ifdef HAVE_PYLON
+#include <pylon/PylonIncludes.h>
+#endif
+
#ifdef WIN32
#pragma comment(lib, "Rpcrt4.lib")
#endif
@@ -194,6 +198,10 @@ static void run(ftl::Configurable *root) {
}
int main(int argc, char **argv) {
+#ifdef HAVE_PYLON
+ Pylon::PylonAutoInitTerm autoInitTerm;
+#endif
+
#ifdef WIN32
SetPriorityClass(GetCurrentProcess(), HIGH_PRIORITY_CLASS);
#endif
diff --git a/cmake/FindPylon.cmake b/cmake/FindPylon.cmake
index ea13d339f..a9f69e004 100644
--- a/cmake/FindPylon.cmake
+++ b/cmake/FindPylon.cmake
@@ -13,34 +13,18 @@ endif()
if (PYLON_DIR)
set(PYLON_FOUND TRUE CACHE BOOL "" FORCE)
set(HAVE_PYLON TRUE)
- # Find lib dir
- # Find include
- find_path(PYLON_LIBRARY_DIRS
- NAMES libpylonbase.so
- PATHS ${PYLON_DIR}
- PATH_SUFFIXES lib
- )
-
- # Find include
- find_path(PYLON_INCLUDE_DIRS
- NAMES pylon/PylonBase.h
- PATHS ${PYLON_DIR}
- PATH_SUFFIXES include
- )
-
include(FindPackageHandleStandardArgs)
find_package_handle_standard_args(Pylon DEFAULT_MSG PYLON_DIR)
mark_as_advanced(PYLON_FOUND)
- mark_as_advanced(PYLON_INCLUDE_DIRS)
- mark_as_advanced(PYLON_LIBRARY_DIRS)
list(APPEND PYLON_LIBRARIES pylonbase pylonutility GenApi_gcc_v3_1_Basler_pylon GCBase_gcc_v3_1_Basler_pylon)
- add_library(Pylon UNKNOWN IMPORTED)
- set_property(TARGET Pylon PROPERTY INTERFACE_INCLUDE_DIRECTORIES ${PYLON_INCLUDE_DIRS})
- set_property(TARGET Pylon PROPERTY INTERFACE_LINK_DIRECTORIES ${PYLON_INCLUDE_DIRS})
+ add_library(Pylon INTERFACE)
+ set_property(TARGET Pylon PROPERTY INTERFACE_INCLUDE_DIRECTORIES ${PYLON_DIR}/include)
+ #set_property(TARGET Pylon PROPERTY INTERFACE_LINK_DIRECTORIES ${PYLON_DIR}/lib)
+ link_directories(${PYLON_DIR}/lib)
set_property(TARGET Pylon PROPERTY INTERFACE_LINK_LIBRARIES ${PYLON_LIBRARIES})
else()
add_library(Pylon INTERFACE)
diff --git a/components/rgbd-sources/CMakeLists.txt b/components/rgbd-sources/CMakeLists.txt
index fafceea69..d15fd8d61 100644
--- a/components/rgbd-sources/CMakeLists.txt
+++ b/components/rgbd-sources/CMakeLists.txt
@@ -17,6 +17,10 @@ if (HAVE_REALSENSE)
list(APPEND RGBDSRC "src/sources/realsense/realsense_source.cpp")
endif()
+if (HAVE_PYLON)
+ list(APPEND RGBDSRC "src/sources/pylon/pylon.cpp")
+endif()
+
if (LibArchive_FOUND)
list(APPEND RGBDSRC
src/sources/snapshot/snapshot.cpp
@@ -38,7 +42,7 @@ if (CUDA_FOUND)
set_property(TARGET ftlrgbd PROPERTY CUDA_SEPARABLE_COMPILATION OFF)
endif()
-target_link_libraries(ftlrgbd ftlcalibration ftlcommon ${OpenCV_LIBS} ${LIBSGM_LIBRARIES} ${CUDA_LIBRARIES} Eigen3::Eigen realsense ftlnet ${LibArchive_LIBRARIES} ftlcodecs ftloperators ftldata ${X11_X11_LIB} ${X11_Xext_LIB})
+target_link_libraries(ftlrgbd ftlcalibration ftlcommon ${OpenCV_LIBS} ${LIBSGM_LIBRARIES} ${CUDA_LIBRARIES} Eigen3::Eigen realsense ftlnet ${LibArchive_LIBRARIES} ftlcodecs ftloperators ftldata ${X11_X11_LIB} ${X11_Xext_LIB} Pylon)
if (BUILD_TESTS)
add_subdirectory(test)
diff --git a/components/rgbd-sources/src/source.cpp b/components/rgbd-sources/src/source.cpp
index d39ea51d5..ea5ff3423 100644
--- a/components/rgbd-sources/src/source.cpp
+++ b/components/rgbd-sources/src/source.cpp
@@ -20,6 +20,11 @@
using ftl::rgbd::detail::RealsenseSource;
#endif
+#ifdef HAVE_PYLON
+#include "sources/pylon/pylon.hpp"
+using ftl::rgbd::detail::PylonSource;
+#endif
+
#include <fstream>
using ftl::rgbd::Source;
@@ -173,6 +178,12 @@ ftl::rgbd::detail::Source *Source::_createNetImpl(const ftl::URI &uri) {
ftl::rgbd::detail::Source *Source::_createDeviceImpl(const ftl::URI &uri) {
if (uri.getPathSegment(0) == "video") {
return new StereoVideoSource(this);
+ } else if (uri.getPathSegment(0) == "pylon") {
+#ifdef HAVE_PYLON
+ return new PylonSource(this);
+#else
+ LOG(ERROR) << "You did not build with 'pylon'";
+#endif
} else if (uri.getPathSegment(0) == "realsense") {
#ifdef HAVE_REALSENSE
return new RealsenseSource(this);
diff --git a/components/rgbd-sources/src/sources/pylon/pylon.cpp b/components/rgbd-sources/src/sources/pylon/pylon.cpp
new file mode 100644
index 000000000..50346b5cb
--- /dev/null
+++ b/components/rgbd-sources/src/sources/pylon/pylon.cpp
@@ -0,0 +1,161 @@
+#include "pylon.hpp"
+
+#include <loguru.hpp>
+#include <ftl/threads.hpp>
+#include <ftl/rgbd/source.hpp>
+
+#include <pylon/PylonIncludes.h>
+
+#include <opencv2/imgproc.hpp>
+
+using ftl::rgbd::detail::PylonSource;
+using std::string;
+using ftl::codecs::Channel;
+using cv::cuda::GpuMat;
+using namespace Pylon;
+
+PylonSource::PylonSource(ftl::rgbd::Source *host)
+ : ftl::rgbd::detail::Source(host), ready_(false), lcam_(nullptr) {
+ capabilities_ = kCapVideo;
+
+ auto &inst = CTlFactory::GetInstance();
+
+ Pylon::DeviceInfoList_t devices;
+ inst.EnumerateDevices(devices);
+
+ if (devices.size() == 0) {
+ LOG(ERROR) << "No Pylon devices attached";
+ return;
+ } else {
+ for (auto d : devices) {
+ LOG(INFO) << " - found Pylon device - " << d.GetModelName();
+ }
+ }
+
+ try {
+ lcam_ = new CInstantCamera( CTlFactory::GetInstance().CreateFirstDevice());
+
+ lcam_->RegisterConfiguration( new Pylon::CSoftwareTriggerConfiguration, Pylon::RegistrationMode_ReplaceAll, Pylon::Cleanup_Delete);
+
+ lcam_->Open();
+
+ lcam_->StartGrabbing( Pylon::GrabStrategy_OneByOne);
+
+ // Get the camera control object.
+ GenApi::INodeMap& nodemap = lcam_->GetNodeMap();
+ // Get the parameters for setting the image area of interest (Image AOI).
+ CIntegerParameter width(nodemap, "Width");
+ CIntegerParameter height(nodemap, "Height");
+ CIntegerParameter offsetX(nodemap, "OffsetX");
+ CIntegerParameter offsetY(nodemap, "OffsetY");
+
+ params_.width = width.GetValue();
+ params_.height = height.GetValue();
+
+ LOG(INFO) << "Camera resolution = " << params_.width << "x" << params_.height;
+
+ // Set the pixel data format.
+ CEnumParameter format(nodemap, "PixelFormat");
+ LOG(INFO) << "Camera format: " << format.GetValue();
+
+ if (format.CanSetValue("BGR8")) {
+ format.SetValue("BGR8");
+ } else {
+ LOG(WARNING) << "Could not change pixel format";
+ }
+
+ ready_ = true;
+ } catch (const Pylon::GenericException &e) {
+ // Error handling.
+ LOG(ERROR) << "Pylon: An exception occurred - " << e.GetDescription();
+ }
+
+ /*params_.width = intrin.width;
+ params_.height = intrin.height;
+ params_.cx = -intrin.ppx;
+ params_.cy = -intrin.ppy;
+ params_.fx = intrin.fx;
+ params_.fy = intrin.fy;
+ params_.maxDepth = 3.0;
+ params_.minDepth = 0.1;
+ params_.doffs = 0.0;*/
+
+ state_.getLeft() = params_;
+}
+
+PylonSource::~PylonSource() {
+
+}
+
+bool PylonSource::capture(int64_t ts) {
+ timestamp_ = ts;
+ if (!lcam_) return false;
+
+ try {
+ if ( lcam_->WaitForFrameTriggerReady( 20, Pylon::TimeoutHandling_ThrowException)) {
+ lcam_->ExecuteSoftwareTrigger();
+ LOG(INFO) << "TRIGGER";
+ }
+ } catch (const GenericException &e) {
+ LOG(ERROR) << "Pylon: Trigger exception - " << e.GetDescription();
+ }
+
+ return true;
+}
+
+bool PylonSource::retrieve() {
+ if (!lcam_) return false;
+
+ auto &frame = frames_[0];
+ frame.reset();
+ frame.setOrigin(&state_);
+
+ try {
+ if ( lcam_->GetGrabResultWaitObject().Wait( 0)) {
+ LOG(INFO) << "Grad result waiting";
+ }
+
+ Pylon::CGrabResultPtr ptrGrabResult;
+
+ int count = 0;
+ if (lcam_->RetrieveResult( 0, ptrGrabResult, Pylon::TimeoutHandling_Return)) ++count;
+
+ if (count == 0 || !ptrGrabResult->GrabSucceeded()) {
+ LOG(ERROR) << "Retrieve failed";
+ return false;
+ }
+
+ cv::Mat wrap(
+ ptrGrabResult->GetHeight(),
+ ptrGrabResult->GetWidth(),
+ CV_8UC3,
+ (uint8_t*)ptrGrabResult->GetBuffer());
+
+ cv::Mat tmp;
+ cv::cvtColor(wrap, tmp, cv::COLOR_BGR2BGRA);
+
+ frame.create<cv::cuda::GpuMat>(ftl::codecs::Channel::Colour).upload(tmp);
+
+ } catch (const GenericException &e) {
+ LOG(ERROR) << "Pylon: An exception occurred - " << e.GetDescription();
+ }
+
+ return true;
+}
+
+void PylonSource::swap() {
+ auto tmp = std::move(frames_[0]);
+ frames_[0] = std::move(frames_[1]);
+ frames_[1] = std::move(tmp);
+}
+
+bool PylonSource::compute(int n, int b) {
+ auto &frame = frames_[1];
+ host_->notify(timestamp_, frame);
+ return true;
+}
+
+bool PylonSource::isReady() {
+ return true;
+}
+
diff --git a/components/rgbd-sources/src/sources/pylon/pylon.hpp b/components/rgbd-sources/src/sources/pylon/pylon.hpp
new file mode 100644
index 000000000..c81bdae21
--- /dev/null
+++ b/components/rgbd-sources/src/sources/pylon/pylon.hpp
@@ -0,0 +1,39 @@
+#pragma once
+#ifndef _FTL_RGBD_PYLON_HPP_
+#define _FTL_RGBD_PYLON_HPP_
+
+#include <ftl/rgbd/detail/source.hpp>
+#include <string>
+
+namespace Pylon {
+class CInstantCamera;
+}
+
+namespace ftl {
+
+namespace rgbd {
+
+namespace detail {
+
+class PylonSource : public ftl::rgbd::detail::Source {
+ public:
+ explicit PylonSource(ftl::rgbd::Source *host);
+ ~PylonSource();
+
+ void swap();
+ bool capture(int64_t ts);
+ bool retrieve();
+ bool compute(int n=-1, int b=-1);
+ bool isReady();
+
+ private:
+ bool ready_;
+ Pylon::CInstantCamera *lcam_;
+ Frame frames_[2];
+};
+
+}
+}
+}
+
+#endif // _FTL_RGBD_PYLON_HPP_
diff --git a/components/rgbd-sources/test/source_unit.cpp b/components/rgbd-sources/test/source_unit.cpp
index c1dbd76f5..29f406272 100644
--- a/components/rgbd-sources/test/source_unit.cpp
+++ b/components/rgbd-sources/test/source_unit.cpp
@@ -120,6 +120,18 @@ class RealsenseSource : public ftl::rgbd::detail::Source {
bool isReady() { return true; };
};
+class PylonSource : public ftl::rgbd::detail::Source {
+ public:
+ explicit PylonSource(ftl::rgbd::Source *host) : ftl::rgbd::detail::Source(host) {
+ last_type = "pylon";
+ }
+
+ bool capture(int64_t ts) { return true; }
+ bool retrieve() { return true; }
+ bool compute(int n, int b) { return true; };
+ bool isReady() { return true; };
+};
+
class MiddleburySource : public ftl::rgbd::detail::Source {
public:
MiddleburySource(ftl::rgbd::Source *host, const std::string &dir) : ftl::rgbd::detail::Source(host) {
@@ -145,6 +157,7 @@ class MiddleburySource : public ftl::rgbd::detail::Source {
#define _FTL_RGBD_SNAPSHOT_SOURCE_HPP_
#define _FTL_RGBD_IMAGE_HPP_
#define _FTL_RGBD_REALSENSE_HPP_
+#define _FTL_RGBD_PYLON_HPP_
#define _FTL_RGBD_SCREENCAPTURE_HPP_
#define _FTL_RGBD_MIDDLEBURY_SOURCE_HPP_
#define _FTL_RGBD_FILE_SOURCE_HPP_
--
GitLab