WIP does grab frames but not fast enough

CMakeLists.txt

@@ -10,7 +10,6 @@ include(GNUInstallDirs)
 include(CTest)
 enable_testing()
 
-option(WITH_PYLON "Use Pylon for Basler cameras" ON)
 option(WITH_OPTFLOW "Use NVIDIA Optical Flow if available" OFF)
 option(WITH_OPENVR "Build with OpenVR support" OFF)
 option(WITH_OPUS "Use Opus audio compression" ON)
@@ -49,9 +48,7 @@ find_package( URIParser REQUIRED )
 find_package( MsgPack REQUIRED )
 find_package( Eigen3 REQUIRED )
 
-if (WITH_PYLON)
-	find_package( Pylon )
-endif()
+find_package( Pylon )
 
 VERSION_STR_TO_INTS(OPENCV_MAJOR OPENCV_MINOR OPENCV_PATCH ${OpenCV_VERSION})
 math(EXPR OPENCV_NUMBER "(${OPENCV_MAJOR} * 10000) + (${OPENCV_MINOR} * 100) + ${OPENCV_PATCH}")

applications/gui/src/main.cpp

@@ -10,8 +10,16 @@
 
 #include <cuda_gl_interop.h>
 
+#ifdef HAVE_PYLON
+#include <pylon/PylonIncludes.h>
+#endif
+
 
 int main(int argc, char **argv) {
+#ifdef HAVE_PYLON
+	Pylon::PylonAutoInitTerm autoInitTerm;
+#endif
+
 	auto root = ftl::configure(argc, argv, "gui_default");
 	ftl::net::Universe *net = ftl::create<ftl::net::Universe>(root, "net");
 

applications/vision/src/main.cpp

@@ -34,6 +34,10 @@
 #include "opencv2/highgui.hpp"
 #include "opencv2/core/utility.hpp"
 
+#ifdef HAVE_PYLON
+#include <pylon/PylonIncludes.h>
+#endif
+
 #ifdef WIN32
 #pragma comment(lib, "Rpcrt4.lib")
 #endif
@@ -194,6 +198,10 @@ static void run(ftl::Configurable *root) {
 }
 
 int main(int argc, char **argv) {
+#ifdef HAVE_PYLON
+	Pylon::PylonAutoInitTerm autoInitTerm;
+#endif
+
 #ifdef WIN32
 	SetPriorityClass(GetCurrentProcess(), HIGH_PRIORITY_CLASS);
 #endif

cmake/FindPylon.cmake

@@ -13,34 +13,18 @@ endif()
 if (PYLON_DIR)
 	set(PYLON_FOUND TRUE CACHE BOOL "" FORCE)
 	set(HAVE_PYLON TRUE)
-	# Find lib dir
 	
-	# Find include
-	find_path(PYLON_LIBRARY_DIRS
-		NAMES libpylonbase.so
-		PATHS ${PYLON_DIR}
-		PATH_SUFFIXES lib
-	)
-
-	# Find include
-	find_path(PYLON_INCLUDE_DIRS
-		NAMES pylon/PylonBase.h
-		PATHS ${PYLON_DIR}
-		PATH_SUFFIXES include
-	)
-
 	include(FindPackageHandleStandardArgs)
 	find_package_handle_standard_args(Pylon DEFAULT_MSG PYLON_DIR)
 
 	mark_as_advanced(PYLON_FOUND)
-	mark_as_advanced(PYLON_INCLUDE_DIRS)
-	mark_as_advanced(PYLON_LIBRARY_DIRS)
 
 	list(APPEND PYLON_LIBRARIES pylonbase pylonutility GenApi_gcc_v3_1_Basler_pylon GCBase_gcc_v3_1_Basler_pylon)
 
-	add_library(Pylon UNKNOWN IMPORTED)
-	set_property(TARGET Pylon PROPERTY INTERFACE_INCLUDE_DIRECTORIES ${PYLON_INCLUDE_DIRS})
-	set_property(TARGET Pylon PROPERTY INTERFACE_LINK_DIRECTORIES ${PYLON_INCLUDE_DIRS})
+	add_library(Pylon INTERFACE)
+	set_property(TARGET Pylon PROPERTY INTERFACE_INCLUDE_DIRECTORIES ${PYLON_DIR}/include)
+	#set_property(TARGET Pylon PROPERTY INTERFACE_LINK_DIRECTORIES ${PYLON_DIR}/lib)
+	link_directories(${PYLON_DIR}/lib)
 	set_property(TARGET Pylon PROPERTY INTERFACE_LINK_LIBRARIES ${PYLON_LIBRARIES})
 else()
 	add_library(Pylon INTERFACE)

components/rgbd-sources/CMakeLists.txt

@@ -17,6 +17,10 @@ if (HAVE_REALSENSE)
 	list(APPEND RGBDSRC "src/sources/realsense/realsense_source.cpp")
 endif()
 
+if (HAVE_PYLON)
+	list(APPEND RGBDSRC "src/sources/pylon/pylon.cpp")
+endif()
+
 if (LibArchive_FOUND)
 	list(APPEND RGBDSRC
 		src/sources/snapshot/snapshot.cpp
@@ -38,7 +42,7 @@ if (CUDA_FOUND)
 set_property(TARGET ftlrgbd PROPERTY CUDA_SEPARABLE_COMPILATION OFF)
 endif()
 
-target_link_libraries(ftlrgbd ftlcalibration ftlcommon ${OpenCV_LIBS} ${LIBSGM_LIBRARIES} ${CUDA_LIBRARIES} Eigen3::Eigen realsense ftlnet ${LibArchive_LIBRARIES} ftlcodecs ftloperators ftldata ${X11_X11_LIB} ${X11_Xext_LIB})
+target_link_libraries(ftlrgbd ftlcalibration ftlcommon ${OpenCV_LIBS} ${LIBSGM_LIBRARIES} ${CUDA_LIBRARIES} Eigen3::Eigen realsense ftlnet ${LibArchive_LIBRARIES} ftlcodecs ftloperators ftldata ${X11_X11_LIB} ${X11_Xext_LIB} Pylon)
 
 if (BUILD_TESTS)
 add_subdirectory(test)

components/rgbd-sources/src/source.cpp

@@ -20,6 +20,11 @@
 using ftl::rgbd::detail::RealsenseSource;
 #endif
 
+#ifdef HAVE_PYLON
+#include "sources/pylon/pylon.hpp"
+using ftl::rgbd::detail::PylonSource;
+#endif
+
 #include <fstream>
 
 using ftl::rgbd::Source;
@@ -173,6 +178,12 @@ ftl::rgbd::detail::Source *Source::_createNetImpl(const ftl::URI &uri) {
 ftl::rgbd::detail::Source *Source::_createDeviceImpl(const ftl::URI &uri) {
 	if (uri.getPathSegment(0) == "video") {
 		return new StereoVideoSource(this);
+	} else if (uri.getPathSegment(0) == "pylon") {
+#ifdef HAVE_PYLON
+		return new PylonSource(this);
+#else
+		LOG(ERROR) << "You did not build with 'pylon'";
+#endif
 	} else if (uri.getPathSegment(0) == "realsense") {
 #ifdef HAVE_REALSENSE
 		return new RealsenseSource(this);

components/rgbd-sources/src/sources/pylon/pylon.cpp

+#include "pylon.hpp"
+
+#include <loguru.hpp>
+#include <ftl/threads.hpp>
+#include <ftl/rgbd/source.hpp>
+
+#include <pylon/PylonIncludes.h>
+
+#include <opencv2/imgproc.hpp>
+
+using ftl::rgbd::detail::PylonSource;
+using std::string;
+using ftl::codecs::Channel;
+using cv::cuda::GpuMat;
+using namespace Pylon;
+
+PylonSource::PylonSource(ftl::rgbd::Source *host)
+        : ftl::rgbd::detail::Source(host), ready_(false), lcam_(nullptr) {
+	capabilities_ = kCapVideo;
+
+	auto &inst = CTlFactory::GetInstance();
+
+	Pylon::DeviceInfoList_t devices;
+	inst.EnumerateDevices(devices);
+
+	if (devices.size() == 0) {
+		LOG(ERROR) << "No Pylon devices attached";
+		return;
+	} else {
+		for (auto d : devices) {
+			LOG(INFO) << " - found Pylon device - " << d.GetModelName();
+		}
+	}
+
+	try {
+    	lcam_ = new CInstantCamera( CTlFactory::GetInstance().CreateFirstDevice());
+
+		lcam_->RegisterConfiguration( new Pylon::CSoftwareTriggerConfiguration, Pylon::RegistrationMode_ReplaceAll, Pylon::Cleanup_Delete);
+
+		lcam_->Open();
+
+		lcam_->StartGrabbing( Pylon::GrabStrategy_OneByOne);
+
+		// Get the camera control object.
+		GenApi::INodeMap& nodemap = lcam_->GetNodeMap();
+		// Get the parameters for setting the image area of interest (Image AOI).
+		CIntegerParameter width(nodemap, "Width");
+		CIntegerParameter height(nodemap, "Height");
+		CIntegerParameter offsetX(nodemap, "OffsetX");
+		CIntegerParameter offsetY(nodemap, "OffsetY");
+
+		params_.width = width.GetValue();
+		params_.height = height.GetValue();
+
+		LOG(INFO) << "Camera resolution = " << params_.width << "x" << params_.height;
+
+		// Set the pixel data format.
+		CEnumParameter format(nodemap, "PixelFormat");
+		LOG(INFO) << "Camera format: " << format.GetValue();
+
+		if (format.CanSetValue("BGR8")) {
+			format.SetValue("BGR8");
+		} else {
+			LOG(WARNING) << "Could not change pixel format";
+		}
+
+		ready_ = true;
+	} catch (const Pylon::GenericException &e) {
+		// Error handling.
+        LOG(ERROR) << "Pylon: An exception occurred - " << e.GetDescription();
+	}
+
+    /*params_.width = intrin.width;
+    params_.height = intrin.height;
+    params_.cx = -intrin.ppx;
+    params_.cy = -intrin.ppy;
+    params_.fx = intrin.fx;
+    params_.fy = intrin.fy;
+    params_.maxDepth = 3.0;
+    params_.minDepth = 0.1;
+	params_.doffs = 0.0;*/
+
+    state_.getLeft() = params_;
+}
+
+PylonSource::~PylonSource() {
+
+}
+
+bool PylonSource::capture(int64_t ts) {
+	timestamp_ = ts;
+	if (!lcam_) return false;
+
+	try {
+		if ( lcam_->WaitForFrameTriggerReady( 20, Pylon::TimeoutHandling_ThrowException)) {
+			lcam_->ExecuteSoftwareTrigger();
+			LOG(INFO) << "TRIGGER";
+		}
+	} catch (const GenericException &e) {
+		LOG(ERROR) << "Pylon: Trigger exception - " << e.GetDescription();
+	}
+
+	return true;
+}
+
+bool PylonSource::retrieve() {
+	if (!lcam_) return false;
+
+	auto &frame = frames_[0];
+	frame.reset();
+	frame.setOrigin(&state_);
+
+	try {
+		if ( lcam_->GetGrabResultWaitObject().Wait( 0)) {
+			LOG(INFO) << "Grad result waiting";
+		}
+
+		Pylon::CGrabResultPtr ptrGrabResult;
+
+		int count = 0;
+		if (lcam_->RetrieveResult( 0, ptrGrabResult, Pylon::TimeoutHandling_Return)) ++count;
+
+		if (count == 0 || !ptrGrabResult->GrabSucceeded()) {
+			LOG(ERROR) << "Retrieve failed";
+			return false;
+		}
+
+		cv::Mat wrap(
+			ptrGrabResult->GetHeight(),
+			ptrGrabResult->GetWidth(),
+			CV_8UC3,
+			(uint8_t*)ptrGrabResult->GetBuffer());
+
+		cv::Mat tmp;
+		cv::cvtColor(wrap, tmp, cv::COLOR_BGR2BGRA);
+
+		frame.create<cv::cuda::GpuMat>(ftl::codecs::Channel::Colour).upload(tmp);
+
+	} catch (const GenericException &e) {
+		LOG(ERROR) << "Pylon: An exception occurred - " << e.GetDescription();
+	}
+
+	return true;
+}
+
+void PylonSource::swap() {
+	auto tmp = std::move(frames_[0]);
+	frames_[0] = std::move(frames_[1]);
+	frames_[1] = std::move(tmp);
+}
+
+bool PylonSource::compute(int n, int b) {
+	auto &frame = frames_[1];
+	host_->notify(timestamp_, frame);
+    return true;
+}
+
+bool PylonSource::isReady() {
+    return true;
+}
+

components/rgbd-sources/src/sources/pylon/pylon.hpp

+#pragma once
+#ifndef _FTL_RGBD_PYLON_HPP_
+#define _FTL_RGBD_PYLON_HPP_
+
+#include <ftl/rgbd/detail/source.hpp>
+#include <string>
+
+namespace Pylon {
+class CInstantCamera;
+}
+
+namespace ftl {
+
+namespace rgbd {
+
+namespace detail {
+
+class PylonSource : public ftl::rgbd::detail::Source {
+	public:
+	explicit PylonSource(ftl::rgbd::Source *host);
+	~PylonSource();
+
+	void swap();
+	bool capture(int64_t ts);
+	bool retrieve();
+	bool compute(int n=-1, int b=-1);
+	bool isReady();
+
+	private:
+	bool ready_;
+	Pylon::CInstantCamera *lcam_;
+	Frame frames_[2];
+};
+
+}
+}
+}
+
+#endif  // _FTL_RGBD_PYLON_HPP_

components/rgbd-sources/test/source_unit.cpp

@@ -120,6 +120,18 @@ class RealsenseSource : public ftl::rgbd::detail::Source {
 	bool isReady() { return true; };
 };
 
+class PylonSource : public ftl::rgbd::detail::Source {
+	public:
+	explicit PylonSource(ftl::rgbd::Source *host) : ftl::rgbd::detail::Source(host) {
+		last_type = "pylon";
+	}
+
+	bool capture(int64_t ts) { return true; }
+	bool retrieve() { return true; }
+	bool compute(int n, int b) { return true; };
+	bool isReady() { return true; };
+};
+
 class MiddleburySource : public ftl::rgbd::detail::Source {
 	public:
 	MiddleburySource(ftl::rgbd::Source *host, const std::string &dir) : ftl::rgbd::detail::Source(host) {
@@ -145,6 +157,7 @@ class MiddleburySource : public ftl::rgbd::detail::Source {
 #define _FTL_RGBD_SNAPSHOT_SOURCE_HPP_
 #define _FTL_RGBD_IMAGE_HPP_
 #define _FTL_RGBD_REALSENSE_HPP_
+#define _FTL_RGBD_PYLON_HPP_
 #define _FTL_RGBD_SCREENCAPTURE_HPP_
 #define _FTL_RGBD_MIDDLEBURY_SOURCE_HPP_
 #define _FTL_RGBD_FILE_SOURCE_HPP_