diff --git a/applications/reconstruct/include/ftl/voxel_scene.hpp b/applications/reconstruct/include/ftl/voxel_scene.hpp index 1babf0145eef0d5c75fc4577571a3d637a72be8c..487cf4b953dc2ea41eda8a645fb3ff56d296eff6 100644 --- a/applications/reconstruct/include/ftl/voxel_scene.hpp +++ b/applications/reconstruct/include/ftl/voxel_scene.hpp @@ -78,9 +78,11 @@ class SceneRep : public ftl::Configurable { void debugHash(); cudaStream_t getIntegrationStream() const { return integ_stream_; } + int getCUDADevice() const { return cuda_device_; } private: + bool _initCUDA(); HashParams _parametersFromConfig(); void _create(const HashParams& params); void _destroy(); @@ -102,6 +104,7 @@ class SceneRep : public ftl::Configurable { std::vector<Cameras> cameras_; cudaStream_t integ_stream_; bool reg_mode_; + int cuda_device_; }; }; // namespace voxhash diff --git a/applications/reconstruct/src/virtual_source.cpp b/applications/reconstruct/src/virtual_source.cpp index 73c80323b9a702964a18bbc7ce68e9c407873b06..245339376c7ded2093de988aa4366c2ef539a325 100644 --- a/applications/reconstruct/src/virtual_source.cpp +++ b/applications/reconstruct/src/virtual_source.cpp @@ -53,6 +53,9 @@ void VirtualSource::setScene(ftl::voxhash::SceneRep *scene) { bool VirtualSource::grab() { if (scene_) { + // Ensure this host thread is using correct GPU. + + cudaSafeCall(cudaSetDevice(scene_->getCUDADevice())); DepthCameraParams params; params.fx = params_.fx; params.fy = params_.fy; diff --git a/applications/reconstruct/src/voxel_scene.cpp b/applications/reconstruct/src/voxel_scene.cpp index 216325dedb2146c1cabacf51746577266fe1264e..831780257adf751999eb397b480349835a0e6cda 100644 --- a/applications/reconstruct/src/voxel_scene.cpp +++ b/applications/reconstruct/src/voxel_scene.cpp @@ -5,10 +5,13 @@ #include <opencv2/core/cuda_stream_accessor.hpp> +#include <vector> + using namespace ftl::voxhash; using ftl::rgbd::Source; using ftl::Configurable; using cv::Mat; +using std::vector; #define SAFE_DELETE_ARRAY(a) { delete [] (a); (a) = NULL; } @@ -23,6 +26,8 @@ extern "C" void integrateDepthMapCUDA(ftl::voxhash::HashData& hashData, const ft SceneRep::SceneRep(nlohmann::json &config) : Configurable(config), do_reset_(false), m_frameCount(0) { + _initCUDA(); + // Allocates voxel structure on GPU _create(_parametersFromConfig()); @@ -62,6 +67,30 @@ SceneRep::~SceneRep() { cudaStreamDestroy(integ_stream_); } +bool SceneRep::_initCUDA() { + // Do an initial CUDA check + int cuda_device_count = 0; + cudaSafeCall(cudaGetDeviceCount(&cuda_device_count)); + CHECK_GE(cuda_device_count, 1) << "No CUDA devices found"; + + LOG(INFO) << "CUDA Devices (" << cuda_device_count << "):"; + + vector<cudaDeviceProp> properties(cuda_device_count); + for (int i=0; i<cuda_device_count; i++) { + cudaSafeCall(cudaGetDeviceProperties(&properties[i], i)); + LOG(INFO) << " - " << properties[i].name; + } + + int desired_device = value("cudaDevice", 0); + cuda_device_ = (desired_device < cuda_device_count) ? desired_device : cuda_device_count-1; + cudaSafeCall(cudaSetDevice(cuda_device_)); + + // TODO(Nick) Check memory is sufficient + // TODO(Nick) Find out what our compute capability should be. + + return true; +} + void SceneRep::addSource(ftl::rgbd::Source *src) { auto &cam = cameras_.emplace_back(); cam.source = src;