diff --git a/applications/reconstruct/include/ftl/scene_rep_hash_sdf.hpp b/applications/reconstruct/include/ftl/scene_rep_hash_sdf.hpp index a28567931b55cf2059687f924ec6efdb08e00451..20072e7f3317ae2ddecea625d08a56be9f1bd977 100644 --- a/applications/reconstruct/include/ftl/scene_rep_hash_sdf.hpp +++ b/applications/reconstruct/include/ftl/scene_rep_hash_sdf.hpp @@ -336,7 +336,7 @@ private: //t.startEvent("compactifyAllInOne"); m_hashParams.m_numOccupiedBlocks = compactifyHashAllInOneCUDA(m_hashData, m_hashParams); //this version uses atomics over prefix sums, which has a much better performance - std::cout << "Occ blocks = " << m_hashParams.m_numOccupiedBlocks << std::endl; + //std::cout << "Occ blocks = " << m_hashParams.m_numOccupiedBlocks << std::endl; m_hashData.updateParams(m_hashParams); //make sure numOccupiedBlocks is updated on the GPU //t.endEvent(); //t.evaluate(); diff --git a/applications/reconstruct/include/ftl/virtual_source.hpp b/applications/reconstruct/include/ftl/virtual_source.hpp index 6bc1022f719863a893d0586d6fd7b6e010873f05..aed3dee506e0f66849797d610f9a538c28a2a1c2 100644 --- a/applications/reconstruct/include/ftl/virtual_source.hpp +++ b/applications/reconstruct/include/ftl/virtual_source.hpp @@ -38,8 +38,7 @@ class VirtualSource : public RGBDSource { ftl::voxhash::SceneRep *scene_; CUDARayCastSDF *rays_; bool ready_; - cv::Mat rgb_; - cv::Mat depth_; + cv::Mat idepth_; }; } diff --git a/applications/reconstruct/src/main.cpp b/applications/reconstruct/src/main.cpp index 9a40bf1ad24e246205c5b02c2e39dc642d6ff030..9bcd659793fedd7ed2532448acd347498c107e80 100644 --- a/applications/reconstruct/src/main.cpp +++ b/applications/reconstruct/src/main.cpp @@ -444,6 +444,7 @@ static void run() { // Get the RGB-Depth frame from input RGBDSource *input = inputs[i].source; Mat rgb, depth; + input->grab(); input->getRGBD(rgb,depth); active += 1; diff --git a/applications/reconstruct/src/virtual_source.cpp b/applications/reconstruct/src/virtual_source.cpp index 7b7bd8299c1a6aed299a48fe00797999f595700f..8e4955a320bc2847f2ca5b358661208c1c5cc7fc 100644 --- a/applications/reconstruct/src/virtual_source.cpp +++ b/applications/reconstruct/src/virtual_source.cpp @@ -6,6 +6,8 @@ #include <glog/logging.h> using ftl::rgbd::VirtualSource; +using std::mutex; +using std::unique_lock; VirtualSource::VirtualSource(nlohmann::json &config, ftl::net::Universe *net) : RGBDSource(config, net) { @@ -22,7 +24,7 @@ VirtualSource::VirtualSource(nlohmann::json &config, ftl::net::Universe *net) params_.minDepth = config.value("min_depth", 0.1f); rgb_ = cv::Mat(cv::Size(params_.width,params_.height), CV_8UC3); - // init depth + idepth_ = cv::Mat(cv::Size(params_.width,params_.height), CV_32SC1); } VirtualSource::~VirtualSource() { @@ -47,13 +49,11 @@ void VirtualSource::grab() { params.m_sensorDepthWorldMax = params_.maxDepth; rays_->render(scene_->getHashData(), scene_->getHashParams(), params, getPose()); - rays_->getRayCastData().download(nullptr, (uchar3*)rgb_.data, rays_->getRayCastParams()); - } -} -void VirtualSource::getRGBD(cv::Mat &rgb, cv::Mat &depth) { - rgb_.copyTo(rgb); - depth_.copyTo(depth); + unique_lock<mutex> lk(mutex_); + rays_->getRayCastData().download((int*)idepth_.data, (uchar3*)rgb_.data, rays_->getRayCastParams()); + idepth_.convertTo(depth_, CV_32FC1, 1.0f / 100.0f); + } } bool VirtualSource::isReady() { diff --git a/applications/vision/src/main.cpp b/applications/vision/src/main.cpp index d3a193ff12302738ca6c1094fcb541b44d71fa3f..35fa22bd125af31a29acc09a55a7257f29548de4 100644 --- a/applications/vision/src/main.cpp +++ b/applications/vision/src/main.cpp @@ -149,7 +149,7 @@ static void run(const string &file) { std::chrono::duration<double> elapsed = std::chrono::high_resolution_clock::now() - start; - LOG(INFO) << "Disparity in " << elapsed.count() << "s"; + //LOG(INFO) << "Disparity in " << elapsed.count() << "s"; }); // Pipeline for stream compression @@ -163,7 +163,7 @@ static void run(const string &file) { std::chrono::duration<double> elapsed = std::chrono::high_resolution_clock::now() - start; - LOG(INFO) << "Stream in " << elapsed.count() << "s"; + //LOG(INFO) << "Stream in " << elapsed.count() << "s"; }); // Send RGB+Depth images for local rendering diff --git a/applications/vision/src/streamer.cpp b/applications/vision/src/streamer.cpp index 0a4d0853583e7ef74d1f2985f56131c64598d9ec..0a05351f7de5cd2a72e08846d2cee03305a1dc71 100644 --- a/applications/vision/src/streamer.cpp +++ b/applications/vision/src/streamer.cpp @@ -53,7 +53,7 @@ void Streamer::send(const Mat &rgb, const Mat &depth) { // d_buf.resize(s); cv::imencode(".png", d2, d_buf); - LOG(INFO) << "Depth Size = " << ((float)d_buf.size() / (1024.0f*1024.0f)); + //LOG(INFO) << "Depth Size = " << ((float)d_buf.size() / (1024.0f*1024.0f)); try { net_.publish(uri_, rgb_buf, d_buf); diff --git a/components/renderers/cpp/src/rgbd_display.cpp b/components/renderers/cpp/src/rgbd_display.cpp index 4eeedd0e9ec76aeb4c14167d8ed097a661504f5f..473878548586a5508e97622fe31bbf1b55f7fe55 100644 --- a/components/renderers/cpp/src/rgbd_display.cpp +++ b/components/renderers/cpp/src/rgbd_display.cpp @@ -72,10 +72,10 @@ void Display::init() { // Keyboard camera controls onKey([this](int key) { - LOG(INFO) << "Key = " << key; + //LOG(INFO) << "Key = " << key; if (key == 81 || key == 83) { // TODO Should rotate around lookAt object, but requires correct depth - Eigen::Quaternion<float> q; q = Eigen::AngleAxis<float>((key == 81) ? 0.01f : -0.01f, Eigen::Vector3f(0.0,1.0f,0.0f)); + Eigen::Quaternion<float> q; q = Eigen::AngleAxis<float>((key == 81) ? 0.01f : -0.01f, up_); eye_ = (q * (eye_ - centre_)) + centre_; } else if (key == 84 || key == 82) { float scalar = (key == 84) ? 0.99f : 1.01f; @@ -85,16 +85,13 @@ void Display::init() { // TODO(Nick) Calculate "camera" properties of viewport. mouseaction_ = [this]( int event, int ux, int uy, int) { - LOG(INFO) << "Mouse " << ux << "," << uy; + //LOG(INFO) << "Mouse " << ux << "," << uy; if (event == 1 && source_) { // click - const auto params = source_->getParameters(); - const float x = ((float)ux-params.width/2) / params.fx; - const float y = ((float)uy-params.height/2) / params.fy; - const float depth = -4.0f; - Eigen::Vector4f camPos(x*depth,y*depth,depth,1.0); + Eigen::Vector4f camPos = source_->point(ux,uy); + camPos *= -1.0f; Eigen::Vector4f worldPos = source_->getPose() * camPos; lookPoint_ = Eigen::Vector3f(worldPos[0],worldPos[1],worldPos[2]); - LOG(INFO) << "Look at: " << worldPos; + LOG(INFO) << "Depth at click = " << -camPos[2]; } }; ::setMouseAction(name_, mouseaction_); @@ -128,6 +125,6 @@ void Display::update() { Mat rgb, depth; source_->grab(); source_->getRGBD(rgb, depth); - cv::imshow(name_, rgb); + if (rgb.rows > 0) cv::imshow(name_, rgb); wait(1); } diff --git a/components/rgbd-sources/include/ftl/net_source.hpp b/components/rgbd-sources/include/ftl/net_source.hpp index 327e95319145252dc7417f7dc9bbf6b4511fc34c..8d7c44a88fc3ddbf813cdf96f5131cb208f4c57b 100644 --- a/components/rgbd-sources/include/ftl/net_source.hpp +++ b/components/rgbd-sources/include/ftl/net_source.hpp @@ -5,7 +5,6 @@ #include <ftl/net/universe.hpp> #include <ftl/rgbd_source.hpp> #include <string> -#include <mutex> namespace ftl { namespace rgbd { @@ -23,7 +22,6 @@ class NetSource : public RGBDSource { ~NetSource(); void grab(); - void getRGBD(cv::Mat &rgb, cv::Mat &depth); bool isReady(); static inline RGBDSource *create(nlohmann::json &config, ftl::net::Universe *net) { @@ -32,9 +30,6 @@ class NetSource : public RGBDSource { private: bool has_calibration_; - cv::Mat rgb_; - cv::Mat depth_; - std::mutex m_; }; } diff --git a/components/rgbd-sources/include/ftl/rgbd_source.hpp b/components/rgbd-sources/include/ftl/rgbd_source.hpp index 5032aa3c59fb1a60e4bf31fa99b3671476ce6378..9944533b257bc35d5cba227a003eaaeb8b791180 100644 --- a/components/rgbd-sources/include/ftl/rgbd_source.hpp +++ b/components/rgbd-sources/include/ftl/rgbd_source.hpp @@ -8,6 +8,7 @@ #include <ftl/net/universe.hpp> #include <opencv2/opencv.hpp> #include <Eigen/Eigen> +#include <mutex> namespace ftl { namespace rgbd { @@ -23,14 +24,22 @@ class RGBDSource : public ftl::Configurable { virtual ~RGBDSource(); virtual void grab()=0; - virtual void getRGBD(cv::Mat &rgb, cv::Mat &depth)=0; virtual bool isReady(); + void getRGBD(cv::Mat &rgb, cv::Mat &depth); + const CameraParameters &getParameters() { return params_; }; - void setPose(const Eigen::Matrix4f &pose) { pose_ = pose; }; + virtual void setPose(const Eigen::Matrix4f &pose) { pose_ = pose; }; const Eigen::Matrix4f &getPose() { return pose_; }; + virtual void reset() {} + + /** + * Get a point in camera coordinates at specified pixel location. + */ + Eigen::Vector4f point(uint x, uint y); + /** * Save the current RGB and Depth images to image files (jpg and png) with * the specified file prefix (excluding file extension). @@ -66,6 +75,9 @@ class RGBDSource : public ftl::Configurable { protected: CameraParameters params_; ftl::net::Universe *net_; + std::mutex mutex_; + cv::Mat rgb_; + cv::Mat depth_; private: Eigen::Matrix4f pose_; diff --git a/components/rgbd-sources/include/ftl/stereovideo_source.hpp b/components/rgbd-sources/include/ftl/stereovideo_source.hpp index d81df41865141f2692f3ba8fda414e3f53ebab5c..47e2d089cbe55f114e6cce283c3504f3dfb35482 100644 --- a/components/rgbd-sources/include/ftl/stereovideo_source.hpp +++ b/components/rgbd-sources/include/ftl/stereovideo_source.hpp @@ -26,7 +26,6 @@ class StereoVideoSource : public RGBDSource { ~StereoVideoSource(); void grab(); - void getRGBD(cv::Mat &rgb, cv::Mat &depth); bool isReady(); const cv::Mat &getRight() const { return right_; } diff --git a/components/rgbd-sources/src/net_source.cpp b/components/rgbd-sources/src/net_source.cpp index 2f4ed19955a8e72a6263aaeced455e1c5d6c4946..0e50d0c915a7dfc60022ea66be2648f46e0a1889 100644 --- a/components/rgbd-sources/src/net_source.cpp +++ b/components/rgbd-sources/src/net_source.cpp @@ -44,7 +44,7 @@ NetSource::NetSource(nlohmann::json &config, ftl::net::Universe *net) has_calibration_ = getCalibration(*net, config["uri"].get<string>(), params_); net->subscribe(config["uri"].get<string>(), [this](const vector<unsigned char> &jpg, const vector<unsigned char> &d) { - unique_lock<mutex> lk(m_); + unique_lock<mutex> lk(mutex_); cv::imdecode(jpg, cv::IMREAD_COLOR, &rgb_); //Mat(rgb_.size(), CV_16UC1); cv::imdecode(d, cv::IMREAD_UNCHANGED, &depth_); @@ -63,9 +63,3 @@ void NetSource::grab() { bool NetSource::isReady() { return has_calibration_ && !rgb_.empty(); } - -void NetSource::getRGBD(cv::Mat &rgb, cv::Mat &depth) { - unique_lock<mutex> lk(m_); - rgb_.copyTo(rgb); - depth_.copyTo(depth); -} diff --git a/components/rgbd-sources/src/rgbd_source.cpp b/components/rgbd-sources/src/rgbd_source.cpp index f4f3e4aede512ab515cca6a4fc5cbb4a11abb3b2..df70d45437127e543c12b573b662c9833d92a103 100644 --- a/components/rgbd-sources/src/rgbd_source.cpp +++ b/components/rgbd-sources/src/rgbd_source.cpp @@ -3,6 +3,8 @@ using ftl::rgbd::RGBDSource; using ftl::Configurable; using std::string; +using std::mutex; +using std::unique_lock; std::map<std::string, std::function<RGBDSource*(nlohmann::json&,ftl::net::Universe*)>> *RGBDSource::sources__ = nullptr; @@ -22,6 +24,22 @@ bool RGBDSource::isReady() { return false; } +void RGBDSource::getRGBD(cv::Mat &rgb, cv::Mat &depth) { + unique_lock<mutex> lk(mutex_); + rgb_.copyTo(rgb); + depth_.copyTo(depth); +} + +Eigen::Vector4f RGBDSource::point(uint ux, uint uy) { + const auto ¶ms = getParameters(); + const float x = ((float)ux-params.width/2) / params.fx; + const float y = ((float)uy-params.height/2) / params.fy; + + unique_lock<mutex> lk(mutex_); + const float depth = depth_.at<float>(uy,ux); + return Eigen::Vector4f(x*depth,y*depth,depth,1.0); +} + bool RGBDSource::snapshot(const std::string &fileprefix) { cv::Mat rgb; cv::Mat depth; diff --git a/components/rgbd-sources/src/stereovideo_source.cpp b/components/rgbd-sources/src/stereovideo_source.cpp index 571981758d806ede860693c5d369861a45af54af..e254099c728371b686566e97107f8f03cc38bcd7 100644 --- a/components/rgbd-sources/src/stereovideo_source.cpp +++ b/components/rgbd-sources/src/stereovideo_source.cpp @@ -4,11 +4,14 @@ #include "calibrate.hpp" #include "local.hpp" #include "disparity.hpp" +#include <mutex> using ftl::Calibrate; using ftl::LocalSource; using ftl::rgbd::StereoVideoSource; using std::string; +using std::mutex; +using std::unique_lock; StereoVideoSource::StereoVideoSource(nlohmann::json &config, ftl::net::Universe *net) : RGBDSource(config, net) { @@ -79,14 +82,6 @@ StereoVideoSource::~StereoVideoSource() { delete lsrc_; } -void StereoVideoSource::grab() { - calib_->rectified(left_, right_); -} - -bool StereoVideoSource::isReady() { - return ready_; -} - static void disparityToDepth(const cv::Mat &disparity, cv::Mat &depth, const cv::Mat &q) { cv::Matx44d _Q; q.convertTo(_Q, CV_64F); @@ -110,10 +105,17 @@ static void disparityToDepth(const cv::Mat &disparity, cv::Mat &depth, const cv: } } -void StereoVideoSource::getRGBD(cv::Mat &rgb, cv::Mat &depth) { +void StereoVideoSource::grab() { + calib_->rectified(left_, right_); + cv::Mat disp; disp_->compute(left_, right_, disp, mask_l_); - rgb = left_; - disparityToDepth(disp, depth, calib_->getQ()); - //calib_->distort(rgb,depth); + + unique_lock<mutex> lk(mutex_); + left_.copyTo(rgb_); + disparityToDepth(disp, depth_, calib_->getQ()); +} + +bool StereoVideoSource::isReady() { + return ready_; }