/* * Copyright 2019 Nicolas Pope. All rights reserved. * * See LICENSE. */ #define LOGURU_WITH_STREAMS 1 #include <loguru.hpp> #include <ftl/config.h> #include <ftl/configuration.hpp> #include <ftl/depth_camera.hpp> #include <ftl/scene_rep_hash_sdf.hpp> #include <ftl/rgbd.hpp> #include <ftl/virtual_source.hpp> // #include <zlib.h> // #include <lz4.h> #include <string> #include <vector> #include <thread> #include <chrono> #include <mutex> #include <opencv2/opencv.hpp> #include <ftl/net/universe.hpp> #include <ftl/rgbd_display.hpp> #include <nlohmann/json.hpp> #include <opencv2/imgproc.hpp> #include <opencv2/imgcodecs.hpp> #include <opencv2/highgui.hpp> #include <opencv2/core/utility.hpp> #include <ftl/utility/opencv_to_pcl.hpp> #include <ftl/registration.hpp> #ifdef WIN32 #pragma comment(lib, "Rpcrt4.lib") #endif #ifdef HAVE_PCL #include <pcl/point_cloud.h> #include <pcl/common/transforms.h> #include <pcl/filters/uniform_sampling.h> #endif using ftl::net::Universe; using ftl::rgbd::Display; using std::string; using std::vector; using ftl::rgbd::RGBDSource; using ftl::config::json_t; using json = nlohmann::json; using std::this_thread::sleep_for; using std::chrono::milliseconds; using std::mutex; using std::unique_lock; using std::vector; using cv::Mat; using ftl::registration::loadTransformations; using ftl::registration::saveTransformations; struct Cameras { RGBDSource *source; DepthCameraData gpu; DepthCameraParams params; }; static void run(ftl::Configurable *root) { Universe *net = ftl::create<Universe>(root, "net"); // Make sure connections are complete // sleep_for(milliseconds(500)); // TODO: possible to do more reliably? net->waitConnections(); std::vector<Cameras> inputs; auto sources = root->get<vector<json_t>>("sources"); if (!sources) { LOG(ERROR) << "No sources configured!"; return; } // TODO Allow for non-net source types for (auto &src : *sources) { RGBDSource *in = ftl::rgbd::RGBDSource::create(src, net); if (!in) { LOG(ERROR) << "Unrecognized source: " << src; } else { auto &cam = inputs.emplace_back(); cam.source = in; cam.params.fx = in->getParameters().fx; cam.params.fy = in->getParameters().fy; cam.params.mx = -in->getParameters().cx; cam.params.my = -in->getParameters().cy; cam.params.m_imageWidth = in->getParameters().width; cam.params.m_imageHeight = in->getParameters().height; cam.params.m_sensorDepthWorldMax = in->getParameters().maxDepth; cam.params.m_sensorDepthWorldMin = in->getParameters().minDepth; cam.gpu.alloc(cam.params); LOG(INFO) << (string) src["uri"] << " loaded " << cam.params.fx; } } // TODO move this block in its own method and add automated tests // for error cases std::optional<json_t> merge = root->get<json_t>("merge"); if (!merge) { LOG(WARNING) << "Input merging not configured, using only first input in configuration"; inputs = { inputs[0] }; } if (inputs.size() > 1) { std::map<std::string, Eigen::Matrix4f> transformations; if ((*merge)["register"]) { LOG(INFO) << "Registration requested"; ftl::registration::Registration *reg = ftl::registration::ChessboardRegistration::create(*merge); for (auto &input : inputs) { while(!input.source->isReady()) { std::this_thread::sleep_for(std::chrono::milliseconds(50)); } reg->addSource(input.source); } reg->run(); if (reg->findTransformations(transformations)) { if (!saveTransformations(string(FTL_LOCAL_CONFIG_ROOT) + "/registration.json", transformations)) { LOG(ERROR) << "Error saving new registration"; }; } else { LOG(ERROR) << "Registration failed"; } free(reg); } else { if (loadTransformations(string(FTL_LOCAL_CONFIG_ROOT) + "/registration.json", transformations)) { LOG(INFO) << "Loaded camera trasformations from file"; } else { LOG(ERROR) << "Error loading camera transformations from file"; } } for (auto &input : inputs) { string uri = input.source->getURI(); auto T = transformations.find(uri); if (T == transformations.end()) { LOG(ERROR) << "Camera pose for " + uri + " not found in transformations"; LOG(WARNING) << "Using only first configured source"; // TODO: use target source if configured and found inputs = { inputs[0] }; inputs[0].source->setPose(Eigen::Matrix4f::Identity()); break; } input.source->setPose(T->second); } } // Displays and Inputs configured LOG(INFO) << "Using sources:"; for (auto &input : inputs) { LOG(INFO) << " " + (string) input.source->getURI(); } ftl::rgbd::Display *display = ftl::create<ftl::rgbd::Display>(root, "display"); ftl::rgbd::VirtualSource *virt = ftl::create<ftl::rgbd::VirtualSource>(root, "virtual", net); ftl::voxhash::SceneRep *scene = ftl::create<ftl::voxhash::SceneRep>(root, "voxelhash"); virt->setScene(scene); display->setSource(virt); unsigned char frameCount = 0; bool paused = false; // Keyboard camera controls display->onKey([&paused](int key) { if (key == 32) paused = !paused; }); int active = inputs.size(); while (active > 0 && display->active()) { active = 0; if (!paused) { //net.broadcast("grab"); // To sync cameras scene->nextFrame(); for (size_t i = 0; i < inputs.size(); i++) { // Get the RGB-Depth frame from input RGBDSource *input = inputs[i].source; Mat rgb, depth; input->grab(); input->getRGBD(rgb,depth); active += 1; if (depth.cols == 0) continue; // Must be in RGBA for GPU Mat rgba; cv::cvtColor(rgb,rgba, cv::COLOR_BGR2BGRA); inputs[i].params.flags = frameCount; // Send to GPU and merge view into scene inputs[i].gpu.updateParams(inputs[i].params); inputs[i].gpu.updateData(depth, rgba); scene->integrate(inputs[i].source->getPose(), inputs[i].gpu, inputs[i].params, nullptr); } } else { active = 1; } frameCount++; display->update(); } } int main(int argc, char **argv) { run(ftl::configure(argc, argv, "reconstruction_default")); }