diff --git a/CMakeLists.txt b/CMakeLists.txt index ef3b984099eeedcc3ce0d98c94e5266bd3239099..5d5a06e1091b155dc1c393de696bbca07dd65453 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -20,7 +20,7 @@ find_package( Threads REQUIRED ) find_package( URIParser REQUIRED ) find_package( MsgPack REQUIRED ) find_package( LibSGM ) -find_package( ZLIB REQUIRED ) +#find_package( ZLIB REQUIRED ) # Readline library is not required on Windows # May also entirely remove dependence on this... it should be optional at least. diff --git a/common/config/config.json b/common/config/config.json index 80d6589a63910e414ff02d2db35e46291eb69130..4c0ec32253deeac8ac77c89a1f6659f8f437ee2d 100644 --- a/common/config/config.json +++ b/common/config/config.json @@ -9,7 +9,8 @@ "flip": false, "nostereo": false, "scale": 1.0, - "flip_vert": false + "flip_vert": false, + "max_fps": 25 }, "calibrate": false, "calibration": { diff --git a/reconstruct/CMakeLists.txt b/reconstruct/CMakeLists.txt index 78ce7d52c79333e4b237c1fe7aa9b1c0a764bcd6..3d851d469c66f19ce084a3bf0bbbf1f390adc636 100644 --- a/reconstruct/CMakeLists.txt +++ b/reconstruct/CMakeLists.txt @@ -12,6 +12,6 @@ add_dependencies(ftl-reconstruct ftlnet) add_dependencies(ftl-reconstruct ftlcommon) #target_include_directories(cv-node PUBLIC ${PROJECT_SOURCE_DIR}/include) -target_link_libraries(ftl-reconstruct ftlcommon Threads::Threads ZLIB::ZLIB ${OpenCV_LIBS} glog::glog ftlnet ftlrender) +target_link_libraries(ftl-reconstruct ftlcommon Threads::Threads ${OpenCV_LIBS} glog::glog ftlnet ftlrender) diff --git a/reconstruct/src/main.cpp b/reconstruct/src/main.cpp index 02f915a938730181a6fe97e1097c5768c3169c82..51c346da25e0f367247cc33234a63bbac957e251 100644 --- a/reconstruct/src/main.cpp +++ b/reconstruct/src/main.cpp @@ -7,7 +7,7 @@ #include <glog/logging.h> #include <ftl/config.h> #include <ftl/configuration.hpp> -#include <zlib.h> +// #include <zlib.h> // #include <lz4.h> #include <string> diff --git a/vision/CMakeLists.txt b/vision/CMakeLists.txt index a0cc3f6b9f57fb1d43d5c56ed035c17b11b0da44..5320fd283d02d6468da3fb2d2192b9819b52a8e6 100644 --- a/vision/CMakeLists.txt +++ b/vision/CMakeLists.txt @@ -47,6 +47,6 @@ set_property(TARGET ftl-vision PROPERTY CUDA_SEPARABLE_COMPILATION ON) endif() #target_include_directories(cv-node PUBLIC ${PROJECT_SOURCE_DIR}/include) -target_link_libraries(ftl-vision ftlcommon ftlrender Threads::Threads ZLIB::ZLIB libelas ${OpenCV_LIBS} ${LIBSGM_LIBRARIES} ${CUDA_LIBRARIES} glog::glog ftlnet) +target_link_libraries(ftl-vision ftlcommon ftlrender Threads::Threads libelas ${OpenCV_LIBS} ${LIBSGM_LIBRARIES} ${CUDA_LIBRARIES} glog::glog ftlnet) diff --git a/vision/include/ftl/local.hpp b/vision/include/ftl/local.hpp index cf18b9928ac8d0e4e13479cced920ae3038b0df2..80b527cc74c50122274f9568c1ce8b5b9ad2b653 100644 --- a/vision/include/ftl/local.hpp +++ b/vision/include/ftl/local.hpp @@ -28,6 +28,7 @@ class LocalSource { private: double timestamp_; + double tps_; bool stereo_; //float fps_; bool flip_; diff --git a/vision/src/local.cpp b/vision/src/local.cpp index ea000e1a07c6550fc5085db59ed6106fe80fb181..d0e542669df9e2e6faba2ba0268fb53b7d99183a 100644 --- a/vision/src/local.cpp +++ b/vision/src/local.cpp @@ -6,6 +6,7 @@ #include <string> #include <chrono> +#include <thread> #include <ftl/local.hpp> #include <opencv2/core.hpp> @@ -19,6 +20,8 @@ using std::string; using std::chrono::duration_cast; using std::chrono::duration; using std::chrono::high_resolution_clock; +using std::chrono::milliseconds; +using std::this_thread::sleep_for; LocalSource::LocalSource(nlohmann::json &config) : timestamp_(0.0), @@ -51,6 +54,8 @@ LocalSource::LocalSource(nlohmann::json &config) } else { stereo_ = true; } + + tps_ = 1.0 / (double)config["max_fps"]; } LocalSource::LocalSource(const string &vid, nlohmann::json &config) @@ -89,6 +94,8 @@ LocalSource::LocalSource(const string &vid, nlohmann::json &config) LOG(INFO) << "Video size : " << frame.cols << "x" << frame.rows; stereo_ = false; } + + tps_ = 1.0 / (double)config["max_fps"]; } bool LocalSource::left(cv::Mat &l) { @@ -177,8 +184,15 @@ bool LocalSource::get(cv::Mat &l, cv::Mat &r) { } // Record timestamp - timestamp_ = duration_cast<duration<double>>( + double timestamp = duration_cast<duration<double>>( high_resolution_clock::now().time_since_epoch()).count(); + + // Limit max framerate + if (timestamp - timestamp_ < tps_) { + sleep_for(milliseconds((int)std::round((tps_ - (timestamp - timestamp_))*1000))); + } + + timestamp_ = timestamp; if (camera_b_ || !stereo_) { if (!camera_a_->retrieve(l)) { diff --git a/vision/src/main.cpp b/vision/src/main.cpp index 0755a65134fdc7407ebd56c238fbc778db763c35..59b61bae91e02c781a00edbce2d1a252081a33be 100644 --- a/vision/src/main.cpp +++ b/vision/src/main.cpp @@ -7,6 +7,7 @@ #include <glog/logging.h> #include <ftl/configuration.hpp> #include <ctpl_stl.h> +#include <zlib.h> #include <string> #include <map> @@ -105,6 +106,9 @@ static void run(const string &file) { Mat l, r, disp; Mat pl, pdisp; + vector<unsigned char> rgb_buf; + vector<unsigned char> d_buf; + string uri = string("ftl://utu.fi/")+(string)config["stream"]["name"]+string("/rgb-d"); Display display(config["display"]); display.setCalibration(Q_32F); @@ -116,6 +120,7 @@ static void run(const string &file) { condition_variable cv; int jobs = 0; + // Pipeline for disparity pool.push([&](int id) { auto start = std::chrono::high_resolution_clock::now(); // Read calibrated images. @@ -141,6 +146,51 @@ static void run(const string &file) { LOG(INFO) << "Disparity in " << elapsed.count() << "s"; }); + // Pipeline for jpeg compression + /*pool.push([&](int id) { + auto start = std::chrono::high_resolution_clock::now(); + if (pl.rows != 0) cv::imencode(".jpg", pl, rgb_buf); + unique_lock<mutex> lk(m); + jobs++; + lk.unlock(); + cv.notify_one(); + + std::chrono::duration<double> elapsed = + std::chrono::high_resolution_clock::now() - start; + LOG(INFO) << "JPG in " << elapsed.count() << "s"; + });*/ + + // Pipeline for zlib compression + /*pool.push([&](int id) { + auto start = std::chrono::high_resolution_clock::now(); + if (pl.rows != 0) { + d_buf.resize(pdisp.step*pdisp.rows); + z_stream defstream; + defstream.zalloc = Z_NULL; + defstream.zfree = Z_NULL; + defstream.opaque = Z_NULL; + defstream.avail_in = pdisp.step*pdisp.rows; + defstream.next_in = (Bytef *)pdisp.data; // input char array + defstream.avail_out = (uInt)pdisp.step*pdisp.rows; // size of output + defstream.next_out = (Bytef *)d_buf.data(); // output char array + + deflateInit(&defstream, Z_BEST_COMPRESSION); + deflate(&defstream, Z_FINISH); + deflateEnd(&defstream); + + d_buf.resize(defstream.total_out); + } + unique_lock<mutex> lk(m); + jobs++; + lk.unlock(); + cv.notify_one(); + + std::chrono::duration<double> elapsed = + std::chrono::high_resolution_clock::now() - start; + LOG(INFO) << "ZLIB in " << elapsed.count() << "s"; + });*/ + + // Pipeline for stream compression pool.push([&](int id) { auto start = std::chrono::high_resolution_clock::now(); if (pl.rows != 0) stream.send(pl, pdisp); @@ -158,11 +208,15 @@ static void run(const string &file) { if (pl.rows > 0) display.render(pl, pdisp); display.wait(1); + // Wait for both pipelines to complete unique_lock<mutex> lk(m); cv.wait(lk, [&jobs]{return jobs == 2;}); + // Store previous frame for next stage compression l.copyTo(pl); disp.copyTo(pdisp); + + //net.publish(uri, rgb_buf, d_buf); } } diff --git a/vision/src/streamer.cpp b/vision/src/streamer.cpp index 6a882949b710b42edafc4986d8576905decfa3c1..d9bef41d43f1b9c9c273eb03a6c2e987ecb46432 100644 --- a/vision/src/streamer.cpp +++ b/vision/src/streamer.cpp @@ -1,7 +1,7 @@ #include <glog/logging.h> #include <ftl/streamer.hpp> #include <vector> -#include <zlib.h> +// #include <zlib.h> // #include <lz4.h> using ftl::Streamer; @@ -39,7 +39,7 @@ void Streamer::send(const Mat &rgb, const Mat &depth) { defstream.avail_out = (uInt)d2.step*d2.rows; // size of output defstream.next_out = (Bytef *)d_buf.data(); // output char array - deflateInit(&defstream, 4); // Z_DEFAULT_COMPRESSION + deflateInit(&defstream, Z_DEFAULT_COMPRESSION); deflate(&defstream, Z_FINISH); deflateEnd(&defstream);