diff --git a/components/rgbd-sources/include/ftl/rgbd/streamer.hpp b/components/rgbd-sources/include/ftl/rgbd/streamer.hpp index b894d40f4d2a2b90026eb201e998d67d0260f951..e832fb546bdead7334367ccf14b2b024ba36e7b7 100644 --- a/components/rgbd-sources/include/ftl/rgbd/streamer.hpp +++ b/components/rgbd-sources/include/ftl/rgbd/streamer.hpp @@ -36,8 +36,11 @@ struct StreamSource { std::atomic<unsigned int> jobs; // Busy or ready to swap? cv::Mat rgb; // Tx buffer cv::Mat depth; // Tx buffer + cv::Mat prev_rgb; + cv::Mat prev_depth; std::vector<detail::StreamClient> clients[10]; // One list per bitrate std::shared_mutex mutex; + unsigned long long frame; }; } diff --git a/components/rgbd-sources/src/net.cpp b/components/rgbd-sources/src/net.cpp index 64ecf1f3e9ddbbce8a3f9854a3d8ab498b1d79e3..a527665c29d4d1ca5f301f1eae05ff7a8618bba7 100644 --- a/components/rgbd-sources/src/net.cpp +++ b/components/rgbd-sources/src/net.cpp @@ -85,7 +85,7 @@ void NetSource::_recvChunk(int frame, int chunk, bool delta, const vector<unsign //LOG(INFO) << "Received chunk " << (int)chunk; - try { + //try { // Decode in temporary buffers to prevent long locks cv::imdecode(jpg, cv::IMREAD_COLOR, &tmp_rgb); cv::imdecode(d, cv::IMREAD_UNCHANGED, &tmp_depth); @@ -96,18 +96,22 @@ void NetSource::_recvChunk(int frame, int chunk, bool delta, const vector<unsign cv::Rect roi(cx,cy,chunk_width_,chunk_height_); cv::Mat chunkRGB = rgb_(roi); + //cv::Mat ichunkDepth = idepth_(roi); cv::Mat chunkDepth = depth_(roi); // Lock host to prevent grab UNIQUE_LOCK(host_->mutex(),lk); tmp_rgb.copyTo(chunkRGB); - tmp_depth.convertTo(chunkDepth, CV_32FC1, 1.0f/(16.0f*100.0f)); + //tmp_depth.convertTo(tmp_depth, CV_16UC1); + //if (delta) ichunkDepth = tmp_depth - ichunkDepth; + //tmp_depth.copyTo(ichunkDepth); + tmp_depth.convertTo(chunkDepth, CV_32FC1, 1.0f/(16.0f*10.0f)); if (chunk == 0) N_--; //lk.unlock(); - } catch(...) { - LOG(ERROR) << "Decode exception"; - return; - } + //} catch(...) { + // LOG(ERROR) << "Decode exception"; + // return; + //} } void NetSource::setPose(const Eigen::Matrix4f &pose) { @@ -163,6 +167,7 @@ void NetSource::_updateURI() { chunk_height_ = params_.height / chunks_dim_; rgb_ = cv::Mat(cv::Size(params_.width, params_.height), CV_8UC3, cv::Scalar(0,0,0)); depth_ = cv::Mat(cv::Size(params_.width, params_.height), CV_32FC1, 0.0f); + //idepth_ = cv::Mat(cv::Size(params_.width, params_.height), CV_16UC1, cv::Scalar(0)); uri_ = *uri; active_ = true; diff --git a/components/rgbd-sources/src/net.hpp b/components/rgbd-sources/src/net.hpp index 5ad0e08d7d71eba003edd69a5a04bebe09225d50..3a986d0647ae465e355dbf887e283f61a43b53b3 100644 --- a/components/rgbd-sources/src/net.hpp +++ b/components/rgbd-sources/src/net.hpp @@ -40,6 +40,7 @@ class NetSource : public detail::Source { int chunks_dim_; int chunk_width_; int chunk_height_; + cv::Mat idepth_; bool _getCalibration(ftl::net::Universe &net, const ftl::UUID &peer, const std::string &src, ftl::rgbd::Camera &p); void _recv(const std::vector<unsigned char> &jpg, const std::vector<unsigned char> &d); diff --git a/components/rgbd-sources/src/streamer.cpp b/components/rgbd-sources/src/streamer.cpp index add5d681d8a312719a6f94c8012916fdbf52ed1b..2e559262a6619719bc869da2313a557d3786a96e 100644 --- a/components/rgbd-sources/src/streamer.cpp +++ b/components/rgbd-sources/src/streamer.cpp @@ -100,7 +100,9 @@ void Streamer::add(Source *src) { StreamSource *s = new StreamSource; s->src = src; + //s->prev_depth = cv::Mat(cv::Size(src->parameters().width, src->parameters().height), CV_16SC1, 0); s->jobs = 0; + s->frame = 0; sources_[src->getID()] = s; } @@ -212,7 +214,12 @@ void Streamer::_swap(StreamSource *src) { } src->src->getFrames(src->rgb, src->depth); + //if (!src->rgb.empty() && src->prev_depth.empty()) { + //src->prev_depth = cv::Mat(src->rgb.size(), CV_16UC1, cv::Scalar(0)); + //LOG(INFO) << "Creating prevdepth: " << src->rgb.cols << "," << src->rgb.rows; + //} src->jobs = 0; + src->frame++; } } @@ -278,6 +285,7 @@ void Streamer::_schedule() { for (int i=0; i<(kChunkDim*kChunkDim); i++) { ftl::pool.push([this,src](int id, int chunk) { if (!src->rgb.empty() && !src->depth.empty()) { + bool delta = (chunk+src->frame) % 8 > 0; int chunk_width = src->rgb.cols / kChunkDim; int chunk_height = src->rgb.rows / kChunkDim; @@ -289,23 +297,27 @@ void Streamer::_schedule() { vector<unsigned char> rgb_buf; cv::Mat chunkRGB = src->rgb(roi); cv::Mat chunkDepth = src->depth(roi); + //cv::Mat chunkDepthPrev = src->prev_depth(roi); cv::imencode(".jpg", chunkRGB, rgb_buf); - cv::Mat d2; + cv::Mat d2, d3; vector<unsigned char> d_buf; - chunkDepth.convertTo(d2, CV_16UC1, 16*100); + chunkDepth.convertTo(d2, CV_16UC1, 16*10); + //if (delta) d3 = (d2 * 2) - chunkDepthPrev; + //else d3 = d2; + //d2.copyTo(chunkDepthPrev); vector<int> pngparams = {cv::IMWRITE_PNG_COMPRESSION, 1}; // Default is 1 for fast, 9 = small but slow. cv::imencode(".png", d2, d_buf, pngparams); - //LOG(INFO) << "Sending chunk " << chunk << " : size = " << (d_buf.size()+rgb_buf.size()) / 1024 << "kb"; + LOG(INFO) << "Sending chunk " << chunk << " : size = " << (d_buf.size()+rgb_buf.size()) / 1024 << "kb"; UNIQUE_LOCK(src->mutex,lk); auto i = src->clients[0].begin(); while (i != src->clients[0].end()) { try { // TODO(Nick) Send pose and timestamp - if (!net_->send((*i).peerid, (*i).uri, 0, chunk, false, rgb_buf, d_buf)) { + if (!net_->send((*i).peerid, (*i).uri, 0, chunk, delta, rgb_buf, d_buf)) { (*i).txcount = (*i).txmax; } } catch(...) {