Experiments with delta didnt work out

components/rgbd-sources/include/ftl/rgbd/streamer.hpp

@@ -36,8 +36,11 @@ struct StreamSource {
 	std::atomic<unsigned int> jobs;				// Busy or ready to swap?
 	cv::Mat rgb;									// Tx buffer
 	cv::Mat depth;									// Tx buffer
+	cv::Mat prev_rgb;
+	cv::Mat prev_depth;
 	std::vector<detail::StreamClient> clients[10];	// One list per bitrate
 	std::shared_mutex mutex;
+	unsigned long long frame;
 };
 
 }

components/rgbd-sources/src/net.cpp

@@ -85,7 +85,7 @@ void NetSource::_recvChunk(int frame, int chunk, bool delta, const vector<unsign
 
 	//LOG(INFO) << "Received chunk " << (int)chunk;
 
-	try {
+	//try {
 	// Decode in temporary buffers to prevent long locks
 	cv::imdecode(jpg, cv::IMREAD_COLOR, &tmp_rgb);
 	cv::imdecode(d, cv::IMREAD_UNCHANGED, &tmp_depth);
@@ -96,18 +96,22 @@ void NetSource::_recvChunk(int frame, int chunk, bool delta, const vector<unsign
 
 	cv::Rect roi(cx,cy,chunk_width_,chunk_height_);
 	cv::Mat chunkRGB = rgb_(roi);
+	//cv::Mat ichunkDepth = idepth_(roi);
 	cv::Mat chunkDepth = depth_(roi);
 
 	// Lock host to prevent grab
 	UNIQUE_LOCK(host_->mutex(),lk);
 	tmp_rgb.copyTo(chunkRGB);
-	tmp_depth.convertTo(chunkDepth, CV_32FC1, 1.0f/(16.0f*100.0f));
+	//tmp_depth.convertTo(tmp_depth, CV_16UC1);
+	//if (delta) ichunkDepth = tmp_depth - ichunkDepth;
+	//tmp_depth.copyTo(ichunkDepth);
+	tmp_depth.convertTo(chunkDepth, CV_32FC1, 1.0f/(16.0f*10.0f));
 	if (chunk == 0) N_--;
 	//lk.unlock();
-	} catch(...) {
-		LOG(ERROR) << "Decode exception";
-		return;
-	}
+	//} catch(...) {
+	//	LOG(ERROR) << "Decode exception";
+	//	return;
+	//}
 }
 
 void NetSource::setPose(const Eigen::Matrix4f &pose) {
@@ -163,6 +167,7 @@ void NetSource::_updateURI() {
 		chunk_height_ = params_.height / chunks_dim_;
 		rgb_ = cv::Mat(cv::Size(params_.width, params_.height), CV_8UC3, cv::Scalar(0,0,0));
 		depth_ = cv::Mat(cv::Size(params_.width, params_.height), CV_32FC1, 0.0f);
+		//idepth_ = cv::Mat(cv::Size(params_.width, params_.height), CV_16UC1, cv::Scalar(0));
 
 		uri_ = *uri;
 		active_ = true;

components/rgbd-sources/src/net.hpp

@@ -40,6 +40,7 @@ class NetSource : public detail::Source {
 	int chunks_dim_;
 	int chunk_width_;
 	int chunk_height_;
+	cv::Mat idepth_;
 
 	bool _getCalibration(ftl::net::Universe &net, const ftl::UUID &peer, const std::string &src, ftl::rgbd::Camera &p);
 	void _recv(const std::vector<unsigned char> &jpg, const std::vector<unsigned char> &d);

components/rgbd-sources/src/streamer.cpp

@@ -100,7 +100,9 @@ void Streamer::add(Source *src) {
 
 		StreamSource *s = new StreamSource;
 		s->src = src;
+		//s->prev_depth = cv::Mat(cv::Size(src->parameters().width, src->parameters().height), CV_16SC1, 0);
 		s->jobs = 0;
+		s->frame = 0;
 		sources_[src->getID()] = s;
 	}
 
@@ -212,7 +214,12 @@ void Streamer::_swap(StreamSource *src) {
 		}
 
 		src->src->getFrames(src->rgb, src->depth);
+		//if (!src->rgb.empty() && src->prev_depth.empty()) {
+			//src->prev_depth = cv::Mat(src->rgb.size(), CV_16UC1, cv::Scalar(0));
+			//LOG(INFO) << "Creating prevdepth: " << src->rgb.cols << "," << src->rgb.rows;
+		//}
 		src->jobs = 0;
+		src->frame++;
 	}
 }
 
@@ -278,6 +285,7 @@ void Streamer::_schedule() {
 		for (int i=0; i<(kChunkDim*kChunkDim); i++) {
 			ftl::pool.push([this,src](int id, int chunk) {
 				if (!src->rgb.empty() && !src->depth.empty()) {
+					bool delta = (chunk+src->frame) % 8 > 0;
 					int chunk_width = src->rgb.cols / kChunkDim;
 					int chunk_height = src->rgb.rows / kChunkDim;
 
@@ -289,23 +297,27 @@ void Streamer::_schedule() {
 					vector<unsigned char> rgb_buf;
 					cv::Mat chunkRGB = src->rgb(roi);
 					cv::Mat chunkDepth = src->depth(roi);
+					//cv::Mat chunkDepthPrev = src->prev_depth(roi);
 
 					cv::imencode(".jpg", chunkRGB, rgb_buf);
 
-					cv::Mat d2;
+					cv::Mat d2, d3;
 					vector<unsigned char> d_buf;
-					chunkDepth.convertTo(d2, CV_16UC1, 16*100);
+					chunkDepth.convertTo(d2, CV_16UC1, 16*10);
+					//if (delta) d3 = (d2 * 2) - chunkDepthPrev;
+					//else d3 = d2;
+					//d2.copyTo(chunkDepthPrev);
 					vector<int> pngparams = {cv::IMWRITE_PNG_COMPRESSION, 1}; // Default is 1 for fast, 9 = small but slow.
 					cv::imencode(".png", d2, d_buf, pngparams);
 
-					//LOG(INFO) << "Sending chunk " << chunk << " : size = " << (d_buf.size()+rgb_buf.size()) / 1024 << "kb";
+					LOG(INFO) << "Sending chunk " << chunk << " : size = " << (d_buf.size()+rgb_buf.size()) / 1024 << "kb";
 
 					UNIQUE_LOCK(src->mutex,lk);
 					auto i = src->clients[0].begin();
 					while (i != src->clients[0].end()) {
 						try {
 							// TODO(Nick) Send pose and timestamp
-							if (!net_->send((*i).peerid, (*i).uri, 0, chunk, false, rgb_buf, d_buf)) {
+							if (!net_->send((*i).peerid, (*i).uri, 0, chunk, delta, rgb_buf, d_buf)) {
 								(*i).txcount = (*i).txmax;
 							}
 						} catch(...) {