WIP: resolution

applications/reconstruct/src/reconstruction.cpp

@@ -50,10 +50,36 @@ Reconstruction::Reconstruction(nlohmann::json &config, const std::string name) :
 
 		ftl::pool.push([this](int id) {
 			UNIQUE_LOCK(fs_align_.mtx, lk);
+			
+			rgb_.resize(fs_align_.frames.size());
+			for (size_t i = 0; i < rgb_.size(); i++) {
+				auto &depth = fs_align_.frames[i].get<cv::cuda::GpuMat>(ftl::codecs::Channel::Depth);
+				auto &color = fs_align_.frames[i].get<cv::cuda::GpuMat>(ftl::codecs::Channel::Colour);
+
+				if (depth.size() != color.size()) {
+					std::swap(rgb_[i], color);
+					cv::cuda::resize(rgb_[i], color, depth.size(), 0.0, 0.0, cv::INTER_LINEAR);
+				}
+			}
+
 			pipeline_->apply(fs_align_, fs_align_, 0);
 			
 			// TODO: To use second GPU, could do a download, swap, device change,
 			// then upload to other device. Or some direct device-2-device copy.
+			/*
+			for (size_t i = 0; i < rgb_.size(); i++) {
+				auto &depth = fs_align_.frames[i].get<cv::cuda::GpuMat>(ftl::codecs::Channel::Depth);
+				auto &color = fs_align_.frames[i].get<cv::cuda::GpuMat>(ftl::codecs::Channel::Colour);
+				auto &tmp = rgb_[i];
+
+				// TODO doesn't always work correctly if resolution changes
+				if (!tmp.empty() && (depth.size() != tmp.size())) {
+					std::swap(tmp, color);
+					fs_align_.frames[i].resetTexture(ftl::codecs::Channel::Colour);
+					fs_align_.frames[i].createTexture<uchar4>(ftl::codecs::Channel::Colour, true);
+				}
+			}*/
+
 			fs_align_.swapTo(fs_render_);
 
 			LOG(INFO) << "Align complete... " << fs_align_.timestamp;

applications/reconstruct/src/reconstruction.hpp

@@ -25,11 +25,14 @@ class Reconstruction : public ftl::Configurable {
 
 	private:
 	bool busy_;
+	
 	ftl::rgbd::FrameSet fs_render_;
 	ftl::rgbd::FrameSet fs_align_;
 	ftl::rgbd::Group *group_;
 	ftl::operators::Graph *pipeline_;
 	ftl::render::Triangular *renderer_;
+
+	std::vector<cv::cuda::GpuMat> rgb_;
 };
 
 }

components/codecs/src/opencv_decoder.cpp

@@ -18,7 +18,7 @@ bool OpenCVDecoder::accepts(const ftl::codecs::Packet &pkt) {
 }
 
 bool OpenCVDecoder::decode(const ftl::codecs::Packet &pkt, cv::cuda::GpuMat &out) {
-
+	//CHECK(cv::Size(ftl::codecs::getWidth(pkt.definition), ftl::codecs::getHeight(pkt.definition)) == out.size()); 
 	int chunk_dim = std::sqrt(pkt.block_total);
 	int chunk_width = out.cols / chunk_dim;
 	int chunk_height = out.rows / chunk_dim;
@@ -37,7 +37,6 @@ bool OpenCVDecoder::decode(const ftl::codecs::Packet &pkt, cv::cuda::GpuMat &out
 	// Apply colour correction to chunk
 	//ftl::rgbd::colourCorrection(tmp_rgb, gamma_, temperature_);
 
-
 	// TODO:(Nick) Decode directly into double buffer if no scaling
 	// Can either check JPG/PNG headers or just use pkt definition.
 

components/codecs/src/opencv_encoder.cpp

@@ -17,7 +17,7 @@ OpenCVEncoder::OpenCVEncoder(ftl::codecs::definition_t maxdef,
 }
 
 OpenCVEncoder::~OpenCVEncoder() {
-    
+	
 }
 
 bool OpenCVEncoder::supports(ftl::codecs::codec_t codec) {
@@ -33,6 +33,7 @@ bool OpenCVEncoder::encode(const cv::cuda::GpuMat &in, definition_t definition,
 	current_definition_ = definition;
 
 	in.download(tmp_);
+	//CHECK(cv::Size(ftl::codecs::getWidth(definition), ftl::codecs::getHeight(definition)) == in.size()); 
 
 	// Scale down image to match requested definition...
 	if (ftl::codecs::getHeight(current_definition_) < in.rows) {
@@ -42,7 +43,7 @@ bool OpenCVEncoder::encode(const cv::cuda::GpuMat &in, definition_t definition,
 	}
 
 	// Represent float at 16bit int
-    if (!is_colour) {
+	if (!is_colour) {
 		tmp_.convertTo(tmp_, CV_16UC1, 1000);
 	}
 

components/rgbd-sources/src/sources/net/net.cpp

@@ -276,8 +276,9 @@ void NetSource::_recvPacket(short ttimeoff, const ftl::codecs::StreamPacket &spk
 		LOG(WARNING) << "Missing calibration, skipping frame";
 		return;
 	}
-
-	NetFrame &frame = queue_.getFrame(spkt.timestamp, cv::Size(params_.width, params_.height), CV_8UC3, (isFloatChannel(chan) ? CV_32FC1 : CV_8UC3));
+	
+	const cv::Size size = cv::Size(ftl::codecs::getWidth(pkt.definition), ftl::codecs::getHeight(pkt.definition));
+	NetFrame &frame = queue_.getFrame(spkt.timestamp, size, CV_8UC3, (isFloatChannel(chan) ? CV_32FC1 : CV_8UC3));
 
 	// Update frame statistics
 	frame.tx_size += pkt.data.size();
@@ -291,6 +292,7 @@ void NetSource::_recvPacket(short ttimeoff, const ftl::codecs::StreamPacket &spk
 			return;
 		}
 
+		frame.channel[channum].create(size, frame.channel[channum].type()); // TODO!!! queue_.getFrame() assumes same size
 		decoder->decode(pkt, frame.channel[channum]);
 	} else if (chan != Channel::None && rchan != Channel::Colour) {
 		// Didn't receive correct second channel so just clear the images