diff --git a/applications/ftl2mkv/src/main.cpp b/applications/ftl2mkv/src/main.cpp
index e02e2de0d0464ab872e2877a48831a2649fe0570..0ebf1a0bfdecd03e1696b3c688744dc761f92862 100644
--- a/applications/ftl2mkv/src/main.cpp
+++ b/applications/ftl2mkv/src/main.cpp
@@ -41,18 +41,21 @@ static AVStream *add_video_stream(AVFormatContext *oc, const ftl::codecs::Packet
//c->codec_id = codec_id;
//c->codec_type = AVMEDIA_TYPE_VIDEO;
- st->time_base.den = 20;
- st->time_base.num = 1;
+ //st->time_base.den = 20;
+ //st->time_base.num = 1;
//st->id = oc->nb_streams-1;
//st->nb_frames = 0;
st->codecpar->codec_id = codec_id;
st->codecpar->codec_type = AVMEDIA_TYPE_VIDEO;
st->codecpar->width = ftl::codecs::getWidth(pkt.definition);
+ //if (pkt.flags & ftl::codecs::kFlagStereo) st->codecpar->width *= 2;
st->codecpar->height = ftl::codecs::getHeight(pkt.definition);
st->codecpar->format = AV_PIX_FMT_NV12;
st->codecpar->bit_rate = 4000000;
- if (pkt.flags & ftl::codecs::kFlagStereo) av_dict_set_int(&st->metadata, "stereo_mode", 1, 0);
+ if (pkt.flags & ftl::codecs::kFlagStereo) av_dict_set(&st->metadata, "stereo_mode", "left_right", 0);
+ //if (pkt.flags & ftl::codecs::kFlagStereo) av_dict_set(&oc->metadata, "stereo_mode", "1", 0);
+ //if (pkt.flags & ftl::codecs::kFlagStereo) av_dict_set_int(&st->metadata, "StereoMode", 1, 0);
/* put sample parameters */
//c->bit_rate = 4000000;
@@ -141,9 +144,11 @@ int main(int argc, char **argv) {
//bool stream_added[10] = {false};
+ int64_t first_ts = 10000000000000ll;
+
// TODO: In future, find a better way to discover number of streams...
// Read entire file to find all streams before reading again to write data
- bool res = r.read(90000000000000, [¤t_stream,¤t_channel,&r,&video_st,oc](const ftl::codecs::StreamPacket &spkt, const ftl::codecs::Packet &pkt) {
+ bool res = r.read(90000000000000, [&first_ts,¤t_stream,¤t_channel,&r,&video_st,oc](const ftl::codecs::StreamPacket &spkt, const ftl::codecs::Packet &pkt) {
if (spkt.channel != static_cast<ftl::codecs::Channel>(current_channel) && current_channel != -1) return;
if (spkt.frame_number == current_stream || current_stream == 255) {
@@ -153,6 +158,8 @@ int main(int argc, char **argv) {
if (spkt.frame_number >= 10) return; // TODO: Allow for more than 10
+ if (spkt.timestamp < first_ts) first_ts = spkt.timestamp;
+
if (video_st[spkt.frame_number][(spkt.channel == Channel::Left) ? 0 : 1] == nullptr) {
video_st[spkt.frame_number][(spkt.channel == Channel::Left) ? 0 : 1] = add_video_stream(oc, pkt);
}
@@ -177,7 +184,7 @@ int main(int argc, char **argv) {
bool seen_key[10] = {false};
- res = r.read(90000000000000, [¤t_stream,¤t_channel,&r,&video_st,oc,&seen_key](const ftl::codecs::StreamPacket &spkt, const ftl::codecs::Packet &pkt) {
+ res = r.read(90000000000000, [first_ts,¤t_stream,¤t_channel,&r,&video_st,oc,&seen_key](const ftl::codecs::StreamPacket &spkt, const ftl::codecs::Packet &pkt) {
if (spkt.channel != static_cast<ftl::codecs::Channel>(current_channel) && current_channel != -1) return;
if (spkt.frame_number == current_stream || current_stream == 255) {
@@ -203,10 +210,14 @@ int main(int argc, char **argv) {
}
if (!seen_key[spkt.frame_number]) return;
+ //if (spkt.timestamp > last_ts) framecount++;
+ //last_ts = spkt.timestamp;
+
AVPacket avpkt;
av_init_packet(&avpkt);
if (keyframe) avpkt.flags |= AV_PKT_FLAG_KEY;
- avpkt.pts = spkt.timestamp - r.getStartTime();
+ //avpkt.pts = framecount*50; //spkt.timestamp - r.getStartTime();
+ avpkt.pts = spkt.timestamp - first_ts;
avpkt.dts = avpkt.pts;
avpkt.stream_index= video_st[spkt.frame_number][(spkt.channel == Channel::Left) ? 0 : 1]->index;
avpkt.data= const_cast<uint8_t*>(pkt.data.data());
diff --git a/applications/gui/src/camera.cpp b/applications/gui/src/camera.cpp
index f7e5e2cc2be16f88941795c9ea0d4324c7b5119c..03306b423c8b0044d8170ee4122c3e59d704d22a 100644
--- a/applications/gui/src/camera.cpp
+++ b/applications/gui/src/camera.cpp
@@ -410,11 +410,6 @@ void ftl::gui::Camera::_draw(std::vector<ftl::rgbd::FrameSet*> &fss) {
// Normalize depth map
frame_.get<cv::cuda::GpuMat>(Channel::Depth).convertTo(frame_.get<cv::cuda::GpuMat>(Channel::Depth), CV_32F, 1.0/8.0);
- // Unmap GL buffer from CUDA and finish updating GL texture
- texture1_.unmap(renderer_->getCUDAStream());
- depth1_.unmap(renderer_->getCUDAStream());
- if (isStereo()) texture2_.unmap(renderer2_->getCUDAStream());
-
width_ = texture1_.width();
height_ = texture1_.height();
@@ -426,16 +421,26 @@ void ftl::gui::Camera::_draw(std::vector<ftl::rgbd::FrameSet*> &fss) {
fs2.mask = 1;
//fs2.stale = false;
fs2.set(ftl::data::FSFlag::STALE);
- frame_.swapTo(Channels<0>(Channel::Colour), f); // Channel::Colour + Channel::Depth
- if (f.hasChannel(Channel::Colour)) {
- f.create<cv::cuda::GpuMat>(Channel::Colour2).create(f.get<cv::cuda::GpuMat>(Channel::Colour).size(), f.get<cv::cuda::GpuMat>(Channel::Colour2).type());
- f.swapChannels(Channel::Colour, Channel::Colour2);
- cv::cuda::flip(f.get<cv::cuda::GpuMat>(Channel::Colour2), f.get<cv::cuda::GpuMat>(Channel::Colour), 0);
+ if (frame_.hasChannel(Channel::Colour2)) {
+ frame_.swapTo(Channels<0>(Channel::Colour) | Channel::Colour2, f);
+ ftl::cuda::flip(f.getTexture<uchar4>(Channel::Colour), 0);
+ ftl::cuda::flip(f.getTexture<uchar4>(Channel::Colour2), 0);
+ } else {
+ frame_.swapTo(Channels<0>(Channel::Colour), f); // Channel::Colour + Channel::Depth
+ ftl::cuda::flip(f.getTexture<uchar4>(Channel::Colour), 0);
}
+
fs2.timestamp = ftl::timer::get_time();
fs2.id = 0;
record_sender_->post(fs2);
record_stream_->select(0, Channels<0>(Channel::Colour));
+ // Reverse the flip
+ if (f.hasChannel(Channel::Colour2)) {
+ ftl::cuda::flip(f.getTexture<uchar4>(Channel::Colour), 0);
+ ftl::cuda::flip(f.getTexture<uchar4>(Channel::Colour2), 0);
+ } else {
+ ftl::cuda::flip(f.getTexture<uchar4>(Channel::Colour), 0);
+ }
f.swapTo(Channels<0>(Channel::Colour), frame_);
} else if (do_snapshot_) {
do_snapshot_ = false;
@@ -451,6 +456,11 @@ void ftl::gui::Camera::_draw(std::vector<ftl::rgbd::FrameSet*> &fss) {
cv::cvtColor(flipped, flipped, cv::COLOR_BGRA2BGR);
cv::imwrite(snapshot_filename_, flipped);
}
+
+ // Unmap GL buffer from CUDA and finish updating GL texture
+ texture1_.unmap(renderer_->getCUDAStream());
+ depth1_.unmap(renderer_->getCUDAStream());
+ if (isStereo()) texture2_.unmap(renderer2_->getCUDAStream());
}
void ftl::gui::Camera::update(int fsid, const ftl::codecs::Channels<0> &c) {
@@ -807,6 +817,7 @@ void ftl::gui::Camera::startVideoRecording(const std::string &filename) {
record_sender_ = ftl::create<ftl::stream::Sender>(screen_->root(), "videoEncode");
record_sender_->setStream(record_stream_);
record_sender_->value("codec", 2); // Default H264
+ record_sender_->value("stereo", true); // If both channels, then default to stereo
}
if (record_stream_->active()) return;
diff --git a/components/codecs/src/nvpipe_encoder.cpp b/components/codecs/src/nvpipe_encoder.cpp
index e4ad3589e20a3c261a3c16a1e61cbb1e01888b41..5e28fd4b0351afd00d62bb293c09a5367847db33 100644
--- a/components/codecs/src/nvpipe_encoder.cpp
+++ b/components/codecs/src/nvpipe_encoder.cpp
@@ -112,8 +112,10 @@ bool NvPipeEncoder::encode(const cv::cuda::GpuMat &in, ftl::codecs::Packet &pkt)
return false;
}
+ bool is_stereo = pkt.flags & ftl::codecs::kFlagStereo;
+
auto [tx,ty] = ftl::codecs::chooseTileConfig(pkt.frame_count);
- pkt.definition = (pkt.definition == definition_t::Any) ? ftl::codecs::findDefinition(in.cols/tx, in.rows/ty) : pkt.definition;
+ pkt.definition = (pkt.definition == definition_t::Any) ? ftl::codecs::findDefinition((is_stereo) ? in.cols/tx/2 : in.cols/tx, in.rows/ty) : pkt.definition;
if (pkt.definition == definition_t::Invalid || pkt.definition == definition_t::Any) {
LOG(ERROR) << "Could not find appropriate definition";
return false;
@@ -133,7 +135,7 @@ bool NvPipeEncoder::encode(const cv::cuda::GpuMat &in, ftl::codecs::Packet &pkt)
return false;
}
- if (tx*width != in.cols || ty*height != in.rows) {
+ if (((is_stereo) ? tx*width*2 : tx*width) != in.cols || ty*height != in.rows) {
// TODO: Resize if lower definition requested...
LOG(ERROR) << "Input size does not match expected: " << in.cols << " != " << tx*width;
pkt.definition = definition_t::Invalid;
diff --git a/components/streams/include/ftl/streams/sender.hpp b/components/streams/include/ftl/streams/sender.hpp
index 2a8afc8ce53ec46fedc26b9537be8b58fbe89b42..70d485172a19962d075b3836b49e324f095b9595 100644
--- a/components/streams/include/ftl/streams/sender.hpp
+++ b/components/streams/include/ftl/streams/sender.hpp
@@ -56,9 +56,9 @@ class Sender : public ftl::Configurable {
//ftl::codecs::Encoder *_getEncoder(int fsid, int fid, ftl::codecs::Channel c);
void _encodeChannel(ftl::rgbd::FrameSet &fs, ftl::codecs::Channel c, bool reset);
- int _generateTiles(const ftl::rgbd::FrameSet &fs, int offset, ftl::codecs::Channel c, cv::cuda::Stream &stream, bool);
+ int _generateTiles(const ftl::rgbd::FrameSet &fs, int offset, ftl::codecs::Channel c, cv::cuda::Stream &stream, bool, bool);
EncodingState &_getTile(int fsid, ftl::codecs::Channel c);
- cv::Rect _generateROI(const ftl::rgbd::FrameSet &fs, ftl::codecs::Channel c, int offset);
+ cv::Rect _generateROI(const ftl::rgbd::FrameSet &fs, ftl::codecs::Channel c, int offset, bool stereo);
float _selectFloatMax(ftl::codecs::Channel c);
};
diff --git a/components/streams/src/sender.cpp b/components/streams/src/sender.cpp
index 236f96f5ec959f124dc04fcea909b362591760fb..204e29e1155381ffabf701c07ad3925b03961825 100644
--- a/components/streams/src/sender.cpp
+++ b/components/streams/src/sender.cpp
@@ -262,6 +262,9 @@ void Sender::_encodeChannel(ftl::rgbd::FrameSet &fs, Channel c, bool reset) {
codec_t codec = static_cast<codec_t>(value("codec", static_cast<int>(codec_t::Any)));
device_t device = static_cast<device_t>(value("encoder_device", static_cast<int>(device_t::Any)));
+ // TODO: Support high res
+ bool is_stereo = value("stereo", false) && c == Channel::Colour && fs.firstFrame().hasChannel(Channel::Colour2);
+
uint32_t offset = 0;
while (offset < fs.frames.size()) {
Channel cc = c;
@@ -303,7 +306,7 @@ void Sender::_encodeChannel(ftl::rgbd::FrameSet &fs, Channel c, bool reset) {
}
}
- int count = _generateTiles(fs, offset, cc, enc->stream(), lossless);
+ int count = _generateTiles(fs, offset, cc, enc->stream(), lossless, is_stereo);
if (count <= 0) {
LOG(ERROR) << "Could not generate tiles.";
break;
@@ -326,12 +329,13 @@ void Sender::_encodeChannel(ftl::rgbd::FrameSet &fs, Channel c, bool reset) {
if (!lossless && ftl::codecs::isFloatChannel(cc)) pkt.flags = ftl::codecs::kFlagFloat | ftl::codecs::kFlagMappedDepth;
else if (lossless && ftl::codecs::isFloatChannel(cc)) pkt.flags = ftl::codecs::kFlagFloat;
else pkt.flags = ftl::codecs::kFlagFlipRGB;
+ if (is_stereo) pkt.flags |= ftl::codecs::kFlagStereo;
// In the event of partial frames, add a flag to indicate that
if (static_cast<size_t>(fs.count) < fs.frames.size()) pkt.flags |= ftl::codecs::kFlagPartial;
// Choose correct region of interest into the surface.
- cv::Rect roi = _generateROI(fs, cc, offset);
+ cv::Rect roi = _generateROI(fs, cc, offset, is_stereo);
cv::cuda::GpuMat sroi = tile.surface(roi);
FTL_Profile("Encoder",0.02);
@@ -357,9 +361,10 @@ void Sender::_encodeChannel(ftl::rgbd::FrameSet &fs, Channel c, bool reset) {
}
}
-cv::Rect Sender::_generateROI(const ftl::rgbd::FrameSet &fs, ftl::codecs::Channel c, int offset) {
+cv::Rect Sender::_generateROI(const ftl::rgbd::FrameSet &fs, ftl::codecs::Channel c, int offset, bool stereo) {
const ftl::rgbd::Frame &cframe = fs.firstFrame();
int rwidth = cframe.get<cv::cuda::GpuMat>(c).cols;
+ if (stereo) rwidth *= 2;
int rheight = cframe.get<cv::cuda::GpuMat>(c).rows;
auto [tx,ty] = ftl::codecs::chooseTileConfig(fs.frames.size()-offset);
return cv::Rect(0, 0, tx*rwidth, ty*rheight);
@@ -393,7 +398,7 @@ float Sender::_selectFloatMax(Channel c) {
}
}
-int Sender::_generateTiles(const ftl::rgbd::FrameSet &fs, int offset, Channel c, cv::cuda::Stream &stream, bool lossless) {
+int Sender::_generateTiles(const ftl::rgbd::FrameSet &fs, int offset, Channel c, cv::cuda::Stream &stream, bool lossless, bool stereo) {
auto &surface = _getTile(fs.id, c);
const ftl::rgbd::Frame *cframe = nullptr; //&fs.frames[offset];
@@ -402,7 +407,7 @@ int Sender::_generateTiles(const ftl::rgbd::FrameSet &fs, int offset, Channel c,
// Choose tile configuration and allocate memory
auto [tx,ty] = ftl::codecs::chooseTileConfig(fs.frames.size());
- int rwidth = m.cols;
+ int rwidth = (stereo) ? m.cols*2 : m.cols;
int rheight = m.rows;
int width = tx * rwidth;
int height = ty * rheight;
@@ -423,7 +428,7 @@ int Sender::_generateTiles(const ftl::rgbd::FrameSet &fs, int offset, Channel c,
if (fs.hasFrame(offset+count)) {
cframe = &fs.frames[offset+count];
auto &m = cframe->get<cv::cuda::GpuMat>(c);
- cv::Rect roi((count % tx)*rwidth, (count / tx)*rheight, rwidth, rheight);
+ cv::Rect roi((count % tx)*rwidth, (count / tx)*rheight, (stereo) ? rwidth/2 : rwidth, rheight);
cv::cuda::GpuMat sroi = surface.surface(roi);
if (m.type() == CV_32F) {
@@ -442,6 +447,30 @@ int Sender::_generateTiles(const ftl::rgbd::FrameSet &fs, int offset, Channel c,
LOG(ERROR) << "Unsupported colour format: " << m.type();
return 0;
}
+
+ // Do the right channel
+ if (stereo) {
+ auto &m = cframe->get<cv::cuda::GpuMat>((c == Channel::Colour) ? Channel::Colour2 : Channel::Colour2HighRes);
+ cv::Rect roi((count % tx)*rwidth + (rwidth/2), (count / tx)*rheight, rwidth/2, rheight);
+ cv::cuda::GpuMat sroi = surface.surface(roi);
+
+ if (m.type() == CV_32F) {
+ if (lossless) {
+ m.convertTo(sroi, CV_16UC1, 1000, stream);
+ } else {
+ ftl::cuda::depth_to_vuya(m, sroi, _selectFloatMax(c), stream);
+ }
+ } else if (m.type() == CV_8UC4) {
+ cv::cuda::cvtColor(m, sroi, cv::COLOR_BGRA2RGBA, 0, stream);
+ } else if (m.type() == CV_8UC3) {
+ cv::cuda::cvtColor(m, sroi, cv::COLOR_BGR2RGBA, 0, stream);
+ } else if (m.type() == CV_8UC1) {
+ m.copyTo(sroi, stream);
+ } else {
+ LOG(ERROR) << "Unsupported colour format: " << m.type();
+ return 0;
+ }
+ }
} else {
cv::Rect roi((count % tx)*rwidth, (count / tx)*rheight, rwidth, rheight);
cv::cuda::GpuMat sroi = surface.surface(roi);