Skip to content
Snippets Groups Projects
Commit 44750fc1 authored by Nicolas Pope's avatar Nicolas Pope
Browse files

Merge branch 'feature/stereovid' into 'master' 

Generate stereo mkv videos

See merge request nicolas.pope/ftl!302
parents b171f76a a7741367
No related branches found
No related tags found
1 merge request!302Generate stereo mkv videos
Pipeline #26487 passed
Stage: all
Stage: deploy
Show whitespace changes
Inline Side-by-side
applications/ftl2mkv/src/main.cpp
+ 17
− 6
View file @ 44750fc1
......@@ -41,18 +41,21 @@ static AVStream *add_video_stream(AVFormatContext *oc, const ftl::codecs::Packet
//c->codec_id = codec_id;
//c->codec_type = AVMEDIA_TYPE_VIDEO;
st->time_base.den = 20;
st->time_base.num = 1;
//st->time_base.den = 20;
//st->time_base.num = 1;
//st->id = oc->nb_streams-1;
//st->nb_frames = 0;
st->codecpar->codec_id = codec_id;
st->codecpar->codec_type = AVMEDIA_TYPE_VIDEO;
st->codecpar->width = ftl::codecs::getWidth(pkt.definition);
//if (pkt.flags & ftl::codecs::kFlagStereo) st->codecpar->width *= 2;
st->codecpar->height = ftl::codecs::getHeight(pkt.definition);
st->codecpar->format = AV_PIX_FMT_NV12;
st->codecpar->bit_rate = 4000000;
if (pkt.flags & ftl::codecs::kFlagStereo) av_dict_set_int(&st->metadata, "stereo_mode", 1, 0);
if (pkt.flags & ftl::codecs::kFlagStereo) av_dict_set(&st->metadata, "stereo_mode", "left_right", 0);
//if (pkt.flags & ftl::codecs::kFlagStereo) av_dict_set(&oc->metadata, "stereo_mode", "1", 0);
//if (pkt.flags & ftl::codecs::kFlagStereo) av_dict_set_int(&st->metadata, "StereoMode", 1, 0);
/* put sample parameters */
//c->bit_rate = 4000000;
......@@ -141,9 +144,11 @@ int main(int argc, char **argv) {
//bool stream_added[10] = {false};
int64_t first_ts = 10000000000000ll;
// TODO: In future, find a better way to discover number of streams...
// Read entire file to find all streams before reading again to write data
bool res = r.read(90000000000000, [&current_stream,&current_channel,&r,&video_st,oc](const ftl::codecs::StreamPacket &spkt, const ftl::codecs::Packet &pkt) {
bool res = r.read(90000000000000, [&first_ts,&current_stream,&current_channel,&r,&video_st,oc](const ftl::codecs::StreamPacket &spkt, const ftl::codecs::Packet &pkt) {
if (spkt.channel != static_cast<ftl::codecs::Channel>(current_channel) && current_channel != -1) return;
if (spkt.frame_number == current_stream || current_stream == 255) {
......@@ -153,6 +158,8 @@ int main(int argc, char **argv) {
if (spkt.frame_number >= 10) return; // TODO: Allow for more than 10
if (spkt.timestamp < first_ts) first_ts = spkt.timestamp;
if (video_st[spkt.frame_number][(spkt.channel == Channel::Left) ? 0 : 1] == nullptr) {
video_st[spkt.frame_number][(spkt.channel == Channel::Left) ? 0 : 1] = add_video_stream(oc, pkt);
}
......@@ -177,7 +184,7 @@ int main(int argc, char **argv) {
bool seen_key[10] = {false};
res = r.read(90000000000000, [&current_stream,&current_channel,&r,&video_st,oc,&seen_key](const ftl::codecs::StreamPacket &spkt, const ftl::codecs::Packet &pkt) {
res = r.read(90000000000000, [first_ts,&current_stream,&current_channel,&r,&video_st,oc,&seen_key](const ftl::codecs::StreamPacket &spkt, const ftl::codecs::Packet &pkt) {
if (spkt.channel != static_cast<ftl::codecs::Channel>(current_channel) && current_channel != -1) return;
if (spkt.frame_number == current_stream || current_stream == 255) {
......@@ -203,10 +210,14 @@ int main(int argc, char **argv) {
}
if (!seen_key[spkt.frame_number]) return;
//if (spkt.timestamp > last_ts) framecount++;
//last_ts = spkt.timestamp;
AVPacket avpkt;
av_init_packet(&avpkt);
if (keyframe) avpkt.flags |= AV_PKT_FLAG_KEY;
avpkt.pts = spkt.timestamp - r.getStartTime();
//avpkt.pts = framecount*50; //spkt.timestamp - r.getStartTime();
avpkt.pts = spkt.timestamp - first_ts;
avpkt.dts = avpkt.pts;
avpkt.stream_index= video_st[spkt.frame_number][(spkt.channel == Channel::Left) ? 0 : 1]->index;
avpkt.data= const_cast<uint8_t*>(pkt.data.data());
......
applications/gui/src/camera.cpp
+ 21
− 10
View file @ 44750fc1
......@@ -410,11 +410,6 @@ void ftl::gui::Camera::_draw(std::vector<ftl::rgbd::FrameSet*> &fss) {
// Normalize depth map
frame_.get<cv::cuda::GpuMat>(Channel::Depth).convertTo(frame_.get<cv::cuda::GpuMat>(Channel::Depth), CV_32F, 1.0/8.0);
// Unmap GL buffer from CUDA and finish updating GL texture
texture1_.unmap(renderer_->getCUDAStream());
depth1_.unmap(renderer_->getCUDAStream());
if (isStereo()) texture2_.unmap(renderer2_->getCUDAStream());
width_ = texture1_.width();
height_ = texture1_.height();
......@@ -426,16 +421,26 @@ void ftl::gui::Camera::_draw(std::vector<ftl::rgbd::FrameSet*> &fss) {
fs2.mask = 1;
//fs2.stale = false;
fs2.set(ftl::data::FSFlag::STALE);
if (frame_.hasChannel(Channel::Colour2)) {
frame_.swapTo(Channels<0>(Channel::Colour) | Channel::Colour2, f);
ftl::cuda::flip(f.getTexture<uchar4>(Channel::Colour), 0);
ftl::cuda::flip(f.getTexture<uchar4>(Channel::Colour2), 0);
} else {
frame_.swapTo(Channels<0>(Channel::Colour), f); // Channel::Colour + Channel::Depth
if (f.hasChannel(Channel::Colour)) {
f.create<cv::cuda::GpuMat>(Channel::Colour2).create(f.get<cv::cuda::GpuMat>(Channel::Colour).size(), f.get<cv::cuda::GpuMat>(Channel::Colour2).type());
f.swapChannels(Channel::Colour, Channel::Colour2);
cv::cuda::flip(f.get<cv::cuda::GpuMat>(Channel::Colour2), f.get<cv::cuda::GpuMat>(Channel::Colour), 0);
ftl::cuda::flip(f.getTexture<uchar4>(Channel::Colour), 0);
}
fs2.timestamp = ftl::timer::get_time();
fs2.id = 0;
record_sender_->post(fs2);
record_stream_->select(0, Channels<0>(Channel::Colour));
// Reverse the flip
if (f.hasChannel(Channel::Colour2)) {
ftl::cuda::flip(f.getTexture<uchar4>(Channel::Colour), 0);
ftl::cuda::flip(f.getTexture<uchar4>(Channel::Colour2), 0);
} else {
ftl::cuda::flip(f.getTexture<uchar4>(Channel::Colour), 0);
}
f.swapTo(Channels<0>(Channel::Colour), frame_);
} else if (do_snapshot_) {
do_snapshot_ = false;
......@@ -451,6 +456,11 @@ void ftl::gui::Camera::_draw(std::vector<ftl::rgbd::FrameSet*> &fss) {
cv::cvtColor(flipped, flipped, cv::COLOR_BGRA2BGR);
cv::imwrite(snapshot_filename_, flipped);
}
// Unmap GL buffer from CUDA and finish updating GL texture
texture1_.unmap(renderer_->getCUDAStream());
depth1_.unmap(renderer_->getCUDAStream());
if (isStereo()) texture2_.unmap(renderer2_->getCUDAStream());
}
void ftl::gui::Camera::update(int fsid, const ftl::codecs::Channels<0> &c) {
......@@ -807,6 +817,7 @@ void ftl::gui::Camera::startVideoRecording(const std::string &filename) {
record_sender_ = ftl::create<ftl::stream::Sender>(screen_->root(), "videoEncode");
record_sender_->setStream(record_stream_);
record_sender_->value("codec", 2); // Default H264
record_sender_->value("stereo", true); // If both channels, then default to stereo
}
if (record_stream_->active()) return;
......
components/codecs/src/nvpipe_encoder.cpp
+ 4
− 2
View file @ 44750fc1
......@@ -112,8 +112,10 @@ bool NvPipeEncoder::encode(const cv::cuda::GpuMat &in, ftl::codecs::Packet &pkt)
return false;
}
bool is_stereo = pkt.flags & ftl::codecs::kFlagStereo;
auto [tx,ty] = ftl::codecs::chooseTileConfig(pkt.frame_count);
pkt.definition = (pkt.definition == definition_t::Any) ? ftl::codecs::findDefinition(in.cols/tx, in.rows/ty) : pkt.definition;
pkt.definition = (pkt.definition == definition_t::Any) ? ftl::codecs::findDefinition((is_stereo) ? in.cols/tx/2 : in.cols/tx, in.rows/ty) : pkt.definition;
if (pkt.definition == definition_t::Invalid || pkt.definition == definition_t::Any) {
LOG(ERROR) << "Could not find appropriate definition";
return false;
......@@ -133,7 +135,7 @@ bool NvPipeEncoder::encode(const cv::cuda::GpuMat &in, ftl::codecs::Packet &pkt)
return false;
}
if (tx*width != in.cols || ty*height != in.rows) {
if (((is_stereo) ? tx*width*2 : tx*width) != in.cols || ty*height != in.rows) {
// TODO: Resize if lower definition requested...
LOG(ERROR) << "Input size does not match expected: " << in.cols << " != " << tx*width;
pkt.definition = definition_t::Invalid;
......
components/streams/include/ftl/streams/sender.hpp
+ 2
− 2
View file @ 44750fc1
......@@ -56,9 +56,9 @@ class Sender : public ftl::Configurable {
//ftl::codecs::Encoder *_getEncoder(int fsid, int fid, ftl::codecs::Channel c);
void _encodeChannel(ftl::rgbd::FrameSet &fs, ftl::codecs::Channel c, bool reset);
int _generateTiles(const ftl::rgbd::FrameSet &fs, int offset, ftl::codecs::Channel c, cv::cuda::Stream &stream, bool);
int _generateTiles(const ftl::rgbd::FrameSet &fs, int offset, ftl::codecs::Channel c, cv::cuda::Stream &stream, bool, bool);
EncodingState &_getTile(int fsid, ftl::codecs::Channel c);
cv::Rect _generateROI(const ftl::rgbd::FrameSet &fs, ftl::codecs::Channel c, int offset);
cv::Rect _generateROI(const ftl::rgbd::FrameSet &fs, ftl::codecs::Channel c, int offset, bool stereo);
float _selectFloatMax(ftl::codecs::Channel c);
};
......
components/streams/src/sender.cpp
+ 35
− 6
View file @ 44750fc1
......@@ -262,6 +262,9 @@ void Sender::_encodeChannel(ftl::rgbd::FrameSet &fs, Channel c, bool reset) {
codec_t codec = static_cast<codec_t>(value("codec", static_cast<int>(codec_t::Any)));
device_t device = static_cast<device_t>(value("encoder_device", static_cast<int>(device_t::Any)));
// TODO: Support high res
bool is_stereo = value("stereo", false) && c == Channel::Colour && fs.firstFrame().hasChannel(Channel::Colour2);
uint32_t offset = 0;
while (offset < fs.frames.size()) {
Channel cc = c;
......@@ -303,7 +306,7 @@ void Sender::_encodeChannel(ftl::rgbd::FrameSet &fs, Channel c, bool reset) {
}
}
int count = _generateTiles(fs, offset, cc, enc->stream(), lossless);
int count = _generateTiles(fs, offset, cc, enc->stream(), lossless, is_stereo);
if (count <= 0) {
LOG(ERROR) << "Could not generate tiles.";
break;
......@@ -326,12 +329,13 @@ void Sender::_encodeChannel(ftl::rgbd::FrameSet &fs, Channel c, bool reset) {
if (!lossless && ftl::codecs::isFloatChannel(cc)) pkt.flags = ftl::codecs::kFlagFloat | ftl::codecs::kFlagMappedDepth;
else if (lossless && ftl::codecs::isFloatChannel(cc)) pkt.flags = ftl::codecs::kFlagFloat;
else pkt.flags = ftl::codecs::kFlagFlipRGB;
if (is_stereo) pkt.flags |= ftl::codecs::kFlagStereo;
// In the event of partial frames, add a flag to indicate that
if (static_cast<size_t>(fs.count) < fs.frames.size()) pkt.flags |= ftl::codecs::kFlagPartial;
// Choose correct region of interest into the surface.
cv::Rect roi = _generateROI(fs, cc, offset);
cv::Rect roi = _generateROI(fs, cc, offset, is_stereo);
cv::cuda::GpuMat sroi = tile.surface(roi);
FTL_Profile("Encoder",0.02);
......@@ -357,9 +361,10 @@ void Sender::_encodeChannel(ftl::rgbd::FrameSet &fs, Channel c, bool reset) {
}
}
cv::Rect Sender::_generateROI(const ftl::rgbd::FrameSet &fs, ftl::codecs::Channel c, int offset) {
cv::Rect Sender::_generateROI(const ftl::rgbd::FrameSet &fs, ftl::codecs::Channel c, int offset, bool stereo) {
const ftl::rgbd::Frame &cframe = fs.firstFrame();
int rwidth = cframe.get<cv::cuda::GpuMat>(c).cols;
if (stereo) rwidth *= 2;
int rheight = cframe.get<cv::cuda::GpuMat>(c).rows;
auto [tx,ty] = ftl::codecs::chooseTileConfig(fs.frames.size()-offset);
return cv::Rect(0, 0, tx*rwidth, ty*rheight);
......@@ -393,7 +398,7 @@ float Sender::_selectFloatMax(Channel c) {
}
}
int Sender::_generateTiles(const ftl::rgbd::FrameSet &fs, int offset, Channel c, cv::cuda::Stream &stream, bool lossless) {
int Sender::_generateTiles(const ftl::rgbd::FrameSet &fs, int offset, Channel c, cv::cuda::Stream &stream, bool lossless, bool stereo) {
auto &surface = _getTile(fs.id, c);
const ftl::rgbd::Frame *cframe = nullptr; //&fs.frames[offset];
......@@ -402,7 +407,7 @@ int Sender::_generateTiles(const ftl::rgbd::FrameSet &fs, int offset, Channel c,
// Choose tile configuration and allocate memory
auto [tx,ty] = ftl::codecs::chooseTileConfig(fs.frames.size());
int rwidth = m.cols;
int rwidth = (stereo) ? m.cols*2 : m.cols;
int rheight = m.rows;
int width = tx * rwidth;
int height = ty * rheight;
......@@ -423,7 +428,30 @@ int Sender::_generateTiles(const ftl::rgbd::FrameSet &fs, int offset, Channel c,
if (fs.hasFrame(offset+count)) {
cframe = &fs.frames[offset+count];
auto &m = cframe->get<cv::cuda::GpuMat>(c);
cv::Rect roi((count % tx)*rwidth, (count / tx)*rheight, rwidth, rheight);
cv::Rect roi((count % tx)*rwidth, (count / tx)*rheight, (stereo) ? rwidth/2 : rwidth, rheight);
cv::cuda::GpuMat sroi = surface.surface(roi);
if (m.type() == CV_32F) {
if (lossless) {
m.convertTo(sroi, CV_16UC1, 1000, stream);
} else {
ftl::cuda::depth_to_vuya(m, sroi, _selectFloatMax(c), stream);
}
} else if (m.type() == CV_8UC4) {
cv::cuda::cvtColor(m, sroi, cv::COLOR_BGRA2RGBA, 0, stream);
} else if (m.type() == CV_8UC3) {
cv::cuda::cvtColor(m, sroi, cv::COLOR_BGR2RGBA, 0, stream);
} else if (m.type() == CV_8UC1) {
m.copyTo(sroi, stream);
} else {
LOG(ERROR) << "Unsupported colour format: " << m.type();
return 0;
}
// Do the right channel
if (stereo) {
auto &m = cframe->get<cv::cuda::GpuMat>((c == Channel::Colour) ? Channel::Colour2 : Channel::Colour2HighRes);
cv::Rect roi((count % tx)*rwidth + (rwidth/2), (count / tx)*rheight, rwidth/2, rheight);
cv::cuda::GpuMat sroi = surface.surface(roi);
if (m.type() == CV_32F) {
......@@ -442,6 +470,7 @@ int Sender::_generateTiles(const ftl::rgbd::FrameSet &fs, int offset, Channel c,
LOG(ERROR) << "Unsupported colour format: " << m.type();
return 0;
}
}
} else {
cv::Rect roi((count % tx)*rwidth, (count / tx)*rheight, rwidth, rheight);
cv::cuda::GpuMat sroi = surface.surface(roi);
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment