diff --git a/components/operators/src/nvopticalflow.cpp b/components/operators/src/nvopticalflow.cpp index 2426c10197a165b02929f444544cdc338de98cf7..ebbda2112285fd4f3cf527fe5db5a77c04057e7f 100644 --- a/components/operators/src/nvopticalflow.cpp +++ b/components/operators/src/nvopticalflow.cpp @@ -19,12 +19,17 @@ using cv::Size; using cv::cuda::GpuMat; NVOpticalFlow::NVOpticalFlow(ftl::Configurable* cfg) : - ftl::operators::Operator(cfg), channel_in_{ftl::codecs::Channel::Colour,ftl::codecs::Channel::Colour2}, channel_out_{ftl::codecs::Channel::Flow,ftl::codecs::Channel::Flow2} { - size_ = Size(0, 0); + ftl::operators::Operator(cfg), + channel_in_{ftl::codecs::Channel::Colour,ftl::codecs::Channel::Colour2}, + channel_out_{ftl::codecs::Channel::Flow,ftl::codecs::Channel::Flow2}, + nvof_(nullptr) { + size_ = Size(0, 0); } -NVOpticalFlow::NVOpticalFlow(ftl::Configurable*cfg, const std::tuple &channels) : ftl::operators::Operator(cfg) { +NVOpticalFlow::NVOpticalFlow(ftl::Configurable*cfg, const std::tuple &channels) : + ftl::operators::Operator(cfg), nvof_(nullptr) { + channel_in_[0] = std::get<0>(channels); channel_out_[0] = std::get<1>(channels); channel_in_[1] = std::get<2>(channels); @@ -38,15 +43,14 @@ NVOpticalFlow::~NVOpticalFlow() { bool NVOpticalFlow::init() { if (!ftl::cuda::hasCompute(7,5)) { config()->set("enabled", false); - //throw FTL_Error("GPU does not support optical flow"); - LOG(ERROR) << "GPU does not support optical flow"; + LOG(WARNING) << "GPU does not support optical flow"; return false; } nvof_ = cv::cuda::NvidiaOpticalFlow_1_0::create( - size_.width, size_.height, + size_.width, size_.height, cv::cuda::NvidiaOpticalFlow_1_0::NV_OF_PERF_LEVEL_SLOW, true, false, false, 0); - + left_gray_.create(size_, CV_8UC1); left_gray_prev_.create(size_, CV_8UC1); right_gray_.create(size_, CV_8UC1); @@ -55,12 +59,22 @@ bool NVOpticalFlow::init() { } bool NVOpticalFlow::apply(Frame &in, Frame &out, cudaStream_t stream) { - bool both_channels = config()->value("both_channels", false); - float scale = config()->value("viz_scale", 200.0f); + if (!nvof_) { + return false; + } + + const bool both_channels = config()->value("both_channels", false); + const bool generate_disparity = config()->value("generate_disparity", false); + const float scale = config()->value("viz_scale", 200.0f); if (!in.hasChannel(channel_in_[0])) return false; if (in.hasChannel(channel_out_[0])) return true; - if (both_channels) { + + if (both_channels || generate_disparity) { + if (!in.hasChannel(channel_in_[1])) return false; + } + + if (both_channels && !generate_disparity) { if (!in.hasChannel(channel_in_[1])) return false; if (in.hasChannel(channel_out_[1])) return true; } @@ -69,31 +83,22 @@ bool NVOpticalFlow::apply(Frame &in, Frame &out, cudaStream_t stream) { size_ = in.get(channel_in_[0]).size(); if (!init()) return false; } - + auto cvstream = cv::cuda::StreamAccessor::wrapStream(stream); - auto &flow1 = out.create(channel_out_[0]); + // convert all necessary channels to gray cv::cuda::cvtColor(in.get(channel_in_[0]), left_gray_, cv::COLOR_BGRA2GRAY, 0, cvstream); - cv::cuda::cvtColor(in.get(channel_in_[1]), right_gray_, cv::COLOR_BGRA2GRAY, 0, cvstream); - - // TODO: Use optical flow confidence output, perhaps combined with a - // sensitivity adjustment - nvof_->calc(left_gray_, right_gray_, flow1, cvstream); - //std::swap(left_gray_, left_gray_prev_); + if (both_channels || generate_disparity) { + cv::cuda::cvtColor(in.get(channel_in_[1]), right_gray_, cv::COLOR_BGRA2GRAY, 0, cvstream); + } - if (both_channels) { + if (generate_disparity) { + auto &flow1 = out.create(channel_out_[0]); auto &flow2 = out.create(channel_out_[1]); - nvof_->calc(right_gray_, left_gray_, flow2, cvstream); - //std::swap(right_gray_, right_gray_prev_); - } - if (config()->value("show_flow", false)) { - ftl::cuda::show_optical_flow(flow1, in.getTexture(channel_in_[0]), scale, stream); - if (both_channels) - ftl::cuda::show_optical_flow(out.get(channel_out_[1]), in.getTexture(channel_in_[1]), scale, stream); - } + nvof_->calc(left_gray_, right_gray_, flow1, cvstream); + nvof_->calc(right_gray_, left_gray_, flow2, cvstream); - if (config()->value("generate_disparity", false)) { if (!in.hasChannel(Channel::Disparity)) { in.create(Channel::Disparity).create(size_, CV_16SC1); } @@ -101,13 +106,36 @@ bool NVOpticalFlow::apply(Frame &in, Frame &out, cudaStream_t stream) { flow1, out.get(channel_out_[1]), in.createTexture(Channel::Disparity), stream); - } - if (config()->value("check_reprojection", false)) { - ftl::cuda::check_reprojection(in.get(Channel::Disparity), in.getTexture(Channel::Colour), - in.createTexture(Channel::Colour2, true), - stream); + + if (config()->value("check_reprojection", false)) { + ftl::cuda::check_reprojection( + in.get(Channel::Disparity), + in.getTexture(Channel::Colour), + in.createTexture(Channel::Colour2, true), + stream); + } + } + else { + auto &flow1 = out.create(channel_out_[0]); + // TODO: Use optical flow confidence output, perhaps combined with a + // sensitivity adjustment + nvof_->calc(left_gray_, left_gray_prev_, flow1, cvstream); + std::swap(left_gray_, left_gray_prev_); + + if (both_channels) { + auto &flow2 = out.create(channel_out_[1]); + + nvof_->calc(right_gray_, right_gray_prev_, flow2, cvstream); + std::swap(right_gray_, right_gray_prev_); + } + if (config()->value("show_flow", false)) { + ftl::cuda::show_optical_flow(flow1, in.getTexture(channel_in_[0]), scale, stream); + if (both_channels) + ftl::cuda::show_optical_flow(out.get(channel_out_[1]), in.getTexture(channel_in_[1]), scale, stream); + } } + return true; -} \ No newline at end of file +}