From ee1954f7a5504b3c3b915408d665211ff9a4f2aa Mon Sep 17 00:00:00 2001
From: Sebastian Hahta <joseha@utu.fi>
Date: Fri, 31 Jan 2020 11:40:08 +0200
Subject: [PATCH] debug info in detecttrack and single camera bugs
---
.../include/ftl/operators/detectandtrack.hpp | 7 ++-
components/operators/src/detectandtrack.cpp | 30 +++++++-----
.../src/sources/stereovideo/stereovideo.cpp | 46 ++++++++++++-------
3 files changed, 54 insertions(+), 29 deletions(-)
diff --git a/components/operators/include/ftl/operators/detectandtrack.hpp b/components/operators/include/ftl/operators/detectandtrack.hpp
index d5b325a29..b769283a4 100644
--- a/components/operators/include/ftl/operators/detectandtrack.hpp
+++ b/components/operators/include/ftl/operators/detectandtrack.hpp
@@ -34,12 +34,17 @@ class DetectAndTrack : public ftl::operators::Operator {
bool track(const cv::Mat &im);
private:
+
ftl::codecs::Channel channel_in_;
ftl::codecs::Channel channel_out_;
+ bool debug_;
+ int id_max_;
+
struct Object {
- cv::Ptr<cv::Tracker> tracker;
+ int id;
cv::Rect2d object;
+ cv::Ptr<cv::Tracker> tracker;
int fail_count;
};
std::vector<Object> tracked_;
diff --git a/components/operators/src/detectandtrack.cpp b/components/operators/src/detectandtrack.cpp
index 6cc0cf92b..97c3dfd4a 100644
--- a/components/operators/src/detectandtrack.cpp
+++ b/components/operators/src/detectandtrack.cpp
@@ -23,7 +23,8 @@ DetectAndTrack::DetectAndTrack(ftl::Configurable *cfg) : ftl::operators::Operato
bool DetectAndTrack::init() {
fname_ = config()->value<string>("filename", "");
-
+ debug_ = config()->value<bool>("debug", false);
+
detect_n_frames_ = config()->value<int>("n_frames", 10);
detect_n_frames_ = detect_n_frames_ < 0.0 ? 0.0 : detect_n_frames_;
@@ -55,6 +56,8 @@ bool DetectAndTrack::init() {
if (min_size_[1] > max_size_[1]) { min_size_[1] = max_size_[1]; }
channel_in_ = ftl::codecs::Channel::Colour;
+ channel_out_ = ftl::codecs::Channel::Data;
+ id_max_ = 0;
bool retval = false;
@@ -75,12 +78,6 @@ bool DetectAndTrack::init() {
return true;
}
-static double distance(Point2i p, Point2i q) {
- double a = (p.x-q.x);
- double b = (p.y-q.y);
- return sqrt(a*a+b*b);
-}
-
static Point2d center(Rect2d obj) {
return Point2d(obj.x+obj.width/2.0, obj.y+obj.height/2.0);
}
@@ -101,7 +98,7 @@ bool DetectAndTrack::detect(const Mat &im) {
bool found = false;
for (auto &tracker : tracked_) {
- if (distance(center(tracker.object), c) < max_distance_) {
+ if (cv::norm(center(tracker.object)-c) < max_distance_) {
// update? (bounding box can be quite different)
// tracker.object = obj;
found = true;
@@ -112,7 +109,7 @@ bool DetectAndTrack::detect(const Mat &im) {
if (!found && (tracked_.size() < max_tracked_)) {
cv::Ptr<cv::Tracker> tracker = cv::TrackerKCF::create();
tracker->init(im, obj);
- tracked_.push_back({ tracker, obj, 0 });
+ tracked_.push_back({ id_max_++, obj, tracker, 0 });
}
}
@@ -171,11 +168,20 @@ bool DetectAndTrack::apply(Frame &in, Frame &out, cudaStream_t stream) {
detect(gray_);
}
- // TODO: save results somewhere
std::vector<Rect2d> result;
result.reserve(tracked_.size());
- for (auto const &tracked : tracked_) { result.push_back(tracked.object); }
- in.create(ftl::codecs::Channel::Data, result);
+ for (auto const &tracked : tracked_) {
+ result.push_back(tracked.object);
+
+ if (debug_) {
+ cv::putText(im, "#" + std::to_string(tracked.id),
+ Point2i(tracked.object.x+5, tracked.object.y+tracked.object.height-5),
+ cv::FONT_HERSHEY_COMPLEX_SMALL, 1.0, cv::Scalar(0,0,255));
+
+ cv::rectangle(im, tracked.object, cv::Scalar(0, 0, 255), 1);
+ }
+ }
+ in.create(channel_out_, result);
// TODO: should be uploaded by operator which requires data on GPU
in.upload(channel_in_);
diff --git a/components/rgbd-sources/src/sources/stereovideo/stereovideo.cpp b/components/rgbd-sources/src/sources/stereovideo/stereovideo.cpp
index 401477a3a..fd9784856 100644
--- a/components/rgbd-sources/src/sources/stereovideo/stereovideo.cpp
+++ b/components/rgbd-sources/src/sources/stereovideo/stereovideo.cpp
@@ -81,7 +81,8 @@ void StereoVideoSource::init(const string &file) {
#ifdef HAVE_OPTFLOW
pipeline_input_->append<ftl::operators::NVOpticalFlow>("optflow", Channel::Colour, Channel::Flow);
#endif
- pipeline_input_->append<ftl::operators::DetectAndTrack>("facedetection")->set("enabled", false);
+ pipeline_input_->append<ftl::operators::DetectAndTrack>("facedetection")->value("enabled", false);
+ pipeline_input_->append<ftl::operators::ColourChannels>("colour");
calib_ = ftl::create<Calibrate>(host_, "calibration", cv::Size(lsrc_->fullWidth(), lsrc_->fullHeight()), stream_);
@@ -192,7 +193,7 @@ void StereoVideoSource::updateParameters() {
// same for left and right
double baseline = 1.0 / calib_->getQ().at<double>(3,2);
- double doff = -calib_->getQ().at<double>(3,3) * baseline;
+ double doff = -calib_->getQ().at<double>(3,3) * baseline;
double min_depth = this->host_->getConfig().value<double>("min_depth", 0.0);
double max_depth = this->host_->getConfig().value<double>("max_depth", 15.0);
@@ -245,12 +246,20 @@ bool StereoVideoSource::retrieve() {
auto &frame = frames_[0];
frame.reset();
frame.setOrigin(&state_);
- auto &left = frame.create<cv::cuda::GpuMat>(Channel::Left);
- auto &right = frame.create<cv::cuda::GpuMat>(Channel::Right);
+
cv::cuda::GpuMat dummy;
- auto &hres = (lsrc_->hasHigherRes()) ? frame.create<cv::cuda::GpuMat>(Channel::ColourHighRes) : dummy;
+ auto &hres = (lsrc_->hasHigherRes()) ? frame.create<cv::cuda::GpuMat>(Channel::ColourHighRes) : dummy;
- lsrc_->get(left, right, hres, calib_, stream2_);
+ if (lsrc_->isStereo()) {
+ cv::cuda::GpuMat &left = frame.create<cv::cuda::GpuMat>(Channel::Left);
+ cv::cuda::GpuMat &right = frame.create<cv::cuda::GpuMat>(Channel::Right);
+ lsrc_->get(left, right, hres, calib_, stream2_);
+ }
+ else {
+ cv::cuda::GpuMat &left = frame.create<cv::cuda::GpuMat>(Channel::Left);
+ cv::cuda::GpuMat right;
+ lsrc_->get(left, right, hres, calib_, stream2_);
+ }
//LOG(INFO) << "Channel size: " << hres.size();
@@ -269,20 +278,25 @@ void StereoVideoSource::swap() {
bool StereoVideoSource::compute(int n, int b) {
auto &frame = frames_[1];
- if (!frame.hasChannel(Channel::Left) || !frame.hasChannel(Channel::Right)) {
- return false;
- }
+ if (lsrc_->isStereo()) {
+ if (!frame.hasChannel(Channel::Left) ||
+ !frame.hasChannel(Channel::Right)) {
+
+ return false;
+ }
- cv::cuda::GpuMat& left = frame.get<cv::cuda::GpuMat>(Channel::Left);
- cv::cuda::GpuMat& right = frame.get<cv::cuda::GpuMat>(Channel::Right);
+ cv::cuda::GpuMat& left = frame.get<cv::cuda::GpuMat>(Channel::Left);
+ cv::cuda::GpuMat& right = frame.get<cv::cuda::GpuMat>(Channel::Right);
- if (left.empty() || right.empty()) {
- return false;
- }
+ if (left.empty() || right.empty()) { return false; }
+ //stream_.waitForCompletion();
- //stream_.waitForCompletion();
+ }
+ else {
+ if (!frame.hasChannel(Channel::Left)) { return false; }
+ }
+
host_->notify(timestamp_, frame);
-
return true;
}
--
GitLab