diff --git a/components/rgbd-sources/include/ftl/rgbd/detail/source.hpp b/components/rgbd-sources/include/ftl/rgbd/detail/source.hpp index 3839fd6c0238965338602dc326cf3e951a59996c..c3aca569e0e4868645239549802707420b3c474a 100644 --- a/components/rgbd-sources/include/ftl/rgbd/detail/source.hpp +++ b/components/rgbd-sources/include/ftl/rgbd/detail/source.hpp @@ -11,6 +11,7 @@ namespace ftl{ namespace rgbd { class Source; +typedef std::function<void(int64_t,ftl::rgbd::Frame&)> FrameCallback; typedef unsigned int capability_t; @@ -28,30 +29,18 @@ class Source { friend class ftl::rgbd::Source; public: - explicit Source(ftl::rgbd::Source *host) : capabilities_(0), host_(host), params_(state_.getLeft()), timestamp_(0) { } + explicit Source(ftl::rgbd::Source *host) : capabilities_(0), host_(host), params_(state_.getLeft()) { } virtual ~Source() {} /** - * Perform hardware data capture. + * Perform hardware data capture. This should be low latency. */ virtual bool capture(int64_t ts)=0; /** - * Perform IO operation to get the data. + * Perform slow IO operation to get the data into the given frame object. */ - virtual bool retrieve()=0; - - /** - * Do any processing from previously captured frames... - * @param n Number of frames to request in batch. Default -1 means automatic (10) - * @param b Bit rate setting. -1 = automatic, 0 = best quality, 9 = lowest quality - */ - virtual bool compute(int64_t ts)=0; - - /** - * Between frames, or before next frame, do any buffer swapping operations. - */ - virtual void swap() {} + virtual bool retrieve(ftl::rgbd::Frame &frame)=0; virtual bool isReady() { return false; }; virtual void setPose(const Eigen::Matrix4d &pose) { state_.setPose(pose); }; @@ -63,9 +52,6 @@ class Source { capability_t capabilities_; ftl::rgbd::Source *host_; ftl::rgbd::Camera ¶ms_; - ftl::rgbd::Frame frame_; - int64_t timestamp_; - //Eigen::Matrix4d &pose_; }; } diff --git a/components/rgbd-sources/include/ftl/rgbd/group.hpp b/components/rgbd-sources/include/ftl/rgbd/group.hpp index 1c283ea59c1657e6298074fa4e7915c510d2de46..eed75bbc5028b07af3e7577a99c9b93e87462822 100644 --- a/components/rgbd-sources/include/ftl/rgbd/group.hpp +++ b/components/rgbd-sources/include/ftl/rgbd/group.hpp @@ -112,7 +112,7 @@ class Group : public ftl::rgbd::Generator { MUTEX mutex_; void _retrieveJob(ftl::rgbd::Source *); - void _computeJob(ftl::rgbd::Source *, int64_t); + void _dispatchJob(ftl::rgbd::Source *, int64_t); }; } diff --git a/components/rgbd-sources/include/ftl/rgbd/source.hpp b/components/rgbd-sources/include/ftl/rgbd/source.hpp index fca5693662c364d8991e5e75b9f29ecf7486f19e..b49dc2116d3ab27c12251aaa3f441bddf06eec90 100644 --- a/components/rgbd-sources/include/ftl/rgbd/source.hpp +++ b/components/rgbd-sources/include/ftl/rgbd/source.hpp @@ -27,13 +27,6 @@ namespace rgbd { static inline bool isValidDepth(float d) { return (d > 0.01f) && (d < 39.99f); } -class SnapshotReader; -class VirtualSource; -class Player; - -typedef std::function<void(ftl::rgbd::Source*, const ftl::codecs::StreamPacket &spkt, const ftl::codecs::Packet &pkt)> RawCallback; -typedef std::function<void(int64_t,ftl::rgbd::Frame&)> FrameCallback; - /** * RGBD Generic data source configurable entity. This class hides the * internal implementation of an RGBD source by providing accessor functions @@ -58,7 +51,6 @@ class Source : public ftl::Configurable { protected: explicit Source(ftl::config::json_t &cfg); - Source(ftl::config::json_t &cfg, ftl::rgbd::SnapshotReader *); Source(ftl::config::json_t &cfg, ftl::net::Universe *net); virtual ~Source(); @@ -89,43 +81,24 @@ class Source : public ftl::Configurable { */ bool retrieve(); + /** + * Generate a thread job using the provided callback for the most recently + * retrieved frame (matching the provided timestamp). If already busy + * dispatching, returns false. + */ + bool dispatch(int64_t ts); + /** * Between frames, do any required buffer swaps. */ - void swap() { if (impl_) impl_->swap(); } + //void swap() { if (impl_) impl_->swap(); } /** * Do any post-grab processing. This function * may take considerable time to return, especially for sources requiring * software stereo correspondance. */ - bool compute(int64_t ts); - - /** - * Wrapper grab that performs capture, swap and computation steps in one. - * It is more optimal to perform capture and compute in parallel. - */ - /*bool grab(int N=-1, int B=-1) { - bool c = capture(0); - c = c && retrieve(); - swap(); - return c && compute(N,B); - }*/ - - /** - * Get a copy of both colour and depth frames. Note that this does a buffer - * swap rather than a copy, so the parameters should be persistent buffers for - * best performance. - */ - [[deprecated]] void getFrames(cv::Mat &c, cv::Mat &d); - - /** - * Directly upload source RGB and Depth to GPU. - */ - void upload(cv::cuda::GpuMat&, cv::cuda::GpuMat&); - - void uploadColour(cv::cuda::GpuMat&); - void uploadDepth(cv::cuda::GpuMat&); + //bool compute(int64_t ts); //bool isVirtual() const { return impl_ == nullptr; } @@ -152,24 +125,19 @@ class Source : public ftl::Configurable { */ virtual void setPose(const Eigen::Matrix4d &pose); - /** - * Get the camera position as a pose matrix. - */ - [[deprecated]] const Eigen::Matrix4d &getPose() const; - /** * Check what features this source has available. */ - bool hasCapabilities(capability_t); + [[deprecated]] bool hasCapabilities(capability_t); - capability_t getCapabilities() const; + [[deprecated]] capability_t getCapabilities() const; /** * Force the internal implementation to be reconstructed. */ void reset(); - ftl::net::Universe *getNet() const { return net_; } + [[deprecated]] ftl::net::Universe *getNet() const { return net_; } std::string getURI() { return value("uri", std::string("")); } @@ -189,42 +157,15 @@ class Source : public ftl::Configurable { void setCallback(const FrameCallback &cb); void removeCallback() { callback_ = nullptr; } - /** - * Add a callback to immediately receive any raw data from this source. - * Currently this only works for a net source since other sources don't - * produce raw encoded data. - */ - void addRawCallback(const RawCallback &); - - /** - * THIS DOES NOT WORK CURRENTLY. - */ - void removeRawCallback(const RawCallback &); - - /** - * INTERNAL. Used to send raw data to callbacks. - */ - void notifyRaw(const ftl::codecs::StreamPacket &spkt, const ftl::codecs::Packet &pkt); - /** * Notify of a decoded or available pair of frames. This calls the source * callback after having verified the correct resolution of the frames. */ //void notify(int64_t ts, cv::cuda::GpuMat &c1, cv::cuda::GpuMat &c2); - void notify(int64_t ts, ftl::rgbd::Frame &f); - - // ==== Inject Data into stream ============================================ - - /** - * Generate a stream packet with arbitrary data. The data is packed using - * msgpack and is given the timestamp of the most recent frame. - */ - template <typename... ARGS> - void inject(ftl::codecs::Channel c, ARGS... args); + //void notify(int64_t ts, ftl::rgbd::Frame &f); - void inject(const Eigen::Matrix4d &pose); - protected: + private: detail::Source *impl_; Eigen::Matrix4d pose_; ftl::net::Universe *net_; @@ -232,55 +173,14 @@ class Source : public ftl::Configurable { ftl::codecs::Channel channel_; cudaStream_t stream_; FrameCallback callback_; - std::list<RawCallback> rawcallbacks_; + ftl::rgbd::Frame frames_[2]; + bool is_dispatching; + bool is_retrieving; - detail::Source *_createImplementation(); - detail::Source *_createFileImpl(const ftl::URI &uri); - detail::Source *_createNetImpl(const ftl::URI &uri); - detail::Source *_createDeviceImpl(const ftl::URI &uri); - - static ftl::rgbd::Player *__createReader(const std::string &path); - - static std::map<std::string, ftl::rgbd::Player*> readers__; + void _swap(); }; } } -class VectorBuffer { - public: - inline explicit VectorBuffer(std::vector<unsigned char> &v) : vector_(v) {} - - inline void write(const char *data, std::size_t size) { - vector_.insert(vector_.end(), (const unsigned char*)data, (const unsigned char*)data+size); - } - - private: - std::vector<unsigned char> &vector_; -}; - -template <typename... ARGS> -void ftl::rgbd::Source::inject(ftl::codecs::Channel c, ARGS... args) { - if (!impl_) return; - auto data = std::make_tuple(args...); - - ftl::codecs::StreamPacket spkt; - ftl::codecs::Packet pkt; - - spkt.timestamp = impl_->timestamp_; - spkt.channel = c; - spkt.frame_number = 0; - spkt.streamID = 0; - pkt.codec = ftl::codecs::codec_t::MSGPACK; - pkt.bitrate = 0; - pkt.frame_count = 1; - //pkt.definition = ftl::codecs::definition_t::Any; - pkt.flags = 0; - - VectorBuffer buf(pkt.data); - msgpack::pack(buf, data); - - notifyRaw(spkt, pkt); -} - #endif // _FTL_RGBD_SOURCE_HPP_ diff --git a/components/rgbd-sources/src/group.cpp b/components/rgbd-sources/src/group.cpp index bbd03207f45e144dc0ee5ee8a7f71fd0b3c58a7b..4a8230c68bab29651e5774d3ff8783d38be8fbbf 100644 --- a/components/rgbd-sources/src/group.cpp +++ b/components/rgbd-sources/src/group.cpp @@ -77,15 +77,15 @@ void Group::_retrieveJob(ftl::rgbd::Source *src) { } } -void Group::_computeJob(ftl::rgbd::Source *src, int64_t ts) { +void Group::_dispatchJob(ftl::rgbd::Source *src, int64_t ts) { try { - src->compute(ts); + src->dispatch(ts); } catch (std::exception &ex) { - LOG(ERROR) << "Exception when computing frame"; + LOG(ERROR) << "Exception when dispatching frame"; LOG(ERROR) << ex.what(); } catch (...) { - LOG(ERROR) << "Unknown exception when computing frame"; + LOG(ERROR) << "Unknown exception when dispatching frame"; } } @@ -135,22 +135,9 @@ void Group::onFrameSet(const ftl::rgbd::VideoCallback &cb) { ftl::pool.push([this,s,ts](int id) { _retrieveJob(s); - //if (jobs_ == 0) LOG(INFO) << "LAST JOB = Retrieve"; --jobs_; - - if (cjobs_ == 0) { - cjobs_++; - s->swap(); - ftl::pool.push([this,s,ts](int id) { - _computeJob(s, ts); - //if (jobs_ == 0) LOG(INFO) << "LAST JOB = Compute"; - - //LOG(INFO) << "Compute time: " << ftl::timer::get_time() - ts; - --cjobs_; - }); - } else { - //LOG(WARNING) << "Frame drop"; - } + //if (jobs_ == 0) LOG(INFO) << "LAST JOB = Retrieve"; + _dispatchJob(s, ts); }); /*ftl::pool.push([this,s](int id) { _computeJob(s); @@ -167,12 +154,6 @@ void Group::onFrameSet(const ftl::rgbd::VideoCallback &cb) { }); } -void Group::addRawCallback(const std::function<void(ftl::rgbd::Source*, const ftl::codecs::StreamPacket &spkt, const ftl::codecs::Packet &pkt)> &f) { - for (auto s : sources_) { - s->addRawCallback(f); - } -} - /*void Group::removeRawCallback(const std::function<void(ftl::rgbd::Source*, const ftl::codecs::StreamPacket &spkt, const ftl::codecs::Packet &pkt)> &f) { for (auto s : sources_) { s->removeRawCallback(f); diff --git a/components/rgbd-sources/src/source.cpp b/components/rgbd-sources/src/source.cpp index 25a4a01b1e8cca1002c6e17e50e1418a2787b42f..c0c728c8ea76381d86ecb46a2217157ee966bb2c 100644 --- a/components/rgbd-sources/src/source.cpp +++ b/components/rgbd-sources/src/source.cpp @@ -39,15 +39,15 @@ using ftl::rgbd::capability_t; using ftl::codecs::Channel; //using ftl::rgbd::detail::FileSource; using ftl::rgbd::Camera; -using ftl::rgbd::RawCallback; using ftl::rgbd::FrameCallback; -std::map<std::string, ftl::rgbd::Player*> Source::readers__; Source::Source(ftl::config::json_t &cfg) : Configurable(cfg), pose_(Eigen::Matrix4d::Identity()), net_(nullptr) { impl_ = nullptr; //params_ = {}; stream_ = 0; + is_dispatching = false; + is_retrieving = false; reset(); on("uri", [this](const ftl::config::Event &e) { @@ -60,6 +60,8 @@ Source::Source(ftl::config::json_t &cfg, ftl::net::Universe *net) : Configurable impl_ = nullptr; //params_ = {}; stream_ = 0; + is_dispatching = false; + is_retrieving = false; reset(); on("uri", [this](const ftl::config::Event &e) { @@ -77,31 +79,7 @@ cv::Mat Source::cameraMatrix() const { return m; } -ftl::rgbd::detail::Source *Source::_createImplementation() { - auto uristr = get<string>("uri"); - if (!uristr) { - //LOG(WARNING) << "Missing URI for source"; - return nullptr; - } - - ftl::URI uri(*uristr); - if (!uri.isValid()) { - LOG(WARNING) << "Invalid URI for source: " << *uristr; - return nullptr; - } - - switch (uri.getScheme()) { - case ftl::URI::SCHEME_FILE : return _createFileImpl(uri); - case ftl::URI::SCHEME_FTL : return _createNetImpl(uri); - case ftl::URI::SCHEME_DEVICE : return _createDeviceImpl(uri); - default: break; - } - - LOG(WARNING) << "Unrecognised source URI: " << *uristr; - return nullptr; -} - -ftl::rgbd::detail::Source *Source::_createFileImpl(const ftl::URI &uri) { +static ftl::rgbd::detail::Source *createFileImpl(const ftl::URI &uri, Source *host) { std::string path = uri.getPath(); // Note: This is non standard if (uri.getHost() == "." || uri.getHost() == "~") path = uri.getHost()+path; @@ -112,7 +90,7 @@ ftl::rgbd::detail::Source *Source::_createFileImpl(const ftl::URI &uri) { // Might be a directory if (ftl::is_directory(path)) { if (ftl::is_file(path + "/video.mp4")) { - return new StereoVideoSource(this, path); + return new StereoVideoSource(host, path); // } else if (ftl::is_file(path + "/im0.png")) { // return new MiddleburySource(this, path); } else { @@ -131,18 +109,9 @@ ftl::rgbd::detail::Source *Source::_createFileImpl(const ftl::URI &uri) { LOG(FATAL) << "File sources not supported"; return nullptr; } else if (ext == "png" || ext == "jpg") { - return new ImageSource(this, path); + return new ImageSource(host, path); } else if (ext == "mp4") { - return new StereoVideoSource(this, path); - } else if (ext == "tar" || ext == "gz") { -#ifdef HAVE_LIBARCHIVE - ftl::rgbd::SnapshotReader reader(path); - auto snapshot = reader.readArchive(); - return new ftl::rgbd::detail::SnapshotSource(this, snapshot, value("index", std::string("0"))); // TODO: Use URI fragment -#else - LOG(ERROR) << "Cannot read snapshots, libarchive not installed"; - return nullptr; -#endif // HAVE_LIBARCHIVE + return new StereoVideoSource(host, path); } else { LOG(WARNING) << "Unrecognised file type: " << path; } @@ -153,45 +122,23 @@ ftl::rgbd::detail::Source *Source::_createFileImpl(const ftl::URI &uri) { return nullptr; } -/*ftl::rgbd::Player *Source::__createReader(const std::string &path) { - if (readers__.find(path) != readers__.end()) { - return readers__[path]; - } - - std::ifstream *file = new std::ifstream; - file->open(path); - - // FIXME: This is a memory leak, must delete ifstream somewhere. - - auto *r = new ftl::rgbd::Player(*file); - readers__[path] = r; - r->begin(); - return r; -}*/ - -ftl::rgbd::detail::Source *Source::_createNetImpl(const ftl::URI &uri) { - LOG(FATAL) << "Net sources no longer supported"; - //return new NetSource(this); - return nullptr; -} - -ftl::rgbd::detail::Source *Source::_createDeviceImpl(const ftl::URI &uri) { +static ftl::rgbd::detail::Source *createDeviceImpl(const ftl::URI &uri, Source *host) { if (uri.getPathSegment(0) == "video") { - return new StereoVideoSource(this); + return new StereoVideoSource(host); } else if (uri.getPathSegment(0) == "pylon") { #ifdef HAVE_PYLON - return new PylonSource(this); + return new PylonSource(host); #else LOG(ERROR) << "You did not build with 'pylon'"; #endif } else if (uri.getPathSegment(0) == "realsense") { #ifdef HAVE_REALSENSE - return new RealsenseSource(this); + return new RealsenseSource(host); #else LOG(ERROR) << "You do not have 'librealsense2' installed"; #endif } else if (uri.getPathSegment(0) == "screen") { - return new ScreenCapture(this); + return new ScreenCapture(host); } else { /*params_.width = value("width", 1280); params_.height = value("height", 720); @@ -207,25 +154,28 @@ ftl::rgbd::detail::Source *Source::_createDeviceImpl(const ftl::URI &uri) { return nullptr; } -void Source::getFrames(cv::Mat &rgb, cv::Mat &depth) { - if (bool(callback_)) LOG(WARNING) << "Cannot use getFrames and callback in source"; - SHARED_LOCK(mutex_,lk); - //rgb_.copyTo(rgb); - //depth_.copyTo(depth); - //rgb = rgb_; - //depth = depth_; -} +static ftl::rgbd::detail::Source *createImplementation(const std::string &uristr, Source *host) { + ftl::URI uri(uristr); + if (!uri.isValid()) { + LOG(WARNING) << "Invalid URI for source: " << uristr; + return nullptr; + } + switch (uri.getScheme()) { + case ftl::URI::SCHEME_FILE : return createFileImpl(uri, host); + case ftl::URI::SCHEME_DEVICE : return createDeviceImpl(uri, host); + default: break; + } + + LOG(WARNING) << "Unrecognised source URI: " << uristr; + return nullptr; +} void Source::setPose(const Eigen::Matrix4d &pose) { pose_ = pose; if (impl_) impl_->setPose(pose); } -const Eigen::Matrix4d &Source::getPose() const { - return pose_; -} - bool Source::hasCapabilities(capability_t c) { return (getCapabilities() & c) == c; } @@ -239,7 +189,11 @@ void Source::reset() { UNIQUE_LOCK(mutex_,lk); channel_ = Channel::None; if (impl_) delete impl_; - impl_ = _createImplementation(); + impl_ = nullptr; + + auto uristr = get<string>("uri"); + if (!uristr) return; + impl_ = createImplementation(*uristr, this); } bool Source::capture(int64_t ts) { @@ -249,13 +203,28 @@ bool Source::capture(int64_t ts) { } bool Source::retrieve() { - if (impl_) return impl_->retrieve(); - else return true; + is_retrieving = true; + bool status = false; + if (impl_) status = impl_->retrieve(frames_[0]); + is_retrieving = false; + return status; } -bool Source::compute(int64_t ts) { - UNIQUE_LOCK(mutex_,lk); - return impl_ && impl_->compute(ts); +bool Source::dispatch(int64_t ts) { + if (!callback_) return false; + if (is_dispatching || is_retrieving) return false; + is_dispatching = true; + _swap(); + ftl::pool.push([this,ts](int id) { + callback_(ts, frames_[1]); + is_dispatching = false; + }); +} + +void Source::_swap() { + auto tmp = std::move(frames_[0]); + frames_[0] = std::move(frames_[1]); + frames_[1] = std::move(tmp); } bool Source::setChannel(ftl::codecs::Channel c) { @@ -273,31 +242,6 @@ void Source::setCallback(const FrameCallback &cb) { callback_ = cb; } -void Source::addRawCallback(const RawCallback &f) { - UNIQUE_LOCK(mutex_,lk); - rawcallbacks_.push_back(f); -} - -void Source::removeRawCallback(const std::function<void(ftl::rgbd::Source*, const ftl::codecs::StreamPacket &spkt, const ftl::codecs::Packet &pkt)> &f) { - UNIQUE_LOCK(mutex_,lk); - for (auto i=rawcallbacks_.begin(); i!=rawcallbacks_.end(); ++i) { - const auto targ = (*i).target<void(*)(ftl::rgbd::Source*, const ftl::codecs::StreamPacket &, const ftl::codecs::Packet &)>(); - if (targ && targ == f.target<void(*)(ftl::rgbd::Source*, const ftl::codecs::StreamPacket &, const ftl::codecs::Packet &)>()) { - rawcallbacks_.erase(i); - LOG(INFO) << "Removing RAW callback"; - return; - } - } -} - -void Source::notifyRaw(const ftl::codecs::StreamPacket &spkt, const ftl::codecs::Packet &pkt) { - SHARED_LOCK(mutex_,lk); - - for (auto &i : rawcallbacks_) { - i(this, spkt, pkt); - } -} - /* * Scale camera parameters to match resolution. */ @@ -320,28 +264,3 @@ Camera Camera::scaled(int width, int height) const { return newcam; } -void Source::notify(int64_t ts, ftl::rgbd::Frame &f) { - //if (impl_) f.setOrigin(&impl_->state_); - if (callback_) callback_(ts, f); -} - -void Source::inject(const Eigen::Matrix4d &pose) { - ftl::codecs::StreamPacket spkt; - ftl::codecs::Packet pkt; - - spkt.timestamp = impl_->timestamp_; - spkt.frame_number = 0; - spkt.channel = Channel::Pose; - spkt.streamID = 0; - pkt.codec = ftl::codecs::codec_t::MSGPACK; - //pkt.definition = ftl::codecs::definition_t::Any; - pkt.bitrate = 0; - pkt.frame_count = 1; - pkt.flags = 0; - - std::vector<double> data(pose.data(), pose.data() + 4*4*sizeof(double)); - VectorBuffer buf(pkt.data); - msgpack::pack(buf, data); - - notifyRaw(spkt, pkt); -} diff --git a/components/rgbd-sources/src/sources/image/image.hpp b/components/rgbd-sources/src/sources/image/image.hpp index 22cfa244db5ebc75fdff491e42fbdf33f3c7f889..f5017cbe3db11a060709303343c869337c6bb1fa 100644 --- a/components/rgbd-sources/src/sources/image/image.hpp +++ b/components/rgbd-sources/src/sources/image/image.hpp @@ -14,9 +14,8 @@ class ImageSource : public ftl::rgbd::detail::Source { } - bool capture(int64_t ts) { timestamp_ = ts; return true; } - bool retrieve() { return true; } - bool compute(int64_t ts) { return false; }; + bool capture(int64_t ts) { return true; } + bool retrieve(ftl::rgbd::Frame &) { return true; } bool isReady() { return false; }; }; diff --git a/components/rgbd-sources/src/sources/middlebury/middlebury_source.cpp b/components/rgbd-sources/src/sources/middlebury/middlebury_source.cpp index 229895049061f2308f1a982eab46005a26940548..372963906a0c03346f88e20811b5ef03fd374f9b 100644 --- a/components/rgbd-sources/src/sources/middlebury/middlebury_source.cpp +++ b/components/rgbd-sources/src/sources/middlebury/middlebury_source.cpp @@ -159,8 +159,4 @@ void MiddleburySource::_performDisparity() { //disparityToDepthTRUE(depth_, depth_, params_); } -bool MiddleburySource::compute(int n, int b) { - //_performDisparity(); - return true; -} diff --git a/components/rgbd-sources/src/sources/middlebury/middlebury_source.hpp b/components/rgbd-sources/src/sources/middlebury/middlebury_source.hpp index d273d23a66d67c6618c0ac4a2062a780d9a3bddb..d1243a17b0a6ca7ac8e1198cda1ca3c47586e37e 100644 --- a/components/rgbd-sources/src/sources/middlebury/middlebury_source.hpp +++ b/components/rgbd-sources/src/sources/middlebury/middlebury_source.hpp @@ -19,9 +19,8 @@ class MiddleburySource : public detail::Source { MiddleburySource(ftl::rgbd::Source *, const std::string &dir); ~MiddleburySource() {}; - bool capture(int64_t ts) { timestamp_ = ts; return true; } - bool retrieve() { return true; } - bool compute(int n, int b); + bool capture(int64_t ts) { return true; } + bool retrieve(ftl::rgbd::Frame &) { return true; } bool isReady() { return ready_; } private: diff --git a/components/rgbd-sources/src/sources/pylon/pylon.cpp b/components/rgbd-sources/src/sources/pylon/pylon.cpp index 76aa5f4e0e7f83411af1dda142c22b0ad0090003..223d201f72cb20879b887a49bfadd9e6be385fa5 100644 --- a/components/rgbd-sources/src/sources/pylon/pylon.cpp +++ b/components/rgbd-sources/src/sources/pylon/pylon.cpp @@ -98,7 +98,6 @@ void PylonSource::_configureCamera(CBaslerUniversalInstantCamera *cam) { } bool PylonSource::capture(int64_t ts) { - timestamp_ = ts; if (!isReady()) return false; try { @@ -114,10 +113,9 @@ bool PylonSource::capture(int64_t ts) { return true; } -bool PylonSource::retrieve() { +bool PylonSource::retrieve(ftl::rgbd::Frame &frame) { if (!isReady()) return false; - auto &frame = frames_[0]; frame.reset(); frame.setOrigin(&state_); @@ -170,18 +168,6 @@ bool PylonSource::retrieve() { return true; } -void PylonSource::swap() { - auto tmp = std::move(frames_[0]); - frames_[0] = std::move(frames_[1]); - frames_[1] = std::move(tmp); -} - -bool PylonSource::compute(int64_t ts) { - auto &frame = frames_[1]; - host_->notify(ts, frame); - return true; -} - bool PylonSource::isReady() { return lcam_ && lcam_->IsOpen(); } diff --git a/components/rgbd-sources/src/sources/pylon/pylon.hpp b/components/rgbd-sources/src/sources/pylon/pylon.hpp index 6065aa402600b4979a197aefb712ae777413813b..2d0a82261de809bd312db5735733a970af136678 100644 --- a/components/rgbd-sources/src/sources/pylon/pylon.hpp +++ b/components/rgbd-sources/src/sources/pylon/pylon.hpp @@ -20,17 +20,14 @@ class PylonSource : public ftl::rgbd::detail::Source { explicit PylonSource(ftl::rgbd::Source *host); ~PylonSource(); - void swap(); bool capture(int64_t ts); - bool retrieve(); - bool compute(int64_t ts); + bool retrieve(ftl::rgbd::Frame &frame); bool isReady(); private: bool ready_; Pylon::CBaslerUniversalInstantCamera *lcam_; Pylon::CBaslerUniversalInstantCamera *rcam_; - Frame frames_[2]; cv::Mat tmp_; void _configureCamera(Pylon::CBaslerUniversalInstantCamera *cam); diff --git a/components/rgbd-sources/src/sources/realsense/realsense_source.cpp b/components/rgbd-sources/src/sources/realsense/realsense_source.cpp index deff4972122c35d625093af796b36d430e6073c3..2f95799bc7d4eeef992f292e07bf6d6c47bf8fc8 100644 --- a/components/rgbd-sources/src/sources/realsense/realsense_source.cpp +++ b/components/rgbd-sources/src/sources/realsense/realsense_source.cpp @@ -45,8 +45,7 @@ RealsenseSource::~RealsenseSource() { } -bool RealsenseSource::retrieve() { - auto &frame = frames_[0]; +bool RealsenseSource::retrieve(ftl::rgbd::Frame &frame) { frame.reset(); frame.setOrigin(&state_); @@ -86,18 +85,6 @@ bool RealsenseSource::retrieve() { return true; } -bool RealsenseSource::compute(int64_t ts) { - auto &frame = frames_[1]; - host_->notify(ts, frame); - return true; -} - -void RealsenseSource::swap() { - auto tmp = std::move(frames_[0]); - frames_[0] = std::move(frames_[1]); - frames_[1] = std::move(tmp); -} - bool RealsenseSource::isReady() { return true; } diff --git a/components/rgbd-sources/src/sources/realsense/realsense_source.hpp b/components/rgbd-sources/src/sources/realsense/realsense_source.hpp index bb4c701e38fd0e213ab94fa293c0a36c8e26478e..a4c7baa49c74e102f287b8a71d8038a99243ed1d 100644 --- a/components/rgbd-sources/src/sources/realsense/realsense_source.hpp +++ b/components/rgbd-sources/src/sources/realsense/realsense_source.hpp @@ -17,11 +17,9 @@ class RealsenseSource : public ftl::rgbd::detail::Source { explicit RealsenseSource(ftl::rgbd::Source *host); ~RealsenseSource(); - bool capture(int64_t ts) { timestamp_ = ts; return true; } - bool retrieve(); - bool compute(int64_t ts); + bool capture(int64_t ts) { return true; } + bool retrieve(ftl::rgbd::Frame &frame); bool isReady(); - void swap(); private: bool ready_; @@ -29,7 +27,6 @@ class RealsenseSource : public ftl::rgbd::detail::Source { rs2::pipeline pipe_; rs2::align align_to_depth_; rs2::frame rscolour_; - Frame frames_[2]; }; } diff --git a/components/rgbd-sources/src/sources/screencapture/screencapture.cpp b/components/rgbd-sources/src/sources/screencapture/screencapture.cpp index 9ef0ee0a20aa370329c6b84b985b19f10a04cfd3..9c7a210528a2b4b06d8312b7a453249373435912 100644 --- a/components/rgbd-sources/src/sources/screencapture/screencapture.cpp +++ b/components/rgbd-sources/src/sources/screencapture/screencapture.cpp @@ -190,14 +190,7 @@ ScreenCapture::~ScreenCapture() { #endif } -void ScreenCapture::swap() { -} - -bool ScreenCapture::retrieve() { - return true; -} - -bool ScreenCapture::compute(int64_t ts) { +bool ScreenCapture::retrieve(ftl::rgbd::Frame &frame) { if (!ready_) return false; cv::Mat img; @@ -206,15 +199,14 @@ bool ScreenCapture::compute(int64_t ts) { img = cv::Mat(params_.height, params_.width, CV_8UC4, impl_state_->ximg->data); #endif - frame_.reset(); - frame_.setOrigin(&state_); + frame.reset(); + frame.setOrigin(&state_); if (!img.empty()) { - frame_.create<cv::Mat>(Channel::Colour) = img; + frame.create<cv::Mat>(Channel::Colour) = img; } - host_->notify(ts, frame_); - return true; + return true; } bool ScreenCapture::isReady() { diff --git a/components/rgbd-sources/src/sources/screencapture/screencapture.hpp b/components/rgbd-sources/src/sources/screencapture/screencapture.hpp index 942db7d56e37047647391050a085ebabb395bc94..f5cb50b4a02b48303dccedeb2b205fa7fe9d3736 100644 --- a/components/rgbd-sources/src/sources/screencapture/screencapture.hpp +++ b/components/rgbd-sources/src/sources/screencapture/screencapture.hpp @@ -22,11 +22,9 @@ class ScreenCapture : public ftl::rgbd::detail::Source { explicit ScreenCapture(ftl::rgbd::Source *host); ~ScreenCapture(); - bool capture(int64_t ts) { timestamp_ = ts; return true; }; - void swap() override; - bool retrieve(); - bool compute(int64_t ts); - bool isReady(); + bool capture(int64_t ts) override { return true; }; + bool retrieve(ftl::rgbd::Frame &frame) override; + bool isReady() override; size_t getOffsetX() const { return (offset_x_ > full_width_-params_.width) ? full_width_-params_.width : offset_x_; } size_t getOffsetY() const { return (offset_y_ > full_height_-params_.height) ? full_height_-params_.height : offset_y_; } diff --git a/components/rgbd-sources/src/sources/stereovideo/stereovideo.cpp b/components/rgbd-sources/src/sources/stereovideo/stereovideo.cpp index e232037033dc5d0ae90359d74f6898d37a3b9e23..0bbbb4aec4a28ade280bca363bae9754a19f515d 100644 --- a/components/rgbd-sources/src/sources/stereovideo/stereovideo.cpp +++ b/components/rgbd-sources/src/sources/stereovideo/stereovideo.cpp @@ -60,7 +60,7 @@ StereoVideoSource::~StereoVideoSource() { void StereoVideoSource::init(const string &file) { capabilities_ = kCapVideo | kCapStereo; - if (ftl::is_video(file)) { + /*if (ftl::is_video(file)) { // Load video file LOG(INFO) << "Using video file..."; //lsrc_ = ftl::create<LocalSource>(host_, "feed", file); @@ -76,14 +76,13 @@ void StereoVideoSource::init(const string &file) { //lsrc_ = ftl::create<LocalSource>(host_, "feed", *vid); } } - else { + else {*/ // Use cameras LOG(INFO) << "Using cameras..."; lsrc_ = ftl::create<OpenCVDevice>(host_, "feed"); - } + //} color_size_ = cv::Size(lsrc_->width(), lsrc_->height()); - frames_ = std::vector<Frame>(2); pipeline_input_ = ftl::config::create<ftl::operators::Graph>(host_, "input"); #ifdef HAVE_OPTFLOW @@ -283,16 +282,13 @@ void StereoVideoSource::updateParameters() { } bool StereoVideoSource::capture(int64_t ts) { - capts_ = timestamp_; - timestamp_ = ts; lsrc_->grab(); return true; } -bool StereoVideoSource::retrieve() { +bool StereoVideoSource::retrieve(ftl::rgbd::Frame &frame) { FTL_Profile("Stereo Retrieve", 0.03); - auto &frame = frames_[0]; frame.reset(); frame.setOrigin(&state_); @@ -320,37 +316,6 @@ bool StereoVideoSource::retrieve() { return true; } -void StereoVideoSource::swap() { - auto tmp = std::move(frames_[0]); - frames_[0] = std::move(frames_[1]); - frames_[1] = std::move(tmp); -} - -bool StereoVideoSource::compute(int64_t ts) { - auto &frame = frames_[1]; - - if (lsrc_->isStereo()) { - if (!frame.hasChannel(Channel::Left) || - !frame.hasChannel(Channel::Right)) { - - return false; - } - - cv::cuda::GpuMat& left = frame.get<cv::cuda::GpuMat>(Channel::Left); - cv::cuda::GpuMat& right = frame.get<cv::cuda::GpuMat>(Channel::Right); - - if (left.empty() || right.empty()) { return false; } - //stream_.waitForCompletion(); - - } - else { - if (!frame.hasChannel(Channel::Left)) { return false; } - } - - host_->notify(ts, frame); - return true; -} - bool StereoVideoSource::isReady() { return ready_; } diff --git a/components/rgbd-sources/src/sources/stereovideo/stereovideo.hpp b/components/rgbd-sources/src/sources/stereovideo/stereovideo.hpp index e7f2ee8877beff5bd924cd692168ac4fa141ab08..4d0c3795ab9c0af6e86b717925ac503a3825ad03 100644 --- a/components/rgbd-sources/src/sources/stereovideo/stereovideo.hpp +++ b/components/rgbd-sources/src/sources/stereovideo/stereovideo.hpp @@ -25,11 +25,9 @@ class StereoVideoSource : public detail::Source { StereoVideoSource(ftl::rgbd::Source*, const std::string &); ~StereoVideoSource(); - void swap(); - bool capture(int64_t ts); - bool retrieve(); - bool compute(int64_t ts); - bool isReady(); + bool capture(int64_t ts) override; + bool retrieve(ftl::rgbd::Frame &frame) override; + bool isReady() override; Camera parameters(ftl::codecs::Channel chan) override; @@ -54,8 +52,6 @@ class StereoVideoSource : public detail::Source { cv::cuda::Stream stream_; cv::cuda::Stream stream2_; - std::vector<Frame> frames_; - cv::Mat mask_l_; void init(const std::string &); diff --git a/components/rgbd-sources/test/source_unit.cpp b/components/rgbd-sources/test/source_unit.cpp index 7c1b3a0e9182a93874de9577752e1a07a3c8cf81..37b9224c0d5bf288a9b16091a71624afdf02dcf4 100644 --- a/components/rgbd-sources/test/source_unit.cpp +++ b/components/rgbd-sources/test/source_unit.cpp @@ -40,8 +40,7 @@ class ImageSource : public ftl::rgbd::detail::Source { } bool capture(int64_t ts) { return true; } - bool retrieve() { return true; } - bool compute(int64_t ts) { return true; }; + bool retrieve(ftl::rgbd::Frame &) { return true; } bool isReady() { return true; }; }; @@ -52,8 +51,7 @@ class ScreenCapture : public ftl::rgbd::detail::Source { } bool capture(int64_t ts) { return true; } - bool retrieve() { return true; } - bool compute(int64_t ts) { return true; }; + bool retrieve(ftl::rgbd::Frame &) { return true; } bool isReady() { return true; }; }; @@ -67,8 +65,7 @@ class StereoVideoSource : public ftl::rgbd::detail::Source { } bool capture(int64_t ts) { return true; } - bool retrieve() { return true; } - bool compute(int64_t ts) { return true; }; + bool retrieve(ftl::rgbd::Frame &) { return true; } bool isReady() { return true; }; }; @@ -79,8 +76,7 @@ class NetSource : public ftl::rgbd::detail::Source { } bool capture(int64_t ts) { return true; } - bool retrieve() { return true; } - bool compute(int64_t ts) { return true; }; + bool retrieve(ftl::rgbd::Frame &) { return true; } bool isReady() { return true; }; }; @@ -91,8 +87,7 @@ class SnapshotSource : public ftl::rgbd::detail::Source { } bool capture(int64_t ts) { return true; } - bool retrieve() { return true; } - bool compute(int64_t ts) { return true; }; + bool retrieve(ftl::rgbd::Frame &) { return true; } bool isReady() { return true; }; }; @@ -103,8 +98,7 @@ class FileSource : public ftl::rgbd::detail::Source { } bool capture(int64_t ts) { return true; } - bool retrieve() { return true; } - bool compute(int64_t ts) { return true; }; + bool retrieve(ftl::rgbd::Frame &) { return true; } bool isReady() { return true; }; }; @@ -115,8 +109,7 @@ class RealsenseSource : public ftl::rgbd::detail::Source { } bool capture(int64_t ts) { return true; } - bool retrieve() { return true; } - bool compute(int64_t ts) { return true; }; + bool retrieve(ftl::rgbd::Frame &) { return true; } bool isReady() { return true; }; }; @@ -127,8 +120,7 @@ class PylonSource : public ftl::rgbd::detail::Source { } bool capture(int64_t ts) { return true; } - bool retrieve() { return true; } - bool compute(int64_t ts) { return true; }; + bool retrieve(ftl::rgbd::Frame &) { return true; } bool isReady() { return true; }; }; @@ -139,8 +131,7 @@ class MiddleburySource : public ftl::rgbd::detail::Source { } bool capture(int64_t ts) { return true; } - bool retrieve() { return true; } - bool compute(int64_t ts) { return true; }; + bool retrieve(ftl::rgbd::Frame &) { return true; } bool isReady() { return true; }; };