diff --git a/applications/gui2/src/modules/camera.cpp b/applications/gui2/src/modules/camera.cpp index e16f381a4c286c04bc7e216b3f774b4c59bde1be..d7ae38c5dcb2fd2c7733a64b7bfa10484ba91c5b 100644 --- a/applications/gui2/src/modules/camera.cpp +++ b/applications/gui2/src/modules/camera.cpp @@ -173,7 +173,7 @@ void Camera::initiate_(ftl::data::Frame &frame) { auto *mod = this->screen->getModule<ftl::gui2::Statistics>(); - mod->getJSON(StatisticsPanel::PERFORMANCE_INFO).erase("FPS"); + mod->getJSON(StatisticsPanel::PERFORMANCE_INFO).clear(); mod->getJSON(StatisticsPanel::MEDIA_STATUS).clear(); mod->getJSON(StatisticsPanel::MEDIA_META).clear(); mod->getJSON(StatisticsPanel::CAMERA_DETAILS).clear(); diff --git a/applications/gui2/src/views/camera3d.cpp b/applications/gui2/src/views/camera3d.cpp index e4432a5031fb31f87a6fce0aed99e0f4d31383ca..542fb0e26f28450619f4d5522a1ebc170ff7e48e 100644 --- a/applications/gui2/src/views/camera3d.cpp +++ b/applications/gui2/src/views/camera3d.cpp @@ -33,6 +33,8 @@ CameraView3D::CameraView3D(ftl::gui2::Screen *parent, ftl::gui2::Camera *ctrl) : delta_ = 0.0; lerp_speed_ = 0.999f; + pose_up_to_date_.test_and_set(); + setZoom(false); setPan(false); } @@ -42,16 +44,19 @@ bool CameraView3D::keyboardEvent(int key, int scancode, int action, int modifier float mag = (modifiers & 0x1) ? 0.01f : 0.1f; float scalar = (key == 263) ? -mag : mag; neye_ += rotmat_*Eigen::Vector4d(scalar, 0.0, 0.0, 1.0); + pose_up_to_date_.clear(); } else if (key == 264 || key == 265) { float mag = (modifiers & 0x1) ? 0.01f : 0.1f; float scalar = (key == 264) ? -mag : mag; neye_ += rotmat_*Eigen::Vector4d(0.0, 0.0, scalar, 1.0); + pose_up_to_date_.clear(); } else if (key == 266 || key == 267) { float mag = (modifiers & 0x1) ? 0.01f : 0.1f; float scalar = (key == 266) ? -mag : mag; neye_ += rotmat_*Eigen::Vector4d(0.0, scalar, 0.0, 1.0); + pose_up_to_date_.clear(); } else if (key >= '0' && key <= '5' && modifiers == 2) { // Ctrl+NUMBER } @@ -71,6 +76,7 @@ bool CameraView3D::mouseMotionEvent(const Eigen::Vector2i &p, const Eigen::Vecto rx_ += rel[0]; ry_ += rel[1]; + pose_up_to_date_.clear(); //LOG(INFO) << "New pose: \n" << getUpdatedPose(); //ctrl_->sendPose(getUpdatedPose()); @@ -87,10 +93,6 @@ bool CameraView3D::keyboardCharacterEvent(unsigned int codepoint) { } Eigen::Matrix4d CameraView3D::getUpdatedPose() { - double now = glfwGetTime(); - delta_ = now - ftime_; - ftime_ = now; - float rrx = ((float)ry_ * 0.2f * delta_); float rry = (float)rx_ * 0.2f * delta_; float rrz = 0.0; @@ -111,7 +113,13 @@ Eigen::Matrix4d CameraView3D::getUpdatedPose() { } void CameraView3D::draw(NVGcontext* ctx) { + double now = glfwGetTime(); + delta_ = now - ftime_; + ftime_ = now; + // poll from ctrl_ or send on event instead? - ctrl_->sendPose(getUpdatedPose()); + if (!pose_up_to_date_.test_and_set()) { + ctrl_->sendPose(getUpdatedPose()); + } CameraView::draw(ctx); } diff --git a/applications/gui2/src/views/camera3d.hpp b/applications/gui2/src/views/camera3d.hpp index 9bd762ee7872320d245a5f21710ca7385183d2af..47132b8c356e236b90cbe9d8ebe39e473c7b0a71 100644 --- a/applications/gui2/src/views/camera3d.hpp +++ b/applications/gui2/src/views/camera3d.hpp @@ -39,6 +39,8 @@ protected: double lerp_speed_; + std::atomic_flag pose_up_to_date_; + public: EIGEN_MAKE_ALIGNED_OPERATOR_NEW }; diff --git a/components/streams/include/ftl/streams/feed.hpp b/components/streams/include/ftl/streams/feed.hpp index e01a20ba0de52bf8c1a7da9a570a30c24eb7c4c0..9054831c20e25fd5653f0dcd76b1eb8602a7b596 100644 --- a/components/streams/include/ftl/streams/feed.hpp +++ b/components/streams/include/ftl/streams/feed.hpp @@ -89,6 +89,7 @@ private: std::unique_ptr<ftl::stream::Sender> recorder_; std::unique_ptr<ftl::stream::Broadcast> record_stream_; ftl::Handle handle_record_; + ftl::Handle record_recv_handle_; Filter *record_filter_; //ftl::Handler<const ftl::data::FrameSetPtr&> frameset_cb_; diff --git a/components/streams/include/ftl/streams/receiver.hpp b/components/streams/include/ftl/streams/receiver.hpp index c54a377e639be64e1d727dd0f48193f8ed276de4..bb78bb3dbbcde50c25c689c8cda5ab3b51124ace 100644 --- a/components/streams/include/ftl/streams/receiver.hpp +++ b/components/streams/include/ftl/streams/receiver.hpp @@ -41,6 +41,8 @@ class Receiver : public ftl::Configurable, public ftl::data::Generator { void registerBuilder(const std::shared_ptr<ftl::streams::BaseBuilder> &b); + void processPackets(const ftl::codecs::StreamPacket &spkt, const ftl::codecs::Packet &pkt); + private: ftl::stream::Stream *stream_; ftl::data::Pool *pool_; diff --git a/components/streams/include/ftl/streams/stream.hpp b/components/streams/include/ftl/streams/stream.hpp index ead565f13e20cb74a668d3b7b53d708a9a97f066..b100e83caf6235c3bc9e3b2c367d64cbe6a1b8ca 100644 --- a/components/streams/include/ftl/streams/stream.hpp +++ b/components/streams/include/ftl/streams/stream.hpp @@ -137,6 +137,7 @@ class Muxer : public Stream { struct StreamEntry { Stream *stream; std::vector<int> maps; + uint32_t original_fsid = 0; }; std::vector<StreamEntry> streams_; diff --git a/components/streams/src/feed.cpp b/components/streams/src/feed.cpp index cd168145965193a63abda763ef2a4ae7b917b492..77e183c867c9875d4bc7460b10b1b76116ea3f06 100644 --- a/components/streams/src/feed.cpp +++ b/components/streams/src/feed.cpp @@ -95,7 +95,9 @@ Feed::Feed(nlohmann::json &config, ftl::net::Universe*net) : "recent_files", "known_hosts", "auto_host_connect", - "auto_host_sources" + "auto_host_sources", + "uri", + "recorder" }); pool_ = std::make_unique<ftl::data::Pool>(3,5); @@ -118,6 +120,11 @@ Feed::Feed(nlohmann::json &config, ftl::net::Universe*net) : (ftl::create<ftl::stream::Broadcast>(this, "record_stream")); recorder_->setStream(record_stream_.get()); + record_recv_handle_ = record_stream_->onPacket([this](const ftl::codecs::StreamPacket &spkt, const ftl::codecs::Packet &pkt) { + receiver_->processPackets(spkt, pkt); + return true; + }); + record_filter_ = nullptr; //interceptor_->setStream(stream_.get()); @@ -645,7 +652,7 @@ uint32_t Feed::add(const std::string &path) { auto *creator = new ftl::streams::ManualSourceBuilder(pool_.get(), fsid, source); if (uri.getBaseURI() == "device::openvr") creator->setFrameRate(1000); else creator->setFrameRate(30); - + std::shared_ptr<ftl::streams::BaseBuilder> creatorptr(creator); lk.lock(); receiver_->registerBuilder(creatorptr); @@ -819,7 +826,7 @@ void Feed::_beginRecord(Filter *f) { record_stream_->select(fs->frameset(), f->channels(), true); for (auto c : f->channels()) { - recorder_->post(*fs.get(), c); + if (fs->hasAnyChanged(c)) recorder_->post(*fs.get(), c); } return true; }); diff --git a/components/streams/src/receiver.cpp b/components/streams/src/receiver.cpp index b1bd910450b26094b271af903859857689634a5c..5b6f6b5756bd821c93f37e1c4c034cc73004810d 100644 --- a/components/streams/src/receiver.cpp +++ b/components/streams/src/receiver.cpp @@ -393,64 +393,68 @@ void Receiver::_processVideo(const StreamPacket &spkt, const Packet &pkt) { } } -void Receiver::setStream(ftl::stream::Stream *s) { - handle_.cancel(); - stream_ = s; +void Receiver::processPackets(const StreamPacket &spkt, const Packet &pkt) { + const unsigned int channum = (unsigned int)spkt.channel; - handle_ = s->onPacket([this](const StreamPacket &spkt, const Packet &pkt) { - const unsigned int channum = (unsigned int)spkt.channel; - - // No data packet means channel is only available. - if (pkt.data.size() == 0) { - if (spkt.streamID < 255 && !(spkt.flags & ftl::codecs::kFlagRequest)) { - // Get the frameset - auto fs = builder(spkt.streamID).get(spkt.timestamp, spkt.frame_number+pkt.frame_count-1); - const auto *cs = stream_; - const auto sel = stream_->selected(spkt.frameSetID()) & cs->available(spkt.frameSetID()); - - for (auto &frame : fs->frames) { - //LOG(INFO) << "MARK " << frame.source() << " " << (int)spkt.channel; - frame.markAvailable(spkt.channel); - - if (spkt.flags & ftl::codecs::kFlagCompleted) { - //UNIQUE_LOCK(vidstate.mutex, lk); // FIXME: Should have a lock here... - timestamp_ = spkt.timestamp; - fs->completed(frame.source()); - } - - //if (frame.availableAll(sel)) { - //LOG(INFO) << "FRAME COMPLETED " << frame.source(); - // fs->completed(frame.source()); - //} + // No data packet means channel is only available. + if (pkt.data.size() == 0) { + if (spkt.streamID < 255 && !(spkt.flags & ftl::codecs::kFlagRequest)) { + // Get the frameset + auto fs = builder(spkt.streamID).get(spkt.timestamp, spkt.frame_number+pkt.frame_count-1); + const auto *cs = stream_; + const auto sel = stream_->selected(spkt.frameSetID()) & cs->available(spkt.frameSetID()); + + for (auto &frame : fs->frames) { + //LOG(INFO) << "MARK " << frame.source() << " " << (int)spkt.channel; + frame.markAvailable(spkt.channel); + + if (spkt.flags & ftl::codecs::kFlagCompleted) { + //UNIQUE_LOCK(vidstate.mutex, lk); // FIXME: Should have a lock here... + timestamp_ = spkt.timestamp; + fs->completed(frame.source()); } + + //if (frame.availableAll(sel)) { + //LOG(INFO) << "FRAME COMPLETED " << frame.source(); + // fs->completed(frame.source()); + //} } - return true; } + return; + } - //LOG(INFO) << "PACKET: " << spkt.timestamp << ", " << (int)spkt.channel << ", " << (int)pkt.codec << ", " << (int)pkt.definition; + //LOG(INFO) << "PACKET: " << spkt.timestamp << ", " << (int)spkt.channel << ", " << (int)pkt.codec << ", " << (int)pkt.definition; - // TODO: Allow for multiple framesets - //if (spkt.frameSetID() > 0) LOG(INFO) << "Frameset " << spkt.frameSetID() << " received: " << (int)spkt.channel; - if (spkt.frameSetID() >= ftl::stream::kMaxStreams) return true; + // TODO: Allow for multiple framesets + //if (spkt.frameSetID() > 0) LOG(INFO) << "Frameset " << spkt.frameSetID() << " received: " << (int)spkt.channel; + if (spkt.frameSetID() >= ftl::stream::kMaxStreams) return; - // Frameset level data channels - if (spkt.frameNumber() == 255 && pkt.data.size() > 0) { - _processData(spkt,pkt); - return true; - } + // Frameset level data channels + if (spkt.frameNumber() == 255 && pkt.data.size() > 0) { + _processData(spkt,pkt); + return; + } - // Too many frames, so ignore. - //if (spkt.frameNumber() >= value("max_frames",32)) return; - if (spkt.frameNumber() >= 32 || ((1 << spkt.frameNumber()) & frame_mask_) == 0) return true; + // Too many frames, so ignore. + //if (spkt.frameNumber() >= value("max_frames",32)) return; + if (spkt.frameNumber() >= 32 || ((1 << spkt.frameNumber()) & frame_mask_) == 0) return; - if (channum >= 64) { - _processData(spkt,pkt); - } else if (channum >= 32 && channum < 64) { - _processAudio(spkt,pkt); - } else { - _processVideo(spkt,pkt); - } + if (channum >= 64) { + _processData(spkt,pkt); + } else if (channum >= 32 && channum < 64) { + _processAudio(spkt,pkt); + } else { + _processVideo(spkt,pkt); + } +} + +void Receiver::setStream(ftl::stream::Stream *s) { + handle_.cancel(); + stream_ = s; + + handle_ = s->onPacket([this](const StreamPacket &spkt, const Packet &pkt) { + processPackets(spkt, pkt); return true; }); } diff --git a/components/streams/src/sender.cpp b/components/streams/src/sender.cpp index b6b807080180cc046a4900c2309f7a47fd891624..4eb69ec59b72d53e2a650ae065b7a48b548796ad 100644 --- a/components/streams/src/sender.cpp +++ b/components/streams/src/sender.cpp @@ -173,11 +173,12 @@ void Sender::post(ftl::data::FrameSet &fs, ftl::codecs::Channel c, bool noencode const auto &packets = frame.getEncoded(cc); if (packets.size() > 0) { if (packets.size() == 1) { - forward_count += packets.front().frame_count; + } else { // PROBLEMS - LOG(WARNING) << "Multi packet send!"; + LOG(WARNING) << "Multi packet send! - Channel = " << int(c) << ", count = " << packets.size(); } + forward_count += packets.back().frame_count; } } else { needs_encoding = false; @@ -200,19 +201,19 @@ void Sender::post(ftl::data::FrameSet &fs, ftl::codecs::Channel c, bool noencode if (!frame.has(c)) continue; const auto &packets = frame.getEncoded(c); - if (packets.size() == 1) { + //if (packets.size() == 1) { StreamPacket spkt; spkt.version = 5; spkt.timestamp = fs.timestamp(); - spkt.streamID = 0; //fs.id; + spkt.streamID = fs.frameset(); //fs.id; spkt.frame_number = i; spkt.channel = c; spkt.flags = (last_flush) ? ftl::codecs::kFlagCompleted : 0; - stream_->post(spkt, packets.front()); - } else if (packets.size() > 1) { + stream_->post(spkt, packets.back()); + //} else if (packets.size() > 1) { // PROBLEMS - } + //} } } @@ -297,7 +298,7 @@ void Sender::post(ftl::data::Frame &frame, ftl::codecs::Channel c) { StreamPacket spkt; spkt.version = 5; spkt.timestamp = frame.timestamp(); - spkt.streamID = 0; //fs.id; + spkt.streamID = frame.frameset(); //fs.id; spkt.frame_number = frame.source(); spkt.channel = c; @@ -393,7 +394,7 @@ void Sender::_encodeVideoChannel(ftl::data::FrameSet &fs, Channel c, bool reset, StreamPacket spkt; spkt.version = 5; spkt.timestamp = fs.timestamp(); - spkt.streamID = 0; // FIXME: fs.id; + spkt.streamID = fs.frameset(); spkt.frame_number = offset; spkt.channel = c; spkt.flags = (last_flush) ? ftl::codecs::kFlagCompleted : 0; diff --git a/components/streams/src/stream.cpp b/components/streams/src/stream.cpp index 9e66f165b110f357727cc21958b8352432bd6749..782ec1fb3d04e7ea76bf83c216edd8f2920470dd 100644 --- a/components/streams/src/stream.cpp +++ b/components/streams/src/stream.cpp @@ -86,12 +86,14 @@ void Muxer::add(Stream *s, size_t fsid) { auto &se = streams_.emplace_back(); int i = streams_.size()-1; se.stream = s; + ftl::stream::Muxer::StreamEntry *ptr = &se; - handles_.push_back(std::move(s->onPacket([this,s,i,fsid](const ftl::codecs::StreamPacket &spkt, const ftl::codecs::Packet &pkt) { + handles_.push_back(std::move(s->onPacket([this,s,i,fsid,ptr](const ftl::codecs::StreamPacket &spkt, const ftl::codecs::Packet &pkt) { //TODO: Allow input streams to have other streamIDs // Same fsid means same streamIDs map together in the end ftl::codecs::StreamPacket spkt2 = spkt; + ptr->original_fsid = spkt.streamID; spkt2.streamID = fsid; if (spkt2.frame_number < 255) { @@ -129,9 +131,9 @@ bool Muxer::post(const ftl::codecs::StreamPacket &spkt, const ftl::codecs::Packe //LOG(INFO) << "POST " << spkt.frame_number; ftl::codecs::StreamPacket spkt2 = spkt; - spkt2.streamID = 0; + spkt2.streamID = se.original_fsid; spkt2.frame_number = ssid; - se.stream->select(0, selected(spkt.frameSetID())); + se.stream->select(spkt2.streamID, selected(spkt.frameSetID())); return se.stream->post(spkt2, pkt); } else { return false; @@ -217,7 +219,7 @@ void Broadcast::add(Stream *s) { streams_.push_back(s); handles_.push_back(std::move(s->onPacket([this,s](const ftl::codecs::StreamPacket &spkt, const ftl::codecs::Packet &pkt) { - LOG(INFO) << "BCAST Request: " << (int)spkt.streamID << " " << (int)spkt.channel << " " << spkt.timestamp; + //LOG(INFO) << "BCAST Request: " << (int)spkt.streamID << " " << (int)spkt.channel << " " << spkt.timestamp; SHARED_LOCK(mutex_, lk); if (spkt.frameSetID() < 255) available(spkt.frameSetID()) += spkt.channel; cb_.trigger(spkt, pkt); diff --git a/components/streams/test/recsend_unit.cpp b/components/streams/test/recsend_unit.cpp index a2a2dc2dfd9deb8972d2351f404fab3cddd08cbd..25ff480a7ed4a03c8f72f304eb8ef1aa757e359c 100644 --- a/components/streams/test/recsend_unit.cpp +++ b/components/streams/test/recsend_unit.cpp @@ -147,6 +147,7 @@ TEST_CASE( "Response via loopback" ) { stream.select(0, {Channel::Control}, true); auto *builder = new ftl::streams::ManualSourceBuilder(&pool, 0, &source); + builder->setFrameRate(10000); std::shared_ptr<ftl::streams::BaseBuilder> builderptr(builder); receiver->registerBuilder(builderptr); diff --git a/components/structures/include/ftl/data/new_frameset.hpp b/components/structures/include/ftl/data/new_frameset.hpp index 229bf974e41ba1f7cd53a9c6ca3426eb095a6c3f..93b6d4ee4003ab59068c0ddc1770561e4cf0fc35 100644 --- a/components/structures/include/ftl/data/new_frameset.hpp +++ b/components/structures/include/ftl/data/new_frameset.hpp @@ -125,6 +125,11 @@ class FrameSet : public ftl::data::Frame { */ static std::shared_ptr<FrameSet> fromFrame(Frame &); + /** + * Check if channel has changed in any frames. + */ + bool hasAnyChanged(ftl::codecs::Channel) const; + private: std::atomic<int> flags_; }; diff --git a/components/structures/src/frameset.cpp b/components/structures/src/frameset.cpp index fe45a6ed34a9bbc3cf6bc10a4eed2924a1853742..330ccf6592025a6c36b3f36da4d1b3e31b6fffb8 100644 --- a/components/structures/src/frameset.cpp +++ b/components/structures/src/frameset.cpp @@ -83,6 +83,13 @@ const ftl::data::Frame &ftl::data::FrameSet::firstFrame() const { throw FTL_Error("No frames in frameset"); } +bool ftl::data::FrameSet::hasAnyChanged(ftl::codecs::Channel c) const { + for (size_t i=0; i<frames.size(); ++i) { + if (frames[i].changed(c)) return true; + } + return false; +} + void FrameSet::store() { if (status() != ftl::data::FrameStatus::CREATED) throw FTL_Error("Cannot store frameset multiple times");