diff --git a/components/streams/src/renderers/openvr_render.cpp b/components/streams/src/renderers/openvr_render.cpp index 6346cf2448a6b52b8268fd25525f1e6022df5368..0f578740eb58f795d58984eab5d8e4be62192a00 100644 --- a/components/streams/src/renderers/openvr_render.cpp +++ b/components/streams/src/renderers/openvr_render.cpp @@ -379,12 +379,25 @@ bool OpenVRRender::retrieve(ftl::data::Frame &frame_out) { for (auto &s : sets) { if (s->frameset() == my_id_) continue; // Skip self - // TODO: Render audio also... - // Use another thread to merge all audio channels along with - // some degree of volume adjustment. Later, do 3D audio. + // Inject and copy data items and mix audio + for (size_t i=0; i<s->frames.size(); ++i) { + auto &f = s->frames[i]; + + // If audio is present, mix with the other frames + if (f.hasChannel(Channel::AudioStereo)) { + // Map a mixer track to this frame + auto &mixmap = mixmap_[f.id().id]; + if (mixmap.track == -1) mixmap.track = tracks_++; + mixer_.resize(tracks_); + + // Do mix but must not mix same frame multiple times + if (mixmap.last_timestamp != f.timestamp()) { + const auto &audio = f.get<std::list<ftl::audio::Audio>>(Channel::AudioStereo).front(); + mixer_.write(mixmap.track, audio.data()); + mixmap.last_timestamp = f.timestamp(); + } + } - // Inject and copy data items - for (const auto &f : s->frames) { // Add pose as a camera shape auto &shape = shapes.list.emplace_back(); shape.id = f.id().id; @@ -401,6 +414,17 @@ bool OpenVRRender::retrieve(ftl::data::Frame &frame_out) { } } + mixer_.mix(); + + // Write mixed audio to frame. + if (mixer_.frames() > 0) { + auto &list = frame_out.create<std::list<ftl::audio::Audio>>(Channel::AudioStereo).list; + list.clear(); + + int fcount = mixer_.frames(); + mixer_.read(list.emplace_front().data(), fcount); + } + // TODO: Blend option renderer_->end(); diff --git a/components/streams/src/renderers/openvr_render.hpp b/components/streams/src/renderers/openvr_render.hpp index bdeadbeb7614d66f552aa806565b556f26c3c940..1423dd8dc667bb4712e78bc3bd44663f938287ab 100644 --- a/components/streams/src/renderers/openvr_render.hpp +++ b/components/streams/src/renderers/openvr_render.hpp @@ -7,6 +7,7 @@ #include <ftl/render/CUDARender.hpp> #include <ftl/streams/feed.hpp> #include <ftl/utility/gltexture.hpp> +#include <ftl/audio/mixer.hpp> #include "../baserender.hpp" @@ -55,6 +56,15 @@ class OpenVRRender : public ftl::render::BaseSourceImpl { vr::TrackedDevicePose_t rTrackedDevicePose_[ vr::k_unMaxTrackedDeviceCount ]; #endif + struct AudioMixerMapping { + int64_t last_timestamp=0; + int track=-1; + }; + + int tracks_=0; + ftl::audio::StereoMixerF<100> mixer_; + std::unordered_map<uint32_t, AudioMixerMapping> mixmap_; + bool initVR(); }; diff --git a/components/streams/src/renderers/screen_render.cpp b/components/streams/src/renderers/screen_render.cpp index 37c46a717a86642dc8f137d0c91c85c7b30cdd53..a125f770c70b69d74501b86403395b79d66fb96e 100644 --- a/components/streams/src/renderers/screen_render.cpp +++ b/components/streams/src/renderers/screen_render.cpp @@ -155,24 +155,24 @@ bool ScreenRender::retrieve(ftl::data::Frame &frame_out) { for (auto &s : sets) { if (s->frameset() == my_id_) continue; // Skip self - // TODO: Render audio also... - // Use another thread to merge all audio channels along with - // some degree of volume adjustment. Later, do 3D audio. - //mixer_.resize(tracks_); - - // Inject and copy data items + // Inject and copy data items and mix audio for (size_t i=0; i<s->frames.size(); ++i) { auto &f = s->frames[i]; - auto &mixmap = mixmap_[f.id().id]; - if (mixmap.track == -1) mixmap.track = tracks_++; - mixer_.resize(tracks_); - - if (mixmap.last_timestamp != f.timestamp() && f.hasChannel(Channel::AudioStereo)) { - const auto &audio = f.get<std::list<ftl::audio::Audio>>(Channel::AudioStereo).front(); - mixer_.write(mixmap.track, audio.data()); + // If audio is present, mix with the other frames + if (f.hasChannel(Channel::AudioStereo)) { + // Map a mixer track to this frame + auto &mixmap = mixmap_[f.id().id]; + if (mixmap.track == -1) mixmap.track = tracks_++; + mixer_.resize(tracks_); + + // Do mix but must not mix same frame multiple times + if (mixmap.last_timestamp != f.timestamp()) { + const auto &audio = f.get<std::list<ftl::audio::Audio>>(Channel::AudioStereo).front(); + mixer_.write(mixmap.track, audio.data()); + mixmap.last_timestamp = f.timestamp(); + } } - mixmap.last_timestamp = f.timestamp(); // Add pose as a camera shape auto &shape = shapes.list.emplace_back();