Add openvr audio mixer

components/streams/src/renderers/openvr_render.cpp

@@ -379,12 +379,25 @@ bool OpenVRRender::retrieve(ftl::data::Frame &frame_out) {
 			for (auto &s : sets) {
 				if (s->frameset() == my_id_) continue;  // Skip self
 
-				// TODO: Render audio also...
-				// Use another thread to merge all audio channels along with
-				// some degree of volume adjustment. Later, do 3D audio.
+				// Inject and copy data items and mix audio
+				for (size_t i=0; i<s->frames.size(); ++i) {
+					auto &f = s->frames[i];
+
+					// If audio is present, mix with the other frames
+					if (f.hasChannel(Channel::AudioStereo)) {
+						// Map a mixer track to this frame
+						auto &mixmap = mixmap_[f.id().id];
+						if (mixmap.track == -1) mixmap.track = tracks_++;
+						mixer_.resize(tracks_);
+
+						// Do mix but must not mix same frame multiple times
+						if (mixmap.last_timestamp != f.timestamp()) {
+							const auto &audio = f.get<std::list<ftl::audio::Audio>>(Channel::AudioStereo).front();
+							mixer_.write(mixmap.track, audio.data());
+							mixmap.last_timestamp = f.timestamp();
+						}
+					}
 
-				// Inject and copy data items
-				for (const auto &f : s->frames) {
 					// Add pose as a camera shape
 					auto &shape = shapes.list.emplace_back();
 					shape.id = f.id().id;
@@ -401,6 +414,17 @@ bool OpenVRRender::retrieve(ftl::data::Frame &frame_out) {
 				}
 			}
 
+			mixer_.mix();
+
+			// Write mixed audio to frame.
+			if (mixer_.frames() > 0) {
+				auto &list = frame_out.create<std::list<ftl::audio::Audio>>(Channel::AudioStereo).list;
+				list.clear();
+
+				int fcount = mixer_.frames();
+				mixer_.read(list.emplace_front().data(), fcount);
+			}
+
 			// TODO: Blend option
 
 			renderer_->end();

components/streams/src/renderers/openvr_render.hpp

@@ -7,6 +7,7 @@
 #include <ftl/render/CUDARender.hpp>
 #include <ftl/streams/feed.hpp>
 #include <ftl/utility/gltexture.hpp>
+#include <ftl/audio/mixer.hpp>
 
 #include "../baserender.hpp"
 
@@ -55,6 +56,15 @@ class OpenVRRender : public ftl::render::BaseSourceImpl {
 	vr::TrackedDevicePose_t rTrackedDevicePose_[ vr::k_unMaxTrackedDeviceCount ];
 	#endif
 
+	struct AudioMixerMapping {
+		int64_t last_timestamp=0;
+		int track=-1;
+	};
+
+	int tracks_=0;
+	ftl::audio::StereoMixerF<100> mixer_;
+	std::unordered_map<uint32_t, AudioMixerMapping> mixmap_;
+
 	bool initVR();
 };
 

components/streams/src/renderers/screen_render.cpp

@@ -155,24 +155,24 @@ bool ScreenRender::retrieve(ftl::data::Frame &frame_out) {
 			for (auto &s : sets) {
 				if (s->frameset() == my_id_) continue;  // Skip self
 
-				// TODO: Render audio also...
-				// Use another thread to merge all audio channels along with
-				// some degree of volume adjustment. Later, do 3D audio.
-				//mixer_.resize(tracks_);
-
-				// Inject and copy data items
+				// Inject and copy data items and mix audio
 				for (size_t i=0; i<s->frames.size(); ++i) {
 					auto &f = s->frames[i];
 
+					// If audio is present, mix with the other frames
+					if (f.hasChannel(Channel::AudioStereo)) {
+						// Map a mixer track to this frame
 						auto &mixmap = mixmap_[f.id().id];
 						if (mixmap.track == -1) mixmap.track = tracks_++;
 						mixer_.resize(tracks_);
 
-					if (mixmap.last_timestamp != f.timestamp() && f.hasChannel(Channel::AudioStereo)) {
+						// Do mix but must not mix same frame multiple times
+						if (mixmap.last_timestamp != f.timestamp()) {
 							const auto &audio = f.get<std::list<ftl::audio::Audio>>(Channel::AudioStereo).front();
 							mixer_.write(mixmap.track, audio.data());
-					}
 							mixmap.last_timestamp = f.timestamp();
+						}
+					}
 
 					// Add pose as a camera shape
 					auto &shape = shapes.list.emplace_back();