Merge branch 'feature/audio-mixer' into 'master'

Audio mixer for screen render

See merge request nicolas.pope/ftl!318

CMakeLists.txt

@@ -222,25 +222,6 @@ else()
 	add_library(realsense INTERFACE)
 endif()
 
-# ==============================================================================
-
-if (BUILD_GUI)
-	set(HAVE_NANOGUI TRUE)
-
-	# Disable building extras we won't need (pure C++ project)
-	set(NANOGUI_BUILD_SHARED OFF CACHE BOOL " " FORCE)
-	set(NANOGUI_BUILD_EXAMPLE OFF CACHE BOOL " " FORCE)
-	set(NANOGUI_BUILD_PYTHON  OFF CACHE BOOL " " FORCE)
-	set(NANOGUI_INSTALL       OFF CACHE BOOL " " FORCE)
-	set(NANOGUI_EIGEN_INCLUDE_DIR ${EIGEN_INCLUDE_DIR} CACHE STRING " " FORCE)
-
-	# Add the configurations from nanogui
-	add_subdirectory(ext/nanogui)
-
-	# For reliability of parallel build, make the NanoGUI targets dependencies
-	set_property(TARGET glfw glfw_objects nanogui PROPERTY FOLDER "dependencies")
-endif()
-
 # ==== Portaudio v19 ===========================================================
 
 # Portaudio v19 library
@@ -390,16 +371,16 @@ include(ftl_paths)
 
 if (WIN32) # TODO(nick) Should do based upon compiler (VS)
 	add_definitions(-DWIN32)
-	set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /MP4 /std:c++17 /wd4996")
+	set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /arch:AVX512 /MP4 /std:c++17 /wd4996")
 	set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} /DFTL_DEBUG /Wall")
 	set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} /O2")
 	set(OS_LIBS "")
 else()
 	add_definitions(-DUNIX)
 	# -fdiagnostics-color
-	set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++17 -fPIC -msse3 -Wall -Werror=unused-result -Werror=return-type")
+	set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++17 -fPIC -march=native -mfpmath=sse -Wall -Werror=unused-result -Werror=return-type")
 	set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -pg")
-	set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} -O3 -mfpmath=sse")
+	set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} -O3")
 	set(OS_LIBS "dl")
 endif()
 
@@ -441,6 +422,27 @@ else()
 	add_library(sgm INTERFACE)
 endif()
 
+# ==============================================================================
+
+if (BUILD_GUI)
+	set(HAVE_NANOGUI TRUE)
+
+	# Disable building extras we won't need (pure C++ project)
+	set(NANOGUI_BUILD_SHARED OFF CACHE BOOL " " FORCE)
+	set(NANOGUI_BUILD_EXAMPLE OFF CACHE BOOL " " FORCE)
+	set(NANOGUI_BUILD_PYTHON  OFF CACHE BOOL " " FORCE)
+	set(NANOGUI_INSTALL       OFF CACHE BOOL " " FORCE)
+	set(NANOGUI_EIGEN_INCLUDE_DIR ${EIGEN_INCLUDE_DIR} CACHE STRING " " FORCE)
+
+	# Add the configurations from nanogui
+	add_subdirectory(ext/nanogui)
+
+	# For reliability of parallel build, make the NanoGUI targets dependencies
+	set_property(TARGET glfw glfw_objects nanogui PROPERTY FOLDER "dependencies")
+endif()
+
+# =============================================================================
+
 
 add_subdirectory(components/common/cpp)
 add_subdirectory(components/codecs)

applications/gui2/src/views/addsource.hpp

@@ -34,7 +34,7 @@ private:
 	nanogui::TabWidget *tabs_;
 
 public:
-	// EIGEN_MAKE_ALIGNED_OPERATOR_NEW
+	EIGEN_MAKE_ALIGNED_OPERATOR_NEW
 };
 
 }

components/audio/CMakeLists.txt

@@ -15,5 +15,6 @@ target_include_directories(ftlaudio PUBLIC
 
 target_link_libraries(ftlaudio ftlcommon Eigen3::Eigen ftlstreams ftldata portaudio Opus)
 
-#add_subdirectory(test)
-
+if (BUILD_TESTS)
+add_subdirectory(test)
+endif()

components/audio/include/ftl/audio/buffer.hpp

@@ -72,7 +72,7 @@ class FixedBuffer : public ftl::audio::Buffer<T> {
 
 	inline void writeFrame(const T *d) {
 		const T *in = d;
-		T *out = &data_[(write_position_++) % SIZE][0];
+		T *out = data_[(write_position_++) % SIZE];
 		for (size_t i=0; i<CHAN*FRAME; ++i) *out++ = *in++;
 		if (write_position_ > 5 && read_position_ < 0) read_position_ = 0;
 	}
@@ -105,11 +105,17 @@ class FixedBuffer : public ftl::audio::Buffer<T> {
 		read_position_ = 0;
 	}
 
+	inline T *data() { return (T*)data_; }
+	inline T *data(int f) { return data_[f]; }
+
+	inline int writePosition() const { return write_position_; }
+	inline void setWritePosition(int p) { write_position_ = p; }
+
 	private:
 	int write_position_;
 	int read_position_;
 	int offset_;
-	T data_[SIZE][CHAN*FRAME];
+	alignas(32) T data_[SIZE][CHAN*FRAME];
 };
 
 // ==== Implementations ========================================================
@@ -153,7 +159,7 @@ void FixedBuffer<T,CHAN,FRAME,SIZE>::write(const std::vector<T> &in) {
 			++write_position_;
 		}
 	}
-	if (write_position_ > 20 && read_position_ < 0) read_position_ = 0;
+	if (write_position_ > 5 && read_position_ < 0) read_position_ = 0;
 }
 
 template <typename T, int CHAN, int FRAME, int SIZE>

components/audio/include/ftl/audio/mixer.hpp

+#ifndef _FTL_AUDIO_MIXER_HPP_
+#define _FTL_AUDIO_MIXER_HPP_
+
+#include <Eigen/Eigen>
+#include <vector>
+#include <cmath>
+#include "buffer.hpp"
+
+namespace ftl {
+namespace audio {
+
+//static constexpr int kBufferCount = 100;
+
+/**
+ * A fast circular buffer to capture, play and manipulate audio data.
+ * This class can be used directly with portaudio. The hardware uses
+ * `readFrame` and `writeFrame` to consume or append audio data. A more
+ * advanced `write` function allows for non-frame aligned data and for time
+ * dilation / shifting, and amplitude control.
+ */
+template <typename T, int CHAN, int FRAME, int SIZE>
+class FixedMixer : public ftl::audio::Buffer<T> {
+	public:
+	FixedMixer() : Buffer<T>(CHAN, FRAME, 44100), write_position_(0), read_position_(0), offset_(0) { resize(1); }
+	explicit FixedMixer(int tracks) : Buffer<T>(CHAN, FRAME, 44100), write_position_(0), read_position_(0), offset_(0) { resize(tracks); }
+	FixedMixer(int tracks, int rate) : Buffer<T>(CHAN, FRAME, rate), write_position_(0), read_position_(0), offset_(0) { resize(tracks); }
+
+
+	inline int maxFrames() const { return SIZE; }
+
+	inline void readFrame(T *d) {
+		T *out = d;
+		if (read_position_ >= write_position_) {
+			for (size_t i=0; i<CHAN*FRAME; ++i) *out++ = 0;
+		} else {
+			T *in = data_[(read_position_++) % SIZE];
+			for (size_t i=0; i<CHAN*FRAME; ++i) *out++ = *in++;
+		}
+	}
+
+	int size() const override { return (read_position_>=0) ? write_position_ - read_position_ : 0; }
+	int frames() const override { return (read_position_>=0) ? write_position_ - read_position_ : 0; }
+
+	/**
+	 * Append sound samples to the end of the buffer. The samples may be over
+	 * or under sampled so as to gradually introduce or remove a requested
+	 * delay and hence change the latency of the audio.
+	 */
+	void write(const std::vector<T> &in) override;
+
+	inline void write(int track, const std::vector<T> &in) {
+		tracks_[track].write(in);
+	}
+
+	void mix();
+
+	void read(std::vector<T> &out, int frames) override;
+
+	void reset() override {
+		Buffer<T>::reset();
+		write_position_ = 0; //int(this->cur_delay_);
+		read_position_ = 0;
+	}
+
+	inline int writePosition() const { return write_position_; }
+	inline int readPosition() const { return read_position_; }
+	inline int tracks() const { return track_num_; }
+
+	inline void setDelay(int track, float d) { tracks_[track].setDelay(d); }
+	inline float delay(int track) const { return tracks_[track].delay(); }
+
+	inline void setGain(float g) { gain_ = g; }
+	inline float gain() const { return gain_; }
+
+	void resize(int tracks);
+
+	private:
+	int track_num_=0;
+	int write_position_;
+	int read_position_;
+	int offset_;
+	float gain_ = 1.0f;
+	alignas(32) T data_[SIZE][CHAN*FRAME];
+	std::vector<ftl::audio::FixedBuffer<T,CHAN,FRAME,SIZE>> tracks_;
+};
+
+// ==== Implementations ========================================================
+
+template <typename T, int CHAN, int FRAME, int SIZE>
+void FixedMixer<T,CHAN,FRAME,SIZE>::write(const std::vector<T> &in) {
+	// Not supported...
+}
+
+template <typename T, int CHAN, int FRAME, int SIZE>
+void FixedMixer<T,CHAN,FRAME,SIZE>::mix() {
+	// Add together up to most recent frame
+	int min_write = std::numeric_limits<int>::max();
+	for (auto &t : tracks_) {
+		min_write = std::min(t.writePosition(), min_write);
+	}
+
+	// For each frame
+	while (write_position_ < min_write) {
+		int wp = write_position_ % SIZE;
+		float *ptr1 = data_[wp];
+
+		// For each block of 8 float samples
+		for (size_t i=0; i<CHAN*FRAME; i+=8) {
+			Eigen::Map<Eigen::Matrix<float,8,1>,Eigen::Aligned32> v1(ptr1+i);
+			v1.setZero();
+
+			// For each track, accumulate the samples
+			for (auto &t : tracks_) {
+				const Eigen::Map<Eigen::Matrix<float,8,1>,Eigen::Aligned32> v2(&t.data(wp)[i]);
+				v1 += v2;
+			}
+
+			v1 *= gain_;
+		}
+
+		++write_position_;
+	}
+}
+
+template <typename T, int CHAN, int FRAME, int SIZE>
+void FixedMixer<T,CHAN,FRAME,SIZE>::read(std::vector<T> &out, int count) {
+	out.resize(FRAME*count*CHAN);
+	T *ptr = out.data();
+	for (int i=0; i<count; ++i) {
+		readFrame(ptr);
+		ptr += FRAME*CHAN;
+	}
+}
+
+template <typename T, int CHAN, int FRAME, int SIZE>
+void FixedMixer<T,CHAN,FRAME,SIZE>::resize(int t) {
+	if (track_num_ == t) return;
+	
+	track_num_ = t;
+	tracks_.reserve(t);
+	while (static_cast<int>(tracks_.size()) < t) {
+		auto &tr = tracks_.emplace_back();
+		tr.setWritePosition(write_position_);
+	}
+}
+
+// ==== Common forms ===========================================================
+
+template <int SIZE>
+using StereoMixerF = ftl::audio::FixedMixer<float,2,960,SIZE>;
+
+template <int SIZE>
+using MonoMixerF = ftl::audio::FixedMixer<float,1,960,SIZE>;
+
+}
+}
+
+#endif  // _FTL_AUDIO_BUFFER_HPP_

components/audio/test/CMakeLists.txt

+### OpenCV Codec Unit ################################################################
+add_executable(mixer_unit
+$<TARGET_OBJECTS:CatchTest>
+	mixer_unit.cpp
+)
+target_include_directories(mixer_unit PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}/../include")
+target_link_libraries(mixer_unit
+	Threads::Threads ${OS_LIBS} ftlcommon)
+
+
+add_test(MixerUnitTest mixer_unit)

components/audio/test/mixer_unit.cpp

+#include "catch.hpp"
+#include <ftl/audio/mixer.hpp>
+
+using ftl::audio::StereoMixerF;
+
+TEST_CASE("Audio Mixer Stereo Float", "") {
+	SECTION("Add two in sync tracks") {
+		StereoMixerF<100> mixer(2);
+
+		// Three 960 sample stereo frames
+		std::vector<float> in1(960*2*3);
+		std::vector<float> in2(960*2*3);
+
+		for (int i=0; i<960*2*3; ++i) in1[i] = float(i)+1.0f;
+		for (int i=0; i<960*2*3; ++i) in2[i] = float(i)+2.0f;
+
+		mixer.write(0, in1);
+		mixer.write(1, in2);
+		mixer.mix();
+
+		REQUIRE( mixer.writePosition() == 3 );
+		REQUIRE( mixer.readPosition() == 0 );
+
+		// Read one of the three valid frames
+		std::vector<float> out;
+		mixer.read(out, 1);
+		bool correct = true;
+
+		// Check all values are correct
+		for (int i=0; i<960*2*1; ++i) {
+			float e = float(i)+1.0f + float(i)+2.0f;
+			correct &= int(e) == int(out[i]);
+		}
+
+		REQUIRE( correct );
+	}
+
+	SECTION("Add two out of sync tracks") {
+		StereoMixerF<100> mixer(2);
+
+		// Three 960 sample stereo frames
+		std::vector<float> in1(960*2*3);
+		std::vector<float> in2(960*2*2);
+
+		for (int i=0; i<960*2*3; ++i) in1[i] = float(i)+1.0f;
+		for (int i=0; i<960*2*2; ++i) in2[i] = float(i)+2.0f;
+
+		mixer.write(0, in1);
+		mixer.write(1, in2);
+		mixer.mix();
+
+		REQUIRE( mixer.writePosition() == 2 );
+		REQUIRE( mixer.readPosition() == 0 );
+
+		// Read one of the three valid frames
+		std::vector<float> out;
+		mixer.read(out, 2);
+		bool correct = true;
+
+		// Check all values are correct
+		for (int i=0; i<960*2*2; ++i) {
+			float e = float(i)+1.0f + float(i)+2.0f;
+			correct &= int(e) == int(out[i]);
+		}
+
+		REQUIRE( correct );
+
+		// Now add final frame
+		std::vector<float> in3(960*2*1);
+		for (int i=0; i<960*2*1; ++i) in3[i] = float(i)+1.0f;
+
+		mixer.write(1, in3);
+		mixer.mix();
+
+		REQUIRE( mixer.writePosition() == 3 );
+		REQUIRE( mixer.readPosition() == 2 );
+
+		mixer.read(out, 1);
+
+		// Check all values are correct
+		for (int i=0; i<960*2*1; ++i) {
+			float e = float(i)+1.0f + float(i+960*2*2)+1.0f;
+			correct &= int(e) == int(out[i]);
+		}
+
+		REQUIRE( correct );
+	}
+}
+
+TEST_CASE("Audio Mixer Stereo Float Dynamic Tracks", "") {
+	SECTION("Add one track after write") {
+		StereoMixerF<100> mixer(1);
+
+		// Three 960 sample stereo frames
+		std::vector<float> in1(960*2*3);
+		for (int i=0; i<960*2*3; ++i) in1[i] = float(i)+1.0f;
+
+		mixer.write(0, in1);
+		mixer.mix();
+
+		REQUIRE( mixer.writePosition() == 3 );
+		REQUIRE( mixer.readPosition() == 0 );
+
+		std::vector<float> in2(960*2*3);
+		for (int i=0; i<960*2*3; ++i) in2[i] = float(i)+2.0f;
+
+		mixer.resize(2);
+		mixer.write(0, in1);
+		mixer.write(1, in2);
+		mixer.mix();
+
+		REQUIRE( mixer.writePosition() == 6 );
+		REQUIRE( mixer.readPosition() == 0 );
+		REQUIRE( mixer.frames() == 6 );
+
+		// Read one of the three valid frames
+		std::vector<float> out;
+		mixer.read(out, mixer.frames());
+		bool correct = true;
+
+		// Check all values are correct
+		for (int i=0; i<960*2*3; ++i) {
+			float e = float(i)+1.0f;
+			correct &= int(e) == int(out[i]);
+		}
+		for (int i=960*2*3; i<960*2*6; ++i) {
+			float e = float(i-960*2*3)+1.0f + float(i-960*2*3)+2.0f;
+			correct &= int(e) == int(out[i]);
+		}
+
+		REQUIRE( correct );
+	}
+}

components/streams/src/renderers/screen_render.cpp

@@ -155,12 +155,24 @@ bool ScreenRender::retrieve(ftl::data::Frame &frame_out) {
 			for (auto &s : sets) {
 				if (s->frameset() == my_id_) continue;  // Skip self
 
+				auto &mixmap = mixmap_[s->id().id];
+				if (mixmap.track == -1) mixmap.track = tracks_++;
+
 				// TODO: Render audio also...
 				// Use another thread to merge all audio channels along with
 				// some degree of volume adjustment. Later, do 3D audio.
+				mixer_.resize(tracks_);
 
 				// Inject and copy data items
-				for (const auto &f : s->frames) {
+				for (size_t i=0; i<s->frames.size(); ++i) {
+					auto &f = s->frames[i];
+
+					if (mixmap.last_timestamp != f.timestamp() && f.hasChannel(Channel::AudioStereo)) {
+						const auto &audio = f.get<std::list<ftl::audio::Audio>>(Channel::AudioStereo).front();
+						mixer_.write(mixmap.track, audio.data());
+					}
+					mixmap.last_timestamp = f.timestamp();
+
 					// Add pose as a camera shape
 					auto &shape = shapes.list.emplace_back();
 					shape.id = f.id().id;
@@ -177,6 +189,17 @@ bool ScreenRender::retrieve(ftl::data::Frame &frame_out) {
 				}
 			}
 
+			mixer_.mix();
+
+			// Write mixed audio to frame.
+			if (mixer_.frames() > 0) {
+				auto &list = frame_out.create<std::list<ftl::audio::Audio>>(Channel::AudioStereo).list;
+				list.clear();
+
+				int fcount = mixer_.frames();
+				mixer_.read(list.emplace_front().data(), fcount);
+			}
+
 			// This waits for GPU also
 			if (!data_only) renderer_->end();
 		} catch (const ftl::exception &e) {

components/streams/src/renderers/screen_render.hpp

@@ -6,6 +6,7 @@
 #include <ftl/render/renderer.hpp>
 #include <ftl/render/CUDARender.hpp>
 #include <ftl/streams/feed.hpp>
+#include <ftl/audio/mixer.hpp>
 
 #include "../baserender.hpp"
 
@@ -35,6 +36,15 @@ class ScreenRender : public ftl::render::BaseSourceImpl {
 	uint32_t my_id_;
 	ftl::operators::Graph *post_pipe_;
 	std::atomic_flag calibration_uptodate_;
+
+	struct AudioMixerMapping {
+		int64_t last_timestamp=0;
+		int track=-1;
+	};
+
+	int tracks_=0;
+	ftl::audio::StereoMixerF<100> mixer_;
+	std::unordered_map<uint32_t, AudioMixerMapping> mixmap_;
 };
 
 }