diff --git a/CMakeLists.txt b/CMakeLists.txt
index 54256ba0e0746e5a48a1df20c0d2ee841326e0b0..3592611681ad0c31c087a38a3af84b8f71cf7d96 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -222,25 +222,6 @@ else()
add_library(realsense INTERFACE)
endif()
-# ==============================================================================
-
-if (BUILD_GUI)
- set(HAVE_NANOGUI TRUE)
-
- # Disable building extras we won't need (pure C++ project)
- set(NANOGUI_BUILD_SHARED OFF CACHE BOOL " " FORCE)
- set(NANOGUI_BUILD_EXAMPLE OFF CACHE BOOL " " FORCE)
- set(NANOGUI_BUILD_PYTHON OFF CACHE BOOL " " FORCE)
- set(NANOGUI_INSTALL OFF CACHE BOOL " " FORCE)
- set(NANOGUI_EIGEN_INCLUDE_DIR ${EIGEN_INCLUDE_DIR} CACHE STRING " " FORCE)
-
- # Add the configurations from nanogui
- add_subdirectory(ext/nanogui)
-
- # For reliability of parallel build, make the NanoGUI targets dependencies
- set_property(TARGET glfw glfw_objects nanogui PROPERTY FOLDER "dependencies")
-endif()
-
# ==== Portaudio v19 ===========================================================
# Portaudio v19 library
@@ -390,16 +371,16 @@ include(ftl_paths)
if (WIN32) # TODO(nick) Should do based upon compiler (VS)
add_definitions(-DWIN32)
- set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /MP4 /std:c++17 /wd4996")
+ set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /arch:AVX512 /MP4 /std:c++17 /wd4996")
set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} /DFTL_DEBUG /Wall")
set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} /O2")
set(OS_LIBS "")
else()
add_definitions(-DUNIX)
# -fdiagnostics-color
- set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++17 -fPIC -msse3 -Wall -Werror=unused-result -Werror=return-type")
+ set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++17 -fPIC -march=native -mfpmath=sse -Wall -Werror=unused-result -Werror=return-type")
set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -pg")
- set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} -O3 -mfpmath=sse")
+ set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} -O3")
set(OS_LIBS "dl")
endif()
@@ -441,6 +422,27 @@ else()
add_library(sgm INTERFACE)
endif()
+# ==============================================================================
+
+if (BUILD_GUI)
+ set(HAVE_NANOGUI TRUE)
+
+ # Disable building extras we won't need (pure C++ project)
+ set(NANOGUI_BUILD_SHARED OFF CACHE BOOL " " FORCE)
+ set(NANOGUI_BUILD_EXAMPLE OFF CACHE BOOL " " FORCE)
+ set(NANOGUI_BUILD_PYTHON OFF CACHE BOOL " " FORCE)
+ set(NANOGUI_INSTALL OFF CACHE BOOL " " FORCE)
+ set(NANOGUI_EIGEN_INCLUDE_DIR ${EIGEN_INCLUDE_DIR} CACHE STRING " " FORCE)
+
+ # Add the configurations from nanogui
+ add_subdirectory(ext/nanogui)
+
+ # For reliability of parallel build, make the NanoGUI targets dependencies
+ set_property(TARGET glfw glfw_objects nanogui PROPERTY FOLDER "dependencies")
+endif()
+
+# =============================================================================
+
add_subdirectory(components/common/cpp)
add_subdirectory(components/codecs)
diff --git a/applications/gui2/src/views/addsource.hpp b/applications/gui2/src/views/addsource.hpp
index 7e6ca31938b57ce063be81c8d0b77dc73e3b92da..34e8353fe496454e8146787f00149e390b002cc2 100644
--- a/applications/gui2/src/views/addsource.hpp
+++ b/applications/gui2/src/views/addsource.hpp
@@ -34,7 +34,7 @@ private:
nanogui::TabWidget *tabs_;
public:
- // EIGEN_MAKE_ALIGNED_OPERATOR_NEW
+ EIGEN_MAKE_ALIGNED_OPERATOR_NEW
};
}
diff --git a/components/audio/CMakeLists.txt b/components/audio/CMakeLists.txt
index cae7b4080b8c8fa96e96327b60762b3049faf603..7a2721087b04834506d0af31671a8e06030fb62f 100644
--- a/components/audio/CMakeLists.txt
+++ b/components/audio/CMakeLists.txt
@@ -15,5 +15,6 @@ target_include_directories(ftlaudio PUBLIC
target_link_libraries(ftlaudio ftlcommon Eigen3::Eigen ftlstreams ftldata portaudio Opus)
-#add_subdirectory(test)
-
+if (BUILD_TESTS)
+add_subdirectory(test)
+endif()
diff --git a/components/audio/include/ftl/audio/buffer.hpp b/components/audio/include/ftl/audio/buffer.hpp
index da39fd9c3802f01c99dce542de3f5c4f6ee5821b..545e15bf129c9a7e429c5f325916a331d29031db 100644
--- a/components/audio/include/ftl/audio/buffer.hpp
+++ b/components/audio/include/ftl/audio/buffer.hpp
@@ -72,7 +72,7 @@ class FixedBuffer : public ftl::audio::Buffer<T> {
inline void writeFrame(const T *d) {
const T *in = d;
- T *out = &data_[(write_position_++) % SIZE][0];
+ T *out = data_[(write_position_++) % SIZE];
for (size_t i=0; i<CHAN*FRAME; ++i) *out++ = *in++;
if (write_position_ > 5 && read_position_ < 0) read_position_ = 0;
}
@@ -105,11 +105,17 @@ class FixedBuffer : public ftl::audio::Buffer<T> {
read_position_ = 0;
}
+ inline T *data() { return (T*)data_; }
+ inline T *data(int f) { return data_[f]; }
+
+ inline int writePosition() const { return write_position_; }
+ inline void setWritePosition(int p) { write_position_ = p; }
+
private:
int write_position_;
int read_position_;
int offset_;
- T data_[SIZE][CHAN*FRAME];
+ alignas(32) T data_[SIZE][CHAN*FRAME];
};
// ==== Implementations ========================================================
@@ -153,7 +159,7 @@ void FixedBuffer<T,CHAN,FRAME,SIZE>::write(const std::vector<T> &in) {
++write_position_;
}
}
- if (write_position_ > 20 && read_position_ < 0) read_position_ = 0;
+ if (write_position_ > 5 && read_position_ < 0) read_position_ = 0;
}
template <typename T, int CHAN, int FRAME, int SIZE>
diff --git a/components/audio/include/ftl/audio/mixer.hpp b/components/audio/include/ftl/audio/mixer.hpp
new file mode 100644
index 0000000000000000000000000000000000000000..b8e41d16e54fb4a476a54ca598c82c11ac155dcd
--- /dev/null
+++ b/components/audio/include/ftl/audio/mixer.hpp
@@ -0,0 +1,158 @@
+#ifndef _FTL_AUDIO_MIXER_HPP_
+#define _FTL_AUDIO_MIXER_HPP_
+
+#include <Eigen/Eigen>
+#include <vector>
+#include <cmath>
+#include "buffer.hpp"
+
+namespace ftl {
+namespace audio {
+
+//static constexpr int kBufferCount = 100;
+
+/**
+ * A fast circular buffer to capture, play and manipulate audio data.
+ * This class can be used directly with portaudio. The hardware uses
+ * `readFrame` and `writeFrame` to consume or append audio data. A more
+ * advanced `write` function allows for non-frame aligned data and for time
+ * dilation / shifting, and amplitude control.
+ */
+template <typename T, int CHAN, int FRAME, int SIZE>
+class FixedMixer : public ftl::audio::Buffer<T> {
+ public:
+ FixedMixer() : Buffer<T>(CHAN, FRAME, 44100), write_position_(0), read_position_(0), offset_(0) { resize(1); }
+ explicit FixedMixer(int tracks) : Buffer<T>(CHAN, FRAME, 44100), write_position_(0), read_position_(0), offset_(0) { resize(tracks); }
+ FixedMixer(int tracks, int rate) : Buffer<T>(CHAN, FRAME, rate), write_position_(0), read_position_(0), offset_(0) { resize(tracks); }
+
+
+ inline int maxFrames() const { return SIZE; }
+
+ inline void readFrame(T *d) {
+ T *out = d;
+ if (read_position_ >= write_position_) {
+ for (size_t i=0; i<CHAN*FRAME; ++i) *out++ = 0;
+ } else {
+ T *in = data_[(read_position_++) % SIZE];
+ for (size_t i=0; i<CHAN*FRAME; ++i) *out++ = *in++;
+ }
+ }
+
+ int size() const override { return (read_position_>=0) ? write_position_ - read_position_ : 0; }
+ int frames() const override { return (read_position_>=0) ? write_position_ - read_position_ : 0; }
+
+ /**
+ * Append sound samples to the end of the buffer. The samples may be over
+ * or under sampled so as to gradually introduce or remove a requested
+ * delay and hence change the latency of the audio.
+ */
+ void write(const std::vector<T> &in) override;
+
+ inline void write(int track, const std::vector<T> &in) {
+ tracks_[track].write(in);
+ }
+
+ void mix();
+
+ void read(std::vector<T> &out, int frames) override;
+
+ void reset() override {
+ Buffer<T>::reset();
+ write_position_ = 0; //int(this->cur_delay_);
+ read_position_ = 0;
+ }
+
+ inline int writePosition() const { return write_position_; }
+ inline int readPosition() const { return read_position_; }
+ inline int tracks() const { return track_num_; }
+
+ inline void setDelay(int track, float d) { tracks_[track].setDelay(d); }
+ inline float delay(int track) const { return tracks_[track].delay(); }
+
+ inline void setGain(float g) { gain_ = g; }
+ inline float gain() const { return gain_; }
+
+ void resize(int tracks);
+
+ private:
+ int track_num_=0;
+ int write_position_;
+ int read_position_;
+ int offset_;
+ float gain_ = 1.0f;
+ alignas(32) T data_[SIZE][CHAN*FRAME];
+ std::vector<ftl::audio::FixedBuffer<T,CHAN,FRAME,SIZE>> tracks_;
+};
+
+// ==== Implementations ========================================================
+
+template <typename T, int CHAN, int FRAME, int SIZE>
+void FixedMixer<T,CHAN,FRAME,SIZE>::write(const std::vector<T> &in) {
+ // Not supported...
+}
+
+template <typename T, int CHAN, int FRAME, int SIZE>
+void FixedMixer<T,CHAN,FRAME,SIZE>::mix() {
+ // Add together up to most recent frame
+ int min_write = std::numeric_limits<int>::max();
+ for (auto &t : tracks_) {
+ min_write = std::min(t.writePosition(), min_write);
+ }
+
+ // For each frame
+ while (write_position_ < min_write) {
+ int wp = write_position_ % SIZE;
+ float *ptr1 = data_[wp];
+
+ // For each block of 8 float samples
+ for (size_t i=0; i<CHAN*FRAME; i+=8) {
+ Eigen::Map<Eigen::Matrix<float,8,1>,Eigen::Aligned32> v1(ptr1+i);
+ v1.setZero();
+
+ // For each track, accumulate the samples
+ for (auto &t : tracks_) {
+ const Eigen::Map<Eigen::Matrix<float,8,1>,Eigen::Aligned32> v2(&t.data(wp)[i]);
+ v1 += v2;
+ }
+
+ v1 *= gain_;
+ }
+
+ ++write_position_;
+ }
+}
+
+template <typename T, int CHAN, int FRAME, int SIZE>
+void FixedMixer<T,CHAN,FRAME,SIZE>::read(std::vector<T> &out, int count) {
+ out.resize(FRAME*count*CHAN);
+ T *ptr = out.data();
+ for (int i=0; i<count; ++i) {
+ readFrame(ptr);
+ ptr += FRAME*CHAN;
+ }
+}
+
+template <typename T, int CHAN, int FRAME, int SIZE>
+void FixedMixer<T,CHAN,FRAME,SIZE>::resize(int t) {
+ if (track_num_ == t) return;
+
+ track_num_ = t;
+ tracks_.reserve(t);
+ while (static_cast<int>(tracks_.size()) < t) {
+ auto &tr = tracks_.emplace_back();
+ tr.setWritePosition(write_position_);
+ }
+}
+
+// ==== Common forms ===========================================================
+
+template <int SIZE>
+using StereoMixerF = ftl::audio::FixedMixer<float,2,960,SIZE>;
+
+template <int SIZE>
+using MonoMixerF = ftl::audio::FixedMixer<float,1,960,SIZE>;
+
+}
+}
+
+#endif // _FTL_AUDIO_BUFFER_HPP_
diff --git a/components/audio/test/CMakeLists.txt b/components/audio/test/CMakeLists.txt
new file mode 100644
index 0000000000000000000000000000000000000000..f1ab14fb0922e8c91e16220f4d8726d82d5e9da9
--- /dev/null
+++ b/components/audio/test/CMakeLists.txt
@@ -0,0 +1,11 @@
+### OpenCV Codec Unit ################################################################
+add_executable(mixer_unit
+$<TARGET_OBJECTS:CatchTest>
+ mixer_unit.cpp
+)
+target_include_directories(mixer_unit PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}/../include")
+target_link_libraries(mixer_unit
+ Threads::Threads ${OS_LIBS} ftlcommon)
+
+
+add_test(MixerUnitTest mixer_unit)
diff --git a/components/audio/test/mixer_unit.cpp b/components/audio/test/mixer_unit.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..2339144469e01fa5f1cc3183f985383c0ad6521b
--- /dev/null
+++ b/components/audio/test/mixer_unit.cpp
@@ -0,0 +1,133 @@
+#include "catch.hpp"
+#include <ftl/audio/mixer.hpp>
+
+using ftl::audio::StereoMixerF;
+
+TEST_CASE("Audio Mixer Stereo Float", "") {
+ SECTION("Add two in sync tracks") {
+ StereoMixerF<100> mixer(2);
+
+ // Three 960 sample stereo frames
+ std::vector<float> in1(960*2*3);
+ std::vector<float> in2(960*2*3);
+
+ for (int i=0; i<960*2*3; ++i) in1[i] = float(i)+1.0f;
+ for (int i=0; i<960*2*3; ++i) in2[i] = float(i)+2.0f;
+
+ mixer.write(0, in1);
+ mixer.write(1, in2);
+ mixer.mix();
+
+ REQUIRE( mixer.writePosition() == 3 );
+ REQUIRE( mixer.readPosition() == 0 );
+
+ // Read one of the three valid frames
+ std::vector<float> out;
+ mixer.read(out, 1);
+ bool correct = true;
+
+ // Check all values are correct
+ for (int i=0; i<960*2*1; ++i) {
+ float e = float(i)+1.0f + float(i)+2.0f;
+ correct &= int(e) == int(out[i]);
+ }
+
+ REQUIRE( correct );
+ }
+
+ SECTION("Add two out of sync tracks") {
+ StereoMixerF<100> mixer(2);
+
+ // Three 960 sample stereo frames
+ std::vector<float> in1(960*2*3);
+ std::vector<float> in2(960*2*2);
+
+ for (int i=0; i<960*2*3; ++i) in1[i] = float(i)+1.0f;
+ for (int i=0; i<960*2*2; ++i) in2[i] = float(i)+2.0f;
+
+ mixer.write(0, in1);
+ mixer.write(1, in2);
+ mixer.mix();
+
+ REQUIRE( mixer.writePosition() == 2 );
+ REQUIRE( mixer.readPosition() == 0 );
+
+ // Read one of the three valid frames
+ std::vector<float> out;
+ mixer.read(out, 2);
+ bool correct = true;
+
+ // Check all values are correct
+ for (int i=0; i<960*2*2; ++i) {
+ float e = float(i)+1.0f + float(i)+2.0f;
+ correct &= int(e) == int(out[i]);
+ }
+
+ REQUIRE( correct );
+
+ // Now add final frame
+ std::vector<float> in3(960*2*1);
+ for (int i=0; i<960*2*1; ++i) in3[i] = float(i)+1.0f;
+
+ mixer.write(1, in3);
+ mixer.mix();
+
+ REQUIRE( mixer.writePosition() == 3 );
+ REQUIRE( mixer.readPosition() == 2 );
+
+ mixer.read(out, 1);
+
+ // Check all values are correct
+ for (int i=0; i<960*2*1; ++i) {
+ float e = float(i)+1.0f + float(i+960*2*2)+1.0f;
+ correct &= int(e) == int(out[i]);
+ }
+
+ REQUIRE( correct );
+ }
+}
+
+TEST_CASE("Audio Mixer Stereo Float Dynamic Tracks", "") {
+ SECTION("Add one track after write") {
+ StereoMixerF<100> mixer(1);
+
+ // Three 960 sample stereo frames
+ std::vector<float> in1(960*2*3);
+ for (int i=0; i<960*2*3; ++i) in1[i] = float(i)+1.0f;
+
+ mixer.write(0, in1);
+ mixer.mix();
+
+ REQUIRE( mixer.writePosition() == 3 );
+ REQUIRE( mixer.readPosition() == 0 );
+
+ std::vector<float> in2(960*2*3);
+ for (int i=0; i<960*2*3; ++i) in2[i] = float(i)+2.0f;
+
+ mixer.resize(2);
+ mixer.write(0, in1);
+ mixer.write(1, in2);
+ mixer.mix();
+
+ REQUIRE( mixer.writePosition() == 6 );
+ REQUIRE( mixer.readPosition() == 0 );
+ REQUIRE( mixer.frames() == 6 );
+
+ // Read one of the three valid frames
+ std::vector<float> out;
+ mixer.read(out, mixer.frames());
+ bool correct = true;
+
+ // Check all values are correct
+ for (int i=0; i<960*2*3; ++i) {
+ float e = float(i)+1.0f;
+ correct &= int(e) == int(out[i]);
+ }
+ for (int i=960*2*3; i<960*2*6; ++i) {
+ float e = float(i-960*2*3)+1.0f + float(i-960*2*3)+2.0f;
+ correct &= int(e) == int(out[i]);
+ }
+
+ REQUIRE( correct );
+ }
+}
diff --git a/components/streams/src/renderers/screen_render.cpp b/components/streams/src/renderers/screen_render.cpp
index d06d57f7f243ac0acdbe0bb11786997b6b6ec932..b9b5bb73721574ab6e0345d7233718f9ce7c2879 100644
--- a/components/streams/src/renderers/screen_render.cpp
+++ b/components/streams/src/renderers/screen_render.cpp
@@ -155,12 +155,24 @@ bool ScreenRender::retrieve(ftl::data::Frame &frame_out) {
for (auto &s : sets) {
if (s->frameset() == my_id_) continue; // Skip self
+ auto &mixmap = mixmap_[s->id().id];
+ if (mixmap.track == -1) mixmap.track = tracks_++;
+
// TODO: Render audio also...
// Use another thread to merge all audio channels along with
// some degree of volume adjustment. Later, do 3D audio.
+ mixer_.resize(tracks_);
// Inject and copy data items
- for (const auto &f : s->frames) {
+ for (size_t i=0; i<s->frames.size(); ++i) {
+ auto &f = s->frames[i];
+
+ if (mixmap.last_timestamp != f.timestamp() && f.hasChannel(Channel::AudioStereo)) {
+ const auto &audio = f.get<std::list<ftl::audio::Audio>>(Channel::AudioStereo).front();
+ mixer_.write(mixmap.track, audio.data());
+ }
+ mixmap.last_timestamp = f.timestamp();
+
// Add pose as a camera shape
auto &shape = shapes.list.emplace_back();
shape.id = f.id().id;
@@ -177,6 +189,17 @@ bool ScreenRender::retrieve(ftl::data::Frame &frame_out) {
}
}
+ mixer_.mix();
+
+ // Write mixed audio to frame.
+ if (mixer_.frames() > 0) {
+ auto &list = frame_out.create<std::list<ftl::audio::Audio>>(Channel::AudioStereo).list;
+ list.clear();
+
+ int fcount = mixer_.frames();
+ mixer_.read(list.emplace_front().data(), fcount);
+ }
+
// This waits for GPU also
if (!data_only) renderer_->end();
} catch (const ftl::exception &e) {
diff --git a/components/streams/src/renderers/screen_render.hpp b/components/streams/src/renderers/screen_render.hpp
index e8a5635a57d9c96bf327625c32ad3e9785d182a6..a24bb747676b8731eb1a472995d85c523d67b7cf 100644
--- a/components/streams/src/renderers/screen_render.hpp
+++ b/components/streams/src/renderers/screen_render.hpp
@@ -6,6 +6,7 @@
#include <ftl/render/renderer.hpp>
#include <ftl/render/CUDARender.hpp>
#include <ftl/streams/feed.hpp>
+#include <ftl/audio/mixer.hpp>
#include "../baserender.hpp"
@@ -35,6 +36,15 @@ class ScreenRender : public ftl::render::BaseSourceImpl {
uint32_t my_id_;
ftl::operators::Graph *post_pipe_;
std::atomic_flag calibration_uptodate_;
+
+ struct AudioMixerMapping {
+ int64_t last_timestamp=0;
+ int track=-1;
+ };
+
+ int tracks_=0;
+ ftl::audio::StereoMixerF<100> mixer_;
+ std::unordered_map<uint32_t, AudioMixerMapping> mixmap_;
};
}