cmake/FindPylon.cmake

+###############################################################################
+# Find Pylon
+#
+
+set(PYLON_FOUND FALSE CACHE BOOL "" FORCE)
+
+if(WIN32)
+find_path(PYLON_DIR NAMES include/pylon/PylonBase.h PATHS "C:/Program Files/Pylon" "C:/Program Files (x86)/Pylon")
+else()
+find_path(PYLON_DIR NAMES include/pylon/PylonBase.h PATHS "/opt/pylon" "/opt/pylon6")
+endif()
+
+if (PYLON_DIR)
+	set(PYLON_FOUND TRUE CACHE BOOL "" FORCE)
+	set(HAVE_PYLON TRUE)
+	
+	include(FindPackageHandleStandardArgs)
+	find_package_handle_standard_args(Pylon DEFAULT_MSG PYLON_DIR)
+
+	mark_as_advanced(PYLON_FOUND)
+
+	if (WIN32)
+	list(APPEND PYLON_LIBRARIES PylonBase_v6_1 PylonUtility_v6_1 GenApi_MD_VC141_v3_1_Basler_pylon GCBase_MD_VC141_v3_1_Basler_pylon)
+	else()
+	list(APPEND PYLON_LIBRARIES pylonbase pylonutility GenApi_gcc_v3_1_Basler_pylon GCBase_gcc_v3_1_Basler_pylon)
+	endif()
+
+	add_library(Pylon INTERFACE)
+	set_property(TARGET Pylon PROPERTY INTERFACE_INCLUDE_DIRECTORIES ${PYLON_DIR}/include)
+	#set_property(TARGET Pylon PROPERTY INTERFACE_LINK_DIRECTORIES ${PYLON_DIR}/lib)
+
+	if (WIN32)
+	link_directories(${PYLON_DIR}/lib/x64)
+	else()
+	link_directories(${PYLON_DIR}/lib)
+	endif()
+
+	set_property(TARGET Pylon PROPERTY INTERFACE_LINK_LIBRARIES ${PYLON_LIBRARIES})
+else()
+	add_library(Pylon INTERFACE)
+endif()

cmake/Findglog.cmake

@@ -43,5 +43,5 @@ if(GLOG_FOUND)
     add_library(glog::glog UNKNOWN IMPORTED)
     set_property(TARGET glog::glog PROPERTY INTERFACE_INCLUDE_DIRECTORIES ${GLOG_INCLUDE_DIRS})
 	set_property(TARGET glog::glog PROPERTY IMPORTED_LOCATION ${GLOG_LIBRARY})
-    message(STATUS "Found glog: ${GLOG_LIBRARY}")
+    message(STATUS "Found glog: ${GLOG_LIBRARY} ${GLOG_INCLUDE_DIRS}")
 endif()

cmake/ftl_CPack.cmake

+# use build date as patch version
+string(TIMESTAMP BUILD_TIME "%Y%m%d")
+set(CPACK_PACKAGE_VERSION_PATCH "${BUILD_TIME}")
+
+set(CPACK_DEBIAN_PACKAGE_MAINTAINER "UTU Future Tech Lab")
+set(CPACK_DEBIAN_PACKAGE_SHLIBDEPS ON)
+set(CPACK_DEBIAN_PACKAGE_GENERATE_SHLIBS ON)
+set(CPACK_DEBIAN_PACKAGE_GENERATE_SHLIBS_POLICY ">=")
+set(CPACK_DEB_PACKAGE_COMPONENT ON)
+set(CPACK_DEBIAN_PACKAGE_SECTION "Miscellaneous")
+
+macro(deb_append_dependency DEPENDS)
+	if ("${CPACK_DEBIAN_PACKAGE_DEPENDS}" STREQUAL "")
+		set(CPACK_DEBIAN_PACKAGE_DEPENDS "${DEPENDS}")
+	else()
+		set(CPACK_DEBIAN_PACKAGE_DEPENDS "${CPACK_DEBIAN_PACKAGE_DEPENDS}, ${DEPENDS}")
+	endif()
+endmacro()
+
+if (HAVE_PYLON)
+	deb_append_dependency("pylon (>= 6.1.1)")
+	set(ENV{LD_LIBRARY_PATH} "=/opt/pylon/lib/")
+endif()
+
+if(WIN32)
+	message(INFO "Copying DLLs: OpenCV")
+	file(GLOB WINDOWS_LIBS "${OpenCV_INSTALL_PATH}/${OpenCV_ARCH}/${OpenCV_RUNTIME}/bin/*.dll")
+	install(FILES ${WINDOWS_LIBS} DESTINATION bin)
+	set(CPACK_GENERATOR "WiX")
+endif()
+
+include(CPack)

components/audio/CMakeLists.txt

 set(AUDIOSRC
 	src/source.cpp
-	src/frame.cpp
 	src/portaudio.cpp
 	src/speaker.cpp
+	src/software_encoder.cpp
+	src/software_decoder.cpp
 )
 
 add_library(ftlaudio ${AUDIOSRC})
@@ -12,7 +13,8 @@ target_include_directories(ftlaudio PUBLIC
 	$<INSTALL_INTERFACE:include>
 	PRIVATE src)
 
-target_link_libraries(ftlaudio ftlcommon Eigen3::Eigen ftlstreams ftldata portaudio)
-
-#add_subdirectory(test)
+target_link_libraries(ftlaudio ftlcommon Eigen3::Eigen ftlstreams ftldata portaudio Opus)
 
+if (BUILD_TESTS)
+add_subdirectory(test)
+endif()

components/audio/include/ftl/audio/audio.hpp

@@ -12,14 +12,24 @@ class Audio {
 
 	size_t size() const { return data_.size()*sizeof(short); }
 
-	std::vector<short> &data() { return data_; }
-	const std::vector<short> &data() const { return data_; }
+	std::vector<float> &data() { return data_; }
+	const std::vector<float> &data() const { return data_; }
 
 	private:
-	std::vector<short> data_;
+	std::vector<float> data_;
 };
 
 }
 }
 
+template <>
+inline bool ftl::data::make_type<std::list<ftl::audio::Audio>>() {
+	return false;
+}
+
+template <>
+inline bool ftl::data::decode_type<std::list<ftl::audio::Audio>>(std::any &a, const std::vector<uint8_t> &data) {
+	return false;
+}
+
 #endif  // _FTL_AUDIO_AUDIO_HPP_

components/audio/include/ftl/audio/buffer.hpp

@@ -3,9 +3,10 @@
 
 #include <vector>
 #include <cmath>
+#include <Eigen/Eigen>
 
-#define LOGURU_REPLACE_GLOG 1
-#include <loguru.hpp>
+//#define LOGURU_REPLACE_GLOG 1
+//#include <loguru.hpp>
 
 namespace ftl {
 namespace audio {
@@ -36,6 +37,9 @@ class Buffer {
 
 	float delay() const { return cur_delay_ / static_cast<float>(rate_); }
 
+	inline void setGain(float g) { gain_ = g; }
+	inline float gain() const { return gain_; }
+
 	virtual void reset() {
 		cur_delay_ = req_delay_;
 	}
@@ -49,6 +53,7 @@ class Buffer {
 	float req_delay_;
 	int channels_;
 	int frame_size_;
+	float gain_ = 1.0f;
 };
 
 //static constexpr int kBufferCount = 100;
@@ -72,18 +77,33 @@ class FixedBuffer : public ftl::audio::Buffer<T> {
 
 	inline void writeFrame(const T *d) {
 		const T *in = d;
-		T *out = &data_[(write_position_++) % SIZE][0];
+		T *out = data_[(write_position_++) % SIZE];
 		for (size_t i=0; i<CHAN*FRAME; ++i) *out++ = *in++;
 		if (write_position_ > 5 && read_position_ < 0) read_position_ = 0;
 	}
 
 	inline void readFrame(T *d) {
-		T *out = d;
+		T* __restrict out = d;
+		//if ((size_t(out) & 0x1f) == 0) out_alignment_ = 32;
+		//else if ((size_t(out) & 0xf) == 0) out_alignment_ = 16;
+		//else if ((size_t(out) & 0x7) == 0) out_alignment_ = 8;
+
 		if (read_position_ < 0 || read_position_ >= write_position_-1) {
 			for (size_t i=0; i<CHAN*FRAME; ++i) *out++ = 0;
 		} else {
-			T *in = &data_[(read_position_++) % SIZE][0];
-			for (size_t i=0; i<CHAN*FRAME; ++i) *out++ = *in++;
+			const T* __restrict in = data_[(read_position_++) % SIZE];
+
+			// 16 byte aligned, use SIMD intrinsics
+			if ((size_t(out) & 0xf) == 0) {
+				for (size_t i=0; i<CHAN*FRAME; i += 4) {
+					Eigen::Map<Eigen::Matrix<float,4,1>,Eigen::Aligned16> vout(out+i);
+					const Eigen::Map<const Eigen::Matrix<float,4,1>,Eigen::Aligned16> vin(in+i);
+					vout = vin*this->gain_;
+				}
+			// Not aligned
+			} else {
+				for (size_t i=0; i<CHAN*FRAME; ++i) *out++ = this->gain_ * (*in++);
+			}
 		}
 	}
 
@@ -102,15 +122,20 @@ class FixedBuffer : public ftl::audio::Buffer<T> {
 	void reset() override {
 		Buffer<T>::reset();
 		write_position_ = 0; //int(this->cur_delay_);
-		LOG(INFO) << "RESET AUDIO: " << write_position_;
 		read_position_ = 0;
 	}
 
+	inline T *data() { return (T*)data_; }
+	inline T *data(int f) { return data_[f]; }
+
+	inline int writePosition() const { return write_position_; }
+	inline void setWritePosition(int p) { write_position_ = p; }
+
 	private:
 	int write_position_;
 	int read_position_;
 	int offset_;
-	T data_[SIZE][CHAN*FRAME];
+	alignas(32) T data_[SIZE][CHAN*FRAME];
 };
 
 // ==== Implementations ========================================================
@@ -120,7 +145,7 @@ static T fracIndex(const std::vector<T> &in, float ix, int c) {
 	const auto i1 = static_cast<unsigned int>(ix);
 	const auto i2 = static_cast<unsigned int>(ix+1.0f);
 	const float alpha = ix - static_cast<float>(i1);
-	return (i2*CHAN+CHAN >= in.size()) ? in[i1*CHAN+c] : in[i1*CHAN+c]*(1.0f-alpha) + in[i2*CHAN+c]*alpha;
+	return static_cast<T>((i2*CHAN+CHAN >= in.size()) ? in[i1*CHAN+c] : in[i1*CHAN+c]*(1.0f-alpha) + in[i2*CHAN+c]*alpha);
 }
 
 inline float clamp(float v, float c) { return (v < -c) ? -c : (v > c) ? c : v; }
@@ -154,7 +179,7 @@ void FixedBuffer<T,CHAN,FRAME,SIZE>::write(const std::vector<T> &in) {
 			++write_position_;
 		}
 	}
-	if (write_position_ > 20 && read_position_ < 0) read_position_ = 0;
+	if (write_position_ > 5 && read_position_ < 0) read_position_ = 0;
 }
 
 template <typename T, int CHAN, int FRAME, int SIZE>
@@ -170,10 +195,16 @@ void FixedBuffer<T,CHAN,FRAME,SIZE>::read(std::vector<T> &out, int count) {
 // ==== Common forms ===========================================================
 
 template <int SIZE>
-using StereoBuffer16 = ftl::audio::FixedBuffer<short,2,256,SIZE>;
+using StereoBuffer16 = ftl::audio::FixedBuffer<short,2,960,SIZE>;
+
+template <int SIZE>
+using MonoBuffer16 = ftl::audio::FixedBuffer<short,1,960,SIZE>;
+
+template <int SIZE>
+using StereoBufferF = ftl::audio::FixedBuffer<float,2,960,SIZE>;
 
 template <int SIZE>
-using MonoBuffer16 = ftl::audio::FixedBuffer<short,1,256,SIZE>;
+using MonoBufferF = ftl::audio::FixedBuffer<float,1,960,SIZE>;
 
 }
 }

components/audio/include/ftl/audio/decoder.hpp

+#ifndef _FTL_AUDIO_DECODER_HPP_
+#define _FTL_AUDIO_DECODER_HPP_
+
+#include <vector>
+#include <ftl/codecs/packet.hpp>
+#include <ftl/codecs/codecs.hpp>
+
+namespace ftl {
+namespace audio {
+
+class Decoder {
+	public:
+	Decoder() { };
+	virtual ~Decoder() { };
+
+	virtual bool decode(const ftl::codecs::Packet &pkt, std::vector<float> &out)=0;
+
+	virtual bool accepts(const ftl::codecs::Packet &)=0;
+};
+
+}
+}
+
+#endif
\ No newline at end of file

components/audio/include/ftl/audio/encoder.hpp

+#ifndef _FTL_AUDIO_ENCODER_HPP_
+#define _FTL_AUDIO_ENCODER_HPP_
+
+#include <vector>
+#include <ftl/codecs/packet.hpp>
+#include <ftl/codecs/codecs.hpp>
+
+namespace ftl {
+namespace audio {
+
+class Encoder {
+	public:
+	Encoder() {};
+	virtual ~Encoder() {};
+
+	virtual bool encode(const std::vector<float> &in, ftl::codecs::Packet &pkt)=0;
+
+	virtual void reset() {}
+
+	virtual bool supports(ftl::codecs::codec_t codec)=0;
+};
+
+}
+}
+
+#endif

components/audio/include/ftl/audio/frame.hpp

@@ -2,47 +2,25 @@
 #ifndef _FTL_AUDIO_FRAME_HPP_
 #define _FTL_AUDIO_FRAME_HPP_
 
-#include <ftl/data/framestate.hpp>
-#include <ftl/data/frame.hpp>
+#include <ftl/data/new_frame.hpp>
 #include <ftl/audio/audio.hpp>
 
 namespace ftl {
 namespace audio {
 
+static constexpr int kFrameSize = 960;
+static constexpr int kSampleRate = 48000;
+
+typedef ftl::data::Frame Frame;
+typedef ftl::audio::Audio AudioFrame;
+
 struct AudioSettings {
 	int sample_rate;
 	int frame_size;
 	int channels;
 };
 
-struct AudioData {
-	template <typename T>
-	const T &as() const {
-		throw FTL_Error("Type not valid for audio channel");
-	}
-
-	template <typename T>
-	T &as() {
-		throw FTL_Error("Type not valid for audio channel");
-	}
-
-	template <typename T>
-	T &make() {
-		throw FTL_Error("Type not valid for audio channel");
-	}
-
-	inline void reset() {}
-
-	Audio data;
-};
-
-// Specialisations for getting Audio data.
-template <> Audio &AudioData::as<Audio>(); 
-template <> const Audio &AudioData::as<Audio>() const;
-template <> Audio &AudioData::make<Audio>();
 
-typedef ftl::data::FrameState<AudioSettings,2> FrameState;
-typedef ftl::data::Frame<32,2,FrameState,AudioData> Frame;
 
 }
 }

components/audio/include/ftl/audio/frameset.hpp

@@ -2,12 +2,12 @@
 #define _FTL_AUDIO_FRAMESET_HPP_
 
 #include <ftl/audio/frame.hpp>
-#include <ftl/data/frameset.hpp>
+#include <ftl/data/new_frameset.hpp>
 
 namespace ftl {
 namespace audio {
 
-typedef ftl::data::FrameSet<ftl::audio::Frame> FrameSet;
+typedef ftl::data::FrameSet FrameSet;
 
 }
 }

components/audio/include/ftl/audio/mixer.hpp

+#ifndef _FTL_AUDIO_MIXER_HPP_
+#define _FTL_AUDIO_MIXER_HPP_
+
+#include <Eigen/Eigen>
+#include <vector>
+#include <cmath>
+#include "buffer.hpp"
+
+namespace ftl {
+namespace audio {
+
+//static constexpr int kBufferCount = 100;
+
+/**
+ * A fast circular buffer to capture, play and manipulate audio data.
+ * This class can be used directly with portaudio. The hardware uses
+ * `readFrame` and `writeFrame` to consume or append audio data. A more
+ * advanced `write` function allows for non-frame aligned data and for time
+ * dilation / shifting, and amplitude control.
+ */
+template <typename T, int CHAN, int FRAME, int SIZE>
+class FixedMixer : public ftl::audio::Buffer<T> {
+	public:
+	FixedMixer() : Buffer<T>(CHAN, FRAME, 44100) { }
+	explicit FixedMixer(int rate) : Buffer<T>(CHAN, FRAME, rate) { }
+
+
+	inline int maxFrames() const { return SIZE; }
+
+	inline void readFrame(T *d) {
+		T* __restrict out = d;
+		if (read_position_ >= write_position_) {
+			std::fill(out, out+CHAN*FRAME, T(0));
+		} else {
+			const T* __restrict in = data_[(read_position_++) % SIZE];
+			std::copy(in, in+CHAN*FRAME, out);
+		}
+	}
+
+	int size() const override { return (read_position_>=0) ? write_position_ - read_position_ : 0; }
+	int frames() const override { return (read_position_>=0) ? write_position_ - read_position_ : 0; }
+
+	/**
+	 * Append sound samples to the end of the buffer. The samples may be over
+	 * or under sampled so as to gradually introduce or remove a requested
+	 * delay and hence change the latency of the audio.
+	 */
+	void write(const std::vector<T> &in) override;
+
+	inline void write(int track, const std::vector<T> &in) {
+		tracks_.at(track).write(in);
+	}
+
+	void mix();
+
+	void read(std::vector<T> &out, int frames) override;
+
+	void reset() override {
+		Buffer<T>::reset();
+		write_position_ = 0;
+		read_position_ = 0;
+		for (auto &t : tracks_) t.reset();
+	}
+
+	inline int writePosition() const { return write_position_; }
+	inline int readPosition() const { return read_position_; }
+	inline int tracks() const { return track_num_; }
+
+	inline void setDelay(int track, float d) { tracks_.at(track).setDelay(d); }
+	inline float delay(int track) const { return tracks_.at(track).delay(); }
+
+	inline void setGain(int track, float g) { tracks_.at(track).setGain(g); }
+	inline float gain(int track) const { return tracks_.at(track).gain(); }
+
+	//void resize(int tracks);
+
+	int add(const std::string &name);
+
+	const std::string &name(int track) const { return names_.at(track); }
+
+	private:
+	int track_num_=0;
+	int write_position_=0;
+	int read_position_=0;
+	alignas(32) T data_[SIZE][CHAN*FRAME];
+	std::vector<ftl::audio::FixedBuffer<T,CHAN,FRAME,SIZE>> tracks_;
+	std::vector<std::string> names_;
+};
+
+// ==== Implementations ========================================================
+
+template <typename T, int CHAN, int FRAME, int SIZE>
+void FixedMixer<T,CHAN,FRAME,SIZE>::write(const std::vector<T> &in) {
+	// Not supported...
+}
+
+template <typename T, int CHAN, int FRAME, int SIZE>
+void FixedMixer<T,CHAN,FRAME,SIZE>::mix() {
+	if (track_num_ == 0) return;
+
+	// Add together up to most recent frame
+	int min_write = std::numeric_limits<int>::max();
+	for (auto &t : tracks_) {
+		min_write = std::min(t.writePosition(), min_write);
+	}
+
+	// For each frame
+	while (write_position_ < min_write) {
+		int wp = write_position_ % SIZE;
+		float *ptr1 = data_[wp];
+
+		// For each block of 8 float samples
+		for (size_t i=0; i<CHAN*FRAME; i+=8) {
+			Eigen::Map<Eigen::Matrix<float,8,1>,Eigen::Aligned32> v1(ptr1+i);
+			v1.setZero();
+
+			// For each track, accumulate the samples
+			for (auto &t : tracks_) {
+				const Eigen::Map<Eigen::Matrix<float,8,1>,Eigen::Aligned32> v2(&t.data(wp)[i]);
+				v1 += t.gain()*v2;
+			}
+
+			v1 *= this->gain_;
+		}
+
+		++write_position_;
+	}
+}
+
+template <typename T, int CHAN, int FRAME, int SIZE>
+void FixedMixer<T,CHAN,FRAME,SIZE>::read(std::vector<T> &out, int count) {
+	out.resize(FRAME*count*CHAN);
+	T *ptr = out.data();
+	for (int i=0; i<count; ++i) {
+		// TODO: Do mix here directly
+		readFrame(ptr);
+		ptr += FRAME*CHAN;
+	}
+}
+
+/*template <typename T, int CHAN, int FRAME, int SIZE>
+void FixedMixer<T,CHAN,FRAME,SIZE>::resize(int t) {
+	if (track_num_ == t) return;
+	
+	track_num_ = t;
+	tracks_.reserve(t);
+	while (static_cast<int>(tracks_.size()) < t) {
+		auto &tr = tracks_.emplace_back();
+		tr.setWritePosition(write_position_);
+	}
+}*/
+
+template <typename T, int CHAN, int FRAME, int SIZE>
+int FixedMixer<T,CHAN,FRAME,SIZE>::add(const std::string &name) {
+	names_.push_back(name);
+	auto &tr = tracks_.emplace_back();
+	tr.setWritePosition(write_position_);
+	return track_num_++;
+}
+
+// ==== Common forms ===========================================================
+
+template <int SIZE>
+using StereoMixerF = ftl::audio::FixedMixer<float,2,960,SIZE>;
+
+template <int SIZE>
+using MonoMixerF = ftl::audio::FixedMixer<float,1,960,SIZE>;
+
+}
+}
+
+#endif  // _FTL_AUDIO_BUFFER_HPP_

components/audio/include/ftl/audio/software_decoder.hpp

+#ifndef _FTL_AUDIO_SOFTWARE_DECODER_HPP_
+#define _FTL_AUDIO_SOFTWARE_DECODER_HPP_
+
+#include <ftl/audio/decoder.hpp>
+
+struct OpusMSDecoder;
+
+namespace ftl {
+namespace audio {
+
+class SoftwareDecoder : public ftl::audio::Decoder {
+	public:
+	SoftwareDecoder();
+	~SoftwareDecoder();
+
+	bool decode(const ftl::codecs::Packet &pkt, std::vector<float> &out) override;
+
+	bool accepts(const ftl::codecs::Packet &) override;
+
+	private:
+	OpusMSDecoder *opus_decoder_;
+	bool cur_stereo_;
+	ftl::codecs::definition_t cur_definition_;
+
+	bool _decodeOpus(const ftl::codecs::Packet &pkt, std::vector<float> &out);
+	bool _decodeRaw(const ftl::codecs::Packet &pkt, std::vector<float> &out);
+	bool _createOpus(const ftl::codecs::Packet &pkt);
+};
+
+}
+}
+
+#endif

components/audio/include/ftl/audio/software_encoder.hpp

+#ifndef _FTL_AUDIO_SOFTWARE_ENCODER_HPP_
+#define _FTL_AUDIO_SOFTWARE_ENCODER_HPP_
+
+#include <ftl/audio/encoder.hpp>
+
+struct OpusMSEncoder;
+
+namespace ftl {
+namespace audio {
+
+class SoftwareEncoder : public ftl::audio::Encoder {
+	public:
+	SoftwareEncoder();
+	~SoftwareEncoder();
+
+	bool encode(const std::vector<float> &in, ftl::codecs::Packet &pkt) override;
+
+	void reset() override;
+
+	bool supports(ftl::codecs::codec_t codec) override;
+
+	private:
+	OpusMSEncoder *opus_encoder_;
+	bool cur_stereo_;
+	ftl::codecs::definition_t cur_definition_;
+	uint8_t cur_bitrate_;
+
+	bool _encodeRaw(const std::vector<float> &in, ftl::codecs::Packet &pkt);
+	bool _encodeOpus(const std::vector<float> &in, ftl::codecs::Packet &pkt);
+	bool _createOpus(ftl::codecs::Packet &pkt);
+};
+
+}
+}
+
+#endif

components/audio/include/ftl/audio/source.hpp

@@ -3,6 +3,7 @@
 
 #include <ftl/audio/buffer.hpp>
 #include <ftl/audio/frameset.hpp>
+#include <ftl/data/creators.hpp>
 #include <ftl/configurable.hpp>
 #include <ftl/config.h>
 
@@ -13,40 +14,23 @@
 namespace ftl {
 namespace audio {
 
-static constexpr int kFrameSize = 256;
-
-typedef ftl::data::Generator<ftl::audio::FrameSet> Generator;
-
-class Source : public ftl::Configurable, public ftl::audio::Generator {
+class Source : public ftl::Configurable, public ftl::data::DiscreteSource {
     public:
     explicit Source(nlohmann::json &config);
     ~Source();
 
-    /** Number of frames in last frameset. This can change over time. */
-	size_t size() override;
+	bool capture(int64_t ts) override;
 
-	/**
-	 * Get the persistent state object for a frame. An exception is thrown
-	 * for a bad index.
-	 */
-	ftl::audio::FrameState &state(size_t ix) override;
-
-	/** Register a callback to receive new frame sets. */
-	void onFrameSet(const ftl::audio::FrameSet::Callback &) override;
+	bool retrieve(ftl::data::Frame &) override;
 
     private:
-    ftl::audio::FrameState state_;
     bool active_;
-    ftl::timer::TimerHandle timer_hp_;
-	ftl::timer::TimerHandle timer_main_;
-	ftl::audio::FrameSet::Callback cb_;
+	ftl::audio::AudioSettings settings_;
 
-	ftl::audio::Buffer<short> *buffer_;
+	ftl::audio::Buffer<float> *buffer_;
 	int to_read_;
 	int64_t latency_;
 
-	ftl::audio::FrameSet frameset_;
-
 	#ifdef HAVE_PORTAUDIO
 	PaStream *stream_;
 	#endif

components/audio/include/ftl/audio/speaker.hpp

@@ -19,13 +19,19 @@ class Speaker : public ftl::Configurable {
 	~Speaker();
 
 	void queue(int64_t ts, ftl::audio::Frame &fs);
+	void queue(int64_t ts, const ftl::audio::Audio &af);
 
 	void setDelay(int64_t ms);
+	void setVolume(float value);
+	float volume();
+
+	void reset() { if (buffer_) buffer_->reset(); }
 
 	private:
-	ftl::audio::Buffer<short> *buffer_;
+	ftl::audio::Buffer<float> *buffer_;
 	bool active_;
 	float extra_delay_;
+	float volume_;
 	int64_t latency_;
 
 	#ifdef HAVE_PORTAUDIO

components/audio/src/portaudio.cpp

 #include <ftl/audio/portaudio.hpp>
 #include <ftl/config.h>
+#include <ftl/threads.hpp>
 #include <loguru.hpp>
 
 #include <atomic>
 
 static std::atomic<int> counter = 0;
+static MUTEX pa_mutex;
 
 #ifdef HAVE_PORTAUDIO
 
 #include <portaudio.h>
 
 void ftl::audio::pa_init() {
-    // TODO: Mutex lock?
+    UNIQUE_LOCK(pa_mutex, lk);
     if (counter == 0) {
         auto err = Pa_Initialize();
         if (err != paNoError) {
@@ -34,7 +36,7 @@ void ftl::audio::pa_init() {
 }
 
 void ftl::audio::pa_final() {
-    // TODO: Mutex lock?
+    UNIQUE_LOCK(pa_mutex, lk);
     --counter;
     if (counter == 0) {
         auto err = Pa_Terminate();

components/audio/src/software_decoder.cpp

+#include <ftl/audio/software_decoder.hpp>
+#include <ftl/config.h>
+
+#ifdef HAVE_OPUS
+#include <opus/opus_multistream.h>
+#else
+struct OpusMSDecoder {};
+#endif
+
+#define LOGURU_REPLACE_GLOG 1
+#include <loguru.hpp>
+
+#define FRAME_SIZE 960
+
+using ftl::audio::SoftwareDecoder;
+using ftl::codecs::codec_t;
+
+SoftwareDecoder::SoftwareDecoder() : opus_decoder_(nullptr) {
+
+}
+
+SoftwareDecoder::~SoftwareDecoder() {
+
+}
+
+bool SoftwareDecoder::_createOpus(const ftl::codecs::Packet &pkt) {
+	#ifdef HAVE_OPUS
+	bool stereo = pkt.flags & ftl::codecs::kFlagStereo;
+	if (opus_decoder_ && stereo == cur_stereo_) return true;
+
+	cur_stereo_ = stereo;
+
+	if (opus_decoder_) {
+		opus_multistream_decoder_destroy(opus_decoder_);
+		opus_decoder_ = nullptr;
+	}
+
+	int sample_rate = 48000;  // TODO: Allow it to be different
+
+	int errcode = 0;
+	int channels = (stereo) ? 2 : 1;
+	const unsigned char mapping[2] = {0,1};
+	opus_decoder_ = opus_multistream_decoder_create(sample_rate, channels, 1, channels-1, mapping, &errcode);
+
+	if (errcode < 0) return false;
+
+	LOG(INFO) << "Created OPUS decoder: " << sample_rate << ", " << channels;
+	#endif
+	return true;
+}
+
+bool SoftwareDecoder::decode(const ftl::codecs::Packet &pkt, std::vector<float> &out) {
+	switch (pkt.codec) {
+	case codec_t::OPUS		: return _decodeOpus(pkt, out);
+	case codec_t::RAW		: return _decodeRaw(pkt, out);
+	default: return false;
+	}
+}
+
+bool SoftwareDecoder::_decodeOpus(const ftl::codecs::Packet &pkt, std::vector<float> &out) {
+	#ifdef HAVE_OPUS
+	if (!_createOpus(pkt)) return false;
+
+	int channels = (cur_stereo_) ? 2 : 1;
+
+	out.resize(10*FRAME_SIZE*channels);
+
+	const unsigned char *inptr = pkt.data.data();
+	float *outptr = out.data();
+	int count = 0;
+	int frames = 0;
+
+	for (size_t i=0; i<pkt.data.size(); ) {
+		const short *len = (const short*)inptr;
+		if (*len == 0) break;
+		if (frames == 10) break;
+
+		inptr += 2;
+		i += (*len)+2;
+		int samples = opus_multistream_decode_float(opus_decoder_, inptr, *len, outptr, FRAME_SIZE, 0);
+
+		if (samples != FRAME_SIZE) {
+			LOG(ERROR) << "Failed to Opus decode: " << samples;
+			//return false;
+			break;
+		}
+
+		inptr += *len;
+		outptr += FRAME_SIZE*channels;
+		count += samples;
+		++frames;
+	}
+
+	out.resize(count*channels);
+	//LOG(INFO) << "Received " << frames << " Opus frames";
+	return true;
+
+	#else
+	LOG(WARNING) << "No Opus decoder installed";
+	return false;
+	#endif
+}
+
+bool SoftwareDecoder::_decodeRaw(const ftl::codecs::Packet &pkt, std::vector<float> &out) {
+	size_t size = pkt.data.size()/sizeof(float);
+	out.resize(size);
+	auto *ptr = (float*)pkt.data.data();
+	for (size_t i=0; i<size; i++) out.data()[i] = ptr[i];
+	return true;
+}
+
+bool SoftwareDecoder::accepts(const ftl::codecs::Packet &) {
+	return false;
+}

components/audio/src/software_encoder.cpp

+#include <ftl/audio/software_encoder.hpp>
+#include <ftl/config.h>
+
+#ifdef HAVE_OPUS
+#include <opus/opus_multistream.h>
+#else
+struct OpusMSEncoder {};
+#endif
+
+#define LOGURU_REPLACE_GLOG 1
+#include <loguru.hpp>
+
+using ftl::audio::SoftwareEncoder;
+using ftl::codecs::codec_t;
+
+#define FRAME_SIZE 960
+#define MAX_PACKET_SIZE (3*2*FRAME_SIZE)
+
+SoftwareEncoder::SoftwareEncoder() : ftl::audio::Encoder(), opus_encoder_(nullptr), cur_stereo_(false), cur_bitrate_(0) {
+
+}
+
+SoftwareEncoder::~SoftwareEncoder() {
+
+}
+
+bool SoftwareEncoder::encode(const std::vector<float> &in, ftl::codecs::Packet &pkt) {
+	auto codec = (pkt.codec == codec_t::Any) ? codec_t::OPUS : pkt.codec;
+
+	// Force RAW if no opus
+	#ifndef HAVE_OPUS
+	codec = codec_t::RAW;
+	#endif
+
+	pkt.codec = codec;
+
+	switch (codec) {
+	case codec_t::OPUS		: return _encodeOpus(in, pkt);
+	case codec_t::RAW		: return _encodeRaw(in, pkt);
+	default: return false;
+	}
+}
+
+bool SoftwareEncoder::_createOpus(ftl::codecs::Packet &pkt) {
+	#ifdef HAVE_OPUS
+	bool stereo = pkt.flags & ftl::codecs::kFlagStereo;
+	if (opus_encoder_ && stereo == cur_stereo_) return true;
+
+	cur_stereo_ = stereo;
+
+	if (opus_encoder_) {
+		opus_multistream_encoder_destroy(opus_encoder_);
+		opus_encoder_ = nullptr;
+	}
+
+	int sample_rate = 48000;  // TODO: Allow it to be different
+
+	int errcode = 0;
+	int channels = (stereo) ? 2 : 1;
+	const unsigned char mapping[2] = {0,1};
+	opus_encoder_ = opus_multistream_encoder_create(sample_rate, channels, 1, channels-1, mapping, OPUS_APPLICATION_VOIP, &errcode);
+
+	if (errcode < 0) return false;
+	LOG(INFO) << "Created OPUS encoder";
+	#endif
+
+	return true;
+}
+
+bool SoftwareEncoder::_encodeOpus(const std::vector<float> &in, ftl::codecs::Packet &pkt) {
+	#ifdef HAVE_OPUS
+	static const float MAX_BITRATE = 128000.0f;
+	static const float MIN_BITRATE = 24000.0f;
+
+	if (!_createOpus(pkt)) return false;
+
+	if (pkt.bitrate != cur_bitrate_) {
+		int bitrate = (MAX_BITRATE-MIN_BITRATE) * (float(pkt.bitrate)/255.0f) + MIN_BITRATE;
+		if (!cur_stereo_) bitrate /= 2;
+		int errcode = opus_multistream_encoder_ctl(opus_encoder_, OPUS_SET_BITRATE(bitrate));
+		if (errcode < 0) return false;
+		LOG(INFO) << "OPUS encoder: bitrate = " << bitrate;
+		cur_bitrate_ = pkt.bitrate;
+	}
+
+	int channels = (cur_stereo_) ? 2 : 1;
+
+	int frame_est = (in.size() / (channels*FRAME_SIZE))+1;
+	size_t insize = pkt.data.size();
+	pkt.data.resize(insize+MAX_PACKET_SIZE*frame_est);
+	int count = 0;
+	int frames = 0;
+
+	unsigned char *outptr = pkt.data.data()+insize;
+
+	//LOG(INFO) << "Encode " << (in.size() / (channels*FRAME_SIZE)) << " audio frames";
+
+	for (unsigned int i=0; i<in.size(); i+=channels*FRAME_SIZE) {
+		short *len = (short*)outptr;
+		outptr += 2;
+		int nbBytes = opus_multistream_encode_float(opus_encoder_, &in.data()[i], FRAME_SIZE, outptr, MAX_PACKET_SIZE);
+		if (nbBytes <= 0) return false;
+
+		//if (nbBytes > 32000) LOG(WARNING) << "Packet exceeds size limit";
+
+		*len = nbBytes;
+
+		count += nbBytes+2;
+		outptr += nbBytes;
+		++frames;
+	}
+
+	pkt.data.resize(insize+count);
+	//LOG(INFO) << "Opus Encode = " << pkt.data.size() << ", " << frames;
+	return true;
+
+	#else
+	return false;
+	#endif
+}
+
+bool SoftwareEncoder::_encodeRaw(const std::vector<float> &in, ftl::codecs::Packet &pkt) {
+	const unsigned char *ptr = (unsigned char*)in.data();
+	pkt.data = std::move(std::vector<unsigned char>(ptr, ptr+in.size()*sizeof(float)));
+	return true;
+}
+
+void SoftwareEncoder::reset() {
+
+}
+
+bool SoftwareEncoder::supports(ftl::codecs::codec_t codec) {
+	return false;
+}

components/audio/src/source.cpp

@@ -7,8 +7,6 @@
 
 using ftl::audio::Source;
 using ftl::audio::Frame;
-using ftl::audio::FrameSet;
-using ftl::audio::FrameState;
 using ftl::audio::Audio;
 using ftl::codecs::Channel;
 
@@ -23,7 +21,7 @@ static int pa_source_callback(const void *input, void *output,
         PaStreamCallbackFlags statusFlags, void *userData) {
 
     auto *buffer = (BUFFER*)userData;
-    short *in = (short*)input;
+    float *in = (float*)input;
 	buffer->writeFrame(in);
     return 0;
 }
@@ -76,16 +74,16 @@ Source::Source(nlohmann::json &config) : ftl::Configurable(config), buffer_(null
 	//}
 
 	if (channels >= 2) {
-		buffer_ = new ftl::audio::StereoBuffer16<100>(48000);
+		buffer_ = new ftl::audio::StereoBufferF<100>(48000);
 	} else {
-		buffer_ = new ftl::audio::MonoBuffer16<100>(48000);
+		buffer_ = new ftl::audio::MonoBufferF<100>(48000);
 	}
 
     PaStreamParameters inputParameters;
     //bzero( &inputParameters, sizeof( inputParameters ) );
     inputParameters.channelCount = channels;
     inputParameters.device = device;
-    inputParameters.sampleFormat = paInt16;
+    inputParameters.sampleFormat = paFloat32;
     inputParameters.suggestedLatency = (device >= 0) ? Pa_GetDeviceInfo(device)->defaultLowInputLatency : 0;
     inputParameters.hostApiSpecificStreamInfo = NULL;
 
@@ -101,7 +99,7 @@ Source::Source(nlohmann::json &config) : ftl::Configurable(config), buffer_(null
 			48000,  // Sample rate
 			ftl::audio::kFrameSize,    // Size of single frame
 			paNoFlag,
-			(buffer_->channels() == 1) ? pa_source_callback<ftl::audio::MonoBuffer16<100>> : pa_source_callback<ftl::audio::StereoBuffer16<100>>,
+			(buffer_->channels() == 1) ? pa_source_callback<ftl::audio::MonoBufferF<100>> : pa_source_callback<ftl::audio::StereoBufferF<100>>,
 			this->buffer_
 		);
 	} else {
@@ -109,10 +107,10 @@ Source::Source(nlohmann::json &config) : ftl::Configurable(config), buffer_(null
 			&stream_,
 			channels,
 			0,
-			paInt16,
+			paFloat32,
 			48000,  // Sample rate
 			ftl::audio::kFrameSize,    // Size of single frame
-			(buffer_->channels() == 1) ? pa_source_callback<ftl::audio::MonoBuffer16<100>> : pa_source_callback<ftl::audio::StereoBuffer16<100>>,
+			(buffer_->channels() == 1) ? pa_source_callback<ftl::audio::MonoBufferF<100>> : pa_source_callback<ftl::audio::StereoBufferF<100>>,
 			this->buffer_
 		);
 	}
@@ -135,51 +133,10 @@ Source::Source(nlohmann::json &config) : ftl::Configurable(config), buffer_(null
 
 	to_read_ = 0;
 
-	ftl::audio::AudioSettings settings;
-	settings.channels = channels;
-	settings.sample_rate = 48000;
-	settings.frame_size = 256;
-	state_.setLeft(settings);
-
-    timer_hp_ = ftl::timer::add(ftl::timer::kTimerHighPrecision, [this](int64_t ts) {
-        if (buffer_) to_read_ = buffer_->size();
-        return true;
-    });
-
-	timer_main_ = ftl::timer::add(ftl::timer::kTimerMain, [this](int64_t ts) {
-
-        // Remove one interval since the audio starts from the last frame
-		frameset_.timestamp = ts - ftl::timer::getInterval() + latency_;
-
-		frameset_.id = 0;
-		frameset_.count = 1;
-		//frameset_.stale = false;
-		frameset_.clear(ftl::data::FSFlag::STALE);
-
-        if (to_read_ < 1 || !buffer_) return true;
-
-		if (frameset_.frames.size() < 1) frameset_.frames.emplace_back();
-
-		auto &frame = frameset_.frames[0];
-		frame.reset();
-		frame.setOrigin(&state_);
-        std::vector<short> &data = frame.create<Audio>((buffer_->channels() == 2) ? Channel::AudioStereo : Channel::AudioMono).data();
-
-		/*data.resize(ftl::audio::kFrameSize*to_read_*channels_);  // For stereo * 2
-		short *ptr = data.data();
-		for (int i=0; i<to_read_; ++i) {
-			if (channels_ == 1) mono_buffer_.readFrame(ptr);
-			else stereo_buffer_.readFrame(ptr);
-			ptr += ftl::audio::kFrameSize*channels_;  // For stereo * 2
-		}*/
-		buffer_->read(data, to_read_);
-
-		// Then do something with the data!
-		//LOG(INFO) << "Audio Frames Sent: " << to_read_ << " - " << ltime;
-		if (cb_) cb_(frameset_);
-
-        return true;
-    }); 
+	settings_.channels = channels;
+	settings_.sample_rate = 48000;
+	settings_.frame_size = 960;
+	//state_.setLeft(settings);
 
 	LOG(INFO) << "Microphone ready.";
 
@@ -196,7 +153,7 @@ Source::~Source() {
         active_ = false;
 
 		#ifdef HAVE_PORTAUDIO
-        auto err = Pa_StopStream(stream_);
+        auto err = Pa_AbortStream(stream_);
 
         if (err != paNoError) {
             LOG(ERROR) << "Portaudio stop stream error: " << Pa_GetErrorText(err);
@@ -216,15 +173,20 @@ Source::~Source() {
 	#endif
 }
 
-size_t Source::size() {
-    return 1;
+bool Source::capture(int64_t ts) {
+	if (buffer_) to_read_ = buffer_->size();
+	return true;
 }
 
-ftl::audio::FrameState &Source::state(size_t ix) {
-    if (ix >= 1) throw FTL_Error("State index out-of-bounds");
-    return state_;
-}
+bool Source::retrieve(ftl::data::Frame &frame) {
+	// Remove one interval since the audio starts from the last frame
+		//frameset_.timestamp = ts - ftl::timer::getInterval() + latency_;
 
-void Source::onFrameSet(const ftl::audio::FrameSet::Callback &cb) {
-	cb_ = cb;
+    if (to_read_ < 1 || !buffer_) return true;
+	auto alist = frame.create<std::list<Audio>>((buffer_->channels() == 2) ? Channel::AudioStereo : Channel::AudioMono);
+	Audio aframe;
+    std::vector<float> &data = aframe.data();
+	buffer_->read(data, to_read_);
+	alist = std::move(aframe);
+	return true;
 }

components/audio/src/speaker.cpp

@@ -8,7 +8,6 @@
 using ftl::audio::Speaker;
 using ftl::audio::Frame;
 using ftl::audio::FrameSet;
-using ftl::audio::FrameState;
 using ftl::audio::Audio;
 using ftl::codecs::Channel;
 
@@ -21,7 +20,7 @@ static int pa_speaker_callback(const void *input, void *output,
 		PaStreamCallbackFlags statusFlags, void *userData) {
 
 	auto *buffer = (BUFFER*)userData;  // ftl::audio::MonoBuffer16<2000>
-	short *out = (short*)output;
+	float *out = (float*)output;
 
 	buffer->readFrame(out);
 
@@ -30,20 +29,20 @@ static int pa_speaker_callback(const void *input, void *output,
 
 #endif
 
-Speaker::Speaker(nlohmann::json &config) : ftl::Configurable(config), buffer_(nullptr) {
+Speaker::Speaker(nlohmann::json &config) : ftl::Configurable(config), buffer_(nullptr), stream_(nullptr) {
 	#ifdef HAVE_PORTAUDIO
 	ftl::audio::pa_init();
 	#else  // No portaudio
-
-	active_ = false;
 	LOG(ERROR) << "No audio support";
-
 	#endif
-
-	extra_delay_ = value("delay",0.0f);
-	on("delay", [this](const ftl::config::Event &e) {
-		extra_delay_ = value("delay",0.0f);
+	volume_ = 1.0f;
+	active_ = false;
+	extra_delay_ = value("delay",0.1f);
+	on("delay", [this]() {
+		extra_delay_ = value("delay",0.1f);
+		setDelay(0);
 	});
+	setDelay(0);
 }
 
 Speaker::~Speaker() {
@@ -51,7 +50,7 @@ Speaker::~Speaker() {
 		active_ = false;
 
 		#ifdef HAVE_PORTAUDIO
-		auto err = Pa_StopStream(stream_);
+		auto err = Pa_AbortStream(stream_);
 
 		if (err != paNoError) {
 			LOG(ERROR) << "Portaudio stop stream error: " << Pa_GetErrorText(err);
@@ -80,16 +79,16 @@ void Speaker::_open(int fsize, int sample, int channels) {
 	if (sample == 0 || channels == 0) return;
 
 	if (channels >= 2) {
-		buffer_ = new ftl::audio::StereoBuffer16<2000>(sample);
+		buffer_ = new ftl::audio::StereoBufferF<2000>(sample);
 	} else {
-		buffer_ = new ftl::audio::MonoBuffer16<2000>(sample);
+		buffer_ = new ftl::audio::MonoBufferF<2000>(sample);
 	}
 
 	PaStreamParameters outputParameters;
 	//bzero( &inputParameters, sizeof( inputParameters ) );
 	outputParameters.channelCount = channels;
 	outputParameters.device = Pa_GetDefaultOutputDevice();
-    outputParameters.sampleFormat = paInt16;
+	outputParameters.sampleFormat = paFloat32;
 	outputParameters.suggestedLatency = Pa_GetDeviceInfo(outputParameters.device)->defaultLowOutputLatency;
 	outputParameters.hostApiSpecificStreamInfo = NULL;
 
@@ -101,9 +100,9 @@ void Speaker::_open(int fsize, int sample, int channels) {
 		NULL,
 		&outputParameters,
 		sample,	// Sample rate
-		256,    // Size of single frame
+		960,	// Size of single frame
 		paNoFlag,
-		(channels == 1) ? pa_speaker_callback<ftl::audio::MonoBuffer16<2000>> : pa_speaker_callback<ftl::audio::StereoBuffer16<2000>>,
+		(channels == 1) ? pa_speaker_callback<ftl::audio::MonoBufferF<2000>> : pa_speaker_callback<ftl::audio::StereoBufferF<2000>>,
 		this->buffer_
 	);
 
@@ -130,15 +129,29 @@ void Speaker::_open(int fsize, int sample, int channels) {
 }
 
 void Speaker::queue(int64_t ts, ftl::audio::Frame &frame) {
-	auto &audio = frame.get<ftl::audio::Audio>((frame.hasChannel(Channel::AudioStereo)) ? Channel::AudioStereo : Channel::AudioMono);
+	const auto &audio = frame.get<std::list<ftl::audio::Audio>>
+		((frame.hasChannel(Channel::AudioStereo)) ? Channel::AudioStereo : Channel::AudioMono);
+
+	if (!buffer_) {
+		_open(960, 48000, (frame.hasChannel(Channel::AudioStereo)) ? 2 : 1);
+	}
+	if (!buffer_) return;
 
+	//LOG(INFO) << "Buffer Fullness (" << ts << "): " << buffer_->size() << " - " << audio.size();
+	for (const auto &d : audio) {
+		buffer_->write(d.data());
+	}
+	//LOG(INFO) << "Audio delay: " << buffer_.delay() << "s";
+}
+
+void Speaker::queue(int64_t ts, const ftl::audio::Audio &d) {
 	if (!buffer_) {
-		_open(256, frame.getSettings().sample_rate, frame.getSettings().channels);
+		_open(960, 48000, 2);
 	}
 	if (!buffer_) return;
 
 	//LOG(INFO) << "Buffer Fullness (" << ts << "): " << buffer_->size() << " - " << audio.size();
-	buffer_->write(audio.data());
+	buffer_->write(d.data());
 	//LOG(INFO) << "Audio delay: " << buffer_.delay() << "s";
 }
 
@@ -148,6 +161,16 @@ void Speaker::setDelay(int64_t ms) {
 	if (d < 0.0f) d = 0.001f;  // Clamp to 0 delay (not ideal to be exactly 0)
 	if (buffer_) {
 		buffer_->setDelay(d);
-		//LOG(INFO) << "Audio delay: " << buffer_->delay();
+		LOG(INFO) << "Audio delay: " << buffer_->delay();
 	}
 }
+
+void Speaker::setVolume(float value) {
+	// TODO: adjust volume using system mixer
+	volume_ = std::max(0.0f, std::min(1.0f, value));
+	if (buffer_) buffer_->setGain(volume_);
+}
+
+float Speaker::volume() {
+	return volume_;
+}