Implements #286 basic sound

.gitlab-ci.yml

@@ -6,7 +6,7 @@
 
 variables:
   GIT_SUBMODULE_STRATEGY: recursive
-  CMAKE_ARGS_WINDOWS: '-DCMAKE_GENERATOR_PLATFORM=x64 -DNVPIPE_DIR="D:/Build/NvPipe" -DEigen3_DIR="C:/Program Files (x86)/Eigen3/share/eigen3/cmake" -DOpenCV_DIR="D:/Build/opencv-4.1.1" -DCUDA_TOOLKIT_ROOT_DIR="C:/Program Files/NVIDIA GPU Computing Toolkit/CUDA/v10.1"'
+  CMAKE_ARGS_WINDOWS: '-DCMAKE_GENERATOR_PLATFORM=x64 -DPORTAUDIO_DIR="D:/Build/portaudio" -DNVPIPE_DIR="D:/Build/NvPipe" -DEigen3_DIR="C:/Program Files (x86)/Eigen3/share/eigen3/cmake" -DOpenCV_DIR="D:/Build/opencv-4.1.1" -DCUDA_TOOLKIT_ROOT_DIR="C:/Program Files/NVIDIA GPU Computing Toolkit/CUDA/v10.1"'
 
 stages:
  - all
@@ -25,7 +25,7 @@ linux:
   script:
     - mkdir build
     - cd build
-    - cmake .. -DWITH_OPTFLOW=TRUE -DBUILD_CALIBRATION=TRUE -DCMAKE_BUILD_TYPE=Release
+    - cmake .. -DWITH_OPTFLOW=TRUE -DUSE_CPPCHECK=FALSE -DBUILD_CALIBRATION=TRUE -DCMAKE_BUILD_TYPE=Release
     - make
     - ctest --output-on-failure
 

CMakeLists.txt

@@ -14,6 +14,7 @@ option(WITH_NVPIPE "Use NvPipe for compression if available" ON)
 option(WITH_OPTFLOW "Use NVIDIA Optical Flow if available" OFF)
 option(WITH_OPENVR "Build with OpenVR support" OFF)
 option(WITH_FIXSTARS "Use Fixstars libSGM if available" ON)
+option(USE_CPPCHECK "Apply cppcheck during build" ON)
 option(BUILD_VISION "Enable the vision component" ON)
 option(BUILD_RECONSTRUCT "Enable the reconstruction component" ON)
 option(BUILD_RENDERER "Enable the renderer component" ON)
@@ -94,25 +95,6 @@ else()
 endif()
 
 if (BUILD_GUI)
-	#find_library( NANOGUI_LIBRARY NAMES nanogui libnanogui PATHS ${NANOGUI_DIR} PATH_SUFFIXES lib)
-	#if (NANOGUI_LIBRARY)
-	#	set(HAVE_NANOGUI TRUE)
-	#	add_library(nanogui UNKNOWN IMPORTED)
-	#	#set_property(TARGET nanogui PROPERTY INTERFACE_INCLUDE_DIRECTORIES ${NANOGUI_EXTRA_INCS})
-	#	set_property(TARGET nanogui PROPERTY IMPORTED_LOCATION ${NANOGUI_LIBRARY})
-	#	message(STATUS "Found NanoGUI: ${NANOGUI_LIBRARY}")
-
-	#	if(WIN32)
-	#		# Find include
-	#		find_path(NANOGUI_INCLUDE_DIRS
-	#			NAMES nanogui/nanogui.h
-	#			PATHS "C:/Program Files/NanoGUI" "C:/Program Files (x86)/NanoGUI" ${NANOGUI_DIR}
-	#			PATH_SUFFIXES include
-	#		)
-	#		set_property(TARGET nanogui PROPERTY INTERFACE_INCLUDE_DIRECTORIES ${NANOGUI_INCLUDE_DIRS})
-	#	endif()
-	#endif()
-
 	set(HAVE_NANOGUI TRUE)
 
 	# Disable building extras we won't need (pure C++ project)
@@ -150,6 +132,30 @@ else()
 	add_library(nvpipe INTERFACE)
 endif()
 
+# Portaudio v19 library
+find_library( PORTAUDIO_LIBRARY NAMES portaudio PATHS ${PORTAUDIO_DIR} PATH_SUFFIXES lib)
+if (PORTAUDIO_LIBRARY)
+	set(HAVE_PORTAUDIO TRUE)
+	add_library(portaudio UNKNOWN IMPORTED)
+	#set_property(TARGET nanogui PROPERTY INTERFACE_INCLUDE_DIRECTORIES ${NANOGUI_EXTRA_INCS})
+	set_property(TARGET portaudio PROPERTY IMPORTED_LOCATION ${PORTAUDIO_LIBRARY})
+	message(STATUS "Found Portaudio: ${PORTAUDIO_LIBRARY}")
+	
+	if(WIN32)
+		# Find include
+		find_path(PORTAUDIO_INCLUDE_DIRS
+		    NAMES portaudio.h
+		    PATHS "C:/Program Files/Portaudio" "C:/Program Files (x86)/Portaudio" ${PORTAUDIO_DIR}
+		    PATH_SUFFIXES include
+		)
+		set_property(TARGET portaudio PROPERTY INTERFACE_INCLUDE_DIRECTORIES ${PORTAUDIO_INCLUDE_DIRS})
+	endif()
+else()
+	set(PORTAUDIO_LIBRARY "")
+	add_library(portaudio INTERFACE)
+	message(WARNING "Portaudio not found - sound disabled")
+endif()
+
 find_program( NODE_NPM NAMES npm )
 if (NODE_NPM)
 	message(STATUS "Found NPM: ${NODE_NPM}")
@@ -209,11 +215,13 @@ check_include_file_cxx("opencv2/viz.hpp" HAVE_VIZ)
 check_include_file_cxx("opencv2/cudastereo.hpp" HAVE_OPENCVCUDA)
 
 # Optional source problem check
+if (USE_CPPCHECK)
 	find_program(CPPCHECK_FOUND cppcheck)
 	if (CPPCHECK_FOUND)
 		message(STATUS "Found cppcheck: will perform source checks")
 		set(CMAKE_CXX_CPPCHECK "cppcheck" "-D__align__(A)" "-DCUDARTAPI" "--enable=warning,performance,style" "--inline-suppr" "--std=c++14" "--suppress=*:*catch.hpp" "--suppress=*:*elas*" "--suppress=*:*nanogui*" "--suppress=*:*json.hpp" "--quiet")
 	endif()
+endif()
 
 # include_directories(${PROJECT_SOURCE_DIR}/common/cpp/include)
 
@@ -239,11 +247,13 @@ set(CMAKE_WINDOWS_EXPORT_ALL_SYMBOLS ON)
 
 add_subdirectory(components/common/cpp)
 add_subdirectory(components/codecs)
+add_subdirectory(components/structures)
 add_subdirectory(components/net)
 add_subdirectory(components/rgbd-sources)
 add_subdirectory(components/control/cpp)
 add_subdirectory(components/operators)
 add_subdirectory(components/streams)
+add_subdirectory(components/audio)
 add_subdirectory(applications/calibration)
 #add_subdirectory(applications/groupview)
 #add_subdirectory(applications/player)

applications/gui/src/src_window.cpp

@@ -80,8 +80,10 @@ SourceWindow::SourceWindow(ftl::gui::Screen *screen)
 
 	cycle_ = 0;
 	receiver_->onFrameSet([this](ftl::rgbd::FrameSet &fs) {
+		fs.swapTo(frameset_);
+
 		// Request the channels required by current camera configuration
-		interceptor_->select(fs.id, _aggregateChannels());
+		interceptor_->select(frameset_.id, _aggregateChannels());
 
 		/*if (fs.frames[0].hasChannel(Channel::Data)) {
 			int data = 0;
@@ -90,30 +92,39 @@ SourceWindow::SourceWindow(ftl::gui::Screen *screen)
 		}*/
 
 		const auto *cstream = interceptor_;
-		_createDefaultCameras(fs, cstream->available(fs.id).has(Channel::Depth));
+		_createDefaultCameras(frameset_, cstream->available(fs.id).has(Channel::Depth));
 
 		//LOG(INFO) << "Channels = " << (unsigned int)cstream->available(fs.id);
 
 		// Enforce interpolated colour
-		for (int i=0; i<fs.frames.size(); ++i) {
-			fs.frames[i].createTexture<uchar4>(Channel::Colour, true);
+		for (int i=0; i<frameset_.frames.size(); ++i) {
+			frameset_.frames[i].createTexture<uchar4>(Channel::Colour, true);
 		}
 
-		pre_pipeline_->apply(fs, fs, 0);
+		pre_pipeline_->apply(frameset_, frameset_, 0);
 
 		int i=0;
 		for (auto cam : cameras_) {
 			// Only update the camera periodically unless the active camera
 			if (screen_->activeCamera() == cam.second.camera ||
-				(screen_->activeCamera() == nullptr && cycle_ % cameras_.size() == i++))  cam.second.camera->update(fs);
+				(screen_->activeCamera() == nullptr && cycle_ % cameras_.size() == i++))  cam.second.camera->update(frameset_);
 
-			cam.second.camera->update(cstream->available(fs.id));
+			cam.second.camera->update(cstream->available(frameset_.id));
 		}
 		++cycle_;
 
 		return true;
 	});
 
+	speaker_ = ftl::create<ftl::audio::Speaker>(screen_->root(), "speaker_test");
+
+	receiver_->onAudio([this](ftl::audio::FrameSet &fs) {
+		//LOG(INFO) << "Audio delay required = " << (ts - frameset_.timestamp) << "ms";
+		speaker_->setDelay(fs.timestamp - frameset_.timestamp + ftl::timer::getInterval());  // Add Xms for local render time
+		speaker_->queue(fs.timestamp, fs.frames[0]);
+		return true;
+	});
+
 	_updateCameras(screen_->control()->getNet()->findAll<string>("list_streams"));
 
 	// Also check for a file on command line.

applications/gui/src/src_window.hpp

@@ -16,6 +16,8 @@
 #include <ftl/streams/receiver.hpp>
 #include <ftl/streams/filestream.hpp>
 
+#include <ftl/audio/speaker.hpp>
+
 class VirtualCameraView;
 
 namespace ftl {
@@ -68,6 +70,10 @@ class SourceWindow : public nanogui::Window {
 	ftl::operators::Graph *pre_pipeline_;
 	MUTEX mutex_;
 
+	ftl::audio::Speaker *speaker_;
+
+	ftl::rgbd::FrameSet frameset_;
+
 	void _updateCameras(const std::vector<std::string> &netcams);
 	void _createDefaultCameras(ftl::rgbd::FrameSet &fs, bool makevirtual);
 	ftl::codecs::Channels<0> _aggregateChannels();

applications/reconstruct/CMakeLists.txt

@@ -37,6 +37,6 @@ set_property(TARGET ftl-reconstruct PROPERTY CUDA_SEPARABLE_COMPILATION ON)
 endif()
 
 #target_include_directories(cv-node PUBLIC ${PROJECT_SOURCE_DIR}/include)
-target_link_libraries(ftl-reconstruct ftlcommon ftlrgbd Threads::Threads ${OpenCV_LIBS} ftlctrl ftlnet ftlrender ftloperators ftlstreams)
+target_link_libraries(ftl-reconstruct ftlcommon ftlrgbd Threads::Threads ${OpenCV_LIBS} ftlctrl ftlnet ftlrender ftloperators ftlstreams ftlaudio)
 
 

applications/reconstruct/src/main.cpp

@@ -52,6 +52,8 @@
 #include <ftl/streams/sender.hpp>
 #include <ftl/streams/netstream.hpp>
 
+#include <ftl/audio/source.hpp>
+
 #include <cuda_profiler_api.h>
 
 #ifdef WIN32
@@ -143,6 +145,8 @@ static void run(ftl::Configurable *root) {
 	outstream->begin();
 	sender->setStream(outstream);
 
+	ftl::audio::Source *audioSrc = nullptr;
+
 	std::vector<Source*> sources;
 	// Create a vector of all input RGB-Depth sources
 	if (root->getConfig()["sources"].size() > 0) {
@@ -194,6 +198,14 @@ static void run(ftl::Configurable *root) {
 			});
 			groups.push_back(reconstr);
 			++i;
+
+			// TODO: Temporary reconstruction local audio source for testing
+			audioSrc = ftl::create<ftl::audio::Source>(root, "audio_test");
+
+			audioSrc->onFrameSet([sender](ftl::audio::FrameSet &fs) {
+				sender->post(fs);
+				return true;
+			});
 		}
 	}
 
@@ -229,6 +241,11 @@ static void run(ftl::Configurable *root) {
 				return reconstr->post(fs);
 			});
 
+			gen->onAudio([sender](ftl::audio::FrameSet &fs) {
+				sender->post(fs);
+				return true;
+			});
+
 			int i = groups.size();
 			reconstr->onFrameSet([sender,i](ftl::rgbd::FrameSet &fs) {
 				fs.id = i;
@@ -242,10 +259,11 @@ static void run(ftl::Configurable *root) {
 		}
 	}
 
-
 	LOG(INFO) << "Start timer";
 	ftl::timer::start(true);
 
+	if (audioSrc) delete audioSrc;
+
 	LOG(INFO) << "Shutting down...";
 	ftl::timer::stop();
 	ctrl.stop();

applications/vision/CMakeLists.txt

@@ -21,6 +21,6 @@ set_property(TARGET ftl-vision PROPERTY CUDA_SEPARABLE_COMPILATION OFF)
 endif()
 
 #target_include_directories(cv-node PUBLIC ${PROJECT_SOURCE_DIR}/include)
-target_link_libraries(ftl-vision ftlrgbd ftlcommon ftlstreams ftlctrl ${OpenCV_LIBS} ${LIBSGM_LIBRARIES} ${CUDA_LIBRARIES} ftlnet)
+target_link_libraries(ftl-vision ftlrgbd ftlcommon ftlstreams ftlctrl ${OpenCV_LIBS} ${LIBSGM_LIBRARIES} ${CUDA_LIBRARIES} ftlnet ftlaudio)
 
 

applications/vision/src/main.cpp

@@ -26,6 +26,8 @@
 #include <ftl/streams/netstream.hpp>
 #include <ftl/streams/sender.hpp>
 
+#include <ftl/audio/source.hpp>
+
 #include "opencv2/imgproc.hpp"
 #include "opencv2/imgcodecs.hpp"
 #include "opencv2/highgui.hpp"
@@ -76,6 +78,13 @@ static void run(ftl::Configurable *root) {
 		return true;
 	});
 
+	// TODO: TEMPORARY
+	ftl::audio::Source *audioSrc = ftl::create<ftl::audio::Source>(root, "audio_test");
+	audioSrc->onFrameSet([sender](ftl::audio::FrameSet &fs) {
+		sender->post(fs);
+		return true;
+	});
+	
 	auto pipeline = ftl::config::create<ftl::operators::Graph>(root, "pipeline");
 	pipeline->append<ftl::operators::DepthChannel>("depth");  // Ensure there is a depth channel
 	grp->addPipeline(pipeline);

components/audio/CMakeLists.txt

+set(AUDIOSRC
+	src/source.cpp
+	src/frame.cpp
+	src/portaudio.cpp
+	src/speaker.cpp
+)
+
+add_library(ftlaudio ${AUDIOSRC})
+
+target_include_directories(ftlaudio PUBLIC
+	$<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/include>
+	$<INSTALL_INTERFACE:include>
+	PRIVATE src)
+
+target_link_libraries(ftlaudio ftlcommon Eigen3::Eigen ftlstreams ftldata portaudio)
+
+#add_subdirectory(test)
+

components/audio/include/ftl/audio/audio.hpp

+#ifndef _FTL_AUDIO_AUDIO_HPP_
+#define _FTL_AUDIO_AUDIO_HPP_
+
+#include <vector>
+
+namespace ftl {
+namespace audio {
+
+class Audio {
+	public:
+	Audio() {};
+
+	size_t size() const { return data_.size()*sizeof(short); }
+
+	std::vector<short> &data() { return data_; }
+	const std::vector<short> &data() const { return data_; }
+
+	private:
+	std::vector<short> data_;
+};
+
+}
+}
+
+#endif  // _FTL_AUDIO_AUDIO_HPP_

components/audio/include/ftl/audio/buffer.hpp

+#ifndef _FTL_AUDIO_BUFFER_HPP_
+#define _FTL_AUDIO_BUFFER_HPP_
+
+#include <vector>
+
+namespace ftl {
+namespace audio {
+
+//static constexpr int kBufferCount = 100;
+
+/**
+ * A fast circular buffer to capture, play and manipulate audio data.
+ * This class can be used directly with portaudio. The hardware uses
+ * `readFrame` and `writeFrame` to consume or append audio data. A more
+ * advanced `write` function allows for non-frame aligned data and for time
+ * dilation / shifting, and amplitude control.
+ */
+template <typename T, int CHAN, int FRAME, int SIZE>
+class FixedBuffer {
+	public:
+	typedef T type;
+
+	FixedBuffer() : write_position_(0), read_position_(-1), offset_(0), rate_(44100),
+			cur_delay_(0.0f), req_delay_(0.0f) {}
+	explicit FixedBuffer(int rate) : write_position_(0), read_position_(-1),
+			offset_(0), rate_(rate), cur_delay_(0.0f), req_delay_(0.0f) {}
+
+	int sampleRate() const { return rate_; }
+
+	inline int channels() const { return CHAN; }
+	inline int frameSize() const { return FRAME; }
+	inline int maxFrames() const { return SIZE; }
+
+	void setDelay(float d) {
+		req_delay_ = d  * static_cast<float>(rate_);
+	}
+
+	float delay() const { return cur_delay_ / static_cast<float>(rate_); }
+
+	inline void writeFrame(const T *d) {
+		const T *in = d;
+		T *out = &data_[(write_position_++) % SIZE][0];
+		for (size_t i=0; i<CHAN*FRAME; ++i) *out++ = *in++;
+		if (write_position_ > 5 && read_position_ < 0) read_position_ = 0;
+	}
+
+	inline void readFrame(T *d) {
+		T *out = d;
+		if (read_position_ < 0 || read_position_ >= write_position_-1) {
+			for (size_t i=0; i<CHAN*FRAME; ++i) *out++ = 0;
+		} else {
+			T *in = &data_[(read_position_++) % SIZE][0];
+			for (size_t i=0; i<CHAN*FRAME; ++i) *out++ = *in++;
+		}
+	}
+
+	int size() const { return (read_position_>=0) ? write_position_ - 2 - read_position_ : 0; }
+	int frames() const { return (read_position_>=0) ? write_position_ - 2 - read_position_ : 0; }
+
+	/**
+	 * Append sound samples to the end of the buffer. The samples may be over
+	 * or under sampled so as to gradually introduce or remove a requested
+	 * delay and hence change the latency of the audio.
+	 */
+	void write(const std::vector<T> &in);
+
+	private:
+	int write_position_;
+	int read_position_;
+	int offset_;
+	T data_[SIZE][CHAN*FRAME];
+	int rate_;
+
+	float cur_delay_;
+	float req_delay_;
+};
+
+// ==== Implementations ========================================================
+
+template <typename T, int CHAN>
+static T fracIndex(const std::vector<T> &in, float ix, int c) {
+	const int i1 = static_cast<int>(ix);
+	const int i2 = static_cast<int>(ix+1.0f);
+	const float alpha = ix - static_cast<float>(i1);
+	return (i2*CHAN+CHAN >= in.size()) ? in[i1*CHAN+c] : in[i1*CHAN+c]*(1.0f-alpha) + in[i2*CHAN+c]*alpha;
+}
+
+inline float clamp(float v, float c) { return (v < -c) ? -c : (v > c) ? c : v; }
+
+template <typename T, int CHAN, int FRAME, int SIZE>
+void FixedBuffer<T,CHAN,FRAME,SIZE>::write(const std::vector<T> &in) {
+	float i=0.0f;
+	float s = static_cast<float>(in.size()) / static_cast<float>(CHAN);
+	
+	while (i <= s-1.0f) {
+		T *ptr = data_[write_position_ % SIZE]+offset_;
+		
+		for (int c=0; c<CHAN; ++c) *ptr++ = fracIndex<T,CHAN>(in, i, c);
+
+		const float d = 0.6f*clamp((req_delay_ - cur_delay_) / static_cast<float>(rate_), 0.5f);
+		i += 1.0f - d;  // FIXME: Is this correct? Seems to function but perhaps not ideal
+
+		/*if (d > 0.0f) {	// Increase delay = oversample with increment < 1.0
+			//i += 1.0f * (1.0f - d);
+			i += 1.0f - d;
+		} else {		// Decrease delay = undersample with increment > 1.0
+			//i += 1.0f / (1.0f + d);
+			i += 1.0f - d;
+		}*/
+		cur_delay_ += d;
+
+		offset_+= CHAN;
+		if (offset_ == CHAN*FRAME) {
+			offset_ = 0;
+			++write_position_;
+		}
+	}
+	if (write_position_ > 20 && read_position_ < 0) read_position_ = 0;
+}
+
+// ==== Common forms ===========================================================
+
+template <int SIZE>
+using StereoBuffer16 = ftl::audio::FixedBuffer<short,2,256,SIZE>;
+
+template <int SIZE>
+using MonoBuffer16 = ftl::audio::FixedBuffer<short,1,256,SIZE>;
+
+}
+}
+
+#endif  // _FTL_AUDIO_BUFFER_HPP_

components/audio/include/ftl/audio/frame.hpp

+#pragma once
+#ifndef _FTL_AUDIO_FRAME_HPP_
+#define _FTL_AUDIO_FRAME_HPP_
+
+#include <ftl/data/framestate.hpp>
+#include <ftl/data/frame.hpp>
+#include <ftl/audio/audio.hpp>
+
+namespace ftl {
+namespace audio {
+
+struct AudioSettings {
+	int sample_rate;
+	int frame_size;
+};
+
+struct AudioData {
+	template <typename T>
+	const T &as() const {
+		throw ftl::exception("Type not valid for audio channel");
+	}
+
+	template <typename T>
+	T &as() {
+		throw ftl::exception("Type not valid for audio channel");
+	}
+
+	template <typename T>
+	T &make() {
+		throw ftl::exception("Type not valid for audio channel");
+	}
+
+	Audio data;
+};
+
+// Specialisations for getting Audio data.
+template <> Audio &AudioData::as<Audio>(); 
+template <> const Audio &AudioData::as<Audio>() const;
+template <> Audio &AudioData::make<Audio>();
+
+typedef ftl::data::FrameState<AudioSettings,2> FrameState;
+typedef ftl::data::Frame<32,2,FrameState,AudioData> Frame;
+
+}
+}
+
+#endif // _FTL_AUDIO_FRAME_HPP_
\ No newline at end of file

components/audio/include/ftl/audio/frameset.hpp

+#ifndef _FTL_AUDIO_FRAMESET_HPP_
+#define _FTL_AUDIO_FRAMESET_HPP_
+
+#include <ftl/audio/frame.hpp>
+#include <ftl/data/frameset.hpp>
+
+namespace ftl {
+namespace audio {
+
+typedef ftl::data::FrameSet<ftl::audio::Frame> FrameSet;
+
+}
+}
+
+#endif  // _FTL_AUDIO_FRAMESET_HPP_

components/audio/include/ftl/audio/portaudio.hpp

+#ifndef _FTL_AUDIO_PORTAUDIO_HPP_
+#define _FTL_AUDIO_PORTAUDIO_HPP_
+
+namespace ftl {
+namespace audio {
+
+void pa_init();
+
+void pa_final();
+
+}
+}
+
+#endif  // _FTL_AUDIO_PORTAUDIO_HPP_

components/audio/include/ftl/audio/source.hpp

+#ifndef _FTL_AUDIO_SOURCE_HPP_
+#define _FTL_AUDIO_SOURCE_HPP_
+
+#include <ftl/audio/buffer.hpp>
+#include <ftl/audio/frameset.hpp>
+#include <ftl/configurable.hpp>
+#include <ftl/config.h>
+
+#ifdef HAVE_PORTAUDIO
+#include <portaudio.h>
+#endif
+
+namespace ftl {
+namespace audio {
+
+static constexpr int kFrameSize = 256;
+
+typedef ftl::data::Generator<ftl::audio::FrameSet> Generator;
+
+class Source : public ftl::Configurable, public ftl::audio::Generator {
+    public:
+    explicit Source(nlohmann::json &config);
+    ~Source();
+
+    /** Number of frames in last frameset. This can change over time. */
+	size_t size() override;
+
+	/**
+	 * Get the persistent state object for a frame. An exception is thrown
+	 * for a bad index.
+	 */
+	ftl::audio::FrameState &state(int ix) override;
+
+	/** Register a callback to receive new frame sets. */
+	void onFrameSet(const ftl::audio::FrameSet::Callback &) override;
+
+    private:
+    ftl::audio::FrameState state_;
+    bool active_;
+    ftl::timer::TimerHandle timer_hp_;
+	ftl::timer::TimerHandle timer_main_;
+	ftl::audio::FrameSet::Callback cb_;
+
+	ftl::audio::StereoBuffer16<100> buffer_;
+	int to_read_;
+
+	ftl::audio::FrameSet frameset_;
+
+	#ifdef HAVE_PORTAUDIO
+	PaStream *stream_;
+	#endif
+};
+
+}
+}
+
+#endif  // _FTL_AUDIO_SOURCE_HPP_

components/audio/include/ftl/audio/speaker.hpp

+#ifndef _FTL_AUDIO_SPEAKER_HPP_
+#define _FTL_AUDIO_SPEAKER_HPP_
+
+#include <ftl/configurable.hpp>
+#include <ftl/audio/buffer.hpp>
+#include <ftl/audio/frameset.hpp>
+#include <ftl/config.h>
+
+#ifdef HAVE_PORTAUDIO
+#include <portaudio.h>
+#endif
+
+namespace ftl {
+namespace audio {
+
+class Speaker : public ftl::Configurable {
+	public:
+	explicit Speaker(nlohmann::json &config);
+	~Speaker();
+
+	void queue(int64_t ts, ftl::audio::Frame &fs);
+
+	void setDelay(int64_t ms);
+
+	private:
+	ftl::audio::StereoBuffer16<2000> buffer_;
+	bool active_;
+	float extra_delay_;
+
+	#ifdef HAVE_PORTAUDIO
+	PaStream *stream_;
+	#endif
+};
+
+}
+}
+
+#endif  // _FTL_SPEAKER_HPP_

components/audio/src/frame.cpp

+#include <ftl/audio/frame.hpp>
+#include <ftl/audio/audio.hpp>
+
+using ftl::audio::Audio;
+using ftl::audio::AudioData;
+
+template <> Audio &AudioData::as<Audio>() {
+	return data;
+}
+
+template <> const Audio &AudioData::as<Audio>() const {
+	return data;
+}
+
+template <> Audio &AudioData::make<Audio>() {
+	return data;
+}

components/audio/src/portaudio.cpp

+#include <ftl/audio/portaudio.hpp>
+#include <ftl/config.h>
+#include <loguru.hpp>
+
+#include <atomic>
+
+static std::atomic<int> counter = 0;
+
+#ifdef HAVE_PORTAUDIO
+
+#include <portaudio.h>
+
+void ftl::audio::pa_init() {
+    // TODO: Mutex lock?
+    if (counter == 0) {
+        auto err = Pa_Initialize();
+        if (err != paNoError) {
+            LOG(ERROR) << "Portaudio failed to initialise: " << Pa_GetErrorText(err);
+            counter = 1000;
+        }
+
+        // List devices
+        int numDevices = Pa_GetDeviceCount();
+
+        if (numDevices == 0) LOG(WARNING) << "No audio devices found";
+        else LOG(INFO) << "Audio devices:";
+
+        for (int i=0; i<numDevices; ++i) {
+            const   PaDeviceInfo *deviceInfo = Pa_GetDeviceInfo(i);
+            LOG(INFO) << " -- (" << i << ") " << deviceInfo->name << " - Inputs=" << deviceInfo->maxInputChannels << " Outputs=" << deviceInfo->maxOutputChannels;
+        }
+    }
+    ++counter;
+}
+
+void ftl::audio::pa_final() {
+    // TODO: Mutex lock?
+    --counter;
+    if (counter == 0) {
+        auto err = Pa_Terminate();
+        if (err != paNoError) {
+            LOG(ERROR) << "Portaudio failed to terminate: " << Pa_GetErrorText(err);
+            counter = -1000;
+        }
+    }
+}
+#endif
\ No newline at end of file

components/audio/src/source.cpp

+#include <ftl/audio/source.hpp>
+#include <ftl/audio/audio.hpp>
+#include <ftl/audio/portaudio.hpp>
+
+using ftl::audio::Source;
+using ftl::audio::Frame;
+using ftl::audio::FrameSet;
+using ftl::audio::FrameState;
+using ftl::audio::Audio;
+using ftl::codecs::Channel;
+
+#ifdef HAVE_PORTAUDIO
+
+//static double ltime = 0.0;
+
+/* Portaudio callback to receive audio data. */
+static int pa_source_callback(const void *input, void *output,
+        unsigned long frameCount, const PaStreamCallbackTimeInfo *timeInfo,
+        PaStreamCallbackFlags statusFlags, void *userData) {
+
+    auto *buffer = (ftl::audio::StereoBuffer16<100>*)userData;
+    short *in = (short*)input;
+
+	//short *out = (short*)output;
+	//buffer->readFrame(out);
+
+	//if (timeInfo->currentTime - ltime < (1.0 / 128.0)) return 0;
+	//ltime = timeInfo->inputBufferAdcTime;
+
+    //int i=0;
+    //while (i < frameCount) {
+	    buffer->writeFrame(in);
+        //i+=2*ftl::audio::kFrameSize;
+    //
+
+    return 0;
+}
+
+#endif
+
+Source::Source(nlohmann::json &config) : ftl::Configurable(config), buffer_(48000) {
+	if (!value("enabled",true)) {
+		active_ = false;
+		return;
+	}
+
+	#ifdef HAVE_PORTAUDIO
+    ftl::audio::pa_init();
+
+    PaStreamParameters inputParameters;
+    //bzero( &inputParameters, sizeof( inputParameters ) );
+    inputParameters.channelCount = 2;
+    inputParameters.device = value("audio_device",-1);
+    //inputParameters.hostApiSpecificStreamInfo = NULL;
+    inputParameters.sampleFormat = paInt16;
+    inputParameters.suggestedLatency = (inputParameters.device >= 0) ? Pa_GetDeviceInfo(inputParameters.device)->defaultLowInputLatency : 0;
+    inputParameters.hostApiSpecificStreamInfo = NULL;
+
+	PaError err;
+
+	if (inputParameters.device >= 0) { 
+		err = Pa_OpenStream(
+			&stream_,
+			&inputParameters,
+			NULL,
+			48000,  // Sample rate
+			ftl::audio::kFrameSize,    // Size of single frame
+			paNoFlag,
+			pa_source_callback,
+			&this->buffer_
+		);
+	} else {
+		err = Pa_OpenDefaultStream(
+			&stream_,
+			2,
+			0,
+			paInt16,
+			48000,  // Sample rate
+			ftl::audio::kFrameSize,    // Size of single frame
+			pa_source_callback,
+			&this->buffer_
+		);
+	}
+
+    if (err != paNoError) {
+        LOG(ERROR) << "Portaudio open stream error: " << Pa_GetErrorText(err);
+        active_ = false;
+        return;
+    } else {
+        active_ = true;
+    }
+
+    err = Pa_StartStream(stream_);
+
+    if (err != paNoError) {
+        LOG(ERROR) << "Portaudio start stream error: " << Pa_GetErrorText(err);
+        //active_ = false;
+		return;
+    }
+
+	to_read_ = 0;
+
+    timer_hp_ = ftl::timer::add(ftl::timer::kTimerHighPrecision, [this](int64_t ts) {
+        to_read_ = buffer_.size();
+        return true;
+    });
+
+	timer_main_ = ftl::timer::add(ftl::timer::kTimerMain, [this](int64_t ts) {
+
+        // Remove one interval since the audio starts from the last frame
+		frameset_.timestamp = ts - ftl::timer::getInterval();
+
+		frameset_.count = 1;
+		frameset_.stale = false;
+
+        if (to_read_ < 1) return true;
+
+		if (frameset_.frames.size() < 1) frameset_.frames.emplace_back();
+
+		auto &frame = frameset_.frames[0];
+		frame.reset();
+        std::vector<short> &data = frame.create<Audio>(Channel::Audio).data();
+
+		data.resize(2*ftl::audio::kFrameSize*to_read_);
+		short *ptr = data.data();
+		for (int i=0; i<to_read_; ++i) {
+			buffer_.readFrame(ptr);
+			ptr += 2*ftl::audio::kFrameSize;
+		}
+
+		// Then do something with the data!
+		//LOG(INFO) << "Audio Frames Sent: " << to_read_ << " - " << ltime;
+		if (cb_) cb_(frameset_);
+
+        return true;
+    }); 
+
+	#else  // No portaudio
+
+	active_ = false;
+	LOG(ERROR) << "No audio support";
+
+	#endif
+}
+
+Source::~Source() {
+    if (active_) {
+        active_ = false;
+
+		#ifdef HAVE_PORTAUDIO
+        auto err = Pa_StopStream(stream_);
+
+        if (err != paNoError) {
+            LOG(ERROR) << "Portaudio stop stream error: " << Pa_GetErrorText(err);
+            //active_ = false;
+        }
+
+        err = Pa_CloseStream(stream_);
+
+        if (err != paNoError) {
+            LOG(ERROR) << "Portaudio close stream error: " << Pa_GetErrorText(err);
+        }
+		#endif
+    }
+
+	#ifdef HAVE_PORTAUDIO
+    ftl::audio::pa_final();
+	#endif
+}
+
+size_t Source::size() {
+    return 1;
+}
+
+ftl::audio::FrameState &Source::state(int ix) {
+    if (ix < 0 || ix > 1) throw ftl::exception("State index out-of-bounds");
+    return state_;
+}
+
+void Source::onFrameSet(const ftl::audio::FrameSet::Callback &cb) {
+	cb_ = cb;
+}

components/audio/src/speaker.cpp

+#include <ftl/audio/speaker.hpp>
+#include <ftl/audio/audio.hpp>
+#include <ftl/audio/portaudio.hpp>
+
+using ftl::audio::Speaker;
+using ftl::audio::Frame;
+using ftl::audio::FrameSet;
+using ftl::audio::FrameState;
+using ftl::audio::Audio;
+using ftl::codecs::Channel;
+
+#ifdef HAVE_PORTAUDIO
+
+/* Portaudio callback to receive audio data. */
+static int pa_speaker_callback(const void *input, void *output,
+        unsigned long frameCount, const PaStreamCallbackTimeInfo *timeInfo,
+        PaStreamCallbackFlags statusFlags, void *userData) {
+
+    auto *buffer = (ftl::audio::StereoBuffer16<2000>*)userData;
+    short *out = (short*)output;
+
+	buffer->readFrame(out);
+
+    return 0;
+}
+
+#endif
+
+Speaker::Speaker(nlohmann::json &config) : ftl::Configurable(config), buffer_(48000) {
+	#ifdef HAVE_PORTAUDIO
+    ftl::audio::pa_init();
+
+    auto err = Pa_OpenDefaultStream(
+        &stream_,
+        0,
+        2,
+        paInt16,
+        48000,  // Sample rate
+        256,    // Size of single frame
+        pa_speaker_callback,
+        &this->buffer_
+    );
+
+    if (err != paNoError) {
+        LOG(ERROR) << "Portaudio open stream error: " << Pa_GetErrorText(err);
+        active_ = false;
+		return;
+    } else {
+        active_ = true;
+    }
+
+    err = Pa_StartStream(stream_);
+
+    if (err != paNoError) {
+        LOG(ERROR) << "Portaudio start stream error: " << Pa_GetErrorText(err);
+        //active_ = false;
+		return;
+    }
+
+	#else  // No portaudio
+
+	active_ = false;
+	LOG(ERROR) << "No audio support";
+
+	#endif
+
+	extra_delay_ = value("delay",0.0f);
+	on("delay", [this](const ftl::config::Event &e) {
+		extra_delay_ = value("delay",0.0f);
+	});
+}
+
+Speaker::~Speaker() {
+    if (active_) {
+        active_ = false;
+
+		#ifdef HAVE_PORTAUDIO
+        auto err = Pa_StopStream(stream_);
+
+        if (err != paNoError) {
+            LOG(ERROR) << "Portaudio stop stream error: " << Pa_GetErrorText(err);
+            //active_ = false;
+        }
+
+        err = Pa_CloseStream(stream_);
+
+        if (err != paNoError) {
+            LOG(ERROR) << "Portaudio close stream error: " << Pa_GetErrorText(err);
+        }
+		#endif
+    }
+
+	#ifdef HAVE_PORTAUDIO
+    ftl::audio::pa_final();
+	#endif
+}
+
+void Speaker::queue(int64_t ts, ftl::audio::Frame &frame) {
+    auto &audio = frame.get<ftl::audio::Audio>(Channel::Audio);
+
+	LOG(INFO) << "Buffer Fullness (" << ts << "): " << buffer_.size();
+	buffer_.write(audio.data());
+	LOG(INFO) << "Audio delay: " << buffer_.delay() << "s";
+}
+
+void Speaker::setDelay(int64_t ms) {
+    float d = static_cast<float>(ms) / 1000.0f + extra_delay_;
+    if (d < 0.0f) d = 0.0f;  // Clamp to 0 delay (not ideal to be exactly 0)
+    buffer_.setDelay(d);
+}