Merge branch 'bug/322/audiolatency' into 'master'

Resolves #322 audio latency problem

Closes #322

See merge request nicolas.pope/ftl!268

applications/gui/src/camera.hpp

@@ -69,6 +69,8 @@ class Camera {
 
 	void draw(std::vector<ftl::rgbd::FrameSet*> &fss);
 
+	inline int64_t getFrameTimeMS() const { return int64_t(delta_ * 1000.0f); }
+
 	/**
 	 * @internal. Used to inform the camera if it is the active camera or not.
 	 */

applications/gui/src/src_window.cpp

@@ -110,8 +110,9 @@ SourceWindow::SourceWindow(ftl::gui::Screen *screen)
 
 	receiver_->onAudio([this](ftl::audio::FrameSet &fs) {
 		if (framesets_.size() == 0) return true;
-		//LOG(INFO) << "Audio delay required = " << (ts - frameset_.timestamp) << "ms";
-		speaker_->setDelay(fs.timestamp - framesets_[0]->timestamp + ftl::timer::getInterval());  // Add Xms for local render time
+		auto *c = screen_->activeCamera();
+		int64_t renddelay = (c) ? c->getFrameTimeMS() : 0;
+		speaker_->setDelay(fs.timestamp - framesets_[0]->timestamp + renddelay);  // Add Xms for local render time
 		speaker_->queue(fs.timestamp, fs.frames[0]);
 		return true;
 	});

components/audio/include/ftl/audio/buffer.hpp

@@ -2,6 +2,7 @@
 #define _FTL_AUDIO_BUFFER_HPP_
 
 #include <vector>
+#include <cmath>
 
 namespace ftl {
 namespace audio {
@@ -23,10 +24,19 @@ class Buffer {
 
 	void setDelay(float d) {
 		req_delay_ = d  * static_cast<float>(rate_);
+		// Big jumps should be instant
+		if (fabs(req_delay_ - cur_delay_) > 0.5f) {
+			//cur_delay_ = req_delay_;
+			reset();
+		}
 	}
 
 	float delay() const { return cur_delay_ / static_cast<float>(rate_); }
 
+	virtual void reset() {
+		cur_delay_ = req_delay_;
+	}
+
 	virtual int size() const=0;
 	virtual int frames() const=0;
 
@@ -86,6 +96,12 @@ class FixedBuffer : public ftl::audio::Buffer<T> {
 
 	void read(std::vector<T> &out, int frames) override;
 
+	void reset() override {
+		Buffer<T>::reset();
+		write_position_ = 0;
+		read_position_ = 0;
+	}
+
 	private:
 	int write_position_;
 	int read_position_;

components/audio/include/ftl/audio/source.hpp

@@ -43,6 +43,7 @@ class Source : public ftl::Configurable, public ftl::audio::Generator {
 
 	ftl::audio::Buffer<short> *buffer_;
 	int to_read_;
+	int64_t latency_;
 
 	ftl::audio::FrameSet frameset_;
 

components/audio/include/ftl/audio/speaker.hpp

@@ -26,6 +26,7 @@ class Speaker : public ftl::Configurable {
 	ftl::audio::Buffer<short> *buffer_;
 	bool active_;
 	float extra_delay_;
+	int64_t latency_;
 
 	#ifdef HAVE_PORTAUDIO
 	PaStream *stream_;

components/audio/src/source.cpp

@@ -89,6 +89,8 @@ Source::Source(nlohmann::json &config) : ftl::Configurable(config), buffer_(null
     inputParameters.suggestedLatency = (device >= 0) ? Pa_GetDeviceInfo(device)->defaultLowInputLatency : 0;
     inputParameters.hostApiSpecificStreamInfo = NULL;
 
+	latency_ = int64_t(inputParameters.suggestedLatency * 1000.0);
+
 	PaError err;
 
 	if (inputParameters.device >= 0) { 
@@ -147,7 +149,7 @@ Source::Source(nlohmann::json &config) : ftl::Configurable(config), buffer_(null
 	timer_main_ = ftl::timer::add(ftl::timer::kTimerMain, [this](int64_t ts) {
 
         // Remove one interval since the audio starts from the last frame
-		frameset_.timestamp = ts - ftl::timer::getInterval();
+		frameset_.timestamp = ts - ftl::timer::getInterval() + latency_;
 
 		frameset_.id = 0;
 		frameset_.count = 1;

components/audio/src/speaker.cpp

@@ -83,14 +83,25 @@ void Speaker::_open(int fsize, int sample, int channels) {
 		buffer_ = new ftl::audio::MonoBuffer16<2000>(sample);
 	}
 
+	PaStreamParameters outputParameters;
+    //bzero( &inputParameters, sizeof( inputParameters ) );
+    outputParameters.channelCount = channels;
+    outputParameters.device = Pa_GetDefaultOutputDevice();
+    outputParameters.sampleFormat = paInt16;
+    outputParameters.suggestedLatency = Pa_GetDeviceInfo(outputParameters.device)->defaultLowOutputLatency;
+    outputParameters.hostApiSpecificStreamInfo = NULL;
+
+	//LOG(INFO) << "OUTPUT LATENCY: " << outputParameters.suggestedLatency;
+	latency_ = int64_t(outputParameters.suggestedLatency * 1000.0);
+
 	#ifdef HAVE_PORTAUDIO
-	auto err = Pa_OpenDefaultStream(
+	auto err = Pa_OpenStream(
 		&stream_,
-		0,
-		channels,
-		paInt16,
+		NULL,
+		&outputParameters,
 		sample,  // Sample rate
 		256,    // Size of single frame
+		paNoFlag,
 		(channels == 1) ? pa_speaker_callback<ftl::audio::MonoBuffer16<2000>> : pa_speaker_callback<ftl::audio::StereoBuffer16<2000>>,
 		this->buffer_
 	);
@@ -132,7 +143,11 @@ void Speaker::queue(int64_t ts, ftl::audio::Frame &frame) {
 }
 
 void Speaker::setDelay(int64_t ms) {
+	ms -= latency_;
 	float d = static_cast<float>(ms) / 1000.0f + extra_delay_;
-	if (d < 0.0f) d = 0.0f;  // Clamp to 0 delay (not ideal to be exactly 0)
-	if (buffer_) buffer_->setDelay(d);
+	if (d < 0.0f) d = 0.001f;  // Clamp to 0 delay (not ideal to be exactly 0)
+	if (buffer_) {
+		buffer_->setDelay(d);
+		//LOG(INFO) << "Audio delay: " << buffer_->delay();
+	}
 }

components/streams/src/netstream.cpp

@@ -17,7 +17,7 @@ using std::optional;
 
 static constexpr int kTallyScale = 10;
 
-Net::Net(nlohmann::json &config, ftl::net::Universe *net) : Stream(config), active_(false), net_(net) {
+Net::Net(nlohmann::json &config, ftl::net::Universe *net) : Stream(config), active_(false), net_(net), clock_adjust_(0) {
 	// TODO: Install "find_stream" binding if not installed...
 	if (!net_->isBound("find_stream")) {
 		net_->bind("find_stream", [this](const std::string &uri) -> optional<ftl::UUID> {
@@ -146,6 +146,7 @@ bool Net::begin() {
 		if (!active_) return;
 
 		StreamPacket spkt = spkt_raw;
+		spkt.timestamp -= clock_adjust_;
 		spkt.version = 4;
 
 		// Manage recuring requests
@@ -269,6 +270,27 @@ bool Net::_sendRequest(Channel c, uint8_t frameset, uint8_t frames, uint8_t coun
 
 	net_->send(peer_, uri_, (short)0, spkt, pkt);
 
+	if (true) {  // TODO: Not every time
+		auto start = std::chrono::high_resolution_clock::now();
+		int64_t mastertime;
+
+		try {
+			mastertime = net_->call<int64_t>(peer_, "__ping__");
+		} catch (...) {
+			LOG(ERROR) << "Ping failed";
+			// Reset time peer and remove timer
+			time_peer_ = ftl::UUID(0);
+			return false;
+		}
+
+		auto elapsed = std::chrono::high_resolution_clock::now() - start;
+		int64_t latency = std::chrono::duration_cast<std::chrono::milliseconds>(elapsed).count();
+		clock_adjust_ = mastertime - (ftl::timer::get_time() + (latency/2));
+
+		if (clock_adjust_ > 0) {
+			LOG(INFO) << "Clock adjustment: " << clock_adjust_;
+		}
+	}
 	return true;
 }