diff --git a/applications/gui/src/camera.hpp b/applications/gui/src/camera.hpp
index 7003f3c96a0264872b65ce4cba867959ba7c002a..d044151da12113c7eff28978c527b557d355bcc6 100644
--- a/applications/gui/src/camera.hpp
+++ b/applications/gui/src/camera.hpp
@@ -69,6 +69,8 @@ class Camera {
void draw(std::vector<ftl::rgbd::FrameSet*> &fss);
+ inline int64_t getFrameTimeMS() const { return int64_t(delta_ * 1000.0f); }
+
/**
* @internal. Used to inform the camera if it is the active camera or not.
*/
diff --git a/applications/gui/src/src_window.cpp b/applications/gui/src/src_window.cpp
index 501f6d4f5939e07c23e289ceb55392f412ee8f75..5b6b84174cf8271ee2348988a0e6a9f6417ee427 100644
--- a/applications/gui/src/src_window.cpp
+++ b/applications/gui/src/src_window.cpp
@@ -110,8 +110,9 @@ SourceWindow::SourceWindow(ftl::gui::Screen *screen)
receiver_->onAudio([this](ftl::audio::FrameSet &fs) {
if (framesets_.size() == 0) return true;
- //LOG(INFO) << "Audio delay required = " << (ts - frameset_.timestamp) << "ms";
- speaker_->setDelay(fs.timestamp - framesets_[0]->timestamp + ftl::timer::getInterval()); // Add Xms for local render time
+ auto *c = screen_->activeCamera();
+ int64_t renddelay = (c) ? c->getFrameTimeMS() : 0;
+ speaker_->setDelay(fs.timestamp - framesets_[0]->timestamp + renddelay); // Add Xms for local render time
speaker_->queue(fs.timestamp, fs.frames[0]);
return true;
});
diff --git a/components/audio/include/ftl/audio/buffer.hpp b/components/audio/include/ftl/audio/buffer.hpp
index fad4f447acde457619476602de88ded940bb2f4a..7d438320d3da0d92f6aa4b5efeb2cd0ec5c2cd80 100644
--- a/components/audio/include/ftl/audio/buffer.hpp
+++ b/components/audio/include/ftl/audio/buffer.hpp
@@ -2,6 +2,7 @@
#define _FTL_AUDIO_BUFFER_HPP_
#include <vector>
+#include <cmath>
namespace ftl {
namespace audio {
@@ -23,10 +24,19 @@ class Buffer {
void setDelay(float d) {
req_delay_ = d * static_cast<float>(rate_);
+ // Big jumps should be instant
+ if (fabs(req_delay_ - cur_delay_) > 0.5f) {
+ //cur_delay_ = req_delay_;
+ reset();
+ }
}
float delay() const { return cur_delay_ / static_cast<float>(rate_); }
+ virtual void reset() {
+ cur_delay_ = req_delay_;
+ }
+
virtual int size() const=0;
virtual int frames() const=0;
@@ -86,6 +96,12 @@ class FixedBuffer : public ftl::audio::Buffer<T> {
void read(std::vector<T> &out, int frames) override;
+ void reset() override {
+ Buffer<T>::reset();
+ write_position_ = 0;
+ read_position_ = 0;
+ }
+
private:
int write_position_;
int read_position_;
diff --git a/components/audio/include/ftl/audio/source.hpp b/components/audio/include/ftl/audio/source.hpp
index 8b810e3e8bae58ba4cab89dfb3d432bc93607579..797aee1e9ac2288a93f60bd674e6003f84e0495c 100644
--- a/components/audio/include/ftl/audio/source.hpp
+++ b/components/audio/include/ftl/audio/source.hpp
@@ -43,6 +43,7 @@ class Source : public ftl::Configurable, public ftl::audio::Generator {
ftl::audio::Buffer<short> *buffer_;
int to_read_;
+ int64_t latency_;
ftl::audio::FrameSet frameset_;
diff --git a/components/audio/include/ftl/audio/speaker.hpp b/components/audio/include/ftl/audio/speaker.hpp
index ea6d6d92d9b11b580fc3de2dcd895ecaa2379ebf..f27795bf40c29577c7de8e46ab27414e8ba699a5 100644
--- a/components/audio/include/ftl/audio/speaker.hpp
+++ b/components/audio/include/ftl/audio/speaker.hpp
@@ -26,6 +26,7 @@ class Speaker : public ftl::Configurable {
ftl::audio::Buffer<short> *buffer_;
bool active_;
float extra_delay_;
+ int64_t latency_;
#ifdef HAVE_PORTAUDIO
PaStream *stream_;
diff --git a/components/audio/src/source.cpp b/components/audio/src/source.cpp
index 48d176eaab6e558e5d66a3bc388b330b8fdbd839..d30d2d045048b70928a6d4624bfe9396d451b2a2 100644
--- a/components/audio/src/source.cpp
+++ b/components/audio/src/source.cpp
@@ -89,6 +89,8 @@ Source::Source(nlohmann::json &config) : ftl::Configurable(config), buffer_(null
inputParameters.suggestedLatency = (device >= 0) ? Pa_GetDeviceInfo(device)->defaultLowInputLatency : 0;
inputParameters.hostApiSpecificStreamInfo = NULL;
+ latency_ = int64_t(inputParameters.suggestedLatency * 1000.0);
+
PaError err;
if (inputParameters.device >= 0) {
@@ -147,7 +149,7 @@ Source::Source(nlohmann::json &config) : ftl::Configurable(config), buffer_(null
timer_main_ = ftl::timer::add(ftl::timer::kTimerMain, [this](int64_t ts) {
// Remove one interval since the audio starts from the last frame
- frameset_.timestamp = ts - ftl::timer::getInterval();
+ frameset_.timestamp = ts - ftl::timer::getInterval() + latency_;
frameset_.id = 0;
frameset_.count = 1;
diff --git a/components/audio/src/speaker.cpp b/components/audio/src/speaker.cpp
index f9ebd5d6e84451cdc61cb5aa4f48de1685328ad1..1e897f33b1d5e7a7dbd3a9d898c58bde985a9585 100644
--- a/components/audio/src/speaker.cpp
+++ b/components/audio/src/speaker.cpp
@@ -83,14 +83,25 @@ void Speaker::_open(int fsize, int sample, int channels) {
buffer_ = new ftl::audio::MonoBuffer16<2000>(sample);
}
+ PaStreamParameters outputParameters;
+ //bzero( &inputParameters, sizeof( inputParameters ) );
+ outputParameters.channelCount = channels;
+ outputParameters.device = Pa_GetDefaultOutputDevice();
+ outputParameters.sampleFormat = paInt16;
+ outputParameters.suggestedLatency = Pa_GetDeviceInfo(outputParameters.device)->defaultLowOutputLatency;
+ outputParameters.hostApiSpecificStreamInfo = NULL;
+
+ //LOG(INFO) << "OUTPUT LATENCY: " << outputParameters.suggestedLatency;
+ latency_ = int64_t(outputParameters.suggestedLatency * 1000.0);
+
#ifdef HAVE_PORTAUDIO
- auto err = Pa_OpenDefaultStream(
+ auto err = Pa_OpenStream(
&stream_,
- 0,
- channels,
- paInt16,
+ NULL,
+ &outputParameters,
sample, // Sample rate
256, // Size of single frame
+ paNoFlag,
(channels == 1) ? pa_speaker_callback<ftl::audio::MonoBuffer16<2000>> : pa_speaker_callback<ftl::audio::StereoBuffer16<2000>>,
this->buffer_
);
@@ -132,7 +143,11 @@ void Speaker::queue(int64_t ts, ftl::audio::Frame &frame) {
}
void Speaker::setDelay(int64_t ms) {
+ ms -= latency_;
float d = static_cast<float>(ms) / 1000.0f + extra_delay_;
- if (d < 0.0f) d = 0.0f; // Clamp to 0 delay (not ideal to be exactly 0)
- if (buffer_) buffer_->setDelay(d);
+ if (d < 0.0f) d = 0.001f; // Clamp to 0 delay (not ideal to be exactly 0)
+ if (buffer_) {
+ buffer_->setDelay(d);
+ //LOG(INFO) << "Audio delay: " << buffer_->delay();
+ }
}
diff --git a/components/streams/src/netstream.cpp b/components/streams/src/netstream.cpp
index 54fc032074f25be186a8a20055f330f6968c7eac..0515b1e2fb3f79859c97a7476ba56d0f15f42cfd 100644
--- a/components/streams/src/netstream.cpp
+++ b/components/streams/src/netstream.cpp
@@ -17,7 +17,7 @@ using std::optional;
static constexpr int kTallyScale = 10;
-Net::Net(nlohmann::json &config, ftl::net::Universe *net) : Stream(config), active_(false), net_(net) {
+Net::Net(nlohmann::json &config, ftl::net::Universe *net) : Stream(config), active_(false), net_(net), clock_adjust_(0) {
// TODO: Install "find_stream" binding if not installed...
if (!net_->isBound("find_stream")) {
net_->bind("find_stream", [this](const std::string &uri) -> optional<ftl::UUID> {
@@ -146,6 +146,7 @@ bool Net::begin() {
if (!active_) return;
StreamPacket spkt = spkt_raw;
+ spkt.timestamp -= clock_adjust_;
spkt.version = 4;
// Manage recuring requests
@@ -269,6 +270,27 @@ bool Net::_sendRequest(Channel c, uint8_t frameset, uint8_t frames, uint8_t coun
net_->send(peer_, uri_, (short)0, spkt, pkt);
+ if (true) { // TODO: Not every time
+ auto start = std::chrono::high_resolution_clock::now();
+ int64_t mastertime;
+
+ try {
+ mastertime = net_->call<int64_t>(peer_, "__ping__");
+ } catch (...) {
+ LOG(ERROR) << "Ping failed";
+ // Reset time peer and remove timer
+ time_peer_ = ftl::UUID(0);
+ return false;
+ }
+
+ auto elapsed = std::chrono::high_resolution_clock::now() - start;
+ int64_t latency = std::chrono::duration_cast<std::chrono::milliseconds>(elapsed).count();
+ clock_adjust_ = mastertime - (ftl::timer::get_time() + (latency/2));
+
+ if (clock_adjust_ > 0) {
+ LOG(INFO) << "Clock adjustment: " << clock_adjust_;
+ }
+ }
return true;
}