Skip to content
Snippets Groups Projects

Compare revisions

Changes are shown as if the source revision was being merged into the target revision. Learn more about comparing revisions.

Source

Select target project
No results found
Select Git revision

Target

Select target project
  • nicolaspope/ftl
1 result
Select Git revision
Show changes
Showing
with 776 additions and 166 deletions
###############################################################################
# Find Pylon
#
set(PYLON_FOUND FALSE CACHE BOOL "" FORCE)
if(WIN32)
find_path(PYLON_DIR NAMES include/pylon/PylonBase.h PATHS "C:/Program Files/Pylon" "C:/Program Files (x86)/Pylon")
else()
find_path(PYLON_DIR NAMES include/pylon/PylonBase.h PATHS "/opt/pylon" "/opt/pylon6")
endif()
if (PYLON_DIR)
set(PYLON_FOUND TRUE CACHE BOOL "" FORCE)
set(HAVE_PYLON TRUE)
include(FindPackageHandleStandardArgs)
find_package_handle_standard_args(Pylon DEFAULT_MSG PYLON_DIR)
mark_as_advanced(PYLON_FOUND)
if (WIN32)
list(APPEND PYLON_LIBRARIES PylonBase_v6_1 PylonUtility_v6_1 GenApi_MD_VC141_v3_1_Basler_pylon GCBase_MD_VC141_v3_1_Basler_pylon)
else()
list(APPEND PYLON_LIBRARIES pylonbase pylonutility GenApi_gcc_v3_1_Basler_pylon GCBase_gcc_v3_1_Basler_pylon)
endif()
add_library(Pylon INTERFACE)
set_property(TARGET Pylon PROPERTY INTERFACE_INCLUDE_DIRECTORIES ${PYLON_DIR}/include)
#set_property(TARGET Pylon PROPERTY INTERFACE_LINK_DIRECTORIES ${PYLON_DIR}/lib)
if (WIN32)
link_directories(${PYLON_DIR}/lib/x64)
else()
link_directories(${PYLON_DIR}/lib)
endif()
set_property(TARGET Pylon PROPERTY INTERFACE_LINK_LIBRARIES ${PYLON_LIBRARIES})
else()
add_library(Pylon INTERFACE)
endif()
......@@ -43,5 +43,5 @@ if(GLOG_FOUND)
add_library(glog::glog UNKNOWN IMPORTED)
set_property(TARGET glog::glog PROPERTY INTERFACE_INCLUDE_DIRECTORIES ${GLOG_INCLUDE_DIRS})
set_property(TARGET glog::glog PROPERTY IMPORTED_LOCATION ${GLOG_LIBRARY})
message(STATUS "Found glog: ${GLOG_LIBRARY}")
message(STATUS "Found glog: ${GLOG_LIBRARY} ${GLOG_INCLUDE_DIRS}")
endif()
# use build date as patch version
string(TIMESTAMP BUILD_TIME "%Y%m%d")
set(CPACK_PACKAGE_VERSION_PATCH "${BUILD_TIME}")
set(CPACK_DEBIAN_PACKAGE_MAINTAINER "UTU Future Tech Lab")
set(CPACK_DEBIAN_PACKAGE_SHLIBDEPS ON)
set(CPACK_DEBIAN_PACKAGE_GENERATE_SHLIBS ON)
set(CPACK_DEBIAN_PACKAGE_GENERATE_SHLIBS_POLICY ">=")
set(CPACK_DEB_PACKAGE_COMPONENT ON)
set(CPACK_DEBIAN_PACKAGE_SECTION "Miscellaneous")
macro(deb_append_dependency DEPENDS)
if ("${CPACK_DEBIAN_PACKAGE_DEPENDS}" STREQUAL "")
set(CPACK_DEBIAN_PACKAGE_DEPENDS "${DEPENDS}")
else()
set(CPACK_DEBIAN_PACKAGE_DEPENDS "${CPACK_DEBIAN_PACKAGE_DEPENDS}, ${DEPENDS}")
endif()
endmacro()
if (HAVE_PYLON)
deb_append_dependency("pylon (>= 6.1.1)")
set(ENV{LD_LIBRARY_PATH} "=/opt/pylon/lib/")
endif()
if(WIN32)
message(INFO "Copying DLLs: OpenCV")
file(GLOB WINDOWS_LIBS "${OpenCV_INSTALL_PATH}/${OpenCV_ARCH}/${OpenCV_RUNTIME}/bin/*.dll")
install(FILES ${WINDOWS_LIBS} DESTINATION bin)
set(CPACK_GENERATOR "WiX")
endif()
include(CPack)
set(AUDIOSRC
src/source.cpp
src/frame.cpp
src/portaudio.cpp
src/speaker.cpp
src/software_encoder.cpp
src/software_decoder.cpp
)
add_library(ftlaudio ${AUDIOSRC})
......@@ -12,7 +13,8 @@ target_include_directories(ftlaudio PUBLIC
$<INSTALL_INTERFACE:include>
PRIVATE src)
target_link_libraries(ftlaudio ftlcommon Eigen3::Eigen ftlstreams ftldata portaudio)
#add_subdirectory(test)
target_link_libraries(ftlaudio ftlcommon Eigen3::Eigen ftlstreams ftldata portaudio Opus)
if (BUILD_TESTS)
add_subdirectory(test)
endif()
......@@ -12,14 +12,24 @@ class Audio {
size_t size() const { return data_.size()*sizeof(short); }
std::vector<short> &data() { return data_; }
const std::vector<short> &data() const { return data_; }
std::vector<float> &data() { return data_; }
const std::vector<float> &data() const { return data_; }
private:
std::vector<short> data_;
std::vector<float> data_;
};
}
}
template <>
inline bool ftl::data::make_type<std::list<ftl::audio::Audio>>() {
return false;
}
template <>
inline bool ftl::data::decode_type<std::list<ftl::audio::Audio>>(std::any &a, const std::vector<uint8_t> &data) {
return false;
}
#endif // _FTL_AUDIO_AUDIO_HPP_
......@@ -3,9 +3,10 @@
#include <vector>
#include <cmath>
#include <Eigen/Eigen>
#define LOGURU_REPLACE_GLOG 1
#include <loguru.hpp>
//#define LOGURU_REPLACE_GLOG 1
//#include <loguru.hpp>
namespace ftl {
namespace audio {
......@@ -36,6 +37,9 @@ class Buffer {
float delay() const { return cur_delay_ / static_cast<float>(rate_); }
inline void setGain(float g) { gain_ = g; }
inline float gain() const { return gain_; }
virtual void reset() {
cur_delay_ = req_delay_;
}
......@@ -49,6 +53,7 @@ class Buffer {
float req_delay_;
int channels_;
int frame_size_;
float gain_ = 1.0f;
};
//static constexpr int kBufferCount = 100;
......@@ -72,18 +77,33 @@ class FixedBuffer : public ftl::audio::Buffer<T> {
inline void writeFrame(const T *d) {
const T *in = d;
T *out = &data_[(write_position_++) % SIZE][0];
T *out = data_[(write_position_++) % SIZE];
for (size_t i=0; i<CHAN*FRAME; ++i) *out++ = *in++;
if (write_position_ > 5 && read_position_ < 0) read_position_ = 0;
}
inline void readFrame(T *d) {
T *out = d;
T* __restrict out = d;
//if ((size_t(out) & 0x1f) == 0) out_alignment_ = 32;
//else if ((size_t(out) & 0xf) == 0) out_alignment_ = 16;
//else if ((size_t(out) & 0x7) == 0) out_alignment_ = 8;
if (read_position_ < 0 || read_position_ >= write_position_-1) {
for (size_t i=0; i<CHAN*FRAME; ++i) *out++ = 0;
} else {
T *in = &data_[(read_position_++) % SIZE][0];
for (size_t i=0; i<CHAN*FRAME; ++i) *out++ = *in++;
const T* __restrict in = data_[(read_position_++) % SIZE];
// 16 byte aligned, use SIMD intrinsics
if ((size_t(out) & 0xf) == 0) {
for (size_t i=0; i<CHAN*FRAME; i += 4) {
Eigen::Map<Eigen::Matrix<float,4,1>,Eigen::Aligned16> vout(out+i);
const Eigen::Map<const Eigen::Matrix<float,4,1>,Eigen::Aligned16> vin(in+i);
vout = vin*this->gain_;
}
// Not aligned
} else {
for (size_t i=0; i<CHAN*FRAME; ++i) *out++ = this->gain_ * (*in++);
}
}
}
......@@ -102,15 +122,20 @@ class FixedBuffer : public ftl::audio::Buffer<T> {
void reset() override {
Buffer<T>::reset();
write_position_ = 0; //int(this->cur_delay_);
LOG(INFO) << "RESET AUDIO: " << write_position_;
read_position_ = 0;
}
inline T *data() { return (T*)data_; }
inline T *data(int f) { return data_[f]; }
inline int writePosition() const { return write_position_; }
inline void setWritePosition(int p) { write_position_ = p; }
private:
int write_position_;
int read_position_;
int offset_;
T data_[SIZE][CHAN*FRAME];
alignas(32) T data_[SIZE][CHAN*FRAME];
};
// ==== Implementations ========================================================
......@@ -120,7 +145,7 @@ static T fracIndex(const std::vector<T> &in, float ix, int c) {
const auto i1 = static_cast<unsigned int>(ix);
const auto i2 = static_cast<unsigned int>(ix+1.0f);
const float alpha = ix - static_cast<float>(i1);
return (i2*CHAN+CHAN >= in.size()) ? in[i1*CHAN+c] : in[i1*CHAN+c]*(1.0f-alpha) + in[i2*CHAN+c]*alpha;
return static_cast<T>((i2*CHAN+CHAN >= in.size()) ? in[i1*CHAN+c] : in[i1*CHAN+c]*(1.0f-alpha) + in[i2*CHAN+c]*alpha);
}
inline float clamp(float v, float c) { return (v < -c) ? -c : (v > c) ? c : v; }
......@@ -154,7 +179,7 @@ void FixedBuffer<T,CHAN,FRAME,SIZE>::write(const std::vector<T> &in) {
++write_position_;
}
}
if (write_position_ > 20 && read_position_ < 0) read_position_ = 0;
if (write_position_ > 5 && read_position_ < 0) read_position_ = 0;
}
template <typename T, int CHAN, int FRAME, int SIZE>
......@@ -170,10 +195,16 @@ void FixedBuffer<T,CHAN,FRAME,SIZE>::read(std::vector<T> &out, int count) {
// ==== Common forms ===========================================================
template <int SIZE>
using StereoBuffer16 = ftl::audio::FixedBuffer<short,2,256,SIZE>;
using StereoBuffer16 = ftl::audio::FixedBuffer<short,2,960,SIZE>;
template <int SIZE>
using MonoBuffer16 = ftl::audio::FixedBuffer<short,1,960,SIZE>;
template <int SIZE>
using StereoBufferF = ftl::audio::FixedBuffer<float,2,960,SIZE>;
template <int SIZE>
using MonoBuffer16 = ftl::audio::FixedBuffer<short,1,256,SIZE>;
using MonoBufferF = ftl::audio::FixedBuffer<float,1,960,SIZE>;
}
}
......
#ifndef _FTL_AUDIO_DECODER_HPP_
#define _FTL_AUDIO_DECODER_HPP_
#include <vector>
#include <ftl/codecs/packet.hpp>
#include <ftl/codecs/codecs.hpp>
namespace ftl {
namespace audio {
class Decoder {
public:
Decoder() { };
virtual ~Decoder() { };
virtual bool decode(const ftl::codecs::Packet &pkt, std::vector<float> &out)=0;
virtual bool accepts(const ftl::codecs::Packet &)=0;
};
}
}
#endif
\ No newline at end of file
#ifndef _FTL_AUDIO_ENCODER_HPP_
#define _FTL_AUDIO_ENCODER_HPP_
#include <vector>
#include <ftl/codecs/packet.hpp>
#include <ftl/codecs/codecs.hpp>
namespace ftl {
namespace audio {
class Encoder {
public:
Encoder() {};
virtual ~Encoder() {};
virtual bool encode(const std::vector<float> &in, ftl::codecs::Packet &pkt)=0;
virtual void reset() {}
virtual bool supports(ftl::codecs::codec_t codec)=0;
};
}
}
#endif
......@@ -2,47 +2,25 @@
#ifndef _FTL_AUDIO_FRAME_HPP_
#define _FTL_AUDIO_FRAME_HPP_
#include <ftl/data/framestate.hpp>
#include <ftl/data/frame.hpp>
#include <ftl/data/new_frame.hpp>
#include <ftl/audio/audio.hpp>
namespace ftl {
namespace audio {
static constexpr int kFrameSize = 960;
static constexpr int kSampleRate = 48000;
typedef ftl::data::Frame Frame;
typedef ftl::audio::Audio AudioFrame;
struct AudioSettings {
int sample_rate;
int frame_size;
int channels;
};
struct AudioData {
template <typename T>
const T &as() const {
throw FTL_Error("Type not valid for audio channel");
}
template <typename T>
T &as() {
throw FTL_Error("Type not valid for audio channel");
}
template <typename T>
T &make() {
throw FTL_Error("Type not valid for audio channel");
}
inline void reset() {}
Audio data;
};
// Specialisations for getting Audio data.
template <> Audio &AudioData::as<Audio>();
template <> const Audio &AudioData::as<Audio>() const;
template <> Audio &AudioData::make<Audio>();
typedef ftl::data::FrameState<AudioSettings,2> FrameState;
typedef ftl::data::Frame<32,2,FrameState,AudioData> Frame;
}
}
......
......@@ -2,12 +2,12 @@
#define _FTL_AUDIO_FRAMESET_HPP_
#include <ftl/audio/frame.hpp>
#include <ftl/data/frameset.hpp>
#include <ftl/data/new_frameset.hpp>
namespace ftl {
namespace audio {
typedef ftl::data::FrameSet<ftl::audio::Frame> FrameSet;
typedef ftl::data::FrameSet FrameSet;
}
}
......
#ifndef _FTL_AUDIO_MIXER_HPP_
#define _FTL_AUDIO_MIXER_HPP_
#include <Eigen/Eigen>
#include <vector>
#include <cmath>
#include "buffer.hpp"
namespace ftl {
namespace audio {
//static constexpr int kBufferCount = 100;
/**
* A fast circular buffer to capture, play and manipulate audio data.
* This class can be used directly with portaudio. The hardware uses
* `readFrame` and `writeFrame` to consume or append audio data. A more
* advanced `write` function allows for non-frame aligned data and for time
* dilation / shifting, and amplitude control.
*/
template <typename T, int CHAN, int FRAME, int SIZE>
class FixedMixer : public ftl::audio::Buffer<T> {
public:
FixedMixer() : Buffer<T>(CHAN, FRAME, 44100) { }
explicit FixedMixer(int rate) : Buffer<T>(CHAN, FRAME, rate) { }
inline int maxFrames() const { return SIZE; }
inline void readFrame(T *d) {
T* __restrict out = d;
if (read_position_ >= write_position_) {
std::fill(out, out+CHAN*FRAME, T(0));
} else {
const T* __restrict in = data_[(read_position_++) % SIZE];
std::copy(in, in+CHAN*FRAME, out);
}
}
int size() const override { return (read_position_>=0) ? write_position_ - read_position_ : 0; }
int frames() const override { return (read_position_>=0) ? write_position_ - read_position_ : 0; }
/**
* Append sound samples to the end of the buffer. The samples may be over
* or under sampled so as to gradually introduce or remove a requested
* delay and hence change the latency of the audio.
*/
void write(const std::vector<T> &in) override;
inline void write(int track, const std::vector<T> &in) {
tracks_.at(track).write(in);
}
void mix();
void read(std::vector<T> &out, int frames) override;
void reset() override {
Buffer<T>::reset();
write_position_ = 0;
read_position_ = 0;
for (auto &t : tracks_) t.reset();
}
inline int writePosition() const { return write_position_; }
inline int readPosition() const { return read_position_; }
inline int tracks() const { return track_num_; }
inline void setDelay(int track, float d) { tracks_.at(track).setDelay(d); }
inline float delay(int track) const { return tracks_.at(track).delay(); }
inline void setGain(int track, float g) { tracks_.at(track).setGain(g); }
inline float gain(int track) const { return tracks_.at(track).gain(); }
//void resize(int tracks);
int add(const std::string &name);
const std::string &name(int track) const { return names_.at(track); }
private:
int track_num_=0;
int write_position_=0;
int read_position_=0;
alignas(32) T data_[SIZE][CHAN*FRAME];
std::vector<ftl::audio::FixedBuffer<T,CHAN,FRAME,SIZE>> tracks_;
std::vector<std::string> names_;
};
// ==== Implementations ========================================================
template <typename T, int CHAN, int FRAME, int SIZE>
void FixedMixer<T,CHAN,FRAME,SIZE>::write(const std::vector<T> &in) {
// Not supported...
}
template <typename T, int CHAN, int FRAME, int SIZE>
void FixedMixer<T,CHAN,FRAME,SIZE>::mix() {
if (track_num_ == 0) return;
// Add together up to most recent frame
int min_write = std::numeric_limits<int>::max();
for (auto &t : tracks_) {
min_write = std::min(t.writePosition(), min_write);
}
// For each frame
while (write_position_ < min_write) {
int wp = write_position_ % SIZE;
float *ptr1 = data_[wp];
// For each block of 8 float samples
for (size_t i=0; i<CHAN*FRAME; i+=8) {
Eigen::Map<Eigen::Matrix<float,8,1>,Eigen::Aligned32> v1(ptr1+i);
v1.setZero();
// For each track, accumulate the samples
for (auto &t : tracks_) {
const Eigen::Map<Eigen::Matrix<float,8,1>,Eigen::Aligned32> v2(&t.data(wp)[i]);
v1 += t.gain()*v2;
}
v1 *= this->gain_;
}
++write_position_;
}
}
template <typename T, int CHAN, int FRAME, int SIZE>
void FixedMixer<T,CHAN,FRAME,SIZE>::read(std::vector<T> &out, int count) {
out.resize(FRAME*count*CHAN);
T *ptr = out.data();
for (int i=0; i<count; ++i) {
// TODO: Do mix here directly
readFrame(ptr);
ptr += FRAME*CHAN;
}
}
/*template <typename T, int CHAN, int FRAME, int SIZE>
void FixedMixer<T,CHAN,FRAME,SIZE>::resize(int t) {
if (track_num_ == t) return;
track_num_ = t;
tracks_.reserve(t);
while (static_cast<int>(tracks_.size()) < t) {
auto &tr = tracks_.emplace_back();
tr.setWritePosition(write_position_);
}
}*/
template <typename T, int CHAN, int FRAME, int SIZE>
int FixedMixer<T,CHAN,FRAME,SIZE>::add(const std::string &name) {
names_.push_back(name);
auto &tr = tracks_.emplace_back();
tr.setWritePosition(write_position_);
return track_num_++;
}
// ==== Common forms ===========================================================
template <int SIZE>
using StereoMixerF = ftl::audio::FixedMixer<float,2,960,SIZE>;
template <int SIZE>
using MonoMixerF = ftl::audio::FixedMixer<float,1,960,SIZE>;
}
}
#endif // _FTL_AUDIO_BUFFER_HPP_
#ifndef _FTL_AUDIO_SOFTWARE_DECODER_HPP_
#define _FTL_AUDIO_SOFTWARE_DECODER_HPP_
#include <ftl/audio/decoder.hpp>
struct OpusMSDecoder;
namespace ftl {
namespace audio {
class SoftwareDecoder : public ftl::audio::Decoder {
public:
SoftwareDecoder();
~SoftwareDecoder();
bool decode(const ftl::codecs::Packet &pkt, std::vector<float> &out) override;
bool accepts(const ftl::codecs::Packet &) override;
private:
OpusMSDecoder *opus_decoder_;
bool cur_stereo_;
ftl::codecs::definition_t cur_definition_;
bool _decodeOpus(const ftl::codecs::Packet &pkt, std::vector<float> &out);
bool _decodeRaw(const ftl::codecs::Packet &pkt, std::vector<float> &out);
bool _createOpus(const ftl::codecs::Packet &pkt);
};
}
}
#endif
#ifndef _FTL_AUDIO_SOFTWARE_ENCODER_HPP_
#define _FTL_AUDIO_SOFTWARE_ENCODER_HPP_
#include <ftl/audio/encoder.hpp>
struct OpusMSEncoder;
namespace ftl {
namespace audio {
class SoftwareEncoder : public ftl::audio::Encoder {
public:
SoftwareEncoder();
~SoftwareEncoder();
bool encode(const std::vector<float> &in, ftl::codecs::Packet &pkt) override;
void reset() override;
bool supports(ftl::codecs::codec_t codec) override;
private:
OpusMSEncoder *opus_encoder_;
bool cur_stereo_;
ftl::codecs::definition_t cur_definition_;
uint8_t cur_bitrate_;
bool _encodeRaw(const std::vector<float> &in, ftl::codecs::Packet &pkt);
bool _encodeOpus(const std::vector<float> &in, ftl::codecs::Packet &pkt);
bool _createOpus(ftl::codecs::Packet &pkt);
};
}
}
#endif
......@@ -3,6 +3,7 @@
#include <ftl/audio/buffer.hpp>
#include <ftl/audio/frameset.hpp>
#include <ftl/data/creators.hpp>
#include <ftl/configurable.hpp>
#include <ftl/config.h>
......@@ -13,40 +14,23 @@
namespace ftl {
namespace audio {
static constexpr int kFrameSize = 256;
typedef ftl::data::Generator<ftl::audio::FrameSet> Generator;
class Source : public ftl::Configurable, public ftl::audio::Generator {
class Source : public ftl::Configurable, public ftl::data::DiscreteSource {
public:
explicit Source(nlohmann::json &config);
~Source();
/** Number of frames in last frameset. This can change over time. */
size_t size() override;
bool capture(int64_t ts) override;
/**
* Get the persistent state object for a frame. An exception is thrown
* for a bad index.
*/
ftl::audio::FrameState &state(size_t ix) override;
/** Register a callback to receive new frame sets. */
void onFrameSet(const ftl::audio::FrameSet::Callback &) override;
bool retrieve(ftl::data::Frame &) override;
private:
ftl::audio::FrameState state_;
bool active_;
ftl::timer::TimerHandle timer_hp_;
ftl::timer::TimerHandle timer_main_;
ftl::audio::FrameSet::Callback cb_;
ftl::audio::AudioSettings settings_;
ftl::audio::Buffer<short> *buffer_;
ftl::audio::Buffer<float> *buffer_;
int to_read_;
int64_t latency_;
ftl::audio::FrameSet frameset_;
#ifdef HAVE_PORTAUDIO
PaStream *stream_;
#endif
......
......@@ -19,13 +19,19 @@ class Speaker : public ftl::Configurable {
~Speaker();
void queue(int64_t ts, ftl::audio::Frame &fs);
void queue(int64_t ts, const ftl::audio::Audio &af);
void setDelay(int64_t ms);
void setVolume(float value);
float volume();
void reset() { if (buffer_) buffer_->reset(); }
private:
ftl::audio::Buffer<short> *buffer_;
ftl::audio::Buffer<float> *buffer_;
bool active_;
float extra_delay_;
float volume_;
int64_t latency_;
#ifdef HAVE_PORTAUDIO
......
#include <ftl/audio/portaudio.hpp>
#include <ftl/config.h>
#include <ftl/threads.hpp>
#include <loguru.hpp>
#include <atomic>
static std::atomic<int> counter = 0;
static MUTEX pa_mutex;
#ifdef HAVE_PORTAUDIO
#include <portaudio.h>
void ftl::audio::pa_init() {
// TODO: Mutex lock?
UNIQUE_LOCK(pa_mutex, lk);
if (counter == 0) {
auto err = Pa_Initialize();
if (err != paNoError) {
......@@ -34,7 +36,7 @@ void ftl::audio::pa_init() {
}
void ftl::audio::pa_final() {
// TODO: Mutex lock?
UNIQUE_LOCK(pa_mutex, lk);
--counter;
if (counter == 0) {
auto err = Pa_Terminate();
......
#include <ftl/audio/software_decoder.hpp>
#include <ftl/config.h>
#ifdef HAVE_OPUS
#include <opus/opus_multistream.h>
#else
struct OpusMSDecoder {};
#endif
#define LOGURU_REPLACE_GLOG 1
#include <loguru.hpp>
#define FRAME_SIZE 960
using ftl::audio::SoftwareDecoder;
using ftl::codecs::codec_t;
SoftwareDecoder::SoftwareDecoder() : opus_decoder_(nullptr) {
}
SoftwareDecoder::~SoftwareDecoder() {
}
bool SoftwareDecoder::_createOpus(const ftl::codecs::Packet &pkt) {
#ifdef HAVE_OPUS
bool stereo = pkt.flags & ftl::codecs::kFlagStereo;
if (opus_decoder_ && stereo == cur_stereo_) return true;
cur_stereo_ = stereo;
if (opus_decoder_) {
opus_multistream_decoder_destroy(opus_decoder_);
opus_decoder_ = nullptr;
}
int sample_rate = 48000; // TODO: Allow it to be different
int errcode = 0;
int channels = (stereo) ? 2 : 1;
const unsigned char mapping[2] = {0,1};
opus_decoder_ = opus_multistream_decoder_create(sample_rate, channels, 1, channels-1, mapping, &errcode);
if (errcode < 0) return false;
LOG(INFO) << "Created OPUS decoder: " << sample_rate << ", " << channels;
#endif
return true;
}
bool SoftwareDecoder::decode(const ftl::codecs::Packet &pkt, std::vector<float> &out) {
switch (pkt.codec) {
case codec_t::OPUS : return _decodeOpus(pkt, out);
case codec_t::RAW : return _decodeRaw(pkt, out);
default: return false;
}
}
bool SoftwareDecoder::_decodeOpus(const ftl::codecs::Packet &pkt, std::vector<float> &out) {
#ifdef HAVE_OPUS
if (!_createOpus(pkt)) return false;
int channels = (cur_stereo_) ? 2 : 1;
out.resize(10*FRAME_SIZE*channels);
const unsigned char *inptr = pkt.data.data();
float *outptr = out.data();
int count = 0;
int frames = 0;
for (size_t i=0; i<pkt.data.size(); ) {
const short *len = (const short*)inptr;
if (*len == 0) break;
if (frames == 10) break;
inptr += 2;
i += (*len)+2;
int samples = opus_multistream_decode_float(opus_decoder_, inptr, *len, outptr, FRAME_SIZE, 0);
if (samples != FRAME_SIZE) {
LOG(ERROR) << "Failed to Opus decode: " << samples;
//return false;
break;
}
inptr += *len;
outptr += FRAME_SIZE*channels;
count += samples;
++frames;
}
out.resize(count*channels);
//LOG(INFO) << "Received " << frames << " Opus frames";
return true;
#else
LOG(WARNING) << "No Opus decoder installed";
return false;
#endif
}
bool SoftwareDecoder::_decodeRaw(const ftl::codecs::Packet &pkt, std::vector<float> &out) {
size_t size = pkt.data.size()/sizeof(float);
out.resize(size);
auto *ptr = (float*)pkt.data.data();
for (size_t i=0; i<size; i++) out.data()[i] = ptr[i];
return true;
}
bool SoftwareDecoder::accepts(const ftl::codecs::Packet &) {
return false;
}
#include <ftl/audio/software_encoder.hpp>
#include <ftl/config.h>
#ifdef HAVE_OPUS
#include <opus/opus_multistream.h>
#else
struct OpusMSEncoder {};
#endif
#define LOGURU_REPLACE_GLOG 1
#include <loguru.hpp>
using ftl::audio::SoftwareEncoder;
using ftl::codecs::codec_t;
#define FRAME_SIZE 960
#define MAX_PACKET_SIZE (3*2*FRAME_SIZE)
SoftwareEncoder::SoftwareEncoder() : ftl::audio::Encoder(), opus_encoder_(nullptr), cur_stereo_(false), cur_bitrate_(0) {
}
SoftwareEncoder::~SoftwareEncoder() {
}
bool SoftwareEncoder::encode(const std::vector<float> &in, ftl::codecs::Packet &pkt) {
auto codec = (pkt.codec == codec_t::Any) ? codec_t::OPUS : pkt.codec;
// Force RAW if no opus
#ifndef HAVE_OPUS
codec = codec_t::RAW;
#endif
pkt.codec = codec;
switch (codec) {
case codec_t::OPUS : return _encodeOpus(in, pkt);
case codec_t::RAW : return _encodeRaw(in, pkt);
default: return false;
}
}
bool SoftwareEncoder::_createOpus(ftl::codecs::Packet &pkt) {
#ifdef HAVE_OPUS
bool stereo = pkt.flags & ftl::codecs::kFlagStereo;
if (opus_encoder_ && stereo == cur_stereo_) return true;
cur_stereo_ = stereo;
if (opus_encoder_) {
opus_multistream_encoder_destroy(opus_encoder_);
opus_encoder_ = nullptr;
}
int sample_rate = 48000; // TODO: Allow it to be different
int errcode = 0;
int channels = (stereo) ? 2 : 1;
const unsigned char mapping[2] = {0,1};
opus_encoder_ = opus_multistream_encoder_create(sample_rate, channels, 1, channels-1, mapping, OPUS_APPLICATION_VOIP, &errcode);
if (errcode < 0) return false;
LOG(INFO) << "Created OPUS encoder";
#endif
return true;
}
bool SoftwareEncoder::_encodeOpus(const std::vector<float> &in, ftl::codecs::Packet &pkt) {
#ifdef HAVE_OPUS
static const float MAX_BITRATE = 128000.0f;
static const float MIN_BITRATE = 24000.0f;
if (!_createOpus(pkt)) return false;
if (pkt.bitrate != cur_bitrate_) {
int bitrate = (MAX_BITRATE-MIN_BITRATE) * (float(pkt.bitrate)/255.0f) + MIN_BITRATE;
if (!cur_stereo_) bitrate /= 2;
int errcode = opus_multistream_encoder_ctl(opus_encoder_, OPUS_SET_BITRATE(bitrate));
if (errcode < 0) return false;
LOG(INFO) << "OPUS encoder: bitrate = " << bitrate;
cur_bitrate_ = pkt.bitrate;
}
int channels = (cur_stereo_) ? 2 : 1;
int frame_est = (in.size() / (channels*FRAME_SIZE))+1;
size_t insize = pkt.data.size();
pkt.data.resize(insize+MAX_PACKET_SIZE*frame_est);
int count = 0;
int frames = 0;
unsigned char *outptr = pkt.data.data()+insize;
//LOG(INFO) << "Encode " << (in.size() / (channels*FRAME_SIZE)) << " audio frames";
for (unsigned int i=0; i<in.size(); i+=channels*FRAME_SIZE) {
short *len = (short*)outptr;
outptr += 2;
int nbBytes = opus_multistream_encode_float(opus_encoder_, &in.data()[i], FRAME_SIZE, outptr, MAX_PACKET_SIZE);
if (nbBytes <= 0) return false;
//if (nbBytes > 32000) LOG(WARNING) << "Packet exceeds size limit";
*len = nbBytes;
count += nbBytes+2;
outptr += nbBytes;
++frames;
}
pkt.data.resize(insize+count);
//LOG(INFO) << "Opus Encode = " << pkt.data.size() << ", " << frames;
return true;
#else
return false;
#endif
}
bool SoftwareEncoder::_encodeRaw(const std::vector<float> &in, ftl::codecs::Packet &pkt) {
const unsigned char *ptr = (unsigned char*)in.data();
pkt.data = std::move(std::vector<unsigned char>(ptr, ptr+in.size()*sizeof(float)));
return true;
}
void SoftwareEncoder::reset() {
}
bool SoftwareEncoder::supports(ftl::codecs::codec_t codec) {
return false;
}
......@@ -7,8 +7,6 @@
using ftl::audio::Source;
using ftl::audio::Frame;
using ftl::audio::FrameSet;
using ftl::audio::FrameState;
using ftl::audio::Audio;
using ftl::codecs::Channel;
......@@ -23,7 +21,7 @@ static int pa_source_callback(const void *input, void *output,
PaStreamCallbackFlags statusFlags, void *userData) {
auto *buffer = (BUFFER*)userData;
short *in = (short*)input;
float *in = (float*)input;
buffer->writeFrame(in);
return 0;
}
......@@ -76,16 +74,16 @@ Source::Source(nlohmann::json &config) : ftl::Configurable(config), buffer_(null
//}
if (channels >= 2) {
buffer_ = new ftl::audio::StereoBuffer16<100>(48000);
buffer_ = new ftl::audio::StereoBufferF<100>(48000);
} else {
buffer_ = new ftl::audio::MonoBuffer16<100>(48000);
buffer_ = new ftl::audio::MonoBufferF<100>(48000);
}
PaStreamParameters inputParameters;
//bzero( &inputParameters, sizeof( inputParameters ) );
inputParameters.channelCount = channels;
inputParameters.device = device;
inputParameters.sampleFormat = paInt16;
inputParameters.sampleFormat = paFloat32;
inputParameters.suggestedLatency = (device >= 0) ? Pa_GetDeviceInfo(device)->defaultLowInputLatency : 0;
inputParameters.hostApiSpecificStreamInfo = NULL;
......@@ -101,7 +99,7 @@ Source::Source(nlohmann::json &config) : ftl::Configurable(config), buffer_(null
48000, // Sample rate
ftl::audio::kFrameSize, // Size of single frame
paNoFlag,
(buffer_->channels() == 1) ? pa_source_callback<ftl::audio::MonoBuffer16<100>> : pa_source_callback<ftl::audio::StereoBuffer16<100>>,
(buffer_->channels() == 1) ? pa_source_callback<ftl::audio::MonoBufferF<100>> : pa_source_callback<ftl::audio::StereoBufferF<100>>,
this->buffer_
);
} else {
......@@ -109,10 +107,10 @@ Source::Source(nlohmann::json &config) : ftl::Configurable(config), buffer_(null
&stream_,
channels,
0,
paInt16,
paFloat32,
48000, // Sample rate
ftl::audio::kFrameSize, // Size of single frame
(buffer_->channels() == 1) ? pa_source_callback<ftl::audio::MonoBuffer16<100>> : pa_source_callback<ftl::audio::StereoBuffer16<100>>,
(buffer_->channels() == 1) ? pa_source_callback<ftl::audio::MonoBufferF<100>> : pa_source_callback<ftl::audio::StereoBufferF<100>>,
this->buffer_
);
}
......@@ -135,51 +133,10 @@ Source::Source(nlohmann::json &config) : ftl::Configurable(config), buffer_(null
to_read_ = 0;
ftl::audio::AudioSettings settings;
settings.channels = channels;
settings.sample_rate = 48000;
settings.frame_size = 256;
state_.setLeft(settings);
timer_hp_ = ftl::timer::add(ftl::timer::kTimerHighPrecision, [this](int64_t ts) {
if (buffer_) to_read_ = buffer_->size();
return true;
});
timer_main_ = ftl::timer::add(ftl::timer::kTimerMain, [this](int64_t ts) {
// Remove one interval since the audio starts from the last frame
frameset_.timestamp = ts - ftl::timer::getInterval() + latency_;
frameset_.id = 0;
frameset_.count = 1;
//frameset_.stale = false;
frameset_.clear(ftl::data::FSFlag::STALE);
if (to_read_ < 1 || !buffer_) return true;
if (frameset_.frames.size() < 1) frameset_.frames.emplace_back();
auto &frame = frameset_.frames[0];
frame.reset();
frame.setOrigin(&state_);
std::vector<short> &data = frame.create<Audio>((buffer_->channels() == 2) ? Channel::AudioStereo : Channel::AudioMono).data();
/*data.resize(ftl::audio::kFrameSize*to_read_*channels_); // For stereo * 2
short *ptr = data.data();
for (int i=0; i<to_read_; ++i) {
if (channels_ == 1) mono_buffer_.readFrame(ptr);
else stereo_buffer_.readFrame(ptr);
ptr += ftl::audio::kFrameSize*channels_; // For stereo * 2
}*/
buffer_->read(data, to_read_);
// Then do something with the data!
//LOG(INFO) << "Audio Frames Sent: " << to_read_ << " - " << ltime;
if (cb_) cb_(frameset_);
return true;
});
settings_.channels = channels;
settings_.sample_rate = 48000;
settings_.frame_size = 960;
//state_.setLeft(settings);
LOG(INFO) << "Microphone ready.";
......@@ -196,7 +153,7 @@ Source::~Source() {
active_ = false;
#ifdef HAVE_PORTAUDIO
auto err = Pa_StopStream(stream_);
auto err = Pa_AbortStream(stream_);
if (err != paNoError) {
LOG(ERROR) << "Portaudio stop stream error: " << Pa_GetErrorText(err);
......@@ -216,15 +173,20 @@ Source::~Source() {
#endif
}
size_t Source::size() {
return 1;
bool Source::capture(int64_t ts) {
if (buffer_) to_read_ = buffer_->size();
return true;
}
ftl::audio::FrameState &Source::state(size_t ix) {
if (ix >= 1) throw FTL_Error("State index out-of-bounds");
return state_;
}
bool Source::retrieve(ftl::data::Frame &frame) {
// Remove one interval since the audio starts from the last frame
//frameset_.timestamp = ts - ftl::timer::getInterval() + latency_;
void Source::onFrameSet(const ftl::audio::FrameSet::Callback &cb) {
cb_ = cb;
if (to_read_ < 1 || !buffer_) return true;
auto alist = frame.create<std::list<Audio>>((buffer_->channels() == 2) ? Channel::AudioStereo : Channel::AudioMono);
Audio aframe;
std::vector<float> &data = aframe.data();
buffer_->read(data, to_read_);
alist = std::move(aframe);
return true;
}
......@@ -8,7 +8,6 @@
using ftl::audio::Speaker;
using ftl::audio::Frame;
using ftl::audio::FrameSet;
using ftl::audio::FrameState;
using ftl::audio::Audio;
using ftl::codecs::Channel;
......@@ -21,7 +20,7 @@ static int pa_speaker_callback(const void *input, void *output,
PaStreamCallbackFlags statusFlags, void *userData) {
auto *buffer = (BUFFER*)userData; // ftl::audio::MonoBuffer16<2000>
short *out = (short*)output;
float *out = (float*)output;
buffer->readFrame(out);
......@@ -30,20 +29,20 @@ static int pa_speaker_callback(const void *input, void *output,
#endif
Speaker::Speaker(nlohmann::json &config) : ftl::Configurable(config), buffer_(nullptr) {
Speaker::Speaker(nlohmann::json &config) : ftl::Configurable(config), buffer_(nullptr), stream_(nullptr) {
#ifdef HAVE_PORTAUDIO
ftl::audio::pa_init();
#else // No portaudio
active_ = false;
LOG(ERROR) << "No audio support";
#endif
extra_delay_ = value("delay",0.0f);
on("delay", [this](const ftl::config::Event &e) {
extra_delay_ = value("delay",0.0f);
volume_ = 1.0f;
active_ = false;
extra_delay_ = value("delay",0.1f);
on("delay", [this]() {
extra_delay_ = value("delay",0.1f);
setDelay(0);
});
setDelay(0);
}
Speaker::~Speaker() {
......@@ -51,7 +50,7 @@ Speaker::~Speaker() {
active_ = false;
#ifdef HAVE_PORTAUDIO
auto err = Pa_StopStream(stream_);
auto err = Pa_AbortStream(stream_);
if (err != paNoError) {
LOG(ERROR) << "Portaudio stop stream error: " << Pa_GetErrorText(err);
......@@ -80,16 +79,16 @@ void Speaker::_open(int fsize, int sample, int channels) {
if (sample == 0 || channels == 0) return;
if (channels >= 2) {
buffer_ = new ftl::audio::StereoBuffer16<2000>(sample);
buffer_ = new ftl::audio::StereoBufferF<2000>(sample);
} else {
buffer_ = new ftl::audio::MonoBuffer16<2000>(sample);
buffer_ = new ftl::audio::MonoBufferF<2000>(sample);
}
PaStreamParameters outputParameters;
//bzero( &inputParameters, sizeof( inputParameters ) );
outputParameters.channelCount = channels;
outputParameters.device = Pa_GetDefaultOutputDevice();
outputParameters.sampleFormat = paInt16;
outputParameters.sampleFormat = paFloat32;
outputParameters.suggestedLatency = Pa_GetDeviceInfo(outputParameters.device)->defaultLowOutputLatency;
outputParameters.hostApiSpecificStreamInfo = NULL;
......@@ -101,9 +100,9 @@ void Speaker::_open(int fsize, int sample, int channels) {
NULL,
&outputParameters,
sample, // Sample rate
256, // Size of single frame
960, // Size of single frame
paNoFlag,
(channels == 1) ? pa_speaker_callback<ftl::audio::MonoBuffer16<2000>> : pa_speaker_callback<ftl::audio::StereoBuffer16<2000>>,
(channels == 1) ? pa_speaker_callback<ftl::audio::MonoBufferF<2000>> : pa_speaker_callback<ftl::audio::StereoBufferF<2000>>,
this->buffer_
);
......@@ -130,15 +129,29 @@ void Speaker::_open(int fsize, int sample, int channels) {
}
void Speaker::queue(int64_t ts, ftl::audio::Frame &frame) {
auto &audio = frame.get<ftl::audio::Audio>((frame.hasChannel(Channel::AudioStereo)) ? Channel::AudioStereo : Channel::AudioMono);
const auto &audio = frame.get<std::list<ftl::audio::Audio>>
((frame.hasChannel(Channel::AudioStereo)) ? Channel::AudioStereo : Channel::AudioMono);
if (!buffer_) {
_open(960, 48000, (frame.hasChannel(Channel::AudioStereo)) ? 2 : 1);
}
if (!buffer_) return;
//LOG(INFO) << "Buffer Fullness (" << ts << "): " << buffer_->size() << " - " << audio.size();
for (const auto &d : audio) {
buffer_->write(d.data());
}
//LOG(INFO) << "Audio delay: " << buffer_.delay() << "s";
}
void Speaker::queue(int64_t ts, const ftl::audio::Audio &d) {
if (!buffer_) {
_open(256, frame.getSettings().sample_rate, frame.getSettings().channels);
_open(960, 48000, 2);
}
if (!buffer_) return;
//LOG(INFO) << "Buffer Fullness (" << ts << "): " << buffer_->size() << " - " << audio.size();
buffer_->write(audio.data());
buffer_->write(d.data());
//LOG(INFO) << "Audio delay: " << buffer_.delay() << "s";
}
......@@ -148,6 +161,16 @@ void Speaker::setDelay(int64_t ms) {
if (d < 0.0f) d = 0.001f; // Clamp to 0 delay (not ideal to be exactly 0)
if (buffer_) {
buffer_->setDelay(d);
//LOG(INFO) << "Audio delay: " << buffer_->delay();
LOG(INFO) << "Audio delay: " << buffer_->delay();
}
}
void Speaker::setVolume(float value) {
// TODO: adjust volume using system mixer
volume_ = std::max(0.0f, std::min(1.0f, value));
if (buffer_) buffer_->setGain(volume_);
}
float Speaker::volume() {
return volume_;
}