From 3b1f043633974b22853eb15213ce076c8d5bae12 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrien=20B=C3=A9raud?= Date: Wed, 26 Feb 2020 11:02:46 -0500 Subject: [PATCH] audiolayer: use Speex AEC Change-Id: Ieb36dc5b97e8a451160b59079353c82f07d049be --- CMakeLists.txt | 70 ++-- compat/msvc/package.json | 3 +- contrib/src/speexdsp/package.json | 15 + contrib/src/speexdsp/speexdsp_vs_proj.patch | 398 +++++++++++++++++++ src/client/videomanager.cpp | 16 + src/dring/videomanager_interface.h | 2 + src/media/audio/alsa/alsalayer.cpp | 6 +- src/media/audio/alsa/alsalayer.h | 1 - src/media/audio/audio_frame_resizer.cpp | 2 +- src/media/audio/audio_input.cpp | 2 +- src/media/audio/audio_receive_thread.cpp | 2 +- src/media/audio/audiolayer.cpp | 136 ++++++- src/media/audio/audiolayer.h | 30 +- src/media/audio/coreaudio/ios/corelayer.h | 2 - src/media/audio/coreaudio/ios/corelayer.mm | 5 +- src/media/audio/coreaudio/osx/corelayer.cpp | 5 +- src/media/audio/coreaudio/osx/corelayer.h | 2 - src/media/audio/jack/jacklayer.cpp | 3 +- src/media/audio/jack/jacklayer.h | 1 - src/media/audio/opensl/audio_player.cpp | 2 - src/media/audio/opensl/audio_player.h | 1 - src/media/audio/opensl/audio_recorder.cpp | 84 +++- src/media/audio/opensl/audio_recorder.h | 3 + src/media/audio/opensl/opensllayer.cpp | 31 +- src/media/audio/opensl/opensllayer.h | 212 +++++----- src/media/audio/portaudio/portaudiolayer.cpp | 9 +- src/media/audio/pulseaudio/pulselayer.cpp | 30 +- src/media/audio/pulseaudio/pulselayer.h | 13 - src/media/audio/ringbuffer.h | 6 +- src/media/audio/ringbufferpool.cpp | 4 +- 30 files changed, 859 insertions(+), 237 deletions(-) create mode 100644 contrib/src/speexdsp/package.json create mode 100644 contrib/src/speexdsp/speexdsp_vs_proj.patch diff --git a/CMakeLists.txt b/CMakeLists.txt index b492285d6..2ddeeec2b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -135,12 +135,12 @@ if(MSVC) "${CMAKE_CURRENT_SOURCE_DIR}/src/jamidht;" "${CMAKE_CURRENT_SOURCE_DIR}/src/security;" "${CMAKE_CURRENT_SOURCE_DIR}/src/sip;" - "${CMAKE_CURRENT_SOURCE_DIR}/compat/msvc;" "${CMAKE_CURRENT_SOURCE_DIR}/src/upnp;" "${CMAKE_CURRENT_SOURCE_DIR}/src/upnp/igd;" "${CMAKE_CURRENT_SOURCE_DIR}/src/upnp/protocol;" "${CMAKE_CURRENT_SOURCE_DIR}/src/upnp/mapping;" "${CMAKE_CURRENT_SOURCE_DIR}/src/jamidht/eth;" + "${CMAKE_CURRENT_SOURCE_DIR}/compat/msvc;" "${CMAKE_CURRENT_SOURCE_DIR}/contrib/msvc;" "${CMAKE_CURRENT_SOURCE_DIR}/contrib/msvc/include;" "${CMAKE_CURRENT_SOURCE_DIR}/contrib/build/msgpack-c/include;" @@ -152,6 +152,7 @@ if(MSVC) "${CMAKE_CURRENT_SOURCE_DIR}/contrib/build/pjproject/pjsip/include;" "${CMAKE_CURRENT_SOURCE_DIR}/contrib/build/pjproject/third_party;" "${CMAKE_CURRENT_SOURCE_DIR}/contrib/build/pjproject/pjmedia/include" + "${CMAKE_CURRENT_SOURCE_DIR}/contrib/build/speexdsp/include;" ) endif() @@ -224,39 +225,40 @@ if(MSVC) # Dependencies ################################################################################ - set(libAdditionalDependencies "${CMAKE_STATIC_LINKER_FLAGS} /LTCG ws2_32.lib - advapi32.lib - ${CMAKE_CURRENT_SOURCE_DIR}/contrib/build/ffmpeg/Build/win32/x64/bin/avcodec.lib - ${CMAKE_CURRENT_SOURCE_DIR}/contrib/build/ffmpeg/Build/win32/x64/bin/avdevice.lib - ${CMAKE_CURRENT_SOURCE_DIR}/contrib/build/ffmpeg/Build/win32/x64/bin/avfilter.lib - ${CMAKE_CURRENT_SOURCE_DIR}/contrib/build/ffmpeg/Build/win32/x64/bin/avformat.lib - ${CMAKE_CURRENT_SOURCE_DIR}/contrib/build/ffmpeg/Build/win32/x64/bin/avutil.lib - ${CMAKE_CURRENT_SOURCE_DIR}/contrib/build/ffmpeg/Build/win32/x64/bin/swresample.lib - ${CMAKE_CURRENT_SOURCE_DIR}/contrib/build/ffmpeg/Build/win32/x64/bin/swscale.lib - ${CMAKE_CURRENT_SOURCE_DIR}/contrib/msvc/lib/x64/libgnutls.lib - ${CMAKE_CURRENT_SOURCE_DIR}/contrib/msvc/lib/x64/lib_json.lib - ${CMAKE_CURRENT_SOURCE_DIR}/contrib/msvc/lib/x64/libopendht.lib - ${CMAKE_CURRENT_SOURCE_DIR}/contrib/build/argon2/vs2015/Argon2Ref/vs2015/build/Argon2Ref.lib - ${CMAKE_CURRENT_SOURCE_DIR}/contrib/msvc/lib/x64/secp256k1.lib - ${CMAKE_CURRENT_SOURCE_DIR}/contrib/build/yaml-cpp/msvc/Release/libyaml-cppmd.lib - ${CMAKE_CURRENT_SOURCE_DIR}/contrib/msvc/lib/x64/portaudio.lib - ${CMAKE_CURRENT_SOURCE_DIR}/contrib/msvc/lib/x64/libupnp.lib - ${CMAKE_CURRENT_SOURCE_DIR}/contrib/build/natpmp/msvc/Release/natpmp.lib - ${CMAKE_CURRENT_SOURCE_DIR}/contrib/build/pjproject/pjsip/lib/pjsip-core-x86_64-x64-vc15-Release.lib - ${CMAKE_CURRENT_SOURCE_DIR}/contrib/build/pjproject/pjsip/lib/pjsip-simple-x86_64-x64-vc15-Release.lib - ${CMAKE_CURRENT_SOURCE_DIR}/contrib/build/pjproject/pjsip/lib/pjsua2-lib-x86_64-x64-vc15-Release.lib - ${CMAKE_CURRENT_SOURCE_DIR}/contrib/build/pjproject/pjsip/lib/pjsua-lib-x86_64-x64-vc15-Release.lib - ${CMAKE_CURRENT_SOURCE_DIR}/contrib/build/pjproject/pjsip/lib/pjsip-ua-x86_64-x64-vc15-Release.lib - ${CMAKE_CURRENT_SOURCE_DIR}/contrib/build/pjproject/pjmedia/lib/pjmedia-x86_64-x64-vc15-Release.lib - ${CMAKE_CURRENT_SOURCE_DIR}/contrib/build/pjproject/pjlib-util/lib/pjlib-util-x86_64-x64-vc15-Release.lib - ${CMAKE_CURRENT_SOURCE_DIR}/contrib/build/pjproject/pjlib/lib/pjlib-x86_64-x64-vc15-Release.lib - ${CMAKE_CURRENT_SOURCE_DIR}/contrib/build/pjproject/pjnath/lib/pjnath-x86_64-x64-vc15-Release.lib - ${CMAKE_CURRENT_SOURCE_DIR}/contrib/build/fmt/msvc/Release/fmt.lib - ${CMAKE_CURRENT_SOURCE_DIR}/contrib/build/http_parser/x64/Release/http-parser.lib - ${CMAKE_CURRENT_SOURCE_DIR}/contrib/build/asio/asio/msvc/x64/Release/asio.lib - ${CMAKE_CURRENT_SOURCE_DIR}/contrib/build/openssl/out32dll/libeay32.lib - ${CMAKE_CURRENT_SOURCE_DIR}/contrib/build/openssl/out32dll/ssleay32.lib - /ignore:4006 + set(libAdditionalDependencies "${CMAKE_STATIC_LINKER_FLAGS} /LTCG ws2_32.lib + advapi32.lib + ${CMAKE_CURRENT_SOURCE_DIR}/contrib/build/ffmpeg/Build/win32/x64/bin/avcodec.lib + ${CMAKE_CURRENT_SOURCE_DIR}/contrib/build/ffmpeg/Build/win32/x64/bin/avdevice.lib + ${CMAKE_CURRENT_SOURCE_DIR}/contrib/build/ffmpeg/Build/win32/x64/bin/avfilter.lib + ${CMAKE_CURRENT_SOURCE_DIR}/contrib/build/ffmpeg/Build/win32/x64/bin/avformat.lib + ${CMAKE_CURRENT_SOURCE_DIR}/contrib/build/ffmpeg/Build/win32/x64/bin/avutil.lib + ${CMAKE_CURRENT_SOURCE_DIR}/contrib/build/ffmpeg/Build/win32/x64/bin/swresample.lib + ${CMAKE_CURRENT_SOURCE_DIR}/contrib/build/ffmpeg/Build/win32/x64/bin/swscale.lib + ${CMAKE_CURRENT_SOURCE_DIR}/contrib/msvc/lib/x64/libgnutls.lib + ${CMAKE_CURRENT_SOURCE_DIR}/contrib/msvc/lib/x64/lib_json.lib + ${CMAKE_CURRENT_SOURCE_DIR}/contrib/msvc/lib/x64/libopendht.lib + ${CMAKE_CURRENT_SOURCE_DIR}/contrib/build/argon2/vs2015/Argon2Ref/vs2015/build/Argon2Ref.lib + ${CMAKE_CURRENT_SOURCE_DIR}/contrib/msvc/lib/x64/secp256k1.lib + ${CMAKE_CURRENT_SOURCE_DIR}/contrib/build/yaml-cpp/msvc/Release/libyaml-cppmd.lib + ${CMAKE_CURRENT_SOURCE_DIR}/contrib/msvc/lib/x64/portaudio.lib + ${CMAKE_CURRENT_SOURCE_DIR}/contrib/msvc/lib/x64/libupnp.lib + ${CMAKE_CURRENT_SOURCE_DIR}/contrib/build/natpmp/msvc/Release/natpmp.lib + ${CMAKE_CURRENT_SOURCE_DIR}/contrib/build/pjproject/pjsip/lib/pjsip-core-x86_64-x64-vc15-Release.lib + ${CMAKE_CURRENT_SOURCE_DIR}/contrib/build/pjproject/pjsip/lib/pjsip-simple-x86_64-x64-vc15-Release.lib + ${CMAKE_CURRENT_SOURCE_DIR}/contrib/build/pjproject/pjsip/lib/pjsua2-lib-x86_64-x64-vc15-Release.lib + ${CMAKE_CURRENT_SOURCE_DIR}/contrib/build/pjproject/pjsip/lib/pjsua-lib-x86_64-x64-vc15-Release.lib + ${CMAKE_CURRENT_SOURCE_DIR}/contrib/build/pjproject/pjsip/lib/pjsip-ua-x86_64-x64-vc15-Release.lib + ${CMAKE_CURRENT_SOURCE_DIR}/contrib/build/pjproject/pjmedia/lib/pjmedia-x86_64-x64-vc15-Release.lib + ${CMAKE_CURRENT_SOURCE_DIR}/contrib/build/pjproject/pjlib-util/lib/pjlib-util-x86_64-x64-vc15-Release.lib + ${CMAKE_CURRENT_SOURCE_DIR}/contrib/build/pjproject/pjlib/lib/pjlib-x86_64-x64-vc15-Release.lib + ${CMAKE_CURRENT_SOURCE_DIR}/contrib/build/pjproject/pjnath/lib/pjnath-x86_64-x64-vc15-Release.lib + ${CMAKE_CURRENT_SOURCE_DIR}/contrib/build/fmt/msvc/Release/fmt.lib + ${CMAKE_CURRENT_SOURCE_DIR}/contrib/build/http_parser/x64/Release/http-parser.lib + ${CMAKE_CURRENT_SOURCE_DIR}/contrib/build/asio/asio/msvc/x64/Release/asio.lib + ${CMAKE_CURRENT_SOURCE_DIR}/contrib/build/openssl/out32dll/libeay32.lib + ${CMAKE_CURRENT_SOURCE_DIR}/contrib/build/openssl/out32dll/ssleay32.lib + ${CMAKE_CURRENT_SOURCE_DIR}/contrib/build/speexdsp/lib/libspeexdsp.lib + /ignore:4006 " ) diff --git a/compat/msvc/package.json b/compat/msvc/package.json index 591abff16..0aea6554e 100644 --- a/compat/msvc/package.json +++ b/compat/msvc/package.json @@ -8,9 +8,10 @@ "pjproject", "portaudio", "secp256k1", + "speexdsp", "upnp", "yaml-cpp" ], "configuration": "ReleaseLib_win32", "project_paths": ["ring-daemon.vcxproj"] -} \ No newline at end of file +} diff --git a/contrib/src/speexdsp/package.json b/contrib/src/speexdsp/package.json new file mode 100644 index 000000000..95d427120 --- /dev/null +++ b/contrib/src/speexdsp/package.json @@ -0,0 +1,15 @@ +{ + "name": "speexdsp", + "version": "SpeexDSP-1.2.0", + "url": "https://github.com/xiph/speexdsp/archive/__VERSION__.tar.gz", + "deps": [], + "patches": ["speexdsp_vs_proj.patch"], + "win_patches": [], + "project_paths": ["win32/msvc/libspeexdsp/libspeexdsp.vcxproj"], + "with_env" : "", + "custom_scripts": { + "pre_build": [], + "build": [], + "post_build": [] + } +} \ No newline at end of file diff --git a/contrib/src/speexdsp/speexdsp_vs_proj.patch b/contrib/src/speexdsp/speexdsp_vs_proj.patch new file mode 100644 index 000000000..8165ab5f6 --- /dev/null +++ b/contrib/src/speexdsp/speexdsp_vs_proj.patch @@ -0,0 +1,398 @@ +From ba88ffed7a018d37661c9574f98f4246cdf6031c Mon Sep 17 00:00:00 2001 +From: Andreas Traczyk +Date: Wed, 19 Feb 2020 16:14:02 -0500 +Subject: [PATCH] b + +--- + win32/msvc/libspeexdsp/libspeexdsp.vcxproj | 378 +++++++++++++++++++++++++++++ + 1 file changed, 378 insertions(+) + create mode 100644 win32/msvc/libspeexdsp/libspeexdsp.vcxproj + +diff --git a/win32/msvc/libspeexdsp/libspeexdsp.vcxproj b/win32/msvc/libspeexdsp/libspeexdsp.vcxproj +new file mode 100644 +index 0000000..071c9b3 +--- /dev/null ++++ b/win32/msvc/libspeexdsp/libspeexdsp.vcxproj +@@ -0,0 +1,378 @@ ++ ++ ++ ++ ++ Debug ++ Win32 ++ ++ ++ Debug ++ x64 ++ ++ ++ Release_Dynamic_SSE ++ Win32 ++ ++ ++ Release_Dynamic_SSE ++ x64 ++ ++ ++ Release_Static_SSE ++ Win32 ++ ++ ++ Release_Static_SSE ++ x64 ++ ++ ++ Release ++ Win32 ++ ++ ++ Release ++ x64 ++ ++ ++ ++ {03207781-0D1C-4DB3-A71D-45C608F28DBD} ++ Win32Proj ++ ++ ++ ++ StaticLibrary ++ v141 ++ MultiByte ++ true ++ ++ ++ StaticLibrary ++ v141 ++ MultiByte ++ true ++ ++ ++ DynamicLibrary ++ v141 ++ MultiByte ++ true ++ ++ ++ DynamicLibrary ++ v141 ++ MultiByte ++ true ++ ++ ++ StaticLibrary ++ v141 ++ MultiByte ++ true ++ ++ ++ StaticLibrary ++ v141 ++ MultiByte ++ true ++ ++ ++ StaticLibrary ++ v141 ++ MultiByte ++ ++ ++ StaticLibrary ++ v141 ++ MultiByte ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ <_ProjectFileVersion>15.0.28127.55 ++ ++ ++ Debug\ ++ Debug\ ++ ++ ++ ++ $(Configuration)\ ++ $(Configuration)\ ++ ++ ++ $(Configuration)\ ++ $(Configuration)\ ++ ++ ++ $(Configuration)\ ++ $(Configuration)\ ++ ++ ++ ++ Disabled ++ ..\..\..\include;..\..;%(AdditionalIncludeDirectories) ++ WIN32;_DEBUG;_LIB;HAVE_CONFIG_H;%(PreprocessorDefinitions) ++ true ++ EnableFastChecks ++ MultiThreadedDebugDLL ++ ++ Level3 ++ EditAndContinue ++ CompileAsC ++ ++ ++ ../../../lib/libspeexdsp.lib ++ ++ ++ ++ ++ Disabled ++ ..\..\..\include;..\..;%(AdditionalIncludeDirectories) ++ WIN32;_DEBUG;_LIB;HAVE_CONFIG_H;%(PreprocessorDefinitions) ++ EnableFastChecks ++ MultiThreadedDebugDLL ++ ++ ++ Level3 ++ ProgramDatabase ++ CompileAsC ++ ++ ++ ../../../lib/libspeexdsp.lib ++ ++ ++ ++ ++ Full ++ AnySuitable ++ true ++ Speed ++ ..\..\..\include;..\..;%(AdditionalIncludeDirectories) ++ WIN32;NDEBUG;_LIB;HAVE_CONFIG_H;%(PreprocessorDefinitions) ++ true ++ ++ MultiThreadedDLL ++ false ++ ++ Level3 ++ ProgramDatabase ++ CompileAsC ++ 4244;4305;4311;4100;4127;%(DisableSpecificWarnings) ++ ++ ++ ../../../lib/libspeexdsp.lib ++ ++ ++ ++ ++ Full ++ AnySuitable ++ true ++ Speed ++ ..\..\..\include;..\..;%(AdditionalIncludeDirectories) ++ WIN32;NDEBUG;_LIB;HAVE_CONFIG_H;%(PreprocessorDefinitions) ++ true ++ ++ ++ MultiThreadedDLL ++ false ++ ++ ++ Level3 ++ ProgramDatabase ++ CompileAsC ++ 4244;4305;4311;4100;4127;%(DisableSpecificWarnings) ++ ++ ++ ../../../lib/libspeexdsp.lib ++ ++ ++ ++ ++ Full ++ AnySuitable ++ true ++ Speed ++ ..\..\..\include;..\..;%(AdditionalIncludeDirectories) ++ USE_SSE;WIN32;NDEBUG;_WINDOWS;_USRDLL;HAVE_CONFIG_H;%(PreprocessorDefinitions) ++ true ++ ++ MultiThreadedDLL ++ false ++ StreamingSIMDExtensions ++ ++ Level3 ++ ProgramDatabase ++ CompileAsC ++ 4244;4305;4311;4100;4127;%(DisableSpecificWarnings) ++ ++ ++ ../../../bin/libspeexdsp.dll ++ ..\..\libspeexdsp.def ++ Windows ++ true ++ true ++ false ++ ++ ../../../lib/libspeexdsp.lib ++ MachineX86 ++ ++ ++ ++ ++ Full ++ AnySuitable ++ true ++ Speed ++ ..\..\..\include;..\..;%(AdditionalIncludeDirectories) ++ USE_SSE;WIN32;NDEBUG;_WINDOWS;_USRDLL;HAVE_CONFIG_H;%(PreprocessorDefinitions) ++ true ++ ++ ++ MultiThreadedDLL ++ false ++ StreamingSIMDExtensions ++ ++ ++ Level3 ++ ProgramDatabase ++ CompileAsC ++ 4244;4305;4311;4100;4127;%(DisableSpecificWarnings) ++ ++ ++ ../../../bin/libspeexdsp.dll ++ ..\..\libspeexdsp.def ++ Windows ++ true ++ true ++ false ++ ++ ++ ../../../lib/libspeexdsp.lib ++ ++ ++ ++ ++ Full ++ AnySuitable ++ true ++ Speed ++ ..\..\..\include;..\..;%(AdditionalIncludeDirectories) ++ USE_SSE;WIN32;NDEBUG;_WINDOWS;_USRDLL;HAVE_CONFIG_H;%(PreprocessorDefinitions) ++ true ++ ++ MultiThreadedDLL ++ false ++ StreamingSIMDExtensions ++ ++ Level3 ++ ProgramDatabase ++ CompileAsC ++ 4244;4305;4311;4100;4127;%(DisableSpecificWarnings) ++ ++ ++ ../../../lib/libspeexdsp.lib ++ ++ ++ ++ ++ Full ++ AnySuitable ++ true ++ Speed ++ ..\..\..\include;..\..;%(AdditionalIncludeDirectories) ++ USE_SSE;WIN32;NDEBUG;_WINDOWS;_USRDLL;HAVE_CONFIG_H;%(PreprocessorDefinitions) ++ true ++ ++ ++ MultiThreadedDLL ++ false ++ StreamingSIMDExtensions ++ ++ ++ Level3 ++ ProgramDatabase ++ CompileAsC ++ 4244;4305;4311;4100;4127;%(DisableSpecificWarnings) ++ ++ ++ ../../../lib/libspeexdsp.lib ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ +\ No newline at end of file +-- +2.7.4 + diff --git a/src/client/videomanager.cpp b/src/client/videomanager.cpp index 2eec2b14a..672283b8c 100644 --- a/src/client/videomanager.cpp +++ b/src/client/videomanager.cpp @@ -107,6 +107,22 @@ AudioFrame::setFormat(const jami::AudioFormat& format) d->format = format.sampleFormat; } +jami::AudioFormat +AudioFrame::getFormat() const +{ + return { + (unsigned)frame_->sample_rate, + (unsigned)frame_->channels, + (AVSampleFormat)frame_->format + }; +} + +size_t +AudioFrame::getFrameSize() const +{ + return frame_->nb_samples; +} + void AudioFrame::reserve(size_t nb_samples) { diff --git a/src/dring/videomanager_interface.h b/src/dring/videomanager_interface.h index aa37b776a..c9e35138f 100644 --- a/src/dring/videomanager_interface.h +++ b/src/dring/videomanager_interface.h @@ -107,6 +107,8 @@ public: ~AudioFrame() {}; void mix(const AudioFrame& o); float calcRMS() const; + jami::AudioFormat getFormat() const; + size_t getFrameSize() const; private: void setFormat(const jami::AudioFormat& format); diff --git a/src/media/audio/alsa/alsalayer.cpp b/src/media/audio/alsa/alsalayer.cpp index ebef086b7..726998789 100644 --- a/src/media/audio/alsa/alsalayer.cpp +++ b/src/media/audio/alsa/alsalayer.cpp @@ -99,7 +99,6 @@ AlsaLayer::AlsaLayer(const AudioPreference &pref) , is_playback_open_(false) , is_capture_open_(false) , audioThread_(nullptr) - , mainRingBuffer_(Manager::instance().getRingBufferPool().getRingBuffer(RingBufferPool::DEFAULT_ID)) {} AlsaLayer::~AlsaLayer() @@ -702,10 +701,7 @@ void AlsaLayer::capture() const int framesPerBufferAlsa = 2048; toGetFrames = std::min(framesPerBufferAlsa, toGetFrames); if (auto r = read(toGetFrames)) { - if (isCaptureMuted_) - libav_utils::fillWithSilence(r->pointer()); - //dcblocker_.process(captureBuff_); - mainRingBuffer_->put(std::move(r)); + putRecorded(std::move(r)); } else JAMI_ERR("ALSA MIC : Couldn't read!"); } diff --git a/src/media/audio/alsa/alsalayer.h b/src/media/audio/alsa/alsalayer.h index 6837fe050..e28aaf2e5 100644 --- a/src/media/audio/alsa/alsalayer.h +++ b/src/media/audio/alsa/alsalayer.h @@ -250,7 +250,6 @@ class AlsaLayer : public AudioLayer { bool is_capture_open_; std::unique_ptr audioThread_; - std::shared_ptr mainRingBuffer_; }; } // namespace jami diff --git a/src/media/audio/audio_frame_resizer.cpp b/src/media/audio/audio_frame_resizer.cpp index 5e05749f1..43dfbf4e4 100644 --- a/src/media/audio/audio_frame_resizer.cpp +++ b/src/media/audio/audio_frame_resizer.cpp @@ -123,7 +123,7 @@ AudioFrameResizer::dequeue() if (samples() < frameSize_) return {}; - auto frame = std::make_unique(format_, frameSize_); + auto frame = std::make_shared(format_, frameSize_); int ret; if ((ret = av_audio_fifo_read(queue_, reinterpret_cast(frame->pointer()->data), frameSize_)) < 0) { JAMI_ERR() << "Could not read samples from queue: " << libav_utils::getError(ret); diff --git a/src/media/audio/audio_input.cpp b/src/media/audio/audio_input.cpp index dfecac6ce..1d07c5d51 100644 --- a/src/media/audio/audio_input.cpp +++ b/src/media/audio/audio_input.cpp @@ -323,7 +323,7 @@ AudioInput::createDecoder() } auto decoder = std::make_unique([this](std::shared_ptr&& frame) { - fileBuf_->put(std::move(std::static_pointer_cast(frame))); + fileBuf_->put(std::static_pointer_cast(frame)); }); // NOTE don't emulate rate, file is read as frames are needed diff --git a/src/media/audio/audio_receive_thread.cpp b/src/media/audio/audio_receive_thread.cpp index 895978bf7..a5937217b 100644 --- a/src/media/audio/audio_receive_thread.cpp +++ b/src/media/audio/audio_receive_thread.cpp @@ -59,7 +59,7 @@ AudioReceiveThread::setup() { audioDecoder_.reset(new MediaDecoder([this](std::shared_ptr&& frame) mutable { notify(frame); - ringbuffer_->put(std::move(std::static_pointer_cast(frame))); + ringbuffer_->put(std::static_pointer_cast(frame)); })); audioDecoder_->setInterruptCallback(interruptCb, this); diff --git a/src/media/audio/audiolayer.cpp b/src/media/audio/audiolayer.cpp index 0cc5f2ec1..80ddb2ef7 100644 --- a/src/media/audio/audiolayer.cpp +++ b/src/media/audio/audiolayer.cpp @@ -28,21 +28,88 @@ #include "tonecontrol.h" #include "client/ring_signal.h" +extern "C" { +#include +} + #include #include namespace jami { +struct AudioLayer::EchoState +{ + EchoState(AudioFormat format, unsigned frameSize, unsigned tailLength) + : state(speex_echo_state_init_mc( + frameSize, frameSize * 16, + format.nb_channels, format.nb_channels), &speex_echo_state_destroy) + , playbackQueue(format, frameSize) + , recordQueue(format, frameSize) { + int sr = format.sample_rate; + speex_echo_ctl(state.get(), SPEEX_ECHO_SET_SAMPLING_RATE, &sr); + } + + void putRecorded(std::shared_ptr&& in) { + // JAMI_DBG("putRecorded %s %d", in->getFormat().toString().c_str(), in->getFrameSize()); + recordQueue.enqueue(std::move(in)); + } + + void putPlayback(const std::shared_ptr& in) { + // JAMI_DBG("putPlayback %s %d", in->getFormat().toString().c_str(), in->getFrameSize()); + auto c = in; + playbackQueue.enqueue(std::move(c)); + } + + std::shared_ptr getRecorded() { + if (playbackQueue.samples() < playbackQueue.frameSize() + or recordQueue.samples() < recordQueue.frameSize()) { + /* JAMI_DBG("getRecorded underflow %d / %d, %d / %d", + playbackQueue.samples(), playbackQueue.frameSize(), + recordQueue.samples(), recordQueue.frameSize()); */ + return {}; + } + if (recordQueue.samples() > 2 * recordQueue.frameSize() && playbackQueue.samples() == 0) { + JAMI_DBG("getRecorded PLAYBACK underflow"); + return recordQueue.dequeue(); + } + while (playbackQueue.samples() > 10 * playbackQueue.frameSize()) { + JAMI_DBG("getRecorded playback underflow"); + playbackQueue.dequeue(); + } + while (recordQueue.samples() > 4 * recordQueue.frameSize()) { + JAMI_DBG("getRecorded record underflow"); + recordQueue.dequeue(); + } + auto playback = playbackQueue.dequeue(); + auto record = recordQueue.dequeue(); + if (playback and record) { + auto ret = std::make_shared(record->getFormat(), record->getFrameSize()); + speex_echo_cancellation(state.get(), + (const int16_t*)record->pointer()->data[0], + (const int16_t*)playback->pointer()->data[0], + (int16_t*)ret->pointer()->data[0]); + return ret; + } + return {}; + } + +private: + using SpeexEchoStatePtr = std::unique_ptr; + SpeexEchoStatePtr state; + AudioFrameResizer playbackQueue; + AudioFrameResizer recordQueue; +}; + AudioLayer::AudioLayer(const AudioPreference &pref) : isCaptureMuted_(pref.getCaptureMuted()) , isPlaybackMuted_(pref.getPlaybackMuted()) , captureGain_(pref.getVolumemic()) , playbackGain_(pref.getVolumespkr()) + , mainRingBuffer_(Manager::instance().getRingBufferPool().getRingBuffer(RingBufferPool::DEFAULT_ID)) , audioFormat_(Manager::instance().getRingBufferPool().getInternalAudioFormat()) , audioInputFormat_(Manager::instance().getRingBufferPool().getInternalAudioFormat()) , urgentRingBuffer_("urgentRingBuffer_id", SIZEBUF, audioFormat_) , resampler_(new Resampler) - , inputResampler_(new Resampler) , lastNotificationTime_() { urgentRingBuffer_.createReadOffset(RingBufferPool::DEFAULT_ID); @@ -51,11 +118,12 @@ AudioLayer::AudioLayer(const AudioPreference &pref) AudioLayer::~AudioLayer() {} -void AudioLayer::hardwareFormatAvailable(AudioFormat playback) +void AudioLayer::hardwareFormatAvailable(AudioFormat playback, size_t bufSize) { - JAMI_DBG("Hardware audio format available : %s", playback.toString().c_str()); + JAMI_DBG("Hardware audio format available : %s %zu", playback.toString().c_str(), bufSize); audioFormat_ = Manager::instance().hardwareAudioFormatChanged(playback); urgentRingBuffer_.setFormat(audioFormat_); + nativeFrameSize_ = bufSize; } void AudioLayer::hardwareInputFormatAvailable(AudioFormat capture) @@ -86,6 +154,37 @@ void AudioLayer::flush() urgentRingBuffer_.flushAll(); } +void AudioLayer::playbackChanged(bool started) +{ + playbackStarted_ = started; + checkAEC(); +} + +void AudioLayer::recordChanged(bool started) +{ + recordStarted_ = started; + checkAEC(); +} + +void AudioLayer::setHasNativeAEC(bool hasEAC) +{ + hasNativeAEC_ = hasEAC; + checkAEC(); +} + +void AudioLayer::checkAEC() +{ + bool shouldSoftAEC = not hasNativeAEC_ and playbackStarted_ and recordStarted_; + + if (not echoState_ and shouldSoftAEC) { + JAMI_WARN("Starting AEC"); + echoState_.reset(new EchoState(audioFormat_, nativeFrameSize_, audioFormat_.sample_rate / 4)); + } else if (echoState_ and not shouldSoftAEC) { + JAMI_WARN("Stopping AEC"); + echoState_.reset(); + } +} + void AudioLayer::putUrgent(AudioBuffer& buffer) { std::lock_guard lock(mutex_); @@ -153,19 +252,44 @@ AudioLayer::getToPlay(AudioFormat format, size_t writableSamples) std::shared_ptr playbackBuf {}; while (!(playbackBuf = playbackQueue_->dequeue())) { + std::shared_ptr resampled; + if (auto urgentSamples = urgentRingBuffer_.get(RingBufferPool::DEFAULT_ID)) { bufferPool.discard(1, RingBufferPool::DEFAULT_ID); - playbackQueue_->enqueue(resampler_->resample(std::move(urgentSamples),format)); + resampled = resampler_->resample(std::move(urgentSamples),format); } else if (auto toneToPlay = Manager::instance().getTelephoneTone()) { - playbackQueue_->enqueue(resampler_->resample(toneToPlay->getNext(), format)); + resampled = resampler_->resample(toneToPlay->getNext(), format); } else if (auto buf = bufferPool.getData(RingBufferPool::DEFAULT_ID)) { - playbackQueue_->enqueue(resampler_->resample(std::move(buf), format)); + resampled = resampler_->resample(std::move(buf), format); } else { break; } + + if (resampled) { + if (echoState_) { + echoState_->putPlayback(resampled); + } + playbackQueue_->enqueue(std::move(resampled)); + } else + break; } return playbackBuf; } +void +AudioLayer::putRecorded(std::shared_ptr&& frame) +{ + //if (isCaptureMuted_) + // libav_utils::fillWithSilence(frame->pointer()); + + if (echoState_) { + echoState_->putRecorded(std::move(frame)); + while (auto rec = echoState_->getRecorded()) + mainRingBuffer_->put(std::move(rec)); + } else { + mainRingBuffer_->put(std::move(frame)); + } +} + } // namespace jami diff --git a/src/media/audio/audiolayer.h b/src/media/audio/audiolayer.h index 039ee5e5a..00b8525a0 100644 --- a/src/media/audio/audiolayer.h +++ b/src/media/audio/audiolayer.h @@ -33,6 +33,11 @@ #include #include +extern "C" { +struct SpeexEchoState_; +typedef struct SpeexEchoState_ SpeexEchoState; +} + /** * @file audiolayer.h * @brief Main sound class. Manages the data transfers between the application and the hardware. @@ -223,7 +228,7 @@ protected: /** * Callback to be called by derived classes when the audio output is opened. */ - void hardwareFormatAvailable(AudioFormat playback); + void hardwareFormatAvailable(AudioFormat playback, size_t bufSize = 0); /** * Set the input format on necessary objects. @@ -232,6 +237,10 @@ protected: void devicesChanged(); + void playbackChanged(bool started); + void recordChanged(bool started); + void setHasNativeAEC(bool hasEAC); + std::shared_ptr getToPlay(AudioFormat format, size_t writableSamples); std::shared_ptr getToRing(AudioFormat format, size_t writableSamples); @@ -242,6 +251,8 @@ protected: return ringBuff ? ringBuff : playBuff; } + void putRecorded(std::shared_ptr&& frame); + void flush(); /** @@ -259,6 +270,10 @@ protected: */ bool isRingtoneMuted_ {false}; + bool playbackStarted_ {false}; + bool recordStarted_ {false}; + bool hasNativeAEC_ {true}; + /** * Gain applied to mic signal */ @@ -272,10 +287,8 @@ protected: /** * Buffers for audio processing */ - AudioBuffer playbackBuffer_; - AudioBuffer playbackResampleBuffer_; + std::shared_ptr mainRingBuffer_; AudioBuffer ringtoneBuffer_; - AudioBuffer ringtoneResampleBuffer_; std::unique_ptr playbackQueue_; /** @@ -294,6 +307,8 @@ protected: */ AudioFormat audioInputFormat_; + size_t nativeFrameSize_ {0}; + /** * Urgent ring buffer used for ringtones */ @@ -314,12 +329,11 @@ protected: */ std::unique_ptr resampler_; - /** - * Manage input sampling rate conversions - */ - std::unique_ptr inputResampler_; + struct EchoState; + std::unique_ptr echoState_; private: + void checkAEC(); /** * Time of the last incoming call notification diff --git a/src/media/audio/coreaudio/ios/corelayer.h b/src/media/audio/coreaudio/ios/corelayer.h index 90fccb49a..55ac4fbfc 100644 --- a/src/media/audio/coreaudio/ios/corelayer.h +++ b/src/media/audio/coreaudio/ios/corelayer.h @@ -165,8 +165,6 @@ class CoreLayer : public AudioLayer { UInt32 inChannelsPerFrame_; Float64 outSampleRate_; UInt32 outChannelsPerFrame_; - - std::shared_ptr mainRingBuffer_; }; } // namespace jami diff --git a/src/media/audio/coreaudio/ios/corelayer.mm b/src/media/audio/coreaudio/ios/corelayer.mm index d547c3f87..2188f27e6 100644 --- a/src/media/audio/coreaudio/ios/corelayer.mm +++ b/src/media/audio/coreaudio/ios/corelayer.mm @@ -43,7 +43,6 @@ CoreLayer::CoreLayer(const AudioPreference &pref) , indexOut_(pref.getAlsaCardout()) , indexRing_(pref.getAlsaCardring()) , playbackBuff_(0, audioFormat_) - , mainRingBuffer_(Manager::instance().getRingBufferPool().getRingBuffer(RingBufferPool::DEFAULT_ID)) {} CoreLayer::~CoreLayer() @@ -440,7 +439,7 @@ CoreLayer::read(AudioUnitRenderActionFlags* ioActionFlags, auto format = audioInputFormat_; format.sampleFormat = AV_SAMPLE_FMT_FLTP; - auto inBuff = std::make_unique(format, inNumberFrames); + auto inBuff = std::make_shared(format, inNumberFrames); if (isCaptureMuted_) { libav_utils::fillWithSilence(inBuff->pointer()); } else { @@ -448,7 +447,7 @@ CoreLayer::read(AudioUnitRenderActionFlags* ioActionFlags, for (unsigned i = 0; i < inChannelsPerFrame_; ++i) std::copy_n((Float32*)captureBuff_->mBuffers[i].mData, inNumberFrames, (Float32*)in.extended_data[i]); } - mainRingBuffer_->put(std::move(inBuff)); + putRecorded(std::move(inBuff)); } void CoreLayer::updatePreference(AudioPreference &preference, int index, DeviceType type) diff --git a/src/media/audio/coreaudio/osx/corelayer.cpp b/src/media/audio/coreaudio/osx/corelayer.cpp index 853a648db..72e8242b9 100644 --- a/src/media/audio/coreaudio/osx/corelayer.cpp +++ b/src/media/audio/coreaudio/osx/corelayer.cpp @@ -38,7 +38,6 @@ CoreLayer::CoreLayer(const AudioPreference &pref) , indexOut_(pref.getAlsaCardout()) , indexRing_(pref.getAlsaCardring()) , playbackBuff_(0, audioFormat_) - , mainRingBuffer_(Manager::instance().getRingBufferPool().getRingBuffer(RingBufferPool::DEFAULT_ID)) {} CoreLayer::~CoreLayer() @@ -345,7 +344,7 @@ CoreLayer::read(AudioUnitRenderActionFlags* ioActionFlags, auto format = audioInputFormat_; format.sampleFormat = AV_SAMPLE_FMT_FLTP; - auto inBuff = std::make_unique(format, inNumberFrames); + auto inBuff = std::make_shared(format, inNumberFrames); if (isCaptureMuted_) { libav_utils::fillWithSilence(inBuff->pointer()); } else { @@ -353,7 +352,7 @@ CoreLayer::read(AudioUnitRenderActionFlags* ioActionFlags, for (unsigned i = 0; i < inChannelsPerFrame_; ++i) std::copy_n((Float32*)captureBuff_->mBuffers[i].mData, inNumberFrames, (Float32*)in.extended_data[i]); } - mainRingBuffer_->put(std::move(inBuff)); + putRecorded(std::move(inBuff)); } void CoreLayer::updatePreference(AudioPreference &preference, int index, DeviceType type) diff --git a/src/media/audio/coreaudio/osx/corelayer.h b/src/media/audio/coreaudio/osx/corelayer.h index 9d447b2ec..e7a827625 100644 --- a/src/media/audio/coreaudio/osx/corelayer.h +++ b/src/media/audio/coreaudio/osx/corelayer.h @@ -172,8 +172,6 @@ class CoreLayer : public AudioLayer { Float64 inSampleRate_; UInt32 inChannelsPerFrame_; - std::shared_ptr mainRingBuffer_; - std::vector getDeviceList(bool getCapture) const; }; diff --git a/src/media/audio/jack/jacklayer.cpp b/src/media/audio/jack/jacklayer.cpp index dc34dd3ca..3f175ba38 100644 --- a/src/media/audio/jack/jacklayer.cpp +++ b/src/media/audio/jack/jacklayer.cpp @@ -209,8 +209,7 @@ createPorts(jack_client_t *client, std::vector &ports, JackLayer::JackLayer(const AudioPreference &p) : AudioLayer(p), captureClient_(nullptr), - playbackClient_(nullptr), - mainRingBuffer_(Manager::instance().getRingBufferPool().getRingBuffer(RingBufferPool::DEFAULT_ID)) + playbackClient_(nullptr) { playbackClient_ = jack_client_open(PACKAGE_NAME, (jack_options_t) (JackNullOption | JackNoStartServer), NULL); diff --git a/src/media/audio/jack/jacklayer.h b/src/media/audio/jack/jacklayer.h index dde32e50e..353ed2ecf 100644 --- a/src/media/audio/jack/jacklayer.h +++ b/src/media/audio/jack/jacklayer.h @@ -49,7 +49,6 @@ private: std::thread ringbuffer_thread_; std::mutex ringbuffer_thread_mutex_; std::condition_variable data_ready_; - std::shared_ptr mainRingBuffer_; static int process_capture(jack_nframes_t frames, void *arg); static int process_playback(jack_nframes_t frames, void *arg); diff --git a/src/media/audio/opensl/audio_player.cpp b/src/media/audio/opensl/audio_player.cpp index dfd7b3be5..b80394e5f 100644 --- a/src/media/audio/opensl/audio_player.cpp +++ b/src/media/audio/opensl/audio_player.cpp @@ -40,8 +40,6 @@ void bqPlayerCallback(SLAndroidSimpleBufferQueueItf bq, void *ctx) { (static_cast(ctx))->processSLCallback(bq); } void AudioPlayer::processSLCallback(SLAndroidSimpleBufferQueueItf bq) { - std::lock_guard lk(m_); - // retrieve the finished device buf and put onto the free queue // so recorder could re-use it sample_buf *buf; diff --git a/src/media/audio/opensl/audio_player.h b/src/media/audio/opensl/audio_player.h index 7fc470b8c..0e9383c89 100644 --- a/src/media/audio/opensl/audio_player.h +++ b/src/media/audio/opensl/audio_player.h @@ -58,7 +58,6 @@ public: void registerCallback(EngineCallback cb) {callback_ = cb;} size_t dbgGetDevBufCount(); - std::mutex m_; std::atomic_bool waiting_ {false}; }; diff --git a/src/media/audio/opensl/audio_recorder.cpp b/src/media/audio/opensl/audio_recorder.cpp index ff6e55d5f..699e1c60b 100644 --- a/src/media/audio/opensl/audio_recorder.cpp +++ b/src/media/audio/opensl/audio_recorder.cpp @@ -59,7 +59,7 @@ void AudioRecorder::processSLCallback(SLAndroidSimpleBufferQueueItf bq) { AudioRecorder::AudioRecorder(jami::AudioFormat sampleFormat, SLEngineItf slEngine) : sampleInfo_(sampleFormat) { - // configure audio source + // configure audio source/ SLDataLocator_IODevice loc_dev = {SL_DATALOCATOR_IODEVICE, SL_IODEVICE_AUDIOINPUT, SL_DEFAULTDEVICEID_AUDIOINPUT, @@ -76,7 +76,13 @@ AudioRecorder::AudioRecorder(jami::AudioFormat sampleFormat, SLEngineItf slEngin // create audio recorder // (requires the RECORD_AUDIO permission) - const SLInterfaceID ids[2] = {SL_IID_ANDROIDSIMPLEBUFFERQUEUE, SL_IID_ANDROIDCONFIGURATION}; + const SLInterfaceID ids[] = { + SL_IID_ANDROIDSIMPLEBUFFERQUEUE, + SL_IID_ANDROIDCONFIGURATION, + SL_IID_ANDROIDACOUSTICECHOCANCELLATION, + SL_IID_ANDROIDAUTOMATICGAINCONTROL, + SL_IID_ANDROIDNOISESUPPRESSION + }; const SLboolean req[1] = {SL_BOOLEAN_TRUE}; SLresult result; result = (*slEngine)->CreateAudioRecorder(slEngine, @@ -91,11 +97,85 @@ AudioRecorder::AudioRecorder(jami::AudioFormat sampleFormat, SLEngineItf slEngin result = (*recObjectItf_)->GetInterface(recObjectItf_, SL_IID_ANDROIDCONFIGURATION, &recordConfig); result = (*recordConfig)->SetConfiguration(recordConfig, SL_ANDROID_KEY_RECORDING_PRESET, &streamType, sizeof(SLint32)); + bool aec{true}, agc(true), ns(true); + + result = (*recObjectItf_)->Realize(recObjectItf_, SL_BOOLEAN_FALSE); SLASSERT(result); result = (*recObjectItf_)->GetInterface(recObjectItf_, SL_IID_RECORD, &recItf_); SLASSERT(result); + + /* Check actual performance mode granted*/ + SLuint32 modeRetrieved = SL_ANDROID_PERFORMANCE_NONE; + SLuint32 modeSize = sizeof(SLuint32); + result = (*recordConfig)->GetConfiguration(recordConfig, SL_ANDROID_KEY_PERFORMANCE_MODE, + &modeSize, (void*)&modeRetrieved); + SLASSERT(result); + JAMI_WARN("Actual performance mode is %u\n", modeRetrieved); + + /* Enable AEC if requested */ + if (aec) { + SLAndroidAcousticEchoCancellationItf aecItf; + result = (*recObjectItf_)->GetInterface(recObjectItf_, SL_IID_ANDROIDACOUSTICECHOCANCELLATION, (void*)&aecItf); + JAMI_WARN("AEC is %savailable\n", SL_RESULT_SUCCESS == result ? "" : "not "); + if (SL_RESULT_SUCCESS == result) { + SLboolean enabled; + result = (*aecItf)->IsEnabled(aecItf, &enabled); + SLASSERT(result); + JAMI_WARN("AEC was %s\n", enabled ? "enabled" : "not enabled"); + + result = (*aecItf)->SetEnabled(aecItf, true); + SLASSERT(result); + + result = (*aecItf)->IsEnabled(aecItf, &enabled); + SLASSERT(result); + JAMI_WARN("AEC is now %s\n", enabled ? "enabled" : "not enabled"); + + hasNativeAEC_ = enabled; + } + } + /* Enable AGC if requested */ + if (agc) { + SLAndroidAutomaticGainControlItf agcItf; + result = (*recObjectItf_)->GetInterface( + recObjectItf_, SL_IID_ANDROIDAUTOMATICGAINCONTROL, (void*)&agcItf); + JAMI_WARN("AGC is %savailable\n", SL_RESULT_SUCCESS == result ? "" : "not "); + if (SL_RESULT_SUCCESS == result) { + SLboolean enabled; + result = (*agcItf)->IsEnabled(agcItf, &enabled); + SLASSERT(result); + JAMI_WARN("AGC was %s\n", enabled ? "enabled" : "not enabled"); + + result = (*agcItf)->SetEnabled(agcItf, true); + SLASSERT(result); + + result = (*agcItf)->IsEnabled(agcItf, &enabled); + SLASSERT(result); + JAMI_WARN("AGC is now %s\n", enabled ? "enabled" : "not enabled"); + } + } + /* Enable NS if requested */ + if (ns) { + SLAndroidNoiseSuppressionItf nsItf; + result = (*recObjectItf_)->GetInterface( + recObjectItf_, SL_IID_ANDROIDNOISESUPPRESSION, (void*)&nsItf); + JAMI_WARN("NS is %savailable\n", SL_RESULT_SUCCESS == result ? "" : "not "); + if (SL_RESULT_SUCCESS == result) { + SLboolean enabled; + result = (*nsItf)->IsEnabled(nsItf, &enabled); + SLASSERT(result); + JAMI_WARN("NS was %s\n", enabled ? "enabled" : "not enabled"); + + result = (*nsItf)->SetEnabled(nsItf, true); + SLASSERT(result); + + result = (*nsItf)->IsEnabled(nsItf, &enabled); + SLASSERT(result); + JAMI_WARN("NS is now %s\n", enabled ? "enabled" : "not enabled"); + } + } + result = (*recObjectItf_)->GetInterface(recObjectItf_, SL_IID_ANDROIDSIMPLEBUFFERQUEUE, &recBufQueueItf_); SLASSERT(result); diff --git a/src/media/audio/opensl/audio_recorder.h b/src/media/audio/opensl/audio_recorder.h index 85cf433f2..7e86bf23c 100644 --- a/src/media/audio/opensl/audio_recorder.h +++ b/src/media/audio/opensl/audio_recorder.h @@ -40,6 +40,7 @@ class AudioRecorder { uint32_t audioBufCount; EngineCallback callback_ {}; + bool hasNativeAEC_ {false}; public: explicit AudioRecorder(jami::AudioFormat, SLEngineItf engineEngine); @@ -52,6 +53,8 @@ public: void processSLCallback(SLAndroidSimpleBufferQueueItf bq); void registerCallback(EngineCallback cb) {callback_ = cb;} size_t dbgGetDevBufCount(); + + bool hasNativeAEC() const { return hasNativeAEC_; } }; } diff --git a/src/media/audio/opensl/opensllayer.cpp b/src/media/audio/opensl/opensllayer.cpp index 79ca4f56f..445b0603f 100644 --- a/src/media/audio/opensl/opensllayer.cpp +++ b/src/media/audio/opensl/opensllayer.cpp @@ -30,25 +30,19 @@ #include "logger.h" #include "array_size.h" +#include + #include #include #include #include #include -#include "SLES/OpenSLES_AndroidConfiguration.h" - -/* available only from api 14 */ -#ifndef SL_ANDROID_RECORDING_PRESET_VOICE_COMMUNICATION -#define SL_ANDROID_RECORDING_PRESET_VOICE_COMMUNICATION ((SLuint32) 0x00000004) -#endif - namespace jami { // Constructor OpenSLLayer::OpenSLLayer(const AudioPreference &pref) - : AudioLayer(pref), - mainRingBuffer_(Manager::instance().getRingBufferPool().getRingBuffer(RingBufferPool::DEFAULT_ID)) + : AudioLayer(pref) {} // Destructor @@ -81,7 +75,7 @@ OpenSLLayer::startStream(AudioStreamType stream) emitSignal(&hw_infos); hardwareFormat_ = AudioFormat(hw_infos[0], 1); // Mono on Android hardwareBuffSize_ = hw_infos[1]; - hardwareFormatAvailable(hardwareFormat_); + hardwareFormatAvailable(hardwareFormat_, hardwareBuffSize_); startThread_ = std::thread([this](){ init(); @@ -216,8 +210,9 @@ OpenSLLayer::engineServicePlay(bool waiting) { break; } else freePlayBufQueue_.pop(); - } else + } else { break; + } } } @@ -284,6 +279,7 @@ OpenSLLayer::initAudioCapture() recorder_.reset(new opensl::AudioRecorder(hardwareFormat_, engineInterface_)); recorder_->setBufQueues(&freeRecBufQueue_, &recBufQueue_); recorder_->registerCallback(std::bind(&OpenSLLayer::engineServiceRec, this, _1)); + setHasNativeAEC(recorder_->hasNativeAEC()); } catch (const std::exception& e) { JAMI_ERR("Error initializing audio capture: %s", e.what()); } @@ -301,24 +297,26 @@ OpenSLLayer::startAudioPlayback() if (ringtone_) ringtone_->start(); playThread = std::thread([&]() { + playbackChanged(true); std::unique_lock lck(playMtx); while (player_ || ringtone_) { playCv.wait_for(lck, std::chrono::seconds(1)); if (player_ && player_->waiting_) { - std::lock_guard lk(player_->m_); + //std::lock_guard lk(player_->m_); engineServicePlay(false); auto n = playBufQueue_.size(); if (n >= PLAY_KICKSTART_BUFFER_COUNT) player_->playAudioBuffers(n); } if (ringtone_ && ringtone_->waiting_) { - std::lock_guard lk(ringtone_->m_); + //std::lock_guard lk(ringtone_->m_); engineServiceRing(false); auto n = ringBufQueue_.size(); if (n >= PLAY_KICKSTART_BUFFER_COUNT) ringtone_->playAudioBuffers(n); } } + playbackChanged(false); }); JAMI_WARN("Audio playback started"); } @@ -332,6 +330,7 @@ OpenSLLayer::startAudioCapture() recorder_->start(); recThread = std::thread([&]() { + recordChanged(true); std::unique_lock lck(recMtx); while (recorder_) { recCv.wait_for(lck, std::chrono::seconds(1)); @@ -342,18 +341,18 @@ OpenSLLayer::startAudioCapture() recBufQueue_.pop(); if (buf->size_ > 0) { auto nb_samples = buf->size_ / hardwareFormat_.getBytesPerFrame(); - auto out = std::make_unique(hardwareFormat_, nb_samples); + auto out = std::make_shared(hardwareFormat_, nb_samples); if (isCaptureMuted_) libav_utils::fillWithSilence(out->pointer()); else std::copy_n((const AudioSample*)buf->buf_, nb_samples, (AudioSample*)out->pointer()->data[0]); - // dcblocker_.process(buffer); - mainRingBuffer_->put(std::move(out)); + putRecorded(std::move(out)); } buf->size_ = 0; freeRecBufQueue_.push(buf); } } + recordChanged(false); }); JAMI_DBG("Audio capture started"); diff --git a/src/media/audio/opensl/opensllayer.h b/src/media/audio/opensl/opensllayer.h index 239537b4f..cdd8dc27c 100644 --- a/src/media/audio/opensl/opensllayer.h +++ b/src/media/audio/opensl/opensllayer.h @@ -18,8 +18,7 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. */ -#ifndef _OPENSL_LAYER_H -#define _OPENSL_LAYER_H +#pragma once #include #include @@ -52,147 +51,144 @@ class RingBuffer; */ class OpenSLLayer : public AudioLayer { - public: - /** - * Constructor - */ - OpenSLLayer(const AudioPreference &pref); +public: + /** + * Constructor + */ + OpenSLLayer(const AudioPreference &pref); - /** - * Destructor - */ - ~OpenSLLayer(); + /** + * Destructor + */ + ~OpenSLLayer(); - /** - * Start the capture stream and prepare the playback stream. - * The playback starts accordingly to its threshold - */ - virtual void startStream(AudioStreamType stream = AudioStreamType::DEFAULT); + /** + * Start the capture stream and prepare the playback stream. + * The playback starts accordingly to its threshold + */ + virtual void startStream(AudioStreamType stream = AudioStreamType::DEFAULT); - /** - * Stop the playback and capture streams. - * Drops the pending frames and put the capture and playback handles to PREPARED state - */ - virtual void stopStream(); + /** + * Stop the playback and capture streams. + * Drops the pending frames and put the capture and playback handles to PREPARED state + */ + virtual void stopStream(); - /** - * Scan the sound card available for capture on the system - * @return std::vector The vector containing the string description of the card - */ - virtual std::vector getCaptureDeviceList() const; + /** + * Scan the sound card available for capture on the system + * @return std::vector The vector containing the string description of the card + */ + virtual std::vector getCaptureDeviceList() const; - /** - * Scan the sound card available for capture on the system - * @return std::vector The vector containing the string description of the card - */ - virtual std::vector getPlaybackDeviceList() const; + /** + * Scan the sound card available for capture on the system + * @return std::vector The vector containing the string description of the card + */ + virtual std::vector getPlaybackDeviceList() const; - void init(); + void init(); - void initAudioEngine(); + void initAudioEngine(); - void shutdownAudioEngine(); + void shutdownAudioEngine(); - void initAudioPlayback(); + void initAudioPlayback(); - void initAudioCapture(); + void initAudioCapture(); - void startAudioPlayback(); + void startAudioPlayback(); - void startAudioCapture(); + void startAudioCapture(); - void stopAudioPlayback(); + void stopAudioPlayback(); - void stopAudioCapture(); + void stopAudioCapture(); - virtual int getAudioDeviceIndex(const std::string&, DeviceType) const { - return 0; - } + virtual int getAudioDeviceIndex(const std::string&, DeviceType) const { + return 0; + } - virtual std::string getAudioDeviceName(int, DeviceType) const { - return ""; - } + virtual std::string getAudioDeviceName(int, DeviceType) const { + return ""; + } - void engineServicePlay(bool waiting); - void engineServiceRing(bool waiting); - void engineServiceRec(bool waiting); + void engineServicePlay(bool waiting); + void engineServiceRing(bool waiting); + void engineServiceRec(bool waiting); - private: - /** - * Get the index of the audio card for capture - * @return int The index of the card used for capture - * 0 for the first available card on the system, 1 ... - */ - virtual int getIndexCapture() const { - return 0; - } +private: + /** + * Get the index of the audio card for capture + * @return int The index of the card used for capture + * 0 for the first available card on the system, 1 ... + */ + virtual int getIndexCapture() const { + return 0; + } - /** - * Get the index of the audio card for playback - * @return int The index of the card used for playback - * 0 for the first available card on the system, 1 ... - */ - virtual int getIndexPlayback() const { - return 0; - } + /** + * Get the index of the audio card for playback + * @return int The index of the card used for playback + * 0 for the first available card on the system, 1 ... + */ + virtual int getIndexPlayback() const { + return 0; + } - /** - * Get the index of the audio card for ringtone (could be differnet from playback) - * @return int The index of the card used for ringtone - * 0 for the first available card on the system, 1 ... - */ - virtual int getIndexRingtone() const { - return 0; - } + /** + * Get the index of the audio card for ringtone (could be differnet from playback) + * @return int The index of the card used for ringtone + * 0 for the first available card on the system, 1 ... + */ + virtual int getIndexRingtone() const { + return 0; + } - uint32_t dbgEngineGetBufCount(); + uint32_t dbgEngineGetBufCount(); - void dumpAvailableEngineInterfaces(); + void dumpAvailableEngineInterfaces(); - NON_COPYABLE(OpenSLLayer); + NON_COPYABLE(OpenSLLayer); - virtual void updatePreference(AudioPreference &pref, int index, DeviceType type); + virtual void updatePreference(AudioPreference &pref, int index, DeviceType type); - /** - * OpenSL standard object interface - */ - SLObjectItf engineObject_ {nullptr}; + /** + * OpenSL standard object interface + */ + SLObjectItf engineObject_ {nullptr}; - /** - * OpenSL sound engine interface - */ - SLEngineItf engineInterface_ {nullptr}; + /** + * OpenSL sound engine interface + */ + SLEngineItf engineInterface_ {nullptr}; - std::unique_ptr player_ {}; - std::unique_ptr ringtone_ {}; - std::unique_ptr recorder_ {}; + std::unique_ptr player_ {}; + std::unique_ptr ringtone_ {}; + std::unique_ptr recorder_ {}; - AudioQueue freePlayBufQueue_ {BUF_COUNT}; - AudioQueue playBufQueue_ {BUF_COUNT}; + AudioQueue freePlayBufQueue_ {BUF_COUNT}; + AudioQueue playBufQueue_ {BUF_COUNT}; - AudioQueue freeRingBufQueue_ {BUF_COUNT}; - AudioQueue ringBufQueue_ {BUF_COUNT}; + AudioQueue freeRingBufQueue_ {BUF_COUNT}; + AudioQueue ringBufQueue_ {BUF_COUNT}; - std::mutex playMtx {}; - std::condition_variable playCv {}; - std::thread playThread {}; + std::mutex playMtx {}; + std::condition_variable playCv {}; + std::thread playThread {}; - AudioQueue freeRecBufQueue_ {BUF_COUNT}; //Owner of the queue - AudioQueue recBufQueue_ {BUF_COUNT}; //Owner of the queue + AudioQueue freeRecBufQueue_ {BUF_COUNT}; //Owner of the queue + AudioQueue recBufQueue_ {BUF_COUNT}; //Owner of the queue - std::mutex recMtx {}; - std::condition_variable recCv {}; - std::thread recThread {}; + std::mutex recMtx {}; + std::condition_variable recCv {}; + std::thread recThread {}; - std::vector bufs_ {}; + std::vector bufs_ {}; - AudioFormat hardwareFormat_ {AudioFormat::MONO()}; - size_t hardwareBuffSize_ {BUFFER_SIZE}; + AudioFormat hardwareFormat_ {AudioFormat::MONO()}; + size_t hardwareBuffSize_ {BUFFER_SIZE}; - std::shared_ptr mainRingBuffer_; - std::thread startThread_; + std::thread startThread_; }; } - -#endif // _OPENSL_LAYER_H_ diff --git a/src/media/audio/portaudio/portaudiolayer.cpp b/src/media/audio/portaudio/portaudiolayer.cpp index d567dad89..d36722d20 100644 --- a/src/media/audio/portaudio/portaudiolayer.cpp +++ b/src/media/audio/portaudio/portaudiolayer.cpp @@ -33,6 +33,8 @@ #include namespace jami { + +struct AudioLayer::EchoState; enum Direction { Input = 0, Output = 1, IO = 2, End = 3 }; @@ -55,8 +57,6 @@ struct PortAudioLayer::PortAudioLayerImpl AudioBuffer playbackBuff_; - std::shared_ptr mainRingBuffer_; - std::array(Direction::End)> streams_; int paOutputCallback(PortAudioLayer& parent, @@ -226,7 +226,6 @@ PortAudioLayer::PortAudioLayerImpl::PortAudioLayerImpl(PortAudioLayer& parent, c , indexOut_ {pref.getAlsaCardout()} , indexRing_ {pref.getAlsaCardring()} , playbackBuff_ {0, parent.audioFormat_} - , mainRingBuffer_ {Manager::instance().getRingBufferPool().getRingBuffer(RingBufferPool::DEFAULT_ID)} { init(parent); } @@ -562,13 +561,13 @@ PortAudioLayer::PortAudioLayerImpl::paInputCallback(PortAudioLayer& parent, return paContinue; } - auto inBuff = std::make_unique(parent.audioInputFormat_, framesPerBuffer); + auto inBuff = std::make_shared(parent.audioInputFormat_, framesPerBuffer); auto nFrames = framesPerBuffer * parent.audioInputFormat_.nb_channels; if (parent.isCaptureMuted_) libav_utils::fillWithSilence(inBuff->pointer()); else std::copy_n(inputBuffer, nFrames, (AudioSample*)inBuff->pointer()->extended_data[0]); - mainRingBuffer_->put(std::move(inBuff)); + parent.putRecorded(std::move(inBuff)); return paContinue; } diff --git a/src/media/audio/pulseaudio/pulselayer.cpp b/src/media/audio/pulseaudio/pulselayer.cpp index 90dea49b2..780d6c7c1 100644 --- a/src/media/audio/pulseaudio/pulselayer.cpp +++ b/src/media/audio/pulseaudio/pulselayer.cpp @@ -65,7 +65,6 @@ PulseLayer::PulseLayer(AudioPreference &pref) , ringtone_() , mainloop_(pa_threaded_mainloop_new(), pa_threaded_mainloop_free) , preference_(pref) - , mainRingBuffer_(Manager::instance().getRingBufferPool().getRingBuffer(RingBufferPool::DEFAULT_ID)) { if (!mainloop_) throw std::runtime_error("Couldn't create pulseaudio mainloop"); @@ -456,7 +455,7 @@ void PulseLayer::readFromMic() size_t sample_size = record_->frameSize(); const size_t samples = bytes / sample_size; - auto out = std::make_unique(record_->format(), samples); + auto out = std::make_shared(record_->format(), samples); if (isCaptureMuted_) libav_utils::fillWithSilence(out->pointer()); else @@ -465,8 +464,7 @@ void PulseLayer::readFromMic() if (pa_stream_drop(record_->stream()) < 0) JAMI_ERR("Capture stream drop failed: %s" , pa_strerror(pa_context_errno(context_))); - //dcblocker_.process(*out); - mainRingBuffer_->put(std::move(out)); + putRecorded(std::move(out)); } void PulseLayer::ringtoneToSpeaker() @@ -710,20 +708,20 @@ void PulseLayer::updatePreference(AudioPreference &preference, int index, Device const std::string devName(getAudioDeviceName(index, type)); switch (type) { - case DeviceType::PLAYBACK: - JAMI_DBG("setting %s for playback", devName.c_str()); - preference.setPulseDevicePlayback(devName); - break; + case DeviceType::PLAYBACK: + JAMI_DBG("setting %s for playback", devName.c_str()); + preference.setPulseDevicePlayback(devName); + break; - case DeviceType::CAPTURE: - JAMI_DBG("setting %s for capture", devName.c_str()); - preference.setPulseDeviceRecord(devName); - break; + case DeviceType::CAPTURE: + JAMI_DBG("setting %s for capture", devName.c_str()); + preference.setPulseDeviceRecord(devName); + break; - case DeviceType::RINGTONE: - JAMI_DBG("setting %s for ringer", devName.c_str()); - preference.setPulseDeviceRingtone(devName); - break; + case DeviceType::RINGTONE: + JAMI_DBG("setting %s for ringer", devName.c_str()); + preference.setPulseDeviceRingtone(devName); + break; } } diff --git a/src/media/audio/pulseaudio/pulselayer.h b/src/media/audio/pulseaudio/pulselayer.h index 12e5601a1..30541bad6 100644 --- a/src/media/audio/pulseaudio/pulselayer.h +++ b/src/media/audio/pulseaudio/pulselayer.h @@ -206,18 +206,6 @@ class PulseLayer : public AudioLayer { */ std::vector sourceList_ {}; - /* - * Buffers used to avoid doing malloc/free in the audio thread - */ - AudioBuffer micBuffer_; - AudioBuffer micResampleBuffer_; - - AudioBuffer playbackBuffer_; - AudioBuffer playbackResampleBuffer_; - - AudioBuffer ringtoneBuffer_; - AudioBuffer ringtoneResampleBuffer_; - /** PulseAudio server defaults */ AudioFormat defaultAudioFormat_ {AudioFormat::MONO()}; std::string defaultSink_ {}; @@ -235,7 +223,6 @@ class PulseLayer : public AudioLayer { std::thread streamStarter_ {}; AudioPreference &preference_; - std::shared_ptr mainRingBuffer_; pa_operation* subscribeOp_ {nullptr}; friend class AudioLayerTest; diff --git a/src/media/audio/ringbuffer.h b/src/media/audio/ringbuffer.h index 46032e254..938e9c5cd 100644 --- a/src/media/audio/ringbuffer.h +++ b/src/media/audio/ringbuffer.h @@ -91,7 +91,7 @@ public: * @param buffer Data to copied * @param toCopy Number of bytes to copy */ - void put(std::shared_ptr&& data); + void put(std::shared_ptr&& data); /** * To get how much samples are available in the buffer to read in @@ -130,6 +130,10 @@ public: return putLength() == 0; } + inline void setFrameSize(int nb_samples) { + resizer_.setFrameSize(nb_samples); + } + /** * Blocks until min_data_length samples of data is available, or until deadline has passed. * diff --git a/src/media/audio/ringbufferpool.cpp b/src/media/audio/ringbufferpool.cpp index a89e55972..71062223e 100644 --- a/src/media/audio/ringbufferpool.cpp +++ b/src/media/audio/ringbufferpool.cpp @@ -271,7 +271,7 @@ RingBufferPool::getData(const std::string& call_id) if (bindings->size() == 1) return (*bindings->cbegin())->get(call_id); - auto mixBuffer = std::make_unique(internalAudioFormat_); + auto mixBuffer = std::make_shared(internalAudioFormat_); for (const auto& rbuf : *bindings) { if (auto b = rbuf->get(call_id)) { mixBuffer->mix(*b); @@ -326,7 +326,7 @@ RingBufferPool::getAvailableData(const std::string& call_id) if (availableFrames == 0) return {}; - auto buf = std::make_unique(internalAudioFormat_); + auto buf = std::make_shared(internalAudioFormat_); for (const auto &rbuf : *bindings) { if (auto b = rbuf->get(call_id)) buf->mix(*b);