From c52e332b0bc20c8576f6a9c18adb65dac00bee94 Mon Sep 17 00:00:00 2001 From: philippegorley Date: Fri, 18 Jan 2019 11:13:32 -0500 Subject: [PATCH] audio: add audio meter Adds a signal that sends the linear RMS level for a given ring buffer. The signal must be turned on via the API and can be turned off when needed. Adds an audio preview so the mic can be read. Call startAudioDevice and stopAudioDevice to initialize and stop the audio layer. Change-Id: I6a71ef87ee805a6d4bfa824fa901dd638e8cbd65 --- .../cx.ring.Ring.ConfigurationManager.xml | 31 ++++++++++++ bin/dbus/cx.ring.Ring.VideoManager.xml | 7 +++ bin/dbus/dbusclient.cpp | 2 + bin/dbus/dbusconfigurationmanager.cpp | 12 +++++ bin/dbus/dbusconfigurationmanager.h | 3 ++ bin/dbus/dbusvideomanager.cpp | 12 +++++ bin/dbus/dbusvideomanager.h | 2 + bin/jni/configurationmanager.i | 6 +++ bin/jni/videomanager.i | 2 + bin/nodejs/configurationmanager.i | 6 +++ bin/nodejs/videomanager.i | 2 + configure.ac | 2 +- src/client/configurationmanager.cpp | 13 +++++ src/client/ring_signal.cpp | 1 + src/client/videomanager.cpp | 47 +++++++++++++++++++ src/client/videomanager.h | 1 + src/dring/configurationmanager_interface.h | 19 ++++++++ src/dring/videomanager_interface.h | 3 ++ src/media/audio/ringbuffer.cpp | 14 +++++- src/media/audio/ringbuffer.h | 8 ++++ src/media/audio/ringbufferpool.cpp | 37 +++++++++++++++ src/media/audio/ringbufferpool.h | 3 ++ 22 files changed, 231 insertions(+), 2 deletions(-) diff --git a/bin/dbus/cx.ring.Ring.ConfigurationManager.xml b/bin/dbus/cx.ring.Ring.ConfigurationManager.xml index 517157d08..2de4955e3 100644 --- a/bin/dbus/cx.ring.Ring.ConfigurationManager.xml +++ b/bin/dbus/cx.ring.Ring.ConfigurationManager.xml @@ -867,6 +867,37 @@ + + + Gets whether or not an audio meter is active for id. If id is empty, returns whether or not there is at least one audio meter active. + + Ring buffer id. + + + If the audio meter is active for the passed in id. + + + + + Sets whether or not the audio meter should be active for id. If id is empty, applies to all ring buffers. + + Ring buffer id. + + + True to enabled the audio meter, false to disable. + + + + + Signal containing volume level. + + Ring buffer id. + + + RMS value for the volume. Conversion to dB can be done with dB=20*log10(level). Level is between 0 and 1. + + + diff --git a/bin/dbus/cx.ring.Ring.VideoManager.xml b/bin/dbus/cx.ring.Ring.VideoManager.xml index c1b648a84..c5a79d3ab 100644 --- a/bin/dbus/cx.ring.Ring.VideoManager.xml +++ b/bin/dbus/cx.ring.Ring.VideoManager.xml @@ -72,6 +72,13 @@ + + Starts the audio layer stream, so the audio device can be read. + + + + + diff --git a/bin/dbus/dbusclient.cpp b/bin/dbus/dbusclient.cpp index d412fe375..ac97d336b 100644 --- a/bin/dbus/dbusclient.cpp +++ b/bin/dbus/dbusclient.cpp @@ -198,8 +198,10 @@ DBusClient::initLibrary(int flags) exportable_callback(bind(&DBusPresenceManager::subscriptionStateChanged, presM, _1, _2, _3)), }; + // Audio event handlers const std::map audioEvHandlers = { exportable_callback(bind(&DBusConfigurationManager::audioDeviceEvent, confM)), + exportable_callback(bind(&DBusConfigurationManager::audioMeter, confM, _1, _2)), }; const std::map dataXferEvHandlers = { diff --git a/bin/dbus/dbusconfigurationmanager.cpp b/bin/dbus/dbusconfigurationmanager.cpp index 248e94198..387b497cb 100644 --- a/bin/dbus/dbusconfigurationmanager.cpp +++ b/bin/dbus/dbusconfigurationmanager.cpp @@ -711,3 +711,15 @@ DBusConfigurationManager::cancelDataTransfer(const uint64_t& id) { return uint32_t(DRing::cancelDataTransfer(id)); } + +bool +DBusConfigurationManager::isAudioMeterActive(const std::string& id) +{ + return DRing::isAudioMeterActive(id); +} + +void +DBusConfigurationManager::setAudioMeterState(const std::string& id, const bool& state) +{ + return DRing::setAudioMeterState(id, state); +} diff --git a/bin/dbus/dbusconfigurationmanager.h b/bin/dbus/dbusconfigurationmanager.h index b12f7e0f5..e7097dbe0 100644 --- a/bin/dbus/dbusconfigurationmanager.h +++ b/bin/dbus/dbusconfigurationmanager.h @@ -168,6 +168,9 @@ class DRING_PUBLIC DBusConfigurationManager : void dataTransferBytesProgress(const uint64_t& id, uint32_t& error, int64_t& total, int64_t& progress); uint32_t acceptFileTransfer(const uint64_t& id, const std::string& file_path, const int64_t& offset); uint32_t cancelDataTransfer(const uint64_t& id); + + bool isAudioMeterActive(const std::string& id); + void setAudioMeterState(const std::string& id, const bool& state); }; #endif // __RING_DBUSCONFIGURATIONMANAGER_H__ diff --git a/bin/dbus/dbusvideomanager.cpp b/bin/dbus/dbusvideomanager.cpp index 8c8ee6d5d..e867e5c1a 100644 --- a/bin/dbus/dbusvideomanager.cpp +++ b/bin/dbus/dbusvideomanager.cpp @@ -73,6 +73,18 @@ DBusVideoManager::stopCamera() DRing::stopCamera(); } +void +DBusVideoManager::startAudioDevice() +{ + DRing::startAudioDevice(); +} + +void +DBusVideoManager::stopAudioDevice() +{ + DRing::stopAudioDevice(); +} + auto DBusVideoManager::switchInput(const std::string& resource) -> decltype(DRing::switchInput(resource)) { diff --git a/bin/dbus/dbusvideomanager.h b/bin/dbus/dbusvideomanager.h index 147114315..c0d0427d3 100644 --- a/bin/dbus/dbusvideomanager.h +++ b/bin/dbus/dbusvideomanager.h @@ -59,6 +59,8 @@ class DRING_PUBLIC DBusVideoManager : std::string getDefaultDevice(); void startCamera(); void stopCamera(); + void startAudioDevice(); + void stopAudioDevice(); bool switchInput(const std::string& resource); bool hasCameraStarted(); bool getDecodingAccelerated(); diff --git a/bin/jni/configurationmanager.i b/bin/jni/configurationmanager.i index 71bb314e4..dcc5ff068 100644 --- a/bin/jni/configurationmanager.i +++ b/bin/jni/configurationmanager.i @@ -59,6 +59,8 @@ public: virtual void hardwareDecodingChanged(bool /*state*/){} virtual void hardwareEncodingChanged(bool /*state*/){} + + virtual void audioMeter(const std::string& /*id*/, float /*level*/){} }; %} @@ -215,6 +217,8 @@ void enableProxyClient(const std::string& accountID, bool enable); void setPushNotificationToken(const std::string& pushDeviceToken); void pushNotificationReceived(const std::string& from, const std::map& data); +bool isAudioMeterActive(const std::string& id); +void setAudioMeterState(const std::string& id, bool state); } class ConfigurationCallback { @@ -253,4 +257,6 @@ public: virtual void hardwareDecodingChanged(bool /*state*/){} virtual void hardwareEncodingChanged(bool /*state*/){} + + virtual void audioMeter(const std::string& /*id*/, float /*level*/){} }; diff --git a/bin/jni/videomanager.i b/bin/jni/videomanager.i index 06f63f462..a84d6c83f 100644 --- a/bin/jni/videomanager.i +++ b/bin/jni/videomanager.i @@ -381,6 +381,8 @@ std::string getDefaultDevice(); void startCamera(); void stopCamera(); bool hasCameraStarted(); +void startAudioDevice(); +void stopAudioDevice(); bool switchInput(const std::string& resource); bool switchToCamera(); std::map getSettings(const std::string& name); diff --git a/bin/nodejs/configurationmanager.i b/bin/nodejs/configurationmanager.i index 100fa81a4..17deafcf1 100644 --- a/bin/nodejs/configurationmanager.i +++ b/bin/nodejs/configurationmanager.i @@ -57,6 +57,8 @@ public: virtual void hardwareDecodingChanged(bool /*state*/){} virtual void hardwareEncodingChanged(bool /*state*/){} }; + + virtual void audioMeter(const std::string& /*id*/, float /*level*/){} %} %feature("director") ConfigurationCallback; @@ -206,6 +208,9 @@ int exportAccounts(std::vector accountIDs, std::string toDir, std:: int importAccounts(std::string archivePath, std::string password); void connectivityChanged(); + +bool isAudioMeterActive(const std::string& id); +void setAudioMeterState(const std::string& id, bool state); } class ConfigurationCallback { @@ -242,4 +247,5 @@ public: virtual void hardwareDecodingChanged(bool /*state*/){} virtual void hardwareEncodingChanged(bool /*state*/){} + virtual void audioMeter(const std::string& /*id*/, float /*level*/){} }; diff --git a/bin/nodejs/videomanager.i b/bin/nodejs/videomanager.i index 5e61935fb..ba119b4d9 100644 --- a/bin/nodejs/videomanager.i +++ b/bin/nodejs/videomanager.i @@ -51,6 +51,8 @@ std::string getDefaultDevice(); void startCamera(); void stopCamera(); bool hasCameraStarted(); +void startAudioDevice(); +void stopAudioDevice(); bool switchInput(const std::string& resource); bool switchToCamera(); std::map getSettings(const std::string& name); diff --git a/configure.ac b/configure.ac index 6781decec..28a045b2a 100644 --- a/configure.ac +++ b/configure.ac @@ -2,7 +2,7 @@ dnl Jami - configure.ac for automake 1.9 and autoconf 2.59 dnl Process this file with autoconf to produce a configure script. AC_PREREQ([2.65]) -AC_INIT([Ring Daemon],[7.3.0],[ring@gnu.org],[ring]) +AC_INIT([Ring Daemon],[7.4.0],[ring@gnu.org],[ring]) AC_COPYRIGHT([[Copyright (c) Savoir-faire Linux 2004-2018]]) AC_REVISION([$Revision$]) diff --git a/src/client/configurationmanager.cpp b/src/client/configurationmanager.cpp index c567bf8a8..e1257106c 100644 --- a/src/client/configurationmanager.cpp +++ b/src/client/configurationmanager.cpp @@ -41,6 +41,7 @@ #include "account_const.h" #include "client/ring_signal.h" #include "upnp/upnp_context.h" +#include "audio/ringbufferpool.h" #ifdef __APPLE__ #include @@ -969,4 +970,16 @@ void pushNotificationReceived(const std::string& from, const std::map(), + exported_callback(), /* DataTransfer */ exported_callback(), diff --git a/src/client/videomanager.cpp b/src/client/videomanager.cpp index c26f07731..b99ae6378 100644 --- a/src/client/videomanager.cpp +++ b/src/client/videomanager.cpp @@ -154,6 +154,38 @@ AudioFrame::mix(const AudioFrame& frame) } } +float +AudioFrame::calcRMS() const +{ + double rms = 0.0; + auto fmt = static_cast(frame_->format); + bool planar = av_sample_fmt_is_planar(fmt); + int perChannel = planar ? frame_->nb_samples : frame_->nb_samples * frame_->channels; + int channels = planar ? frame_->channels : 1; + if (fmt == AV_SAMPLE_FMT_S16 || fmt == AV_SAMPLE_FMT_S16P) { + for (int c = 0; c < channels; ++c) { + auto buf = reinterpret_cast(frame_->extended_data[c]); + for (int i = 0; i < perChannel; ++i) { + auto sample = buf[i] * 0.000030517578125f; + rms += sample * sample; + } + } + } else if (fmt == AV_SAMPLE_FMT_FLT || fmt == AV_SAMPLE_FMT_FLTP) { + for (int c = 0; c < channels; ++c) { + auto buf = reinterpret_cast(frame_->extended_data[c]); + for (int i = 0; i < perChannel; ++i) { + rms += buf[i] * buf[i]; + } + } + } else { + // Should not happen + RING_ERR() << "Unsupported format for getting volume level: " << av_get_sample_fmt_name(fmt); + return 0.0; + } + // divide by the number of multi-byte samples + return sqrt(rms / (frame_->nb_samples * frame_->channels)); +} + VideoFrame::~VideoFrame() { if (releaseBufferCb_) @@ -359,6 +391,21 @@ stopCamera() ring::Manager::instance().getVideoManager().videoPreview.reset(); } +void +startAudioDevice() +{ + ring::Manager::instance().initAudioDriver(); + ring::Manager::instance().startAudioDriverStream(); + ring::Manager::instance().getVideoManager().audioPreview = ring::getAudioInput(ring::RingBufferPool::DEFAULT_ID); +} + +void +stopAudioDevice() +{ + ring::Manager::instance().getVideoManager().audioPreview.reset(); + ring::Manager::instance().checkAudio(); // stops audio layer if no calls +} + std::string startLocalRecorder(const bool& audioOnly, const std::string& filepath) { diff --git a/src/client/videomanager.h b/src/client/videomanager.h index fe65f591e..98e8912b8 100644 --- a/src/client/videomanager.h +++ b/src/client/videomanager.h @@ -59,6 +59,7 @@ struct VideoManager */ std::map> audioInputs; std::mutex audioMutex; + std::shared_ptr audioPreview; }; std::shared_ptr getVideoCamera(); diff --git a/src/dring/configurationmanager_interface.h b/src/dring/configurationmanager_interface.h index e0cca93be..1eabe1bcb 100644 --- a/src/dring/configurationmanager_interface.h +++ b/src/dring/configurationmanager_interface.h @@ -215,11 +215,30 @@ DRING_PUBLIC void setPushNotificationToken(const std::string& pushDeviceToken); */ DRING_PUBLIC void pushNotificationReceived(const std::string& from, const std::map& data); +/** + * Returns whether or not the audio meter is enabled for ring buffer @id. + * + * NOTE If @id is empty, returns true if at least 1 audio meter is enabled. + */ +DRING_PUBLIC bool isAudioMeterActive(const std::string& id); + +/** + * Enables/disables an audio meter for the specified @id. + * + * NOTE If @id is empty, applies to all ring buffers. + */ +DRING_PUBLIC void setAudioMeterState(const std::string& id, bool state); + struct DRING_PUBLIC AudioSignal { struct DRING_PUBLIC DeviceEvent { constexpr static const char* name = "audioDeviceEvent"; using cb_type = void(void); }; + // Linear audio level (between 0 and 1). To get level in dB: dB=20*log10(level) + struct DRING_PUBLIC AudioMeter { + constexpr static const char* name = "AudioMeter"; + using cb_type = void(const std::string& id, float level); + }; }; // Configuration signal type definitions diff --git a/src/dring/videomanager_interface.h b/src/dring/videomanager_interface.h index e53c01135..44bb0d144 100644 --- a/src/dring/videomanager_interface.h +++ b/src/dring/videomanager_interface.h @@ -102,6 +102,7 @@ public: AudioFrame(const ring::AudioFormat& format, size_t nb_samples = 0); ~AudioFrame() {}; void mix(const AudioFrame& o); + float calcRMS() const; private: void setFormat(const ring::AudioFormat& format); @@ -165,6 +166,8 @@ DRING_PUBLIC std::string getCurrentCodecName(const std::string& callID); DRING_PUBLIC void startCamera(); DRING_PUBLIC void stopCamera(); DRING_PUBLIC bool hasCameraStarted(); +DRING_PUBLIC void startAudioDevice(); +DRING_PUBLIC void stopAudioDevice(); DRING_PUBLIC bool switchInput(const std::string& resource); DRING_PUBLIC bool switchToCamera(); DRING_PUBLIC void registerSinkTarget(const std::string& sinkId, const SinkTarget& target); diff --git a/src/media/audio/ringbuffer.cpp b/src/media/audio/ringbuffer.cpp index f9374ba48..9a10c1dd1 100644 --- a/src/media/audio/ringbuffer.cpp +++ b/src/media/audio/ringbuffer.cpp @@ -25,7 +25,7 @@ #include "ringbuffer.h" #include "logger.h" - +#include "client/ring_signal.h" #include "media_buffer.h" #include "libav_deps.h" @@ -39,6 +39,8 @@ namespace ring { // corresponds to 160 ms (about 5 rtp packets) static const size_t MIN_BUFFER_SIZE = 1024; +static constexpr const int RMS_SIGNAL_INTERVAL = 5; + RingBuffer::RingBuffer(const std::string& rbuf_id, size_t /*size*/, AudioFormat format) : id(rbuf_id) , endPos_(0) @@ -175,6 +177,16 @@ void RingBuffer::putToBuffer(std::shared_ptr&& data) endPos_ = pos; + if (rmsSignal_) { + ++rmsFrameCount_; + rmsLevel_ += newBuf->calcRMS(); + if (rmsFrameCount_ == RMS_SIGNAL_INTERVAL) { + emitSignal(id, rmsLevel_ / RMS_SIGNAL_INTERVAL); + rmsLevel_ = 0; + rmsFrameCount_ = 0; + } + } + for (auto& offset : readoffsets_) { if (offset.second.callback) offset.second.callback(newBuf); diff --git a/src/media/audio/ringbuffer.h b/src/media/audio/ringbuffer.h index 9a1593477..9eba33d64 100644 --- a/src/media/audio/ringbuffer.h +++ b/src/media/audio/ringbuffer.h @@ -28,6 +28,7 @@ #include "audio_frame_resizer.h" #include "resampler.h" +#include #include #include #include @@ -144,6 +145,9 @@ public: */ void debug(); + bool isAudioMeterActive() const { return rmsSignal_; } + void setAudioMeterState(bool state) { rmsSignal_ = state; } + private: struct ReadOffset { size_t offset; @@ -197,6 +201,10 @@ private: Resampler resampler_; AudioFrameResizer resizer_; + + std::atomic_bool rmsSignal_ {false}; + double rmsLevel_ {0}; + int rmsFrameCount_ {0}; }; } // namespace ring diff --git a/src/media/audio/ringbufferpool.cpp b/src/media/audio/ringbufferpool.cpp index 01086b47b..bf7630c25 100644 --- a/src/media/audio/ringbufferpool.cpp +++ b/src/media/audio/ringbufferpool.cpp @@ -404,4 +404,41 @@ RingBufferPool::flushAllBuffers() } } +bool +RingBufferPool::isAudioMeterActive(const std::string& id) +{ + std::lock_guard lk(stateLock_); + if (!id.empty()) { + if (auto rb = getRingBuffer(id)) { + return rb->isAudioMeterActive(); + } + } else { + for (auto item = ringBufferMap_.begin(); item != ringBufferMap_.end(); ++item) { + if (const auto rb = item->second.lock()) { + if (rb->isAudioMeterActive()) { + return true; + } + } + } + } + return false; +} + +void +RingBufferPool::setAudioMeterState(const std::string& id, bool state) +{ + std::lock_guard lk(stateLock_); + if (!id.empty()) { + if (auto rb = getRingBuffer(id)) { + rb->setAudioMeterState(state); + } + } else { + for (auto item = ringBufferMap_.begin(); item != ringBufferMap_.end(); ++item) { + if (const auto rb = item->second.lock()) { + rb->setAudioMeterState(state); + } + } + } +} + } // namespace ring diff --git a/src/media/audio/ringbufferpool.h b/src/media/audio/ringbufferpool.h index f4fc5e91e..4234406f7 100644 --- a/src/media/audio/ringbufferpool.h +++ b/src/media/audio/ringbufferpool.h @@ -117,6 +117,9 @@ public: */ std::shared_ptr getRingBuffer(const std::string& id) const; + bool isAudioMeterActive(const std::string& id); + void setAudioMeterState(const std::string& id, bool state); + private: NON_COPYABLE(RingBufferPool);