diff --git a/src/call.h b/src/call.h index 49b77bcf6..55caa1430 100644 --- a/src/call.h +++ b/src/call.h @@ -357,18 +357,6 @@ public: virtual std::vector getMediaAttributeList() const = 0; #ifdef ENABLE_VIDEO - /** - * Add a dummy video stream with the attached sink. - * Typically needed in conference to display infos for participants - * that have joined the conference without video (audio only). - */ - virtual bool addDummyVideoRtpSession() = 0; - - /** - * Remove all dummy video streams. - */ - virtual void removeDummyVideoRtpSessions() = 0; - virtual std::shared_ptr>> getReceiveVideoFrameActiveWriter() = 0; virtual void createSinks(const ConfInfo& infos) = 0; diff --git a/src/conference.cpp b/src/conference.cpp index 5054a2f7e..be023394b 100644 --- a/src/conference.cpp +++ b/src/conference.cpp @@ -106,52 +106,81 @@ Conference::Conference(const std::shared_ptr& account) std::unique_lock lk(shared->videoToCallMtx_); for (const auto& info : infos) { std::string uri {}; - std::string deviceId {}; - auto it = shared->videoToCall_.find(info.source); - if (it == shared->videoToCall_.end()) - it = shared->videoToCall_.emplace_hint(it, info.source, std::string()); bool isLocalMuted = false; - // If not local - if (!it->second.empty()) { - // Retrieve calls participants - // TODO: this is a first version, we assume that the peer is not - // a master of a conference and there is only one remote - // In the future, we should retrieve confInfo from the call - // To merge layouts informations - if (auto call = std::dynamic_pointer_cast(getCall(it->second))) { + std::string deviceId {}; + auto active = false; + if (!info.id.empty()) { + if (auto call = std::dynamic_pointer_cast(getCall(info.id))) { uri = call->getPeerNumber(); isLocalMuted = call->isPeerMuted(); if (auto* transport = call->getTransport()) deviceId = transport->deviceId(); } + if (auto videoMixer = shared->videoMixer_) + active = videoMixer->verifyActive(info.id); + std::string_view peerId = string_remove_suffix(uri, '@'); + auto isModerator = shared->isModerator(peerId); + auto isHandRaised = shared->isHandRaised(peerId); + auto isModeratorMuted = shared->isMuted(peerId); + auto sinkId = shared->getConfId() + peerId; + newInfo.emplace_back(ParticipantInfo {std::move(uri), + deviceId, + std::move(sinkId), + active, + info.x, + info.y, + info.w, + info.h, + !info.hasVideo, + isLocalMuted, + isModeratorMuted, + isModerator, + isHandRaised}); + } else { + auto it = shared->videoToCall_.find(info.source); + if (it == shared->videoToCall_.end()) + it = shared->videoToCall_.emplace_hint(it, info.source, std::string()); + // If not local + if (!it->second.empty()) { + // Retrieve calls participants + // TODO: this is a first version, we assume that the peer is not + // a master of a conference and there is only one remote + // In the future, we should retrieve confInfo from the call + // To merge layouts informations + if (auto call = std::dynamic_pointer_cast(getCall(it->second))) { + uri = call->getPeerNumber(); + isLocalMuted = call->isPeerMuted(); + if (auto* transport = call->getTransport()) + deviceId = transport->deviceId(); + } + } + if (auto videoMixer = shared->videoMixer_) + active = videoMixer->verifyActive(info.source); + std::string_view peerId = string_remove_suffix(uri, '@'); + auto isModerator = shared->isModerator(peerId); + if (uri.empty() && !hostAdded) { + hostAdded = true; + peerId = "host"sv; + deviceId = acc->currentDeviceId(); + isLocalMuted = shared->isMediaSourceMuted(MediaType::MEDIA_AUDIO); + } + auto isHandRaised = shared->isHandRaised(peerId); + auto isModeratorMuted = shared->isMuted(peerId); + auto sinkId = shared->getConfId() + peerId; + newInfo.emplace_back(ParticipantInfo {std::move(uri), + deviceId, + std::move(sinkId), + active, + info.x, + info.y, + info.w, + info.h, + !info.hasVideo, + isLocalMuted, + isModeratorMuted, + isModerator, + isHandRaised}); } - auto active = false; - if (auto videoMixer = shared->videoMixer_) - active = info.source == videoMixer->getActiveParticipant(); - std::string_view peerId = string_remove_suffix(uri, '@'); - auto isModerator = shared->isModerator(peerId); - if (uri.empty()) { - hostAdded = true; - peerId = "host"sv; - deviceId = acc->currentDeviceId(); - isLocalMuted = shared->isMediaSourceMuted(MediaType::MEDIA_AUDIO); - } - auto isHandRaised = shared->isHandRaised(peerId); - auto isModeratorMuted = shared->isMuted(peerId); - auto sinkId = shared->getConfId() + peerId; - newInfo.emplace_back(ParticipantInfo {std::move(uri), - deviceId, - std::move(sinkId), - active, - info.x, - info.y, - info.w, - info.h, - !info.hasVideo, - isLocalMuted, - isModeratorMuted, - isModerator, - isHandRaised}); } if (auto videoMixer = shared->videoMixer_) { newInfo.h = videoMixer->getHeight(); @@ -537,12 +566,12 @@ Conference::handleMediaChangeRequest(const std::shared_ptr& call, JAMI_DBG("Conf [%s] Answer to media change request", getConfId().c_str()); #ifdef ENABLE_VIDEO - // If the new media list has video, remove existing dummy - // video sessions if any. + // If the new media list has video, remove the participant from audioonlylist. if (MediaAttribute::hasMediaType(MediaAttribute::buildMediaAttributesList(remoteMediaList, false), MediaType::MEDIA_VIDEO)) { - call->removeDummyVideoRtpSessions(); + if (videoMixer_) + videoMixer_->removeAudioOnlySource(call->getCallId()); } #endif @@ -627,13 +656,12 @@ Conference::addParticipant(const std::string& participant_id) } #ifdef ENABLE_VIDEO if (auto call = getCall(participant_id)) { - // In conference, all participants need to have video session - // (with a sink) in order to display the participant info in - // the layout. So, if a participant joins with an audio only - // call, a dummy video stream is added to the call. + // In conference, if a participant joins with an audio only + // call, it must be listed in the audioonlylist. auto mediaList = call->getMediaAttributeList(); if (not MediaAttribute::hasMediaType(mediaList, MediaType::MEDIA_VIDEO)) { - call->addDummyVideoRtpSession(); + if (videoMixer_) + videoMixer_->addAudioOnlySource(call->getCallId()); } call->enterConference(shared_from_this()); // Continue the recording for the conference if one participant was recording @@ -667,6 +695,8 @@ Conference::setActiveParticipant(const std::string& participant_id) if (auto call = getCallFromPeerID(participant_id)) { if (auto videoRecv = call->getReceiveVideoFrameActiveWriter()) videoMixer_->setActiveParticipant(videoRecv.get()); + else + videoMixer_->setActiveParticipant(call->getCallId()); return; } @@ -677,7 +707,7 @@ Conference::setActiveParticipant(const std::string& participant_id) return; } // Unset active participant by default - videoMixer_->setActiveParticipant(nullptr); + videoMixer_->resetActiveParticipant(); #endif } @@ -689,8 +719,7 @@ Conference::setLayout(int layout) case 0: videoMixer_->setVideoLayout(video::Layout::GRID); // The layout shouldn't have an active participant - if (videoMixer_->getActiveParticipant()) - videoMixer_->setActiveParticipant(nullptr); + videoMixer_->resetActiveParticipant(); break; case 1: videoMixer_->setVideoLayout(video::Layout::ONE_BIG_WITH_SMALL); @@ -802,6 +831,8 @@ Conference::removeParticipant(const std::string& participant_id) return; } if (auto call = getCall(participant_id)) { + if (videoMixer_->verifyActive(call->getCallId())) + videoMixer_->resetActiveParticipant(); participantsMuted_.erase(std::string(string_remove_suffix(call->getPeerNumber(), '@'))); handsRaised_.erase(std::string(string_remove_suffix(call->getPeerNumber(), '@'))); #ifdef ENABLE_VIDEO diff --git a/src/manager.cpp b/src/manager.cpp index b07364e48..4ce503fa2 100644 --- a/src/manager.cpp +++ b/src/manager.cpp @@ -3044,7 +3044,7 @@ Manager::createSinkClients(const std::string& callId, sinkId = callId; sinkId += string_remove_suffix(participant.uri, '@') + participant.device; } - if (participant.w && participant.h) { + if (participant.w && participant.h && !participant.videoMuted) { auto currentSink = getSinkClient(sinkId); if (currentSink) { currentSink->setCrop(participant.x, participant.y, participant.w, participant.h); diff --git a/src/media/rtp_session.h b/src/media/rtp_session.h index 5eedaef20..715b8841b 100644 --- a/src/media/rtp_session.h +++ b/src/media/rtp_session.h @@ -62,8 +62,12 @@ public: receive_ = receive; } - bool isSending() const noexcept { return send_.enabled; } - bool isReceiving() const noexcept { return receive_.enabled; } + bool isReceiving() const noexcept + { + return receive_.enabled + && (receive_.direction_ == MediaDirection::RECVONLY + || receive_.direction_ == MediaDirection::SENDRECV); + } void setMtu(uint16_t mtu) { mtu_ = mtu; } diff --git a/src/media/video/video_mixer.cpp b/src/media/video/video_mixer.cpp index f21dd169f..e6506d7ce 100644 --- a/src/media/video/video_mixer.cpp +++ b/src/media/video/video_mixer.cpp @@ -179,6 +179,7 @@ VideoMixer::stopInput() void VideoMixer::setActiveHost() { + activeAudioOnly_ = ""; activeSource_ = videoLocalSecondary_ ? videoLocalSecondary_.get() : videoLocal_.get(); updateLayout(); } @@ -186,10 +187,19 @@ VideoMixer::setActiveHost() void VideoMixer::setActiveParticipant(Observable>* ob) { + activeAudioOnly_ = ""; activeSource_ = ob; updateLayout(); } +void +VideoMixer::setActiveParticipant(const std::string& id) +{ + activeAudioOnly_ = id; + activeSource_ = nullptr; + updateLayout(); +} + void VideoMixer::updateLayout() { @@ -219,9 +229,7 @@ VideoMixer::detached(Observable>* ob) if (x->source == ob) { // Handle the case where the current shown source leave the conference if (activeSource_ == ob) { - currentLayout_ = Layout::GRID; - activeSource_ = videoLocalSecondary_ ? videoLocalSecondary_.get() - : videoLocal_.get(); + resetActiveParticipant(); } JAMI_DBG("Remove source [%p]", x.get()); sources_.remove(x); @@ -283,12 +291,24 @@ VideoMixer::process() libav_utils::fillWithBlack(output.pointer()); { + std::lock_guard lk(audioOnlySourcesMtx_); auto lock(rwMutex_.read()); int i = 0; bool activeFound = false; bool needsUpdate = layoutUpdated_ > 0; bool successfullyRendered = false; + std::vector sourcesInfo; + sourcesInfo.reserve(sources_.size() + audioOnlySources_.size()); + // add all audioonlysources + for (auto& id : audioOnlySources_) { + if (currentLayout_ != Layout::ONE_BIG or activeAudioOnly_ == id) { + sourcesInfo.emplace_back(SourceInfo {{}, 0, 0, 10, 10, false, id}); + } + if (currentLayout_ == Layout::ONE_BIG and activeAudioOnly_ == id) + successfullyRendered = true; + } + // add video sources for (auto& x : sources_) { /* thread stop pending? */ if (!loop_.isRunning()) @@ -359,11 +379,9 @@ VideoMixer::process() if (needsUpdate and successfullyRendered) { layoutUpdated_ -= 1; if (layoutUpdated_ == 0) { - std::vector sourcesInfo; - sourcesInfo.reserve(sources_.size()); for (auto& x : sources_) { sourcesInfo.emplace_back( - SourceInfo {x->source, x->x, x->y, x->w, x->h, x->hasVideo}); + SourceInfo {x->source, x->x, x->y, x->w, x->h, x->hasVideo, {}}); } if (onSourcesUpdated_) onSourcesUpdated_(std::move(sourcesInfo)); diff --git a/src/media/video/video_mixer.h b/src/media/video/video_mixer.h index 591f4d092..4f391b7e7 100644 --- a/src/media/video/video_mixer.h +++ b/src/media/video/video_mixer.h @@ -45,6 +45,7 @@ struct SourceInfo int w; int h; bool hasVideo; + std::string id; }; using OnSourcesUpdatedCb = std::function&&)>; @@ -73,9 +74,23 @@ public: void stopInput(); void setActiveParticipant(Observable>* ob); + void setActiveParticipant(const std::string& id); + void resetActiveParticipant() { + activeAudioOnly_ = ""; + activeSource_ = nullptr; + updateLayout(); + } void setActiveHost(); - Observable>* getActiveParticipant() { return activeSource_; } + bool verifyActive(const std::string& id) + { + return id == activeAudioOnly_; + } + + bool verifyActive(Observable>* ob) + { + return ob == activeSource_; + } void setVideoLayout(Layout newLayout) { @@ -93,6 +108,20 @@ public: std::shared_ptr& getSink() { return sink_; } + void addAudioOnlySource(const std::string& id) + { + std::lock_guard lk(audioOnlySourcesMtx_); + audioOnlySources_.insert(id); + updateLayout(); + } + + void removeAudioOnlySource(const std::string& id) + { + std::lock_guard lk(audioOnlySourcesMtx_); + audioOnlySources_.erase(id); + updateLayout(); + } + private: NON_COPYABLE(VideoMixer); struct VideoMixerSource; @@ -130,6 +159,10 @@ private: Observable>* activeSource_ {nullptr}; std::list> sources_; + std::mutex audioOnlySourcesMtx_; + std::set audioOnlySources_; + std::string activeAudioOnly_{}; + std::atomic_int layoutUpdated_ {0}; OnSourcesUpdatedCb onSourcesUpdated_ {}; diff --git a/src/media/video/video_rtp_session.cpp b/src/media/video/video_rtp_session.cpp index 86286a045..193d2006d 100644 --- a/src/media/video/video_rtp_session.cpp +++ b/src/media/video/video_rtp_session.cpp @@ -255,10 +255,23 @@ VideoRtpSession::startReceiver() receiveThread_->startLoop(); receiveThread_->setRequestKeyFrameCallback([this]() { cbKeyFrameRequest_(); }); receiveThread_->setRotation(rotation_.load()); + if (videoMixer_) { + auto activeParticipant = videoMixer_->verifyActive(receiveThread_.get()) + || videoMixer_->verifyActive(callID_); + videoMixer_->removeAudioOnlySource(callID_); + if (activeParticipant) + videoMixer_->setActiveParticipant(receiveThread_.get()); + } + } else { JAMI_DBG("[%p] Video receiver disabled", this); if (receiveThread_ and videoMixer_) { + auto activeParticipant = videoMixer_->verifyActive(receiveThread_.get()) + || videoMixer_->verifyActive(callID_); + videoMixer_->addAudioOnlySource(callID_); receiveThread_->detach(videoMixer_.get()); + if (activeParticipant) + videoMixer_->setActiveParticipant(callID_); } } if (socketPair_) @@ -276,7 +289,11 @@ VideoRtpSession::stopReceiver() return; if (videoMixer_) { + auto activeParticipant = videoMixer_->verifyActive(receiveThread_.get()) || videoMixer_->verifyActive(callID_); + videoMixer_->addAudioOnlySource(callID_); receiveThread_->detach(videoMixer_.get()); + if (activeParticipant) + videoMixer_->setActiveParticipant(callID_); } // We need to disable the read operation, otherwise the @@ -463,7 +480,6 @@ VideoRtpSession::setupConferenceVideoPipeline(Conference& conference, Direction conference.getConfId().c_str(), callID_.c_str()); if (receiveThread_) { - conference.detachVideo(dummyVideoReceive_.get()); receiveThread_->stopSink(); conference.attachVideo(receiveThread_.get(), callID_); } else { @@ -472,6 +488,14 @@ VideoRtpSession::setupConferenceVideoPipeline(Conference& conference, Direction } } +std::shared_ptr +VideoRtpSession::getReceiveVideoFrameActiveWriter() +{ + if (isReceiving() && receiveThread_ && conference_) + return std::static_pointer_cast(receiveThread_); + return {}; +} + void VideoRtpSession::enterConference(Conference& conference) { @@ -521,10 +545,11 @@ VideoRtpSession::exitConference() videoMixer_->detach(sender_.get()); if (receiveThread_) { + auto activetParticipant = videoMixer_->verifyActive(receiveThread_.get()); conference_->detachVideo(receiveThread_.get()); receiveThread_->startSink(); - } else { - conference_->detachVideo(dummyVideoReceive_.get()); + if (activetParticipant) + videoMixer_->setActiveParticipant(callID_); } videoMixer_.reset(); diff --git a/src/media/video/video_rtp_session.h b/src/media/video/video_rtp_session.h index 687102b07..16277f51a 100644 --- a/src/media/video/video_rtp_session.h +++ b/src/media/video/video_rtp_session.h @@ -108,12 +108,7 @@ public: std::shared_ptr& getVideoReceive() { return receiveThread_; } - std::shared_ptr getReceiveVideoFrameActiveWriter() - { - if (isReceiving() && receiveThread_) - return std::static_pointer_cast(receiveThread_); - return dummyVideoReceive_; - } + std::shared_ptr getReceiveVideoFrameActiveWriter(); private: void setupConferenceVideoPipeline(Conference& conference, Direction dir); @@ -129,8 +124,6 @@ private: std::unique_ptr sender_; std::shared_ptr receiveThread_; - std::shared_ptr dummyVideoReceive_ - = std::make_shared(); Conference* conference_ {nullptr}; std::shared_ptr videoMixer_; std::shared_ptr videoLocal_; diff --git a/src/sip/sipcall.cpp b/src/sip/sipcall.cpp index a253ed344..a12f98f62 100644 --- a/src/sip/sipcall.cpp +++ b/src/sip/sipcall.cpp @@ -54,6 +54,7 @@ #include #include #include