mirror of
https://git.jami.net/savoirfairelinux/jami-daemon.git
synced 2025-08-12 22:09:25 +08:00
autoadapt: Add REMB feedback
Change-Id: I2b1c37414f60d4d87e23539cbcb57ca4b4350754
This commit is contained in:
Pierre Lespagnol
committed by
Adrien Béraud
Adrien Béraud
parent
0f348a8f09
commit
cc4f10cedf
@ -22,7 +22,8 @@ libmedia_la_SOURCES = \
|
||||
media_filter.cpp \
|
||||
media_recorder.cpp \
|
||||
localrecorder.cpp \
|
||||
localrecordermanager.cpp
|
||||
localrecordermanager.cpp \
|
||||
remb.cpp
|
||||
|
||||
noinst_HEADERS = \
|
||||
rtp_session.h \
|
||||
@ -43,7 +44,8 @@ noinst_HEADERS = \
|
||||
media_stream.h \
|
||||
media_recorder.h \
|
||||
localrecorder.h \
|
||||
localrecordermanager.h
|
||||
localrecordermanager.h \
|
||||
remb.h
|
||||
|
||||
libmedia_la_LIBADD = \
|
||||
./audio/libaudio.la
|
||||
|
||||
122
src/media/remb.cpp
Normal file
122
src/media/remb.cpp
Normal file
@ -0,0 +1,122 @@
|
||||
/*
|
||||
* Copyright (C) 2004-2019 Savoir-faire Linux Inc.
|
||||
* Copyright (c) 2016 The WebRTC project authors. All Rights Reserved.
|
||||
*
|
||||
* Author: Pierre Lespagnol <pierre.lespagnol@savoirfairelinux.com>
|
||||
*
|
||||
* This program is free software; you can redistribute it and/or modify
|
||||
* it under the terms of the GNU General Public License as published by
|
||||
* the Free Software Foundation; either version 3 of the License, or
|
||||
* (at your option) any later version.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License
|
||||
* along with this program; if not, write to the Free Software
|
||||
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
||||
*/
|
||||
|
||||
#include "logger.h"
|
||||
#include "media/remb.h"
|
||||
|
||||
#include <cstdint>
|
||||
#include <utility>
|
||||
|
||||
|
||||
static constexpr uint8_t packetVersion = 2;
|
||||
static constexpr uint8_t packetFMT = 15;
|
||||
static constexpr uint8_t packetType = 206;
|
||||
static constexpr uint32_t uniqueIdentifier = 0x52454D42; // 'R' 'E' 'M' 'B'.
|
||||
|
||||
|
||||
namespace jami { namespace video {
|
||||
// Receiver Estimated Max Bitrate (REMB) (draft-alvestrand-rmcat-remb).
|
||||
//
|
||||
// 0 1 2 3
|
||||
// 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
|
||||
// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
|
||||
// |V=2|P| FMT=15 | PT=206 | length |
|
||||
// +=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+
|
||||
// 0 | SSRC of packet sender |
|
||||
// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
|
||||
// 4 | Unused = 0 |
|
||||
// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
|
||||
// 8 | Unique identifier 'R' 'E' 'M' 'B' |
|
||||
// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
|
||||
// 12 | Num SSRC | BR Exp | BR Mantissa |
|
||||
// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
|
||||
// 16 | SSRC feedback |
|
||||
// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
|
||||
// : ... :
|
||||
|
||||
Remb::Remb()
|
||||
{}
|
||||
|
||||
Remb::~Remb()
|
||||
{}
|
||||
|
||||
static void
|
||||
insert2Byte(std::vector<uint8_t> *v, uint16_t val)
|
||||
{
|
||||
v->insert(v->end(), val >> 8);
|
||||
v->insert(v->end(), val & 0xff);
|
||||
}
|
||||
|
||||
static void
|
||||
insert4Byte(std::vector<uint8_t> *v, uint32_t val)
|
||||
{
|
||||
v->insert(v->end(), val >> 24);
|
||||
v->insert(v->end(), (val >> 16) & 0x00ff);
|
||||
v->insert(v->end(), (val >> 8) & 0x0000ff);
|
||||
v->insert(v->end(), val & 0x000000ff);
|
||||
}
|
||||
|
||||
|
||||
uint64_t
|
||||
Remb::parseREMB(rtcpREMBHeader* packet)
|
||||
{
|
||||
if(packet->fmt != 15 || packet->pt != 206) {
|
||||
JAMI_ERR("Unable to parse REMB packet.");
|
||||
return 0;
|
||||
}
|
||||
uint64_t bitrate_bps = (packet->br_mantis << packet->br_exp);
|
||||
bool shift_overflow = (bitrate_bps >> packet->br_exp) != packet->br_mantis;
|
||||
if (shift_overflow) {
|
||||
JAMI_ERR("Invalid remb bitrate value : %u*2^%u", packet->br_mantis, packet->br_exp);
|
||||
return false;
|
||||
}
|
||||
return bitrate_bps;
|
||||
}
|
||||
|
||||
std::vector<uint8_t>
|
||||
Remb::createREMB(uint64_t bitrate_bps)
|
||||
{
|
||||
std::vector<uint8_t> remb;
|
||||
|
||||
remb.insert(remb.end(), packetVersion << 4 | packetFMT);
|
||||
remb.insert(remb.end(), packetType);
|
||||
insert2Byte(&remb, 5); // (sizeof(rtcpREMBHeader)/4)-1 -> not safe
|
||||
insert4Byte(&remb, 0x12345678); //ssrc
|
||||
insert4Byte(&remb, 0x0); //ssrc source
|
||||
insert4Byte(&remb, uniqueIdentifier); //uid
|
||||
remb.insert(remb.end(), 1); //n_ssrc
|
||||
|
||||
const uint32_t maxMantissa = 0x3ffff; // 18 bits.
|
||||
uint64_t mantissa = bitrate_bps;
|
||||
uint8_t exponenta = 0;
|
||||
while (mantissa > maxMantissa) {
|
||||
mantissa >>= 1;
|
||||
++exponenta;
|
||||
}
|
||||
|
||||
remb.insert(remb.end(), (exponenta << 2) | (mantissa >> 16));
|
||||
insert2Byte(&remb, mantissa & 0xffff);
|
||||
insert4Byte(&remb, 0x2345678b);
|
||||
|
||||
return std::move(remb);
|
||||
}
|
||||
|
||||
}} // namespace jami::video
|
||||
46
src/media/remb.h
Normal file
46
src/media/remb.h
Normal file
@ -0,0 +1,46 @@
|
||||
/*
|
||||
* Copyright (C) 2004-2019 Savoir-faire Linux Inc.
|
||||
* Copyright (c) 2016 The WebRTC project authors. All Rights Reserved.
|
||||
*
|
||||
* Author: Pierre Lespagnol <pierre.lespagnol@savoirfairelinux.com>
|
||||
*
|
||||
* This program is free software; you can redistribute it and/or modify
|
||||
* it under the terms of the GNU General Public License as published by
|
||||
* the Free Software Foundation; either version 3 of the License, or
|
||||
* (at your option) any later version.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License
|
||||
* along with this program; if not, write to the Free Software
|
||||
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
||||
*/
|
||||
|
||||
#ifndef MODULES_RTP_RTCP_SOURCE_RTCP_PACKET_REMB_H_
|
||||
#define MODULES_RTP_RTCP_SOURCE_RTCP_PACKET_REMB_H_
|
||||
|
||||
#include <vector>
|
||||
#include <cstdint>
|
||||
|
||||
#include "socket_pair.h"
|
||||
|
||||
namespace jami { namespace video {
|
||||
|
||||
// Receiver Estimated Max Bitrate (REMB) (draft-alvestrand-rmcat-remb).
|
||||
class Remb {
|
||||
public:
|
||||
Remb();
|
||||
~Remb();
|
||||
|
||||
static uint64_t parseREMB(rtcpREMBHeader* packet);
|
||||
std::vector<uint8_t> createREMB(uint64_t bitrate_bps);
|
||||
|
||||
private:
|
||||
|
||||
};
|
||||
|
||||
}} // namespace jami::video
|
||||
#endif
|
||||
@ -84,6 +84,8 @@ static constexpr int RTP_MAX_PACKET_LENGTH = 2048;
|
||||
static constexpr auto UDP_HEADER_SIZE = 8;
|
||||
static constexpr auto SRTP_OVERHEAD = 10;
|
||||
static constexpr uint32_t RTCP_RR_FRACTION_MASK = 0xFF000000;
|
||||
static constexpr unsigned MINIMUM_RTP_HEADER_SIZE = 16;
|
||||
|
||||
|
||||
enum class DataType : unsigned { RTP=1<<0, RTCP=1<<1 };
|
||||
|
||||
@ -218,12 +220,12 @@ SocketPair::waitForRTCP(std::chrono::seconds interval)
|
||||
{
|
||||
std::unique_lock<std::mutex> lock(rtcpInfo_mutex_);
|
||||
return cvRtcpPacketReadyToRead_.wait_for(lock, interval, [this]{
|
||||
return interrupted_ or not listRtcpHeader_.empty();
|
||||
return interrupted_ or not listRtcpRRHeader_.empty() or not listRtcpREMBHeader_.empty();
|
||||
});
|
||||
}
|
||||
|
||||
void
|
||||
SocketPair::saveRtcpPacket(uint8_t* buf, size_t len)
|
||||
SocketPair::saveRtcpRRPacket(uint8_t* buf, size_t len)
|
||||
{
|
||||
if (len < sizeof(rtcpRRHeader))
|
||||
return;
|
||||
@ -234,20 +236,49 @@ SocketPair::saveRtcpPacket(uint8_t* buf, size_t len)
|
||||
|
||||
std::lock_guard<std::mutex> lock(rtcpInfo_mutex_);
|
||||
|
||||
if (listRtcpHeader_.size() >= MAX_LIST_SIZE) {
|
||||
listRtcpHeader_.pop_front();
|
||||
if (listRtcpRRHeader_.size() >= MAX_LIST_SIZE) {
|
||||
listRtcpRRHeader_.pop_front();
|
||||
}
|
||||
|
||||
listRtcpHeader_.push_back(*header);
|
||||
listRtcpRRHeader_.emplace_back(*header);
|
||||
|
||||
cvRtcpPacketReadyToRead_.notify_one();
|
||||
}
|
||||
|
||||
|
||||
void
|
||||
SocketPair::saveRtcpREMBPacket(uint8_t* buf, size_t len)
|
||||
{
|
||||
if (len < sizeof(rtcpREMBHeader))
|
||||
return;
|
||||
|
||||
auto header = reinterpret_cast<rtcpREMBHeader*>(buf);
|
||||
if(header->pt != 206) //206 = REMB PT
|
||||
return;
|
||||
|
||||
std::lock_guard<std::mutex> lock(rtcpInfo_mutex_);
|
||||
|
||||
if (listRtcpREMBHeader_.size() >= MAX_LIST_SIZE) {
|
||||
listRtcpREMBHeader_.pop_front();
|
||||
}
|
||||
|
||||
listRtcpREMBHeader_.push_back(*header);
|
||||
|
||||
cvRtcpPacketReadyToRead_.notify_one();
|
||||
}
|
||||
|
||||
std::list<rtcpRRHeader>
|
||||
SocketPair::getRtcpInfo()
|
||||
SocketPair::getRtcpRR()
|
||||
{
|
||||
std::lock_guard<std::mutex> lock(rtcpInfo_mutex_);
|
||||
return std::move(listRtcpHeader_);
|
||||
return std::move(listRtcpRRHeader_);
|
||||
}
|
||||
|
||||
std::list<rtcpREMBHeader>
|
||||
SocketPair::getRtcpREMB()
|
||||
{
|
||||
std::lock_guard<std::mutex> lock(rtcpInfo_mutex_);
|
||||
return std::move(listRtcpREMBHeader_);
|
||||
}
|
||||
|
||||
void
|
||||
@ -430,18 +461,21 @@ SocketPair::readCallback(uint8_t* buf, int buf_size)
|
||||
int len = 0;
|
||||
bool fromRTCP = false;
|
||||
|
||||
auto header = reinterpret_cast<rtcpRRHeader*>(buf);
|
||||
if(header->pt == 201) //201 = RR PT
|
||||
{
|
||||
lastDLSR_ = Swap4Bytes(header->dlsr);
|
||||
//JAMI_WARN("Read RR, lastDLSR : %d", lastDLSR_);
|
||||
lastRR_time = std::chrono::steady_clock::now();
|
||||
}
|
||||
|
||||
// Priority to RTCP as its less invasive in bandwidth
|
||||
if (datatype & static_cast<int>(DataType::RTCP)) {
|
||||
len = readRtcpData(buf, buf_size);
|
||||
saveRtcpPacket(buf, len);
|
||||
auto header = reinterpret_cast<rtcpRRHeader*>(buf);
|
||||
if(header->pt == 201) //201 = RR PT
|
||||
{
|
||||
lastDLSR_ = Swap4Bytes(header->dlsr);
|
||||
//JAMI_WARN("Read RR, lastDLSR : %d", lastDLSR_);
|
||||
lastRR_time = std::chrono::steady_clock::now();
|
||||
saveRtcpRRPacket(buf, len);
|
||||
}
|
||||
else if(header->pt == 206) //206 = REMB PT
|
||||
{
|
||||
JAMI_ERR("Receive Remb !!");
|
||||
saveRtcpREMBPacket(buf, len);
|
||||
}
|
||||
fromRTCP = true;
|
||||
}
|
||||
|
||||
@ -454,11 +488,25 @@ SocketPair::readCallback(uint8_t* buf, int buf_size)
|
||||
if (len <= 0)
|
||||
return len;
|
||||
|
||||
if (not fromRTCP && (buf_size < MINIMUM_RTP_HEADER_SIZE))
|
||||
return len;
|
||||
|
||||
// SRTP decrypt
|
||||
if (not fromRTCP and srtpContext_ and srtpContext_->srtp_in.aes) {
|
||||
uint32_t curentSendTS = buf[4] << 24 | buf[5] << 16 | buf[6] << 8 | buf[7];
|
||||
int32_t delay = 0;
|
||||
float abs = 0.0f;
|
||||
bool marker = (buf[1] & 0x80) >> 7;
|
||||
bool res = getOneWayDelayGradient2(abs, marker, &delay);
|
||||
|
||||
if (res)
|
||||
rtpDelayCallback_(delay);
|
||||
|
||||
auto err = ff_srtp_decrypt(&srtpContext_->srtp_in, buf, &len);
|
||||
if(packetLossCallback_ and (buf[2] << 8 | buf[3]) != lastSeqNum_+1)
|
||||
packetLossCallback_();
|
||||
if(packetLossCallback_ and (buf[2] << 8 | buf[3]) != lastSeqNum_+1) {
|
||||
JAMI_ERR("RTP missed !");
|
||||
// packetLossCallback_();
|
||||
}
|
||||
lastSeqNum_ = buf[2] << 8 | buf[3];
|
||||
if (err < 0)
|
||||
JAMI_WARN("decrypt error %d", err);
|
||||
@ -590,4 +638,105 @@ SocketPair::setPacketLossCallback(std::function<void(void)> cb)
|
||||
packetLossCallback_ = std::move(cb);
|
||||
}
|
||||
|
||||
void
|
||||
SocketPair::setRtpDelayCallback(std::function<void(int)> cb)
|
||||
{
|
||||
rtpDelayCallback_ = std::move(cb);
|
||||
}
|
||||
|
||||
int32_t
|
||||
SocketPair::getOneWayDelayGradient(uint32_t sendTS)
|
||||
{
|
||||
if (not lastSendTS_) {
|
||||
lastSendTS_ = sendTS;
|
||||
lastReceiveTS_ = std::chrono::steady_clock::now();
|
||||
return 0;
|
||||
}
|
||||
|
||||
if (sendTS == lastSendTS_)
|
||||
return 0;
|
||||
|
||||
uint32_t deltaS = ((sendTS - lastSendTS_) / 90000.0) * 1000000; // microseconds
|
||||
lastSendTS_ = sendTS;
|
||||
|
||||
std::chrono::steady_clock::time_point arrival_TS = std::chrono::steady_clock::now();
|
||||
auto deltaR = std::chrono::duration_cast<std::chrono::microseconds>(arrival_TS - lastReceiveTS_).count();
|
||||
lastReceiveTS_ = arrival_TS;
|
||||
|
||||
return deltaR - deltaS;
|
||||
}
|
||||
|
||||
bool
|
||||
SocketPair::getOneWayDelayGradient2(float sendTS, bool marker, int32_t* gradient)
|
||||
{
|
||||
// Keep only last packet of each frame
|
||||
if (not marker) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
// 1st frame
|
||||
if (not lastSendTS_) {
|
||||
lastSendTS_ = sendTS;
|
||||
lastReceiveTS_ = std::chrono::steady_clock::now();
|
||||
return 0;
|
||||
}
|
||||
|
||||
uint32_t deltaS = (sendTS - lastSendTS_) * 1000; // milliseconds
|
||||
if(deltaS < 0)
|
||||
deltaS += 64000;
|
||||
lastSendTS_ = sendTS;
|
||||
|
||||
std::chrono::steady_clock::time_point arrival_TS = std::chrono::steady_clock::now();
|
||||
auto deltaR = std::chrono::duration_cast<std::chrono::milliseconds>(arrival_TS - lastReceiveTS_).count();
|
||||
lastReceiveTS_ = arrival_TS;
|
||||
|
||||
*gradient = deltaR - deltaS;
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
bool
|
||||
SocketPair::getOneWayDelayGradient3(uint32_t sendTS, int32_t* gradient)
|
||||
{
|
||||
// First sample, fill Ts0 and Tr0
|
||||
if(not svgTS.send_ts) {
|
||||
svgTS.send_ts = sendTS;
|
||||
svgTS.receive_ts = std::chrono::steady_clock::now();
|
||||
return false;
|
||||
}
|
||||
|
||||
// new frame
|
||||
if(svgTS.send_ts != sendTS) {
|
||||
// Second sample, fill Ts1 and Tr1 and replace Ts0 and Tr0
|
||||
if(not svgTS.last_send_ts) {
|
||||
svgTS.last_send_ts = svgTS.send_ts;
|
||||
svgTS.send_ts = sendTS;
|
||||
|
||||
svgTS.last_receive_ts = svgTS.receive_ts;
|
||||
svgTS.receive_ts = std::chrono::steady_clock::now();
|
||||
return false;
|
||||
}
|
||||
uint32_t deltaS = ((svgTS.send_ts - svgTS.last_send_ts) / 90000.0) * 1000000; // microseconds
|
||||
auto deltaR = std::chrono::duration_cast<std::chrono::microseconds>(svgTS.receive_ts - svgTS.last_receive_ts).count();
|
||||
|
||||
*gradient = deltaR - deltaS;
|
||||
|
||||
// JAMI_ERR("[PL] delR:%ld, delS:%ld, gradient:%d", deltaR, deltaS, *gradient);
|
||||
|
||||
// fill Ts1 and Tr1 and replace Ts0 and Tr0
|
||||
svgTS.last_send_ts = svgTS.send_ts;
|
||||
svgTS.send_ts = sendTS;
|
||||
|
||||
svgTS.last_receive_ts = svgTS.receive_ts;
|
||||
svgTS.receive_ts = std::chrono::steady_clock::now();
|
||||
}
|
||||
else {
|
||||
// Update receive TS
|
||||
svgTS.receive_ts = std::chrono::steady_clock::now();
|
||||
return false;
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
} // namespace jami
|
||||
|
||||
@ -97,6 +97,37 @@ typedef struct {
|
||||
} rtcpSRHeader;
|
||||
|
||||
|
||||
typedef struct {
|
||||
#ifdef WORDS_BIGENDIAN
|
||||
uint32_t version:2; /* protocol version */
|
||||
uint32_t p:1; /* padding flag always 0 */
|
||||
uint32_t fmt:5; /* Feedback message type always 15 */
|
||||
|
||||
#else
|
||||
uint32_t fmt:5; /* Feedback message type always 15 */
|
||||
uint32_t p:1; /* padding flag always 0 */
|
||||
uint32_t version:2; /* protocol version */
|
||||
#endif
|
||||
uint32_t pt:8; /* payload type */
|
||||
uint32_t len:16; /* length of RTCP packet */
|
||||
uint32_t ssrc; /* synchronization source identifier of packet sender */
|
||||
uint32_t ssrc_source; /* synchronization source identifier of first source alway 0*/
|
||||
uint32_t uid; /* Unique identifier Always ‘R’ ‘E’ ‘M’ ‘B’ (4 ASCII characters). */
|
||||
uint32_t n_ssrc:8; /* Number of SSRCs in this message. */
|
||||
uint32_t br_exp:6; /* BR Exp */
|
||||
uint32_t br_mantis:18; /* BR Mantissa */
|
||||
uint32_t f_ssrc; /* SSRC feedback */
|
||||
} rtcpREMBHeader;
|
||||
|
||||
|
||||
|
||||
typedef struct {
|
||||
uint64_t last_send_ts;
|
||||
std::chrono::steady_clock::time_point last_receive_ts;
|
||||
uint64_t send_ts;
|
||||
std::chrono::steady_clock::time_point receive_ts;
|
||||
} TS_Frame;
|
||||
|
||||
class SocketPair {
|
||||
public:
|
||||
SocketPair(const char* uri, int localPort);
|
||||
@ -131,12 +162,16 @@ class SocketPair {
|
||||
const char* in_suite, const char* in_params);
|
||||
|
||||
void stopSendOp(bool state = true);
|
||||
std::list<rtcpRRHeader> getRtcpInfo();
|
||||
std::list<rtcpRRHeader> getRtcpRR();
|
||||
std::list<rtcpREMBHeader> getRtcpREMB();
|
||||
|
||||
bool waitForRTCP(std::chrono::seconds interval);
|
||||
double getLastLatency();
|
||||
|
||||
void setPacketLossCallback(std::function<void (void)> cb);
|
||||
void setRtpDelayCallback(std::function<void (int)> cb);
|
||||
|
||||
int writeData(uint8_t* buf, int buf_size);
|
||||
|
||||
private:
|
||||
NON_COPYABLE(SocketPair);
|
||||
@ -147,8 +182,8 @@ class SocketPair {
|
||||
int waitForData();
|
||||
int readRtpData(void* buf, int buf_size);
|
||||
int readRtcpData(void* buf, int buf_size);
|
||||
int writeData(uint8_t* buf, int buf_size);
|
||||
void saveRtcpPacket(uint8_t* buf, size_t len);
|
||||
void saveRtcpRRPacket(uint8_t* buf, size_t len);
|
||||
void saveRtcpREMBPacket(uint8_t* buf, size_t len);
|
||||
|
||||
std::mutex dataBuffMutex_;
|
||||
std::condition_variable cv_;
|
||||
@ -166,8 +201,13 @@ class SocketPair {
|
||||
std::atomic_bool noWrite_ {false};
|
||||
std::unique_ptr<SRTPProtoContext> srtpContext_;
|
||||
std::function<void(void)> packetLossCallback_;
|
||||
std::function<void(int)> rtpDelayCallback_;
|
||||
int32_t getOneWayDelayGradient(uint32_t sendTS);
|
||||
bool getOneWayDelayGradient2(float sendTS, bool marker, int32_t* gradient);
|
||||
bool getOneWayDelayGradient3(uint32_t sendTS, int32_t* gradient);
|
||||
|
||||
std::list<rtcpRRHeader> listRtcpHeader_;
|
||||
std::list<rtcpRRHeader> listRtcpRRHeader_;
|
||||
std::list<rtcpREMBHeader> listRtcpREMBHeader_;
|
||||
std::mutex rtcpInfo_mutex_;
|
||||
std::condition_variable cvRtcpPacketReadyToRead_;
|
||||
static constexpr unsigned MAX_LIST_SIZE {10};
|
||||
@ -180,6 +220,13 @@ class SocketPair {
|
||||
|
||||
std::chrono::steady_clock::time_point lastRR_time;
|
||||
uint16_t lastSeqNum_ {0};
|
||||
uint32_t lastSendTS_ {0};
|
||||
bool lastMarker_ {false};
|
||||
std::chrono::steady_clock::time_point lastReceiveTS_ {};
|
||||
std::chrono::steady_clock::time_point arrival_TS {};
|
||||
|
||||
TS_Frame svgTS = {};
|
||||
|
||||
};
|
||||
|
||||
|
||||
|
||||
@ -54,7 +54,8 @@ enum RTCPType {
|
||||
RTCP_FIR = 192,
|
||||
RTCP_IJ = 195,
|
||||
RTCP_SR = 200,
|
||||
RTCP_TOKEN = 210
|
||||
RTCP_TOKEN = 210,
|
||||
RTCP_REMB = 206
|
||||
};
|
||||
|
||||
#define RTP_PT_IS_RTCP(x) (((x) >= RTCP_FIR && (x) <= RTCP_IJ) || \
|
||||
|
||||
@ -33,6 +33,7 @@
|
||||
#include "string_utils.h"
|
||||
#include "call.h"
|
||||
#include "conference.h"
|
||||
#include "remb.h"
|
||||
|
||||
#include "account_const.h"
|
||||
|
||||
@ -46,8 +47,17 @@ namespace jami { namespace video {
|
||||
|
||||
using std::string;
|
||||
|
||||
static constexpr unsigned MAX_SIZE_HISTO_QUALITY {30};
|
||||
static constexpr unsigned MAX_SIZE_HISTO_BITRATE {100};
|
||||
static constexpr unsigned MAX_SIZE_HISTO_JITTER {50};
|
||||
static constexpr unsigned MAX_SIZE_HISTO_DELAY {25};
|
||||
|
||||
constexpr auto DELAY_AFTER_RESTART = std::chrono::milliseconds(1000);
|
||||
constexpr auto EXPIRY_TIME_RTCP = std::chrono::milliseconds(1000);
|
||||
constexpr auto DELAY_AFTER_INCREASE = std::chrono::seconds(5);
|
||||
constexpr auto EXPIRY_TIME_RTCP = std::chrono::seconds(2);
|
||||
constexpr auto DELAY_AFTER_REMB = std::chrono::milliseconds(300);
|
||||
|
||||
constexpr int THRESHOLD_CONGESTION {1000};
|
||||
|
||||
VideoRtpSession::VideoRtpSession(const string &callID,
|
||||
const DeviceParams& localVideoParams) :
|
||||
@ -75,7 +85,7 @@ VideoRtpSession::updateMedia(const MediaDescription& send, const MediaDescriptio
|
||||
void
|
||||
VideoRtpSession::setRequestKeyFrameCallback(std::function<void(void)> cb)
|
||||
{
|
||||
if(socketPair_)
|
||||
if (socketPair_)
|
||||
socketPair_->setPacketLossCallback(std::move(cb));
|
||||
else
|
||||
JAMI_ERR("No socket pair, keyframe request callback not possible");
|
||||
@ -133,12 +143,18 @@ void VideoRtpSession::startSender()
|
||||
send_.enabled = false;
|
||||
}
|
||||
lastMediaRestart_ = clock::now();
|
||||
lastIncrease_ = clock::now();
|
||||
lastREMB_ = clock::now();
|
||||
auto codecVideo = std::static_pointer_cast<jami::AccountVideoCodecInfo>(send_.codec);
|
||||
auto autoQuality = codecVideo->isAutoQualityEnabled;
|
||||
if (autoQuality and not rtcpCheckerThread_.isRunning())
|
||||
rtcpCheckerThread_.start();
|
||||
else if (not autoQuality and rtcpCheckerThread_.isRunning())
|
||||
rtcpCheckerThread_.join();
|
||||
|
||||
socketPair_->setRtpDelayCallback([&](int delay) {
|
||||
this->delayMonitor(delay);
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
@ -345,9 +361,9 @@ void VideoRtpSession::exitConference()
|
||||
}
|
||||
|
||||
bool
|
||||
VideoRtpSession::checkMediumRCTPInfo(RTCPInfo& rtcpi)
|
||||
VideoRtpSession::check_RCTP_Info_RR(RTCPInfo& rtcpi)
|
||||
{
|
||||
auto rtcpInfoVect = socketPair_->getRtcpInfo();
|
||||
auto rtcpInfoVect = socketPair_->getRtcpRR();
|
||||
unsigned totalLost = 0;
|
||||
unsigned totalJitter = 0;
|
||||
unsigned nbDropNotNull = 0;
|
||||
@ -355,7 +371,7 @@ VideoRtpSession::checkMediumRCTPInfo(RTCPInfo& rtcpi)
|
||||
|
||||
if (vectSize != 0) {
|
||||
for (const auto& it : rtcpInfoVect) {
|
||||
if(it.fraction_lost != 0) // Exclude null drop
|
||||
if (it.fraction_lost != 0) // Exclude null drop
|
||||
nbDropNotNull++;
|
||||
totalLost += it.fraction_lost;
|
||||
totalJitter += ntohl(it.jitter);
|
||||
@ -371,6 +387,20 @@ VideoRtpSession::checkMediumRCTPInfo(RTCPInfo& rtcpi)
|
||||
return false;
|
||||
}
|
||||
|
||||
bool
|
||||
VideoRtpSession::check_RCTP_Info_REMB(uint64_t* br)
|
||||
{
|
||||
auto rtcpInfoVect = socketPair_->getRtcpREMB();
|
||||
|
||||
if (!rtcpInfoVect.empty()) {
|
||||
for (auto& it : rtcpInfoVect) {
|
||||
*br = Remb::parseREMB(&it);
|
||||
}
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
unsigned
|
||||
VideoRtpSession::getLowerQuality()
|
||||
{
|
||||
@ -410,41 +440,89 @@ VideoRtpSession::adaptQualityAndBitrate()
|
||||
{
|
||||
setupVideoBitrateInfo();
|
||||
|
||||
RTCPInfo rtcpi {};
|
||||
if (not checkMediumRCTPInfo(rtcpi)) {
|
||||
JAMI_DBG("[AutoAdapt] Sample not ready");
|
||||
return;
|
||||
uint64_t br;
|
||||
if (check_RCTP_Info_REMB(&br)) {
|
||||
JAMI_WARN("[AutoAdapt] New REMB !");
|
||||
delayProcessing(br);
|
||||
}
|
||||
|
||||
RTCPInfo rtcpi {};
|
||||
if (check_RCTP_Info_RR(rtcpi)) {
|
||||
// JAMI_WARN("[AutoAdapt] New RR !");
|
||||
dropProcessing(&rtcpi);
|
||||
}
|
||||
}
|
||||
|
||||
void
|
||||
VideoRtpSession::dropProcessing(RTCPInfo* rtcpi)
|
||||
{
|
||||
// If bitrate has changed, let time to receive fresh RTCP packets
|
||||
auto now = clock::now();
|
||||
auto restartTimer = now - lastMediaRestart_;
|
||||
auto increaseTimer = now - lastIncrease_;
|
||||
if (restartTimer < DELAY_AFTER_RESTART) {
|
||||
//JAMI_DBG("[AutoAdapt] Waiting for delay %ld ms", std::chrono::duration_cast<std::chrono::milliseconds>(restartTimer));
|
||||
return;
|
||||
}
|
||||
|
||||
if (rtcpi.jitter > 1000) {
|
||||
//JAMI_DBG("[AutoAdapt] Jitter too high");
|
||||
//Do nothing if jitter is more than 1 second
|
||||
if (rtcpi->jitter > 1000) {
|
||||
return;
|
||||
}
|
||||
|
||||
auto pondLoss = getPonderateLoss(rtcpi->packetLoss);
|
||||
auto oldBitrate = videoBitrateInfo_.videoBitrateCurrent;
|
||||
int newBitrate = oldBitrate;
|
||||
|
||||
// JAMI_DBG("[AutoAdapt] pond loss: %f%, last loss: %f%, last jitter: %dms, jitterAvg: %fms, jitterDev: %fms" , pondLoss, rtcpi->packetLoss, rtcpi->jitter, jitterAvg, jitterDev);
|
||||
|
||||
// Fill histoLoss and histoJitter_ with samples
|
||||
if (restartTimer < DELAY_AFTER_RESTART + std::chrono::seconds(1)) {
|
||||
return;
|
||||
}
|
||||
else {
|
||||
// If ponderate drops are inferior to 10% that mean drop are not from congestion but from network...
|
||||
// ... we can increase
|
||||
if (pondLoss >= 10.0f && rtcpi->packetLoss > 0.0f) {
|
||||
newBitrate *= 1.0f - rtcpi->packetLoss/150.0f;
|
||||
histoLoss_.clear();
|
||||
JAMI_DBG("[AutoAdapt] pondLoss: %f%%, packet loss rate: %f%%, decrease bitrate from %d Kbps to %d Kbps, ratio %f", pondLoss, rtcpi->packetLoss, oldBitrate, newBitrate, (float) newBitrate / oldBitrate);
|
||||
}
|
||||
else {
|
||||
if (increaseTimer < DELAY_AFTER_INCREASE)
|
||||
return;
|
||||
|
||||
newBitrate += 50.0f;
|
||||
JAMI_DBG("[AutoAdapt] pondLoss: %f%%, packet loss rate: %f%%, increase bitrate from %d Kbps to %d Kbps, ratio %f", pondLoss, rtcpi->packetLoss, oldBitrate, newBitrate, (float) newBitrate / oldBitrate);
|
||||
histoLoss_.clear();
|
||||
}
|
||||
}
|
||||
|
||||
setNewBitrate(newBitrate);
|
||||
}
|
||||
|
||||
void
|
||||
VideoRtpSession::delayProcessing(int br)
|
||||
{
|
||||
// If bitrate has changed, let time to receive fresh RTCP packets
|
||||
|
||||
auto oldBitrate = videoBitrateInfo_.videoBitrateCurrent;
|
||||
int newBitrate = oldBitrate;
|
||||
|
||||
//Take action only when two successive drop superior to 5% are catched...
|
||||
//and when jitter is less than 1 seconds
|
||||
auto pondLoss = getPonderateLoss(rtcpi.packetLoss);
|
||||
//JAMI_DBG("[AutoAdapt] Pondloss: %f%%, last loss: %f%%", pondLoss, rtcpi.packetLoss);
|
||||
if(pondLoss >= 5.0f) {
|
||||
videoBitrateInfo_.videoBitrateCurrent = videoBitrateInfo_.videoBitrateCurrent * (1.0f - rtcpi.packetLoss/200.0f);
|
||||
JAMI_DBG("[AutoAdapt] pondLoss: %f%%, packet loss rate: %f%%, decrease bitrate from %d Kbps to %d Kbps, ratio %f", pondLoss, rtcpi.packetLoss, oldBitrate, videoBitrateInfo_.videoBitrateCurrent, (float) videoBitrateInfo_.videoBitrateCurrent / oldBitrate);
|
||||
histoLoss_.clear();
|
||||
}
|
||||
newBitrate *= 0.90f;
|
||||
setNewBitrate(newBitrate);
|
||||
}
|
||||
|
||||
videoBitrateInfo_.videoBitrateCurrent = std::max(videoBitrateInfo_.videoBitrateCurrent, videoBitrateInfo_.videoBitrateMin);
|
||||
videoBitrateInfo_.videoBitrateCurrent = std::min(videoBitrateInfo_.videoBitrateCurrent, videoBitrateInfo_.videoBitrateMax);
|
||||
|
||||
if(oldBitrate != videoBitrateInfo_.videoBitrateCurrent) {
|
||||
void
|
||||
VideoRtpSession::setNewBitrate(unsigned int newBR)
|
||||
{
|
||||
newBR = std::max(newBR, videoBitrateInfo_.videoBitrateMin);
|
||||
newBR = std::min(newBR, videoBitrateInfo_.videoBitrateMax);
|
||||
|
||||
auto now = clock::now();
|
||||
|
||||
if (videoBitrateInfo_.videoBitrateCurrent != newBR) {
|
||||
videoBitrateInfo_.videoBitrateCurrent = newBR;
|
||||
storeVideoBitrateInfo();
|
||||
|
||||
#if __ANDROID__
|
||||
@ -453,8 +531,11 @@ VideoRtpSession::adaptQualityAndBitrate()
|
||||
#endif
|
||||
|
||||
// If encoder no longer exist do nothing
|
||||
if(sender_ && sender_->setBitrate(videoBitrateInfo_.videoBitrateCurrent) == 0)
|
||||
if (sender_ && sender_->setBitrate(newBR) == 0) {
|
||||
lastMediaRestart_ = now;
|
||||
// Reset increase timer for each bitrate change
|
||||
lastIncrease_ = now;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@ -494,10 +575,10 @@ VideoRtpSession::storeVideoBitrateInfo() {
|
||||
{DRing::Account::ConfProperties::CodecInfo::MAX_QUALITY, std::to_string(videoBitrateInfo_.videoQualityMax)}
|
||||
});
|
||||
|
||||
if (histoQuality_.size() > MAX_SIZE_HISTO_QUALITY_)
|
||||
if (histoQuality_.size() > MAX_SIZE_HISTO_QUALITY)
|
||||
histoQuality_.pop_front();
|
||||
|
||||
if (histoBitrate_.size() > MAX_SIZE_HISTO_BITRATE_)
|
||||
if (histoBitrate_.size() > MAX_SIZE_HISTO_BITRATE)
|
||||
histoBitrate_.pop_front();
|
||||
|
||||
histoQuality_.push_back(videoBitrateInfo_.videoQualityCurrent);
|
||||
@ -556,8 +637,8 @@ float
|
||||
VideoRtpSession::getPonderateLoss(float lastLoss)
|
||||
{
|
||||
float pond = 0.0f, pondLoss = 0.0f, totalPond = 0.0f;
|
||||
constexpr float coefficient_a = -1/1000.0f;
|
||||
constexpr float coefficient_b = 1.0f;
|
||||
constexpr float coefficient_a = -1/100.0f;
|
||||
constexpr float coefficient_b = 100.0f;
|
||||
|
||||
auto now = clock::now();
|
||||
|
||||
@ -569,10 +650,12 @@ VideoRtpSession::getPonderateLoss(float lastLoss)
|
||||
//JAMI_WARN("now - it.first: %ld", std::chrono::duration_cast<std::chrono::milliseconds>(delay));
|
||||
|
||||
// 1ms -> 100%
|
||||
// 1000ms -> 1
|
||||
if(delay <= EXPIRY_TIME_RTCP)
|
||||
{
|
||||
pond = std::min(delay.count() * coefficient_a + coefficient_b, 1.0f);
|
||||
// 2000ms -> 80%
|
||||
if (delay <= EXPIRY_TIME_RTCP) {
|
||||
if (it->second == 0.0f)
|
||||
pond = 20.0f; // Reduce weight of null drop
|
||||
else
|
||||
pond = std::min(delay.count() * coefficient_a + coefficient_b, 100.0f);
|
||||
totalPond += pond;
|
||||
pondLoss += it->second * pond;
|
||||
++it;
|
||||
@ -580,8 +663,89 @@ VideoRtpSession::getPonderateLoss(float lastLoss)
|
||||
else
|
||||
it = histoLoss_.erase(it);
|
||||
}
|
||||
if (totalPond == 0)
|
||||
return 0.0f;
|
||||
|
||||
return pondLoss / totalPond;
|
||||
}
|
||||
|
||||
std::pair<float, float>
|
||||
VideoRtpSession::getDelayAvg()
|
||||
{
|
||||
if (histoDelay_.size() != MAX_SIZE_HISTO_DELAY)
|
||||
return std::make_pair(0.0f, 0.0f);
|
||||
|
||||
auto middle = std::next(histoDelay_.begin(), histoDelay_.size() / 2);
|
||||
float totDelayInf = 0.0f;
|
||||
float totDelaySup = 0.0f;
|
||||
unsigned cntInf = 0;
|
||||
unsigned cntSup = 0;
|
||||
|
||||
for (auto it = histoDelay_.begin(); it != middle; ++it) {
|
||||
totDelayInf += *it;
|
||||
cntInf++;
|
||||
}
|
||||
|
||||
for (auto it = middle; it != histoDelay_.end(); ++it) {
|
||||
totDelaySup += *it;
|
||||
cntSup++;
|
||||
}
|
||||
|
||||
return std::make_pair(totDelayInf/cntInf, totDelaySup/cntSup);
|
||||
}
|
||||
|
||||
std::pair<float, float>
|
||||
VideoRtpSession::getDelayMedian()
|
||||
{
|
||||
if (histoDelay_.size() != MAX_SIZE_HISTO_DELAY)
|
||||
return std::make_pair(0.0f, 0.0f);
|
||||
|
||||
auto middle = std::next(histoDelay_.begin(), histoDelay_.size() / 2);
|
||||
std::list<int> lst2( histoDelay_.begin(), middle );
|
||||
std::list<int> lst3( middle, histoDelay_.end() );
|
||||
|
||||
lst2.sort(); lst3.sort();
|
||||
|
||||
auto mediumInf = *(std::next(lst2.begin(), lst2.size() / 2));
|
||||
auto mediumSup = *(std::next(lst3.begin(), lst3.size() / 2));
|
||||
|
||||
JAMI_WARN("Last medium delay: %d, current medium delay: %d", mediumInf, mediumSup);
|
||||
|
||||
return std::make_pair(mediumInf, mediumSup);
|
||||
}
|
||||
|
||||
float
|
||||
VideoRtpSession::getRollingAvg()
|
||||
{
|
||||
if (histoDelay_.size() < MAX_SIZE_HISTO_DELAY)
|
||||
return 0.0f;
|
||||
|
||||
float totDelay = 0;
|
||||
|
||||
for (auto it = histoDelay_.begin(); it != histoDelay_.end(); ++it) {
|
||||
totDelay += (float)*it;
|
||||
}
|
||||
|
||||
return totDelay / histoDelay_.size();
|
||||
}
|
||||
|
||||
int
|
||||
VideoRtpSession::getRollingMedian()
|
||||
{
|
||||
if (histoDelay_.size() < MAX_SIZE_HISTO_DELAY)
|
||||
return 0;
|
||||
|
||||
auto delay_bkp = histoDelay_;
|
||||
delay_bkp.sort();
|
||||
auto middle = *(std::next(delay_bkp.begin(), delay_bkp.size() / 2));
|
||||
|
||||
return middle;
|
||||
}
|
||||
|
||||
void
|
||||
VideoRtpSession::delayMonitor(int delay)
|
||||
{
|
||||
JAMI_WARN("delay: %d", delay);
|
||||
}
|
||||
|
||||
}} // namespace jami::video
|
||||
|
||||
@ -121,7 +121,8 @@ private:
|
||||
|
||||
std::function<void (void)> requestKeyFrameCallback_;
|
||||
|
||||
bool checkMediumRCTPInfo(RTCPInfo&);
|
||||
bool check_RCTP_Info_RR(RTCPInfo&);
|
||||
bool check_RCTP_Info_REMB(uint64_t*);
|
||||
unsigned getLowerQuality();
|
||||
unsigned getLowerBitrate();
|
||||
void adaptQualityAndBitrate();
|
||||
@ -129,17 +130,22 @@ private:
|
||||
void setupVideoBitrateInfo();
|
||||
void checkReceiver();
|
||||
float getPonderateLoss(float lastLoss);
|
||||
void delayMonitor(int delay);
|
||||
void dropProcessing(RTCPInfo* rtcpi);
|
||||
void delayProcessing(int br);
|
||||
void setNewBitrate(unsigned int newBR);
|
||||
|
||||
// no packet loss can be calculated as no data in input
|
||||
static constexpr float NO_INFO_CALCULATED {-1.0};
|
||||
// bitrate and quality info struct
|
||||
VideoBitrateInfo videoBitrateInfo_;
|
||||
// previous quality and bitrate used if quality or bitrate need to be decreased
|
||||
// previous quality, bitrate, jitter and loss used if quality or bitrate need to be decreased
|
||||
std::list<unsigned> histoQuality_ {};
|
||||
std::list<unsigned> histoBitrate_ {};
|
||||
std::list<unsigned> histoJitter_ {};
|
||||
std::list<int> histoDelay_ {};
|
||||
std::list< std::pair<time_point, float> > histoLoss_;
|
||||
// max size of quality and bitrate historic
|
||||
static constexpr unsigned MAX_SIZE_HISTO_QUALITY_ {30};
|
||||
static constexpr unsigned MAX_SIZE_HISTO_BITRATE_ {100};
|
||||
|
||||
// 5 tries in a row
|
||||
static constexpr unsigned MAX_ADAPTATIVE_BITRATE_ITERATION {5};
|
||||
@ -156,8 +162,16 @@ private:
|
||||
std::chrono::seconds rtcp_checking_interval {4};
|
||||
|
||||
time_point lastMediaRestart_ {time_point::min()};
|
||||
time_point lastIncrease_ {time_point::min()};
|
||||
time_point lastREMB_ {time_point::min()};
|
||||
|
||||
int lastDelayAVG_ {0};
|
||||
unsigned cnt_delay_callback_ {0};
|
||||
std::pair<float, float> getDelayAvg();
|
||||
std::pair<float, float> getDelayMedian();
|
||||
float getRollingAvg();
|
||||
int getRollingMedian();
|
||||
|
||||
std::list< std::pair<time_point, float> > histoLoss_;
|
||||
};
|
||||
|
||||
}} // namespace jami::video
|
||||
|
||||
Reference in New Issue
Block a user