autoadapt: Add RTP Absolute Send Time extension

Change-Id: I81a9a4cfc3addaa4be00f45ddc79b5e9642ed4ae

contrib/src/ffmpeg/rtp_ext_abs_send_time.patch

@@ -0,0 +1,71 @@
+diff --git a/libavformat/rtpenc.c b/libavformat/rtpenc.c
+index 63047beccc..d59ec3dc8c 100644
+--- a/libavformat/rtpenc.c
++++ b/libavformat/rtpenc.c
+@@ -28,6 +28,8 @@
+ 
+ #include "rtpenc.h"
+ 
++#define EXT_ABS_SEND_TIME
++
+ static const AVOption options[] = {
+     FF_RTP_FLAG_OPTS(RTPMuxContext, flags),
+     { "payload_type", "Specify RTP payload type", offsetof(RTPMuxContext, payload_type), AV_OPT_TYPE_INT, {.i64 = -1 }, -1, 127, AV_OPT_FLAG_ENCODING_PARAM },
+@@ -146,7 +148,11 @@ static int rtp_write_header(AVFormatContext *s1)
+                                     s1->pb->max_packet_size);
+     } else
+         s1->packet_size = s1->pb->max_packet_size;
++#ifdef EXT_ABS_SEND_TIME
++    if (s1->packet_size <= 20) {
++#else
+     if (s1->packet_size <= 12) {
++#endif
+         av_log(s1, AV_LOG_ERROR, "Max packet size %u too low\n", s1->packet_size);
+         return AVERROR(EIO);
+     }
+@@ -154,7 +160,11 @@ static int rtp_write_header(AVFormatContext *s1)
+     if (!s->buf) {
+         return AVERROR(ENOMEM);
+     }
++#ifdef EXT_ABS_SEND_TIME
++    s->max_payload_size = s1->packet_size - 20;
++#else
+     s->max_payload_size = s1->packet_size - 12;
++#endif
+ 
+     if (st->codecpar->codec_type == AVMEDIA_TYPE_AUDIO) {
+         avpriv_set_pts_info(st, 32, 1, st->codecpar->sample_rate);
+@@ -332,16 +342,34 @@ static void rtcp_send_sr(AVFormatContext *s1, int64_t ntp_time, int bye)
+ void ff_rtp_send_data(AVFormatContext *s1, const uint8_t *buf1, int len, int m)
+ {
+     RTPMuxContext *s = s1->priv_data;
++    uint64_t ntp64_time;
++    uint32_t absoluteSendTime;
+ 
+     av_log(s1, AV_LOG_TRACE, "rtp_send_data size=%d\n", len);
+ 
+     /* build the RTP header */
++#ifdef EXT_ABS_SEND_TIME
++    avio_w8(s1->pb, RTP_VERSION << 6 | 0x10); // extention bit
++#else
+     avio_w8(s1->pb, RTP_VERSION << 6);
++#endif
+     avio_w8(s1->pb, (s->payload_type & 0x7f) | ((m & 0x01) << 7));
+     avio_wb16(s1->pb, s->seq);
+     avio_wb32(s1->pb, s->timestamp);
+     avio_wb32(s1->pb, s->ssrc);
+ 
++#ifdef EXT_ABS_SEND_TIME
++    avio_wb16(s1->pb, 0xBEDE);      // magic word
++    avio_wb16(s1->pb, 0x0001);      // length=1
++    avio_w8(s1->pb, 0x32);         // ID=3 and lenght=2
++    ntp64_time = ff_get_formatted_ntp_time(ff_ntp_time());
++    absoluteSendTime = (uint32_t)((ntp64_time>> 14) & 0x00ffffff);
++    av_log(s1, AV_LOG_TRACE, "ntp64:%lu, abs_time:%u\n", ntp64_time, absoluteSendTime);
++    avio_w8(s1->pb, (uint8_t)(absoluteSendTime >> 16));
++    avio_w8(s1->pb, (uint8_t)(absoluteSendTime >> 8 & 0xff));
++    avio_w8(s1->pb, (uint8_t)(absoluteSendTime & 0xff));
++#endif
++
+     avio_write(s1->pb, buf1, len);
+     avio_flush(s1->pb);

contrib/src/ffmpeg/rules.mak

@@ -325,6 +325,7 @@ ffmpeg: ffmpeg-$(FFMPEG_HASH).tar.gz
 	(cd $@-$(FFMPEG_HASH) && tar x $(if ${BATCH_MODE},,-v) --strip-components=1 -f ../$<)
 	$(APPLY) $(SRC)/ffmpeg/remove-mjpeg-log.patch
 	$(APPLY) $(SRC)/ffmpeg/change-RTCP-ratio.patch
+	$(APPLY) $(SRC)/ffmpeg/rtp_ext_abs_send_time.patch
 	$(UPDATE_AUTOCONFIG)
 	$(MOVE)
 

src/media/socket_pair.cpp

@@ -493,14 +493,21 @@ SocketPair::readCallback(uint8_t* buf, int buf_size)
 
     // SRTP decrypt
     if (not fromRTCP and srtpContext_ and srtpContext_->srtp_in.aes) {
-        uint32_t curentSendTS = buf[4] << 24 | buf[5] << 16 | buf[6] << 8 | buf[7];
-        int32_t delay = 0;
+        int32_t gradient = 0;
+        int32_t deltaT = 0;
         float abs = 0.0f;
-        bool marker = (buf[1] & 0x80) >> 7;
-        bool res = getOneWayDelayGradient2(abs, marker, &delay);
+        bool res_parse = false;
+        bool res_delay = false;
 
-        if (res)
-            rtpDelayCallback_(delay);
+        res_parse = parse_RTP_ext(buf, &abs);
+        bool marker = (buf[1] & 0x80) >> 7;
+
+        if(res_parse)
+            res_delay = getOneWayDelayGradient2(abs, marker, &gradient, &deltaT);
+
+        // rtpDelayCallback_ is not set for audio
+        if (rtpDelayCallback_ && res_delay)
+                rtpDelayCallback_(gradient);
 
         auto err = ff_srtp_decrypt(&srtpContext_->srtp_in, buf, &len);
         if(packetLossCallback_ and (buf[2] << 8 | buf[3]) != lastSeqNum_+1) {
@@ -667,7 +674,7 @@ SocketPair::getOneWayDelayGradient(uint32_t sendTS)
 }
 
 bool
-SocketPair::getOneWayDelayGradient2(float sendTS, bool marker, int32_t* gradient)
+SocketPair::getOneWayDelayGradient2(float sendTS, bool marker, int32_t* gradient, int32_t* deltaT)
 {
     // Keep only last packet of each frame
     if (not marker) {
@@ -681,7 +688,7 @@ SocketPair::getOneWayDelayGradient2(float sendTS, bool marker, int32_t* gradient
         return 0;
     }
 
-    uint32_t deltaS = (sendTS - lastSendTS_) * 1000;            // milliseconds
+    int32_t deltaS = (sendTS - lastSendTS_) * 1000;            // milliseconds
     if(deltaS < 0)
         deltaS += 64000;
     lastSendTS_ = sendTS;
@@ -691,6 +698,7 @@ SocketPair::getOneWayDelayGradient2(float sendTS, bool marker, int32_t* gradient
     lastReceiveTS_ = arrival_TS;
 
     *gradient = deltaR - deltaS;
+    *deltaT = deltaR; 
 
     return true;
 }
@@ -739,4 +747,22 @@ SocketPair::getOneWayDelayGradient3(uint32_t sendTS, int32_t* gradient)
     return true;
 }
 
+bool
+SocketPair::parse_RTP_ext(uint8_t* buf, float* abs)
+{
+    if(not(buf[0] & 0x10))
+        return false;
+
+    uint16_t magic_word = (buf[12] << 8) + buf[13];
+    if(magic_word != 0xBEDE)
+        return false;
+
+    uint8_t sec = buf[17] >> 2;
+    uint32_t fract = ((buf[17] & 0x3) << 16 | (buf[18] << 8) | buf[19]) << 14;
+    float milli = fract / pow(2,32);
+
+    *abs = sec + (milli);
+    return true;
+}
+
 } // namespace jami

src/media/socket_pair.h

@@ -203,8 +203,9 @@ class SocketPair {
         std::function<void(void)> packetLossCallback_;
         std::function<void(int)> rtpDelayCallback_;
         int32_t getOneWayDelayGradient(uint32_t sendTS);
-        bool getOneWayDelayGradient2(float sendTS, bool marker, int32_t* gradient);
+        bool getOneWayDelayGradient2(float sendTS, bool marker, int32_t* gradient, int32_t* deltaR);
         bool getOneWayDelayGradient3(uint32_t sendTS, int32_t* gradient);
+        bool parse_RTP_ext(uint8_t* buf, float* abs);
 
         std::list<rtcpRRHeader> listRtcpRRHeader_;
         std::list<rtcpREMBHeader> listRtcpREMBHeader_;
@@ -220,8 +221,7 @@ class SocketPair {
 
         std::chrono::steady_clock::time_point lastRR_time;
         uint16_t lastSeqNum_ {0};
-        uint32_t lastSendTS_ {0};
-        bool lastMarker_ {false};
+        float lastSendTS_ {0.0f};
         std::chrono::steady_clock::time_point lastReceiveTS_ {};
         std::chrono::steady_clock::time_point arrival_TS {};
 

src/media/video/video_rtp_session.cpp

@@ -151,10 +151,6 @@ void VideoRtpSession::startSender()
             rtcpCheckerThread_.start();
         else if (not autoQuality and rtcpCheckerThread_.isRunning())
             rtcpCheckerThread_.join();
-
-        socketPair_->setRtpDelayCallback([&](int delay) {
-            this->delayMonitor(delay);
-        });
     }
 }
 
@@ -207,6 +203,8 @@ void VideoRtpSession::start(std::unique_ptr<IceSocket> rtp_sock,
         else
             socketPair_.reset(new SocketPair(getRemoteRtpUri().c_str(), receive_.addr.getPort()));
 
+        socketPair_->setRtpDelayCallback([&](int delay) {delayMonitor(delay);});
+
         if (send_.crypto and receive_.crypto) {
             socketPair_->createSRTP(receive_.crypto.getCryptoSuite().c_str(),
                                     receive_.crypto.getSrtpKeyInfo().c_str(),