video_mixer: support multiple video layouts in conference

This patch aims to improve the conference management for the host.
Now, the host is able to switch between 3 conferences layout:
1. The grid view (actual one) where all participants are shown
at the same height/width
2. The One big/Other in small which show one participant bigger than
the others
3. One participant in big

The daemon's API got two new methods:
+ setConferenceLayout() to switch between these layouts
+ setActiveParticipant() used in the 2 last layouts.

Change-Id: I3c16569e24d1b63331ffe9d79e35790a6ac47a0c

bin/dbus/cx.ring.Ring.CallManager.xml

@@ -228,6 +228,30 @@
             <arg type="as" name="participants" direction="in"/>
         </method>
 
+        <method name="setConferenceLayout" tp:name-for-bindings="setConferenceLayout">
+            <tp:added version="9.4.0"/>
+            <tp:docstring>
+                <p>Change displayed layout for a conference. We currently support 3 layouts:
+                0. (default) = grid view
+                1. One big with small previews
+                2. One participant</p>
+            </tp:docstring>
+            <arg type="s" name="confId" direction="in"/>
+            <arg type="u" name="layout" direction="in"/>
+        </method>
+
+        <method name="setActiveParticipant" tp:name-for-bindings="setActiveParticipant">
+            <tp:added version="9.4.0"/>
+            <tp:docstring>
+                <p>Depending the layout of the conference, someone can be shown bigger than the others.
+                This methods is here to set which call is shown. Note if the active participant leave while
+                shown, the layout will change to the default one (grid view).</p>
+            </tp:docstring>
+            <arg type="s" name="confId" direction="in"/>
+            <arg type="s" name="callId" direction="in"/>
+        </method>
+
+
         <method name="isConferenceParticipant" tp:name-for-bindings="isConferenceParticipant">
             <arg type="s" name="callID" direction="in"/>
             <arg type="b" name="isParticipant" direction="out"/>

bin/dbus/dbuscallmanager.cpp

@@ -117,6 +117,19 @@ DBusCallManager::createConfFromParticipantList(const std::vector< std::string >&
     DRing::createConfFromParticipantList(participants);
 }
 
+void
+DBusCallManager::setConferenceLayout(const std::string& confId, const uint32_t& layout)
+{
+    DRing::setConferenceLayout(confId, layout);
+}
+
+void
+DBusCallManager::setActiveParticipant(const std::string& confId, const std::string& callId)
+{
+    DRing::setActiveParticipant(confId, callId);
+}
+
+
 auto
 DBusCallManager::isConferenceParticipant(const std::string& call_id) -> decltype(DRing::isConferenceParticipant(call_id))
 {

bin/dbus/dbuscallmanager.h

@@ -71,6 +71,8 @@ class DRING_PUBLIC DBusCallManager :
         void removeConference(const std::string& conference_id);
         bool joinParticipant(const std::string& sel_callID, const std::string& drag_callID);
         void createConfFromParticipantList(const std::vector< std::string >& participants);
+        void setConferenceLayout(const std::string& confId, const uint32_t& layout);
+        void setActiveParticipant(const std::string& confId, const std::string& callId);
         bool isConferenceParticipant(const std::string& call_id);
         bool addParticipant(const std::string& callID, const std::string& confID);
         bool addMainParticipant(const std::string& confID);

bin/jni/callmanager.i

@@ -74,6 +74,8 @@ std::vector<std::string> getCallList();
 void removeConference(const std::string& conference_id);
 bool joinParticipant(const std::string& sel_callID, const std::string& drag_callID);
 void createConfFromParticipantList(const std::vector<std::string>& participants);
+void setConferenceLayout(const std::string& confId, int layout);
+void setActiveParticipant(const std::string& confId, const std::string& callId);
 bool isConferenceParticipant(const std::string& call_id);
 bool addParticipant(const std::string& callID, const std::string& confID);
 bool addMainParticipant(const std::string& confID);

bin/nodejs/callmanager.i

@@ -73,6 +73,8 @@ std::vector<std::string> getCallList();
 void removeConference(const std::string& conference_id);
 bool joinParticipant(const std::string& sel_callID, const std::string& drag_callID);
 void createConfFromParticipantList(const std::vector<std::string>& participants);
+void setConferenceLayout(const std::string& confId, int layout);
+void setActiveParticipant(const std::string& confId, const std::string& callId);
 bool isConferenceParticipant(const std::string& call_id);
 bool addParticipant(const std::string& callID, const std::string& confID);
 bool addMainParticipant(const std::string& confID);

configure.ac

@@ -2,7 +2,7 @@ dnl Jami - configure.ac for automake 1.9 and autoconf 2.59
 
 dnl Process this file with autoconf to produce a configure script.
 AC_PREREQ([2.65])
-AC_INIT([Jami Daemon],[9.3.0],[ring@gnu.org],[jami])
+AC_INIT([Jami Daemon],[9.4.0],[ring@gnu.org],[jami])
 
 AC_COPYRIGHT([[Copyright (c) Savoir-faire Linux 2004-2019]])
 AC_REVISION([$Revision$])

meson.build

@@ -1,5 +1,5 @@
 project('jami-daemon', ['c', 'cpp'],
-        version: '9.2.0',
+        version: '9.4.0',
         license: 'GPL3+',
         default_options: ['cpp_std=gnu++17', 'buildtype=debugoptimized'],
         meson_version:'>= 0.54'

src/client/callmanager.cpp

@@ -135,6 +135,18 @@ createConfFromParticipantList(const std::vector<std::string>& participants)
    jami::Manager::instance().createConfFromParticipantList(participants);
 }
 
+void
+setConferenceLayout(const std::string& confId, uint32_t layout)
+{
+    jami::Manager::instance().setConferenceLayout(confId, layout);
+}
+
+void
+setActiveParticipant(const std::string& confId, const std::string& callId)
+{
+    jami::Manager::instance().setActiveParticipant(confId, callId);
+}
+
 bool
 isConferenceParticipant(const std::string& callID)
 {

src/conference.cpp

@@ -62,12 +62,14 @@ Conference::getState() const
     return confState_;
 }
 
-void Conference::setState(State state)
+void
+Conference::setState(State state)
 {
     confState_ = state;
 }
 
-void Conference::add(const std::string &participant_id)
+void
+Conference::add(const std::string &participant_id)
 {
     if (participants_.insert(participant_id).second) {
 #ifdef ENABLE_VIDEO
@@ -79,7 +81,23 @@ void Conference::add(const std::string &participant_id)
     }
 }
 
-void Conference::remove(const std::string &participant_id)
+void
+Conference::setActiveParticipant(const std::string &participant_id)
+{
+    for (const auto &item : participants_) {
+        if (participant_id == item) {
+            if (auto call = Manager::instance().callFactory.getCall<SIPCall>(participant_id)) {
+                videoMixer_->setActiveParticipant(call->getVideoRtp().getVideoReceive().get());
+                return;
+            }
+        }
+    }
+    // Set local by default
+    videoMixer_->setActiveParticipant(nullptr);
+}
+
+void
+Conference::remove(const std::string &participant_id)
 {
     if (participants_.erase(participant_id)) {
 #ifdef ENABLE_VIDEO
@@ -128,7 +146,8 @@ Conference::detach()
     }
 }
 
-void Conference::bindParticipant(const std::string &participant_id)
+void
+Conference::bindParticipant(const std::string &participant_id)
 {
     auto &rbPool = Manager::instance().getRingBufferPool();
 

src/conference.h

@@ -134,6 +134,8 @@ public:
 
     void switchInput(const std::string& input);
 
+    void setActiveParticipant(const std::string &participant_id);
+
 #ifdef ENABLE_VIDEO
     std::shared_ptr<video::VideoMixer> getVideoMixer();
     std::string getVideoInput() const { return mediaInput_; }

src/dring/callmanager_interface.h

@@ -57,6 +57,8 @@ DRING_PUBLIC std::vector<std::string> getCallList();
 DRING_PUBLIC void removeConference(const std::string& conference_id);
 DRING_PUBLIC bool joinParticipant(const std::string& sel_callID, const std::string& drag_callID);
 DRING_PUBLIC void createConfFromParticipantList(const std::vector<std::string>& participants);
+DRING_PUBLIC void setConferenceLayout(const std::string& confId, uint32_t layout);
+DRING_PUBLIC void setActiveParticipant(const std::string& confId, const std::string& callId);
 DRING_PUBLIC bool isConferenceParticipant(const std::string& call_id);
 DRING_PUBLIC bool addParticipant(const std::string& callID, const std::string& confID);
 DRING_PUBLIC bool addMainParticipant(const std::string& confID);

src/manager.cpp

@@ -83,6 +83,7 @@ using random_device = dht::crypto::random_device;
 
 #include "libav_utils.h"
 #include "video/sinkclient.h"
+#include "media/video/video_mixer.h"
 #include "audio/tonecontrol.h"
 
 #include "data_transfer.h"
@@ -1450,6 +1451,36 @@ Manager::createConfFromParticipantList(const std::vector< std::string > &partici
     }
 }
 
+void
+Manager::setConferenceLayout(const std::string& confId, int layout)
+{
+    if (auto conf = getConferenceFromID(confId)) {
+        auto videoMixer = conf->getVideoMixer();
+        switch (layout)
+        {
+        case 0:
+            videoMixer->setVideoLayout(video::Layout::GRID);
+            break;
+        case 1:
+            videoMixer->setVideoLayout(video::Layout::ONE_BIG_WITH_SMALL);
+            break;
+        case 2:
+            videoMixer->setVideoLayout(video::Layout::ONE_BIG);
+            break;
+        default:
+            break;
+        }
+    }
+}
+
+void
+Manager::setActiveParticipant(const std::string& confId, const std::string& callId)
+{
+    if (auto conf = getConferenceFromID(confId)) {
+        conf->setActiveParticipant(callId);
+    }
+}
+
 bool
 Manager::detachLocalParticipant(const std::string& conf_id)
 {

src/manager.h

@@ -293,6 +293,20 @@ class DRING_TESTABLE Manager {
          */
         void createConfFromParticipantList(const std::vector< std::string > &);
 
+        /**
+         * Change the conference layout
+         * @param confId
+         * @param layout    0 = matrix, 1 = one big, others in small, 2 = one in big
+         */
+        void setConferenceLayout(const std::string& confId, int layout);
+
+        /**
+         * Change the active participant (used in layout != matrix)
+         * @param confId
+         * @param callId    If callId not found, the local video will be shown
+         */
+        void setActiveParticipant(const std::string& confId, const std::string& callId);
+
         /**
          * Detach a participant from a conference, put the call on hold, do not hangup it
          * @param call id

src/media/video/video_mixer.cpp

@@ -121,6 +121,12 @@ VideoMixer::stopInput()
     }
 }
 
+void
+VideoMixer::setActiveParticipant(Observable<std::shared_ptr<MediaFrame>>* ob)
+{
+    activeSource_ = ob;
+}
+
 void
 VideoMixer::attached(Observable<std::shared_ptr<MediaFrame>>* ob)
 {
@@ -138,6 +144,11 @@ VideoMixer::detached(Observable<std::shared_ptr<MediaFrame>>* ob)
 
     for (const auto& x : sources_) {
         if (x->source == ob) {
+            // Handle the case where the current shown source leave the conference
+            if (activeSource_ == ob) {
+                currentLayout_ = Layout::GRID;
+                activeSource_ = nullptr;
+            }
             sources_.remove(x);
             break;
         }
@@ -187,20 +198,42 @@ VideoMixer::process()
         auto lock(rwMutex_.read());
 
         int i = 0;
+        bool activeFound = false;
         for (const auto& x : sources_) {
             /* thread stop pending? */
             if (!loop_.isRunning())
                 return;
 
+            if (currentLayout_ != Layout::ONE_BIG
+                or activeSource_ == x->source
+                or (not activeSource_ and not activeFound) /* By default ONE_BIG will show the first source */) {
+
                 // make rendered frame temporarily unavailable for update()
                 // to avoid concurrent access.
                 std::unique_ptr<VideoFrame> input;
                 x->atomic_swap_render(input);
 
+                auto wantedIndex = i;
+                if (currentLayout_ == Layout::ONE_BIG) {
+                    wantedIndex = 0;
+                    activeFound = true;
+                } else if (currentLayout_ == Layout::ONE_BIG_WITH_SMALL) {
+                    if (!activeSource_ && i == 0) {
+                        activeFound = true;
+                    } if (activeSource_ == x->source) {
+                        wantedIndex = 0;
+                        activeFound = true;
+                    } else if (not activeFound) {
+                        wantedIndex += 1;
+                    }
+                }
+
                 if (input)
-                render_frame(output, *input, x, i);
+                    render_frame(output, *input, x, wantedIndex);
 
                 x->atomic_swap_render(input);
+            }
+
             ++i;
         }
     }
@@ -221,12 +254,29 @@ VideoMixer::render_frame(VideoFrame& output, const VideoFrame& input,
     std::shared_ptr<VideoFrame> frame = input;
 #endif
 
-    const int n = sources_.size();
-    const int zoom = ceil(sqrt(n));
+    const int n = currentLayout_ == Layout::ONE_BIG? 1 : sources_.size();
+    const int zoom = currentLayout_ == Layout::ONE_BIG_WITH_SMALL? std::max(6,n) : ceil(sqrt(n));
     int cell_width = width_ / zoom;
     int cell_height = height_ / zoom;
+    if (currentLayout_ == Layout::ONE_BIG_WITH_SMALL && index == 0) {
+        // In ONE_BIG_WITH_SMALL, the first line at the top is the previews
+        // The rest is the active source
+        cell_width  = width_;
+        cell_height = height_ - cell_height;
+    }
     int xoff = (index % zoom) * cell_width;
     int yoff = (index / zoom) * cell_height;
+    if (currentLayout_ == Layout::ONE_BIG_WITH_SMALL) {
+        if (index == 0) {
+            xoff = 0;
+            yoff = height_ / zoom; // First line height
+        } else {
+            xoff = (index-1) * cell_width;
+            // Show sources in center
+            xoff += (width_ - (n - 1) * cell_width) / 2;
+            yoff = 0;
+        }
+    }
 
     AVFrameSideData* sideData = av_frame_get_side_data(frame->pointer(), AV_FRAME_DATA_DISPLAYMATRIX);
     int angle = 0;

src/media/video/video_mixer.h

@@ -35,6 +35,13 @@ namespace jami { namespace video {
 
 class SinkClient;
 
+
+enum class Layout {
+    GRID,
+    ONE_BIG_WITH_SMALL,
+    ONE_BIG
+};
+
 class VideoMixer:
         public VideoGenerator,
         public VideoFramePassiveReader
@@ -57,6 +64,12 @@ public:
     void switchInput(const std::string& input);
     void stopInput();
 
+    void setActiveParticipant(Observable<std::shared_ptr<MediaFrame>>* ob);
+
+    void setVideoLayout(Layout newLayout) {
+        currentLayout_ = newLayout;
+    }
+
 private:
     NON_COPYABLE(VideoMixer);
 
@@ -74,7 +87,6 @@ private:
     int width_ = 0;
     int height_ = 0;
     AVPixelFormat format_ = AV_PIX_FMT_YUV422P;
-    std::list<std::unique_ptr<VideoMixerSource>> sources_;
     rw_mutex rwMutex_;
 
     std::shared_ptr<SinkClient> sink_;
@@ -84,6 +96,10 @@ private:
     VideoScaler scaler_;
 
     ThreadLoop loop_; // as to be last member
+
+    Layout currentLayout_ {Layout::GRID};
+    Observable<std::shared_ptr<MediaFrame>>* activeSource_ {nullptr};
+    std::list<std::unique_ptr<VideoMixerSource>> sources_;
 };
 
 }} // namespace jami::video

src/sip/sipvoiplink.h

@@ -56,8 +56,6 @@ class SIPAccountBase;
 class SIPVoIPLink;
 class SipTransportBroker;
 
-typedef std::map<std::string, std::shared_ptr<SIPCall> > SipCallMap;
-
 /**
  * @file sipvoiplink.h
  * @brief Specific VoIPLink for SIP (SIP core for incoming and outgoing events).