jami/jami-daemon
1
0
Fork 0
mirror of https://git.jami.net/savoirfairelinux/jami-daemon.git synced 2025-08-12 22:09:25 +08:00
Code Issues Projects Releases Wiki Activity

ffmpeg: add pipewire support

Browse Source 
This patch enables screen and window sharing on Wayland.
Based on Abhishek's FFmpeg patch:
https://patchwork.ffmpeg.org/project/ffmpeg/list/?submitter=1658
Screen area sharing is not supported for now.

GitLab: #13
Change-Id: Ia54dbc512aa87ae1cb1df7c1ffe71c153a4937a2
This commit is contained in:
Sébastien Blin
2023-11-09 14:58:01 -05:00
committed by François-Simon Fauteux-Chapleau
parent b50677ae8b
commit c5c3afae9a
10 changed files with 1595 additions and 71 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
Dockerfile
View File

@ -24,6 +24,7 @@ RUN apt-get update && apt-get install -y \
libmsgpack-dev \
libnatpmp-dev \
libopus-dev \
libpipewire-0.3-dev \
libpulse-dev \
libspeex-dev \
libspeexdsp-dev \

11
contrib/bootstrap
View File

@ -279,6 +279,17 @@ case "${OS}" in
;;
esac
# If the DISABLE_PIPEWIRE variable is set, then we build FFmpeg
# without pipewiregrab (see contrib/src/ffmpeg/rules.mak)
# This is currently needed because some of the Linux distributions
# we support don't have a recent enough version of PipeWire:
# - Debian 11
# - Ubuntu 20.04
# - openSUSE Leap 15.4
# However, we should be able to get rid of this in the future once
# we stop supporting the above platforms.
test "$DISABLE_PIPEWIRE" && add_make_enabled "DISABLE_PIPEWIRE"
#
# Results output
#

1436
contrib/src/ffmpeg/pipewiregrab-source-filter.patch Normal file
View File

File diff suppressed because it is too large Load Diff

12
contrib/src/ffmpeg/rules.mak
View File

@ -26,6 +26,17 @@ FFMPEGCONF += \
--disable-programs \
--disable-postproc
ifdef HAVE_LINUX
ifndef HAVE_ANDROID
ifndef DISABLE_PIPEWIRE
FFMPEGCONF += --enable-libpipewire \
--enable-filter=pipewiregrab \
--enable-indev=lavfi \
--enable-decoder=wrapped_avframe
endif
endif
endif
FFMPEGCONF += \
--disable-protocols \
--enable-protocol=crypto \
@ -422,6 +433,7 @@ ffmpeg: ffmpeg-$(FFMPEG_HASH).tar.xz
$(APPLY) $(SRC)/ffmpeg/libopusenc-reload-packet-loss-at-encode.patch
$(APPLY) $(SRC)/ffmpeg/ios-disable-b-frames.patch
$(APPLY) $(SRC)/ffmpeg/screen-sharing-x11-fix.patch
$(APPLY) $(SRC)/ffmpeg/pipewiregrab-source-filter.patch
$(UPDATE_AUTOCONFIG)
$(MOVE)

119
src/media/media_decoder.cpp
View File

@ -97,66 +97,89 @@ MediaDemuxer::openInput(const DeviceParams& params)
if (!iformat && !params.format.empty())
JAMI_WARN("Cannot find format \"%s\"", params.format.c_str());
if (params.width and params.height) {
auto sizeStr = fmt::format("{}x{}", params.width, params.height);
av_dict_set(&options_, "video_size", sizeStr.c_str(), 0);
}
std::string input;
if (params.framerate) {
if (params.input == "pipewiregrab") {
//
// We rely on pipewiregrab for screen/window sharing on Wayland.
// Because pipewiregrab is a "video source filter" (part of FFmpeg's libavfilter
// library), its options must all be passed as part of the `input` string.
//
input = fmt::format("pipewiregrab=draw_mouse=1:fd={}:node={}", params.fd, params.node);
JAMI_LOG("Trying to open input {}", input);
//
// In all other cases, we use the `options_` AVDictionary to pass options to FFmpeg.
//
// NOTE: We rely on the "lavfi" virtual input device to read pipewiregrab's output
// and create a corresponding stream (cf. the getDeviceParams function in
// daemon/src/media/video/v4l2/video_device_impl.cpp). The `options_` dictionary
// could be used to set lavfi's parameters if that was ever needed, but it isn't at
// the moment. (Doc: https://ffmpeg.org/ffmpeg-devices.html#lavfi)
//
} else {
if (params.width and params.height) {
auto sizeStr = fmt::format("{}x{}", params.width, params.height);
av_dict_set(&options_, "video_size", sizeStr.c_str(), 0);
}
if (params.framerate) {
#ifdef _WIN32
// On windows, framerate settings don't reduce to avrational values
// that correspond to valid video device formats.
// e.g. A the rational<double>(10000000, 333333) or 30.000030000
// will be reduced by av_reduce to 999991/33333 or 30.00003000003
// which cause the device opening routine to fail.
// So we treat this imprecise reduction and adjust the value,
// or let dshow choose the framerate, which is, unfortunately,
// NOT the highest according to our experimentations.
auto framerate {params.framerate.real()};
framerate = params.framerate.numerator() / (params.framerate.denominator() + 0.5);
if (params.framerate.denominator() != 4999998)
av_dict_set(&options_, "framerate", jami::to_string(framerate).c_str(), 0);
// On windows, framerate settings don't reduce to avrational values
// that correspond to valid video device formats.
// e.g. A the rational<double>(10000000, 333333) or 30.000030000
// will be reduced by av_reduce to 999991/33333 or 30.00003000003
// which cause the device opening routine to fail.
// So we treat this imprecise reduction and adjust the value,
// or let dshow choose the framerate, which is, unfortunately,
// NOT the highest according to our experimentations.
auto framerate {params.framerate.real()};
framerate = params.framerate.numerator() / (params.framerate.denominator() + 0.5);
if (params.framerate.denominator() != 4999998)
av_dict_set(&options_, "framerate", jami::to_string(framerate).c_str(), 0);
#else
av_dict_set(&options_, "framerate", jami::to_string(params.framerate.real()).c_str(), 0);
av_dict_set(&options_, "framerate", jami::to_string(params.framerate.real()).c_str(), 0);
#endif
}
}
if (params.offset_x || params.offset_y) {
av_dict_set(&options_, "offset_x", std::to_string(params.offset_x).c_str(), 0);
av_dict_set(&options_, "offset_y", std::to_string(params.offset_y).c_str(), 0);
}
if (params.channel)
av_dict_set(&options_, "channel", std::to_string(params.channel).c_str(), 0);
av_dict_set(&options_, "loop", params.loop.c_str(), 0);
av_dict_set(&options_, "sdp_flags", params.sdp_flags.c_str(), 0);
if (params.offset_x || params.offset_y) {
av_dict_set(&options_, "offset_x", std::to_string(params.offset_x).c_str(), 0);
av_dict_set(&options_, "offset_y", std::to_string(params.offset_y).c_str(), 0);
}
if (params.channel)
av_dict_set(&options_, "channel", std::to_string(params.channel).c_str(), 0);
av_dict_set(&options_, "loop", params.loop.c_str(), 0);
av_dict_set(&options_, "sdp_flags", params.sdp_flags.c_str(), 0);
// Set jitter buffer options
av_dict_set(&options_, "reorder_queue_size", std::to_string(jitterBufferMaxSize_).c_str(), 0);
auto us = std::chrono::duration_cast<std::chrono::microseconds>(jitterBufferMaxDelay_).count();
av_dict_set(&options_, "max_delay", std::to_string(us).c_str(), 0);
// Set jitter buffer options
av_dict_set(&options_, "reorder_queue_size", std::to_string(jitterBufferMaxSize_).c_str(), 0);
auto us = std::chrono::duration_cast<std::chrono::microseconds>(jitterBufferMaxDelay_).count();
av_dict_set(&options_, "max_delay", std::to_string(us).c_str(), 0);
if (!params.pixel_format.empty()) {
av_dict_set(&options_, "pixel_format", params.pixel_format.c_str(), 0);
}
if (!params.window_id.empty()) {
av_dict_set(&options_, "window_id", params.window_id.c_str(), 0);
}
av_dict_set(&options_, "is_area", std::to_string(params.is_area).c_str(), 0);
if (!params.pixel_format.empty()) {
av_dict_set(&options_, "pixel_format", params.pixel_format.c_str(), 0);
}
if (!params.window_id.empty()) {
av_dict_set(&options_, "window_id", params.window_id.c_str(), 0);
}
av_dict_set(&options_, "draw_mouse", "1", 0);
av_dict_set(&options_, "is_area", std::to_string(params.is_area).c_str(), 0);
#if defined(__APPLE__) && TARGET_OS_MAC
std::string input = params.name;
input = params.name;
#else
std::string input = params.input;
input = params.input;
#endif
JAMI_LOG("Trying to open input {} with format {}, pixel format {}, size {}x{}, rate {}",
input,
params.format,
params.pixel_format,
params.width,
params.height,
params.framerate.real());
JAMI_LOG("Trying to open input {} with format {}, pixel format {}, size {}x{}, rate {}",
input,
params.format,
params.pixel_format,
params.width,
params.height,
params.framerate.real());
}
// Ask FFmpeg to open the input using the options set above
av_opt_set_int(
inputCtx_,
"fpsprobesize",
@ -166,7 +189,7 @@ MediaDemuxer::openInput(const DeviceParams& params)
if (ret) {
JAMI_ERROR("avformat_open_input failed: {}", libav_utils::getError(ret));
} else {
} else if (inputCtx_->nb_streams > 0 && inputCtx_->streams[0]->codecpar) {
baseWidth_ = inputCtx_->streams[0]->codecpar->width;
baseHeight_ = inputCtx_->streams[0]->codecpar->height;
JAMI_LOG("Opened input Using format {:s} and resolution {:d}x{:d}",

2
src/media/media_device.h
View File

@ -52,6 +52,8 @@ struct DeviceParams
int offset_y {};
int orientation {};
std::string window_id {};
int fd {}; // file descriptor for PipeWire (only relevant on Wayland)
std::string node {}; // node id for PipeWire
int is_area {};
};

9
src/media/video/v4l2/video_device_impl.cpp
View File

@ -574,7 +574,14 @@ VideoDeviceImpl::getDeviceParams() const
params.unique_id = unique_id;
params.input = path;
if (unique_id == DEVICE_DESKTOP) {
params.format = "x11grab";
const auto* env = std::getenv("WAYLAND_DISPLAY");
params.format = !env || strlen(env) == 0? "x11grab" : "pipewiregrab";
if (!env || strlen(env) == 0) {
params.format = "x11grab";
} else {
params.format = "lavfi";
params.input = "pipewiregrab";
}
params.framerate = rate_.frame_rate;
return params;
}

61
src/media/video/video_input.cpp
View File

@ -43,6 +43,7 @@
#ifdef _MSC_VER
#include <io.h> // for access
#else
#include <sys/syscall.h>
#include <unistd.h>
#endif
extern "C" {
@ -309,13 +310,13 @@ VideoInput::createDecoder()
[](void* data) -> int { return not static_cast<VideoInput*>(data)->isCapturing(); }, this);
bool ready = false, restartSink = false;
if ((decOpts_.format == "x11grab" || decOpts_.format == "dxgigrab") && !decOpts_.is_area) {
if ((decOpts_.format == "x11grab" || decOpts_.format == "dxgigrab" || decOpts_.format == "pipewiregrab") && !decOpts_.is_area) {
decOpts_.width = 0;
decOpts_.height = 0;
}
while (!ready && !isStopped_) {
// Retry to open the video till the input is opened
auto ret = decoder->openInput(decOpts_);
int ret = decoder->openInput(decOpts_);
ready = ret >= 0;
if (ret < 0 && -ret != EBUSY) {
JAMI_ERR("Could not open input \"%s\" with status %i", decOpts_.input.c_str(), ret);
@ -427,13 +428,17 @@ round2pow(unsigned i, unsigned n)
return (i >> n) << n;
}
#if !defined(WIN32) && !defined(__APPLE__)
bool
VideoInput::initX11(const std::string& display)
VideoInput::initLinuxGrab(const std::string& display)
{
// Patterns
// full screen sharing : :1+0,0 2560x1440 - SCREEN 1, POSITION 0X0, RESOLUTION 2560X1440
// area sharing : :1+882,211 1532x779 - SCREEN 1, POSITION 882x211, RESOLUTION 1532x779
// window sharing : :+1,0 0x0 window-id:0x0340021e - POSITION 0X0
// Patterns (all platforms except Linux with Wayland)
// full screen sharing: :1+0,0 2560x1440 (SCREEN 1, POSITION 0X0, RESOLUTION 2560X1440)
// area sharing: :1+882,211 1532x779 (SCREEN 1, POSITION 882x211, RESOLUTION 1532x779)
// window sharing: :+1,0 0x0 window-id:0x0340021e (POSITION 0X0)
//
// Pattern (Linux with Wayland)
// full screen or window sharing: pipewire pid:2861 fd:23 node:68
size_t space = display.find(' ');
std::string windowIdStr = "window-id:";
size_t winIdPos = display.find(windowIdStr);
@ -443,7 +448,34 @@ VideoInput::initX11(const std::string& display)
p.window_id = display.substr(winIdPos + windowIdStr.size()); // "0x0340021e";
p.is_area = 0;
}
if (space != std::string::npos) {
if (display.find("pipewire") != std::string::npos) {
std::string pidStr = "pid:";
std::string fdStr = "fd:";
std::string nodeStr = "node:";
size_t pidPos = display.find(pidStr) + pidStr.size();
size_t fdPos = display.find(fdStr) + fdStr.size();
size_t nodePos = display.find(nodeStr) + nodeStr.size();
pid_t pid = std::stol(display.substr(pidPos));
int fd = std::stoi(display.substr(fdPos));
if (pid != getpid()) {
// We can't directly use a file descriptor that was opened in a different
// process, so we try to duplicate it in the current process.
int pidfd = syscall(SYS_pidfd_open, pid, 0);
if (pidfd < 0) {
JAMI_ERROR("Can't duplicate PipeWire fd: call to pidfd_open failed (errno = {})", errno);
return false;
}
fd = syscall(SYS_pidfd_getfd, pidfd, fd, 0);
if (fd < 0) {
JAMI_ERROR("Can't duplicate PipeWire fd: call to pidfd_getfd failed (errno = {})", errno);
return false;
}
}
p.fd = fd;
p.node = display.substr(nodePos);
} else if (space != std::string::npos) {
p.input = display.substr(1, space);
if (p.window_id.empty()) {
p.input = display.substr(0, space);
@ -460,18 +492,14 @@ VideoInput::initX11(const std::string& display)
p.is_area = 1;
}
auto dec = std::make_unique<MediaDecoder>();
if (dec->openInput(p) < 0 || dec->setupVideo() < 0)
return initCamera(jami::getVideoDeviceMonitor().getDefaultDevice());
clearOptions();
decOpts_ = p;
decOpts_.width = round2pow(dec->getStream().width, 3);
decOpts_.height = round2pow(dec->getStream().height, 3);
emulateRate_ = false;
return true;
}
#endif
#ifdef __APPLE__
bool
VideoInput::initAVFoundation(const std::string& display)
{
@ -497,6 +525,7 @@ VideoInput::initAVFoundation(const std::string& display)
}
return true;
}
#endif
#ifdef WIN32
bool
@ -650,7 +679,7 @@ VideoInput::switchInput(const std::string& resource)
#elif defined(WIN32)
ready = initWindowsGrab(suffix);
#else
ready = initX11(suffix);
ready = initLinuxGrab(suffix);
#endif
} else if (prefix == libjami::Media::VideoProtocolPrefix::FILE) {
/* Pathname */

9
src/media/video/video_input.h
View File

@ -133,11 +133,14 @@ private:
// true if decOpts_ is ready to use, false if using promise/future
bool initCamera(const std::string& device);
bool initX11(const std::string& display);
bool initAVFoundation(const std::string& display);
bool initFile(std::string path);
#ifdef WIN32
#ifdef __APPLE__
bool initAVFoundation(const std::string& display);
#elif defined(WIN32)
bool initWindowsGrab(const std::string& display);
#else
bool initLinuxGrab(const std::string& display);
#endif
bool isCapturing() const noexcept;

6
src/media/video/video_rtp_session.cpp
View File

@ -182,7 +182,7 @@ VideoRtpSession::startSender()
// Current implementation does not handle resolution change
// (needed by window sharing feature) with HW codecs, so HW
// codecs will be disabled for now.
bool allowHwAccel = (localVideoParams_.format != "x11grab" && localVideoParams_.format != "dxgigrab");
bool allowHwAccel = (localVideoParams_.format != "x11grab" && localVideoParams_.format != "dxgigrab" && localVideoParams_.format != "pipewiregrab");
if (socketPair_)
initSeqVal_ = socketPair_->lastSeqValOut();
@ -195,8 +195,8 @@ VideoRtpSession::startSender()
? MediaStream("video sender",
AV_PIX_FMT_YUV420P,
1 / static_cast<rational<int>>(localVideoParams_.framerate),
localVideoParams_.width,
localVideoParams_.height,
localVideoParams_.width == 0 ? 1080u : localVideoParams_.width,
localVideoParams_.height == 0 ? 720u : localVideoParams_.height,
send_.bitrate,
static_cast<rational<int>>(localVideoParams_.framerate))
: videoMixer_->getStream("Video Sender");