From 921146f553fcda8111a21bd89b4fcc2ba8f741f6 Mon Sep 17 00:00:00 2001 From: Philippe Gorley Date: Wed, 22 Jun 2016 17:16:29 -0400 Subject: [PATCH] video: hardware acceleration Contains code common to all hardware accelerations to be implemented. Enables the VAAPI acceleration for intel linux systems, with support for H.264, H.263 and MPEG4. To use VAAPI, you need libva, libva-x11 and libva-drm. Hardware acceleration is enabled by default, and can be disabled with "./configure --disable-accel". Change-Id: Id0696465b785de0735bbce9750932ac38efe0713 Reviewed-by: Guillaume Roguez --- configure.ac | 23 +- ...tream_info-not-considering-extradata.patch | 41 ++++ contrib/src/ffmpeg/rules.mak | 25 +- contrib/src/libav/rules.mak | 2 + src/media/media_decoder.cpp | 17 +- src/media/media_decoder.h | 11 + src/media/media_device.h | 1 + src/media/video/Makefile.am | 4 + src/media/video/accel.cpp | 197 +++++++++++++++ src/media/video/accel.h | 86 +++++++ src/media/video/v4l2/Makefile.am | 8 +- src/media/video/v4l2/vaapi.cpp | 230 ++++++++++++++++++ src/media/video/v4l2/vaapi.h | 83 +++++++ src/media/video/video_base.cpp | 10 +- src/media/video/video_base.h | 1 + src/media/video/video_device.h | 4 + 16 files changed, 735 insertions(+), 8 deletions(-) create mode 100644 contrib/src/ffmpeg/0004-avformat-fix-find_stream_info-not-considering-extradata.patch create mode 100644 src/media/video/accel.cpp create mode 100644 src/media/video/accel.h create mode 100644 src/media/video/v4l2/vaapi.cpp create mode 100644 src/media/video/v4l2/vaapi.h diff --git a/configure.ac b/configure.ac index 189958728..2c0cdc027 100644 --- a/configure.ac +++ b/configure.ac @@ -120,7 +120,6 @@ AS_IF([test "$SYS" = linux],[ AC_MSG_RESULT([no]) ]) ]) - AM_CONDITIONAL(HAVE_ANDROID, test "${HAVE_ANDROID}" = "1") dnl override platform specific check for dependent libraries @@ -430,6 +429,28 @@ AS_IF([test "x$enable_video" != "xno"], dnl check for GnuTLS PKG_CHECK_MODULES([GNUTLS], [gnutls >= 3.4.14], [HAVE_GNUTLS=1], [HAVE_GNUTLS=0]) +dnl hardware acceleration is enabled by default +AC_ARG_ENABLE([accel], AS_HELP_STRING([--disable-accel], [Disable hardware acceleration])) +AS_IF([test "x$enable_accel" != "xno"], [ + AC_DEFINE([RING_ACCEL], [1], [Allows use of hardware acceleration]) + AM_CONDITIONAL(RING_ACCEL, true) + AS_IF([test "$SYS" = linux && test -z "${HAVE_ANDROID_FALSE}"],[ + PKG_CHECK_MODULES(LIBVA, [libva], , + [AC_MSG_ERROR([Missing libva package])]) + PKG_CHECK_MODULES([LIBVA_DRM], [libva-drm], [ + AC_DEFINE([HAVE_VAAPI_ACCEL_DRM], [1], [Have vaapi drm]) + ], [AC_MSG_ERROR([Could not find libva-drm]) + ]) + AC_CHECK_HEADER([X11/Xlib.h], [ + AC_CHECK_LIB(X11, XOpenDisplay, [], [AC_MSG_ERROR([Could not find X11])]) + AC_DEFINE([HAVE_X11], [1], [Have x11]) + PKG_CHECK_MODULES(LIBVA_X11, [libva-x11], [ + AC_DEFINE([HAVE_VAAPI_ACCEL_X11], [1], [Have vaapi x11]) + ], [AC_MSG_ERROR([Could not find libva-x11]) + ]) + ]) + ]) +], AM_CONDITIONAL(RING_ACCEL, false)) # PTHREAD # required dependency(ies): libxpat diff --git a/contrib/src/ffmpeg/0004-avformat-fix-find_stream_info-not-considering-extradata.patch b/contrib/src/ffmpeg/0004-avformat-fix-find_stream_info-not-considering-extradata.patch new file mode 100644 index 000000000..dda9e0c4d --- /dev/null +++ b/contrib/src/ffmpeg/0004-avformat-fix-find_stream_info-not-considering-extradata.patch @@ -0,0 +1,41 @@ +From 60873bf992eab1d3bad8dd0fd11336363d44854d Mon Sep 17 00:00:00 2001 +From: Anssi Hannula +Date: Tue, 26 Jul 2016 13:23:43 +0300 +Subject: [PATCH 1572/1572] avformat/utils: Fix find_stream_info not + considering the extradata it found + +Commit 9200514ad8717c6 ("lavf: replace AVStream.codec with +AVStream.codecpar") merged in commit 6f69f7a8bf6a0d01 changed +avformat_find_stream_info() to put the extradata it got from +st->parser->parser->split() to st->internal->avctx instead of st->codec +(extradata in st->internal->avctx will be later copied to st->codecpar). + +However, in the same function, the "is stream ready?" check was changed +to check for extradata in st->codecpar instead of st->codec, even +though st->codecpar is not yet updated at that point. + +Extradata retrieved from split() is therefore not considered anymore, +and avformat_find_stream_info() will therefore needlessly continue +probing in some cases. + +Fix that by checking for the extradata at st->internal->avctx where it +is actually put. +--- + libavformat/utils.c | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/libavformat/utils.c b/libavformat/utils.c +index e5a99ff..5a902ea 100644 +--- a/libavformat/utils.c ++++ b/libavformat/utils.c +@@ -3432,7 +3432,7 @@ FF_ENABLE_DEPRECATION_WARNINGS + break; + } + if (st->parser && st->parser->parser->split && +- !st->codecpar->extradata) ++ !st->internal->avctx->extradata) + break; + if (st->first_dts == AV_NOPTS_VALUE && + !(ic->iformat->flags & AVFMT_NOTIMESTAMPS) && +-- +2.7.4 diff --git a/contrib/src/ffmpeg/rules.mak b/contrib/src/ffmpeg/rules.mak index 91f3fd35d..007aed9fe 100644 --- a/contrib/src/ffmpeg/rules.mak +++ b/contrib/src/ffmpeg/rules.mak @@ -1,10 +1,14 @@ -FFMPEG_HASH := c40983a6f631d22fede713d535bb9c31d5c9740c +FFMPEG_HASH := c46d22a4a58467bdc7885685b06a2114dd181c43 FFMPEG_URL := https://git.ffmpeg.org/gitweb/ffmpeg.git/snapshot/$(FFMPEG_HASH).tar.gz ifdef HAVE_WIN32 PKGS += ffmpeg endif +ifdef HAVE_LINUX +PKGS += ffmpeg +endif + FFMPEGCONF = \ --cc="$(CC)" \ --pkg-config="$(PKG_CONFIG)" @@ -89,7 +93,19 @@ FFMPEGCONF += \ --enable-dxva2 endif -DEPS_ffmpeg = iconv zlib x264 vpx opus speex $(DEPS_vpx) +ifdef HAVE_LINUX +FFMPEGCONF += \ + --enable-vaapi \ + --enable-hwaccel=h264_vaapi \ + --enable-hwaccel=mpeg4_vaapi \ + --enable-hwaccel=h263_vaapi +endif + +ifdef HAVE_MACOSX +FFMPEGCONF += \ + --enable-indev=avfcapture \ + --enable-indev=avfgrab +endif ifdef HAVE_IOS FFMPEGCONF += \ @@ -100,6 +116,8 @@ FFMPEGCONF += \ --enable-indev=avfoundation endif +DEPS_ffmpeg = iconv zlib x264 vpx opus speex $(DEPS_vpx) + # Linux ifdef HAVE_LINUX FFMPEGCONF += --target-os=linux --enable-pic @@ -152,7 +170,7 @@ FFMPEGCONF += --target-os=mingw32 --enable-memalign-hack FFMPEGCONF += --enable-w32threads --disable-decoder=dca endif -ifeq ($(call need_pkg,"ffmpeg >= 2.6.1"),) +ifeq ($(call need_pkg,"ffmpeg >= 3.1.3"),) PKGS_FOUND += ffmpeg endif @@ -168,6 +186,7 @@ ffmpeg: ffmpeg-$(FFMPEG_HASH).tar.xz .sum-ffmpeg mkdir -p $@-$(FFMPEG_HASH) (cd $@-$(FFMPEG_HASH) && tar xv --strip-components=1 -f ../$<) $(UPDATE_AUTOCONFIG) + $(APPLY) $(SRC)/ffmpeg/0004-avformat-fix-find_stream_info-not-considering-extradata.patch $(MOVE) .ffmpeg: ffmpeg diff --git a/contrib/src/libav/rules.mak b/contrib/src/libav/rules.mak index 815d8fcb2..c0de28ac0 100644 --- a/contrib/src/libav/rules.mak +++ b/contrib/src/libav/rules.mak @@ -2,9 +2,11 @@ LIBAV_HASH := f851477889ae48e2f17073cf7486e1d5561b7ae4 LIBAV_URL := https://git.libav.org/?p=libav.git;a=snapshot;h=$(LIBAV_HASH);sf=tgz +ifndef HAVE_LINUX ifndef HAVE_WIN32 PKGS += libav endif +endif #disable everything #ensure to add this option first diff --git a/src/media/media_decoder.cpp b/src/media/media_decoder.cpp index 952b184b0..fdc54f795 100644 --- a/src/media/media_decoder.cpp +++ b/src/media/media_decoder.cpp @@ -27,6 +27,10 @@ #include "audio/ringbuffer.h" #include "audio/resampler.h" +#if defined(RING_VIDEO) && defined(RING_ACCEL) +#include "video/accel.h" +#endif + #include "string_utils.h" #include "logger.h" @@ -91,6 +95,9 @@ int MediaDecoder::openInput(const DeviceParams& params) } RING_DBG("Trying to open device %s with format %s, pixel format %s, size %dx%d, rate %lf", params.input.c_str(), params.format.c_str(), params.pixel_format.c_str(), params.width, params.height, params.framerate.real()); + + enableAccel_ = (params.enableAccel == "1"); + int ret = avformat_open_input( &inputCtx_, params.input.c_str(), @@ -259,6 +266,11 @@ int MediaDecoder::setupFromVideoData() decoderCtx_->thread_count = std::thread::hardware_concurrency(); +#ifdef RING_ACCEL + accel_ = video::makeHardwareAccel(decoderCtx_); + decoderCtx_->opaque = accel_.get(); +#endif // RING_ACCEL + // find the decoder for the video stream inputDecoder_ = avcodec_find_decoder(decoderCtx_->codec_id); if (!inputDecoder_) { @@ -314,7 +326,6 @@ MediaDecoder::decode(VideoFrame& result) int frameFinished = 0; int len = avcodec_decode_video2(decoderCtx_, frame, &frameFinished, &inpacket); - av_packet_unref(&inpacket); if (len <= 0) @@ -322,6 +333,10 @@ MediaDecoder::decode(VideoFrame& result) if (frameFinished) { frame->format = (AVPixelFormat) correctPixFmt(frame->format); +#if defined(RING_VIDEO) && defined(RING_ACCEL) + if (accel_ && !accel_->extractData(decoderCtx_, result)) + return Status::DecodeError; +#endif // RING_ACCEL if (emulateRate_ and frame->pkt_pts != AV_NOPTS_VALUE) { auto frame_time = getTimeBase()*(frame->pkt_pts - avStream_->start_time); auto target = startTime_ + static_cast(frame_time.real() * 1e6); diff --git a/src/media/media_decoder.h b/src/media/media_decoder.h index 7012bc38b..fa22ab2b2 100644 --- a/src/media/media_decoder.h +++ b/src/media/media_decoder.h @@ -44,6 +44,12 @@ class AVCodec; namespace ring { +#if defined(RING_VIDEO) && defined(RING_ACCEL) +namespace video { +class HardwareAccel; +} +#endif + class AudioFrame; class AudioFormat; class RingBuffer; @@ -114,6 +120,11 @@ class MediaDecoder { // maximum time a packet can be queued (in ms) const unsigned jitterBufferMaxDelay_ {100000}; + bool enableAccel_ = true; +#if defined(RING_VIDEO) && defined(RING_ACCEL) + std::unique_ptr accel_; +#endif + protected: AVDictionary *options_ = nullptr; }; diff --git a/src/media/media_device.h b/src/media/media_device.h index 9fb685943..abf4826e0 100644 --- a/src/media/media_device.h +++ b/src/media/media_device.h @@ -45,6 +45,7 @@ struct DeviceParams { std::string sdp_flags {}; unsigned offset_x {}; unsigned offset_y {}; + std::string enableAccel {}; }; } diff --git a/src/media/video/Makefile.am b/src/media/video/Makefile.am index 80e846948..a0c53124c 100644 --- a/src/media/video/Makefile.am +++ b/src/media/video/Makefile.am @@ -35,6 +35,10 @@ libvideo_la_SOURCES = \ video_rtp_session.cpp video_rtp_session.h \ sinkclient.cpp sinkclient.h +if RING_ACCEL +libvideo_la_SOURCES += accel.cpp accel.h +endif + libvideo_la_LIBADD = @LIBAVCODEC_LIBS@ @LIBAVFORMAT_LIBS@ @LIBAVDEVICE_LIBS@ @LIBSWSCALE_LIBS@ @LIBAVUTIL_LIBS@ AM_CXXFLAGS=@LIBAVCODEC_CFLAGS@ @LIBAVFORMAT_CFLAGS@ @LIBAVDEVICE_CFLAGS@ @LIBSWSCALE_CFLAGS@ diff --git a/src/media/video/accel.cpp b/src/media/video/accel.cpp new file mode 100644 index 000000000..118a28e07 --- /dev/null +++ b/src/media/video/accel.cpp @@ -0,0 +1,197 @@ +/* + * Copyright (C) 2016 Savoir-faire Linux Inc. + * + * Author: Philippe Gorley + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#include "libav_deps.h" // MUST BE INCLUDED FIRST +#include "media_buffer.h" + +#include "accel.h" + +#if defined(HAVE_VAAPI_ACCEL_X11) || defined(HAVE_VAAPI_ACCEL_DRM) +#include "v4l2/vaapi.h" +#endif + +#include "string_utils.h" +#include "logger.h" + +#include +#include + +namespace ring { namespace video { + +static constexpr const unsigned MAX_ACCEL_FAILURES { 5 }; + +static AVPixelFormat +getFormatCb(AVCodecContext* codecCtx, const AVPixelFormat* formats) +{ + auto accel = static_cast(codecCtx->opaque); + if (!accel) { + // invalid state, try to recover + return avcodec_default_get_format(codecCtx, formats); + } + + for (int i = 0; formats[i] != AV_PIX_FMT_NONE; i++) { + if (formats[i] == accel->format()) { + accel->setWidth(codecCtx->coded_width); + accel->setHeight(codecCtx->coded_height); + accel->setProfile(codecCtx->profile); + if (accel->init(codecCtx)) + return accel->format(); + break; + } + } + + accel->fail(true); + RING_WARN("Falling back to software decoding"); + codecCtx->get_format = avcodec_default_get_format; + codecCtx->get_buffer2 = avcodec_default_get_buffer2; + for (int i = 0; formats[i] != AV_PIX_FMT_NONE; i++) { + auto desc = av_pix_fmt_desc_get(formats[i]); + if (desc && !(desc->flags & AV_PIX_FMT_FLAG_HWACCEL)) { + return formats[i]; + } + } + + return AV_PIX_FMT_NONE; +} + +static int +allocateBufferCb(AVCodecContext* codecCtx, AVFrame* frame, int flags) +{ + if (auto accel = static_cast(codecCtx->opaque)) { + if (!accel->hasFailed() && accel->allocateBuffer(codecCtx, frame, flags) == 0) { + accel->succeed(); + return 0; + } + + accel->fail(); + } + + return avcodec_default_get_buffer2(codecCtx, frame, flags); +} + +template +static std::unique_ptr +makeHardwareAccel(const AccelInfo& info) { + return std::unique_ptr(new T(info)); +} + +static const AccelInfo* +getAccelInfo(std::initializer_list codecAccels) +{ + /* Each item in this array reprensents a fully implemented hardware acceleration in Ring. + * Each item should be enclosed in an #ifdef to prevent its compilation on an + * unsupported platform (VAAPI for Linux Intel won't compile on a Mac). + * A new item should be added when support for an acceleration has been added to Ring, + * which is also supported by FFmpeg. + * Steps to add an acceleration (after its implementation): + * - If it doesn't yet exist, add a unique AccelID + * - Specify its AVPixelFormat (the one used by FFmpeg) + * - Give it a name (this is used for the daemon logs) + * - Add a function pointer that returns an instance (makeHardwareAccel does this already) + * Note: the acceleration's header file must be guarded by the same #ifdef as + * in this array. + */ + static const AccelInfo accels[] = { +#if defined(HAVE_VAAPI_ACCEL_X11) || defined(HAVE_VAAPI_ACCEL_DRM) + { AccelID::Vaapi, AV_PIX_FMT_VAAPI, "vaapi", makeHardwareAccel }, +#endif + }; + + for (auto& accel : accels) { + for (auto& ca : codecAccels) { + if (accel.type == ca) { + RING_DBG("Found '%s' hardware acceleration", accel.name.c_str()); + return &accel; + } + } + } + + RING_DBG("Did not find a matching hardware acceleration"); + return nullptr; +} + +HardwareAccel::HardwareAccel(const AccelInfo& info) + : type_(info.type) + , format_(info.format) + , name_(info.name) +{ + failCount_ = 0; + fallback_ = false; + width_ = -1; + height_ = -1; + profile_ = -1; +} + +void +HardwareAccel::fail(bool forceFallback) +{ + ++failCount_; + if (failCount_ >= MAX_ACCEL_FAILURES || forceFallback) { + fallback_ = true; + failCount_ = 0; + // force reinit of media decoder to correctly set thread count + } +} + +std::unique_ptr +makeHardwareAccel(AVCodecContext* codecCtx) +{ + const AccelInfo* info = nullptr; + + switch (codecCtx->codec_id) { + case AV_CODEC_ID_H264: + info = getAccelInfo({ + AccelID::Vdpau, + AccelID::VideoToolbox, + AccelID::Dxva2, + AccelID::Vaapi, + AccelID::Vda + }); + break; + case AV_CODEC_ID_MPEG4: + case AV_CODEC_ID_H263: + case AV_CODEC_ID_H263P: + info = getAccelInfo({ + AccelID::Vdpau, + AccelID::VideoToolbox, + AccelID::Vaapi + }); + break; + default: + break; + } + + if (info && info->type != AccelID::NoAccel) { + if (auto accel = info->create(*info)) { + codecCtx->get_format = getFormatCb; + codecCtx->get_buffer2 = allocateBufferCb; + codecCtx->thread_safe_callbacks = 1; + codecCtx->thread_count = 1; + RING_DBG("Hardware acceleration setup has succeeded"); + return accel; + } else + RING_ERR("Failed to create %s hardware acceleration", info->name.c_str()); + } + + RING_WARN("Not using hardware acceleration"); + return nullptr; +} + +}} // namespace ring::video diff --git a/src/media/video/accel.h b/src/media/video/accel.h new file mode 100644 index 000000000..f0507b6d2 --- /dev/null +++ b/src/media/video/accel.h @@ -0,0 +1,86 @@ +/* + * Copyright (C) 2016 Savoir-faire Linux Inc. + * + * Author: Philippe Gorley + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#pragma once + +#include "libav_deps.h" +#include "media_buffer.h" +#include "config.h" + +#include +#include + +namespace ring { namespace video { + +class HardwareAccel; + +enum class AccelID { + NoAccel = 0, + Vdpau, + VideoToolbox, + Dxva2, + Vaapi, + Vda +}; + +struct AccelInfo { + AccelID type; + AVPixelFormat format; + std::string name; + std::unique_ptr (*create)(const AccelInfo& info); +}; + +class HardwareAccel { + public: + HardwareAccel(const AccelInfo& info); + virtual ~HardwareAccel() {}; + + AVPixelFormat format() const { return format_; } + std::string name() const { return name_; } + bool hasFailed() const { return fallback_; } + + void setWidth(int width) { width_ = width; } + void setHeight(int height) { height_ = height; } + void setProfile(int profile) { profile_ = profile; } + + void fail(bool forceFallback = false); + void succeed() { failCount_ = 0; } // call on success of allocateBuffer or extractData + + public: // must be implemented by derived classes + virtual bool init(AVCodecContext* codecCtx) = 0; + virtual int allocateBuffer(AVCodecContext* codecCtx, AVFrame* frame, int flags) = 0; + virtual bool extractData(AVCodecContext* codecCtx, VideoFrame& container) = 0; + + protected: + AccelID type_; + AVPixelFormat format_; + std::string name_; + unsigned failCount_; // how many failures in a row, reset on success + bool fallback_; // true when failCount_ exceeds a certain number + int width_; + int height_; + int profile_; +}; + +// HardwareAccel factory +// Checks if codec acceleration is possible +std::unique_ptr makeHardwareAccel(AVCodecContext* codecCtx); + +}} // namespace ring::video diff --git a/src/media/video/v4l2/Makefile.am b/src/media/video/v4l2/Makefile.am index 99339d305..3c3626d83 100644 --- a/src/media/video/v4l2/Makefile.am +++ b/src/media/video/v4l2/Makefile.am @@ -6,5 +6,9 @@ libv4l2_la_SOURCES = \ video_device_impl.cpp \ video_device_monitor_impl.cpp -AM_CXXFLAGS = @UDEV_CFLAGS@ -libv4l2_la_LIBADD = @UDEV_LIBS@ +if RING_ACCEL +libv4l2_la_SOURCES += vaapi.h vaapi.cpp +endif + +AM_CXXFLAGS = @UDEV_CFLAGS@ @LIBVA_CFLAGS@ @LIBVA_DRM_CFLAGS@ @LIBVA_X11_CFLAGS@ +libv4l2_la_LIBADD = @UDEV_LIBS@ @X11_LIBS@ @LIBVA_LIBS@ @LIBVA_DRM_LIBS@ @LIBVA_X11_LIBS@ diff --git a/src/media/video/v4l2/vaapi.cpp b/src/media/video/v4l2/vaapi.cpp new file mode 100644 index 000000000..6139e5aba --- /dev/null +++ b/src/media/video/v4l2/vaapi.cpp @@ -0,0 +1,230 @@ +/* + * Copyright (C) 2016 Savoir-faire Linux Inc. + * + * Author: Philippe Gorley + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#include "libav_deps.h" // MUST BE INCLUDED FIRST + +#include "config.h" + +#if defined(RING_VIDEO) && defined(RING_ACCEL) + +#include "video/v4l2/vaapi.h" +#include "video/accel.h" + +#include +#include +#include +#include +#include + +#include "logger.h" + +namespace ring { namespace video { + +static auto avBufferRefDeleter = [](AVBufferRef* buf){ av_buffer_unref(&buf); }; + +VaapiAccel::VaapiAccel(AccelInfo info) : HardwareAccel(info) + , deviceBufferRef_(nullptr, avBufferRefDeleter) + , framesBufferRef_(nullptr, avBufferRefDeleter) +{ +} + +VaapiAccel::~VaapiAccel() +{ +} + +int +VaapiAccel::allocateBuffer(AVCodecContext* codecCtx, AVFrame* frame, int flags) +{ + return av_hwframe_get_buffer(framesBufferRef_.get(), frame, 0); +} + +bool +VaapiAccel::extractData(AVCodecContext* codecCtx, VideoFrame& container) +{ + try { + auto input = container.pointer(); + + if (input->format != format_) { + std::stringstream buf; + buf << "Frame format mismatch: expected " << av_get_pix_fmt_name(format_); + buf << ", got " << av_get_pix_fmt_name((AVPixelFormat)input->format); + throw std::runtime_error(buf.str()); + } + + auto outContainer = new VideoFrame(); + auto output = outContainer->pointer(); + output->format = AV_PIX_FMT_YUV420P; + + if (av_hwframe_transfer_data(output, input, 0) < 0) { + throw std::runtime_error("Unable to extract data from VAAPI frame"); + } + + if (av_frame_copy_props(output, input) < 0 ) { + av_frame_unref(output); + } + + av_frame_unref(input); + av_frame_move_ref(input, output); + } catch (const std::runtime_error& e) { + fail(); + RING_ERR("%s", e.what()); + return false; + } + + succeed(); + return true; +} + +bool +VaapiAccel::init(AVCodecContext* codecCtx) +{ + vaProfile_ = VAProfileNone; + vaEntryPoint_ = VAEntrypointVLD; + using ProfileMap = std::map; + ProfileMap h264 = { + { FF_PROFILE_H264_CONSTRAINED_BASELINE, VAProfileH264ConstrainedBaseline }, + { FF_PROFILE_H264_BASELINE, VAProfileH264Baseline }, + { FF_PROFILE_H264_MAIN, VAProfileH264Main }, + { FF_PROFILE_H264_HIGH, VAProfileH264High } + }; + ProfileMap mpeg4 = { + { FF_PROFILE_MPEG4_SIMPLE, VAProfileMPEG4Simple }, + { FF_PROFILE_MPEG4_ADVANCED_SIMPLE, VAProfileMPEG4AdvancedSimple }, + { FF_PROFILE_MPEG4_MAIN, VAProfileMPEG4Main } + }; + ProfileMap h263 = { + { FF_PROFILE_UNKNOWN, VAProfileH263Baseline } + }; + + std::map profileMap = { + { AV_CODEC_ID_H264, h264 }, + { AV_CODEC_ID_MPEG4, mpeg4 }, + { AV_CODEC_ID_H263, h263 }, + { AV_CODEC_ID_H263P, h263 } // no clue if this'll work, #ffmpeg isn't answering me + }; + + VAStatus status; + +#ifdef HAVE_VAAPI_ACCEL_DRM + const char* deviceName = "/dev/dri/card0"; // check for renderDX first? +#else + const char* deviceName = nullptr; // use default device +#endif + + AVBufferRef* hardwareDeviceCtx; + if (av_hwdevice_ctx_create(&hardwareDeviceCtx, AV_HWDEVICE_TYPE_VAAPI, deviceName, nullptr, 0) < 0) { + RING_ERR("Failed to create VAAPI device"); + av_buffer_unref(&hardwareDeviceCtx); + return false; + } + + deviceBufferRef_.reset(av_buffer_ref(hardwareDeviceCtx)); + + auto device = reinterpret_cast(deviceBufferRef_->data); + vaConfig_ = VA_INVALID_ID; + vaContext_ = VA_INVALID_ID; + auto hardwareContext = static_cast(device->hwctx); + + int numProfiles = vaMaxNumProfiles(hardwareContext->display); + auto profiles = std::vector(numProfiles); + status = vaQueryConfigProfiles(hardwareContext->display, profiles.data(), &numProfiles); + if (status != VA_STATUS_SUCCESS) { + RING_ERR("Failed to query profiles: %s", vaErrorStr(status)); + return false; + } + + VAProfile codecProfile; + auto itOuter = profileMap.find(codecCtx->codec_id); + if (itOuter != profileMap.end()) { + auto innerMap = itOuter->second; + auto itInner = innerMap.find(codecCtx->profile); + if (itInner != innerMap.end()) { + codecProfile = itInner->second; + } + } + + auto iter = std::find_if(std::begin(profiles), + std::end(profiles), + [codecProfile](const VAProfile& p){ return p == codecProfile; }); + + if (iter == std::end(profiles)) { + RING_ERR("VAAPI does not support selected codec"); + return false; + } + + vaProfile_ = *iter; + + status = vaCreateConfig(hardwareContext->display, vaProfile_, vaEntryPoint_, 0, 0, &vaConfig_); + if (status != VA_STATUS_SUCCESS) { + RING_ERR("Failed to create VAAPI configuration: %s", vaErrorStr(status)); + return false; + } + + auto hardwareConfig = static_cast(av_hwdevice_hwconfig_alloc(deviceBufferRef_.get())); + hardwareConfig->config_id = vaConfig_; + + auto constraints = av_hwdevice_get_hwframe_constraints(deviceBufferRef_.get(), hardwareConfig); + if (width_ < constraints->min_width + || width_ > constraints->max_width + || height_ < constraints->min_height + || height_ > constraints->max_height) { + av_hwframe_constraints_free(&constraints); + av_freep(&hardwareConfig); + RING_ERR("Hardware does not support image size with VAAPI: %dx%d", width_, height_); + return false; + } + + int numSurfaces = 16; // based on codec instead? + if (codecCtx->active_thread_type & FF_THREAD_FRAME) + numSurfaces += codecCtx->thread_count; // need extra surface per thread + + framesBufferRef_.reset(av_hwframe_ctx_alloc(deviceBufferRef_.get())); + auto frames = reinterpret_cast(framesBufferRef_->data); + frames->format = AV_PIX_FMT_VAAPI; + frames->sw_format = AV_PIX_FMT_YUV420P; + frames->width = width_; + frames->height = height_; + frames->initial_pool_size = numSurfaces; + + if (av_hwframe_ctx_init(framesBufferRef_.get()) < 0) { + RING_ERR("Failed to initialize VAAPI frame context"); + return false; + } + + auto framesContext = static_cast(frames->hwctx); + status = vaCreateContext(hardwareContext->display, vaConfig_, width_, height_, + VA_PROGRESSIVE, framesContext->surface_ids, framesContext->nb_surfaces, &vaContext_); + if (status != VA_STATUS_SUCCESS) { + RING_ERR("Failed to create VAAPI context: %s", vaErrorStr(status)); + return false; + } + + RING_DBG("VAAPI decoder initialized"); + + ffmpegAccelCtx_.display = hardwareContext->display; + ffmpegAccelCtx_.config_id = vaConfig_; + ffmpegAccelCtx_.context_id = vaContext_; + codecCtx->hwaccel_context = (void*)&ffmpegAccelCtx_; + return true; +} + +}} + +#endif // defined(RING_VIDEO) && defined(RING_ACCEL) diff --git a/src/media/video/v4l2/vaapi.h b/src/media/video/v4l2/vaapi.h new file mode 100644 index 000000000..989f26731 --- /dev/null +++ b/src/media/video/v4l2/vaapi.h @@ -0,0 +1,83 @@ +/* + * Copyright (C) 2016 Savoir-faire Linux Inc. + * + * Author: Philippe Gorley + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#pragma once + +#include "libav_deps.h" // MUST BE INCLUDED FIRST + +#include "config.h" + +#if defined(RING_VIDEO) && defined(RING_ACCEL) + +extern "C" { +#include +#include +#include +#include + +#include +#ifdef HAVE_VAAPI_ACCEL_DRM +# include +#endif +#ifdef HAVE_VAAPI_ACCEL_X11 +# include +#endif + +#include +#include +#include +#include +#include + +#include +} + +#include "video/accel.h" + +#include +#include + +namespace ring { namespace video { + +class VaapiAccel : public HardwareAccel { + public: + VaapiAccel(AccelInfo info); + ~VaapiAccel(); + + bool init(AVCodecContext* codecCtx) override; + int allocateBuffer(AVCodecContext* codecCtx, AVFrame* frame, int flags) override; + bool extractData(AVCodecContext* codecCtx, VideoFrame& container) override; + + private: + using AVBufferRefPtr = std::unique_ptr>; + AVBufferRefPtr deviceBufferRef_; + AVBufferRefPtr framesBufferRef_; + + VAProfile vaProfile_; + VAEntrypoint vaEntryPoint_; + VAConfigID vaConfig_; + VAContextID vaContext_; + + struct vaapi_context ffmpegAccelCtx_; +}; + +}} // namespace ring::video + +#endif // defined(RING_VIDEO) && defined(RING_ACCEL) diff --git a/src/media/video/video_base.cpp b/src/media/video/video_base.cpp index 63d1e0145..366447a18 100644 --- a/src/media/video/video_base.cpp +++ b/src/media/video/video_base.cpp @@ -81,6 +81,7 @@ VideoSettings::VideoSettings(const std::map& settings) channel = extractString(settings, "channel"); video_size = extractString(settings, "size"); framerate = extractString(settings, "rate"); + enableAccel = extractString(settings, "enableAccel"); } std::map @@ -90,7 +91,8 @@ VideoSettings::to_map() const {"name", name}, {"size", video_size}, {"channel", channel}, - {"rate", framerate} + {"rate", framerate}, + {"enableAccel", enableAccel} }; } @@ -105,6 +107,7 @@ convert::encode(const ring::video::VideoSettings& rh node["video_size"] = rhs.video_size; node["channel"] = rhs.channel; node["framerate"] = rhs.framerate; + node["enableAccel"] = rhs.enableAccel; return node; } @@ -118,6 +121,11 @@ convert::decode(const Node& node, ring::video::Video rhs.video_size = node["video_size"].as(); rhs.channel = node["channel"].as(); rhs.framerate = node["framerate"].as(); + // optional setting that may or may not be there + try { + rhs.enableAccel = node["enableAccel"].as(); + } catch (...) {} + return true; } diff --git a/src/media/video/video_base.h b/src/media/video/video_base.h index 46d287040..54e3d531f 100644 --- a/src/media/video/video_base.h +++ b/src/media/video/video_base.h @@ -159,6 +159,7 @@ struct VideoSettings std::string channel {}; std::string video_size {}; std::string framerate {}; + std::string enableAccel {}; }; }} // namespace ring::video diff --git a/src/media/video/video_device.h b/src/media/video/video_device.h index 504501f81..b4e9f6911 100644 --- a/src/media/video/video_device.h +++ b/src/media/video/video_device.h @@ -128,6 +128,8 @@ public: settings.framerate.c_str()); } + settings.enableAccel = "1"; + return settings; } @@ -141,6 +143,7 @@ public: settings.channel = params.channel_name; settings.video_size = sizeToString(params.width, params.height); settings.framerate = ring::to_string(params.framerate.real()); + settings.enableAccel = params.enableAccel; return settings; } @@ -159,6 +162,7 @@ public: params.width = size.first; params.height = size.second; params.framerate = rateFromString(settings.channel, size, settings.framerate); + params.enableAccel = settings.enableAccel; setDeviceParams(params); }