jami/jami-daemon
1
0
Fork 0
mirror of https://git.jami.net/savoirfairelinux/jami-daemon.git synced 2025-08-12 22:09:25 +08:00
Code Issues Projects Releases Wiki Activity

accel: refactor to fit new hwaccel api

Browse Source 
libavutil now covers the simple cases for creating and managing devices.
As Ring does not need to fine-tune these processes, most of the accel
code can be and has been removed.

Most hardware decoders output NV12, so skip extra conversions by
outputting NV12. Said pixel format is supported by everything that isn't
excessively old.

Change-Id: I10c440026fc3b289dbba7ecbca47e55c57147207
Reviewed-by: Anthony Léonard <anthony.leonard@savoirfairelinux.com>
This commit is contained in:
philippegorley
2017-10-16 14:47:16 -04:00
committed by Anthony Léonard
parent 64322318dc
commit ea8c592136
12 changed files with 134 additions and 902 deletions
Download Patch File Download Diff File Expand all files Collapse all files

57
src/media/media_decoder.cpp
View File

@ -48,6 +48,8 @@ namespace ring {
const unsigned jitterBufferMaxSize_ {1500};
// maximum time a packet can be queued
const constexpr auto jitterBufferMaxDelay_ = std::chrono::milliseconds(50);
// maximum number of times accelerated decoding can fail in a row before falling back to software
const constexpr unsigned MAX_ACCEL_FAILURES { 5 };
MediaDecoder::MediaDecoder() :
inputCtx_(avformat_alloc_context()),
@ -57,6 +59,8 @@ MediaDecoder::MediaDecoder() :
MediaDecoder::~MediaDecoder()
{
if (decoderCtx_->hw_device_ctx)
av_buffer_unref(&decoderCtx_->hw_device_ctx);
if (decoderCtx_)
avcodec_close(decoderCtx_);
if (inputCtx_)
@ -106,7 +110,7 @@ int MediaDecoder::openInput(const DeviceParams& params)
#ifdef RING_ACCEL
// if there was a fallback to software decoding, do not enable accel
// it has been disabled already by the video_receive_thread
// it has been disabled already by the video_receive_thread/video_input
enableAccel_ &= Manager::instance().getDecodingAccelerated();
#endif
@ -301,21 +305,21 @@ int MediaDecoder::setupFromVideoData()
decoderCtx_->thread_count = std::max(1u, std::min(8u, std::thread::hardware_concurrency()/2));
if (emulateRate_) {
RING_DBG("Using framerate emulation");
startTime_ = av_gettime();
}
#ifdef RING_ACCEL
if (enableAccel_) {
accel_ = video::makeHardwareAccel(decoderCtx_);
decoderCtx_->opaque = accel_.get();
accel_ = video::setupHardwareDecoding(decoderCtx_);
decoderCtx_->opaque = &accel_;
} else if (Manager::instance().getDecodingAccelerated()) {
RING_WARN("Hardware accelerated decoding disabled because of previous failure");
} else {
RING_WARN("Hardware accelerated decoding disabled by user preference");
}
#endif // RING_ACCEL
if (emulateRate_) {
RING_DBG("Using framerate emulation");
startTime_ = av_gettime();
}
#endif
ret = avcodec_open2(decoderCtx_, inputDecoder_, NULL);
if (ret) {
@ -353,18 +357,10 @@ MediaDecoder::decode(VideoFrame& result)
int frameFinished = 0;
ret = avcodec_send_packet(decoderCtx_, &inpacket);
if (ret < 0) {
#ifdef RING_ACCEL
if (accel_ && accel_->hasFailed())
return Status::RestartRequired;
#endif
return ret == AVERROR_EOF ? Status::Success : Status::DecodeError;
}
ret = avcodec_receive_frame(decoderCtx_, frame);
if (ret < 0 && ret != AVERROR(EAGAIN) && ret != AVERROR_EOF) {
#ifdef RING_ACCEL
if (accel_ && accel_->hasFailed())
return Status::RestartRequired;
#endif
return Status::DecodeError;
}
if (ret >= 0)
@ -375,13 +371,18 @@ MediaDecoder::decode(VideoFrame& result)
if (frameFinished) {
frame->format = (AVPixelFormat) correctPixFmt(frame->format);
#ifdef RING_ACCEL
if (accel_) {
if (!accel_->hasFailed())
accel_->extractData(result);
else
return Status::RestartRequired;
if (!accel_.name.empty()) {
ret = video::transferFrameData(accel_, decoderCtx_, result);
if (ret < 0) {
++accelFailures_;
if (accelFailures_ >= MAX_ACCEL_FAILURES) {
RING_ERR("Hardware decoding failure");
accelFailures_ = 0; // reset error count for next time
return Status::RestartRequired;
}
}
}
#endif // RING_ACCEL
#endif
if (emulateRate_ and frame->pts != AV_NOPTS_VALUE) {
auto frame_time = getTimeBase()*(frame->pts - avStream_->start_time);
auto target = startTime_ + static_cast<std::int64_t>(frame_time.real() * 1e6);
@ -457,7 +458,9 @@ MediaDecoder::enableAccel(bool enableAccel)
{
enableAccel_ = enableAccel;
if (!enableAccel) {
accel_.reset();
accel_ = {};
if (decoderCtx_->hw_device_ctx)
av_buffer_unref(&decoderCtx_->hw_device_ctx);
if (decoderCtx_)
decoderCtx_->opaque = nullptr;
}
@ -487,9 +490,9 @@ MediaDecoder::flush(VideoFrame& result)
#ifdef RING_ACCEL
// flush is called when closing the stream
// so don't restart the media decoder
if (accel_ && !accel_->hasFailed())
accel_->extractData(result);
#endif // RING_ACCEL
if (!accel_.name.empty() && accelFailures_ < MAX_ACCEL_FAILURES)
video::transferFrameData(accel_, decoderCtx_, result);
#endif
return Status::FrameFinished;
}

13
src/media/media_decoder.h
View File

@ -26,6 +26,10 @@
#include "video/video_scaler.h"
#endif // RING_VIDEO
#ifdef RING_ACCEL
#include "video/accel.h"
#endif
#include "audio/audiobuffer.h"
#include "rational.h"
@ -44,12 +48,6 @@ class AVCodec;
namespace ring {
#ifdef RING_ACCEL
namespace video {
class HardwareAccel;
}
#endif
struct AudioFrame;
class AudioFormat;
class RingBuffer;
@ -120,7 +118,8 @@ class MediaDecoder {
#ifdef RING_ACCEL
bool enableAccel_ = true;
std::unique_ptr<video::HardwareAccel> accel_;
video::HardwareAccel accel_;
unsigned short accelFailures_ = 0;
#endif
protected:

283
src/media/video/accel.cpp
View File

@ -18,232 +18,129 @@
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
*/
#include "libav_deps.h" // MUST BE INCLUDED FIRST
#include "media_buffer.h"
extern "C" {
#include <libavutil/hwcontext.h>
}
#include "accel.h"
#ifdef RING_VAAPI
#include "v4l2/vaapi.h"
#endif
#ifdef RING_VDPAU
#include "v4l2/vdpau.h"
#endif
#ifdef RING_VIDEOTOOLBOX
#include "osxvideo/videotoolbox.h"
#endif
#include "string_utils.h"
#include "logger.h"
#include <sstream>
#include <algorithm>
namespace ring { namespace video {
#include "media_buffer.h"
#include "string_utils.h"
#include "fileutils.h"
#include "logger.h"
#include "accel.h"
#include "config.h"
static constexpr const unsigned MAX_ACCEL_FAILURES { 5 };
namespace ring { namespace video {
static AVPixelFormat
getFormatCb(AVCodecContext* codecCtx, const AVPixelFormat* formats)
{
auto accel = static_cast<HardwareAccel*>(codecCtx->opaque);
if (!accel) {
// invalid state, try to recover
return avcodec_default_get_format(codecCtx, formats);
}
AVPixelFormat fallback = AV_PIX_FMT_NONE;
for (int i = 0; formats[i] != AV_PIX_FMT_NONE; ++i) {
fallback = formats[i];
if (formats[i] == accel->format()) {
accel->setWidth(codecCtx->coded_width);
accel->setHeight(codecCtx->coded_height);
accel->setProfile(codecCtx->profile);
accel->setCodecCtx(codecCtx);
if (accel->init())
return accel->format();
if (formats[i] == accel->format) {
return formats[i];
}
}
accel->fail(true);
RING_WARN("Falling back to software decoding");
codecCtx->get_format = avcodec_default_get_format;
codecCtx->get_buffer2 = avcodec_default_get_buffer2;
RING_WARN("'%s' acceleration not supported, falling back to software decoding", accel->name.c_str());
accel->name = {}; // don't use accel
return fallback;
}
int
transferFrameData(HardwareAccel accel, AVCodecContext* codecCtx, VideoFrame& frame)
{
if (accel.name.empty())
return -1;
auto input = frame.pointer();
if (input->format != accel.format) {
RING_ERR("Frame format mismatch: expected %s, got %s",
av_get_pix_fmt_name(static_cast<AVPixelFormat>(accel.format)),
av_get_pix_fmt_name(static_cast<AVPixelFormat>(input->format)));
return -1;
}
// FFmpeg requires a second frame in which to transfer the data from the GPU buffer to the main memory
auto container = std::unique_ptr<VideoFrame>(new VideoFrame());
auto output = container->pointer();
// most hardware accelerations output NV12, so skip extra conversions
output->format = AV_PIX_FMT_NV12;
int ret = av_hwframe_transfer_data(output, input, 0);
// move output into input so the caller receives extracted image data
// but we have to delete input's data first
av_frame_unref(input);
av_frame_move_ref(input, output);
return ret;
}
static int
allocateBufferCb(AVCodecContext* codecCtx, AVFrame* frame, int flags)
openDevice(HardwareAccel accel, AVBufferRef** hardwareDeviceCtx)
{
if (auto accel = static_cast<HardwareAccel*>(codecCtx->opaque)) {
if (!accel->hasFailed() && accel->allocateBuffer(frame, flags) == 0) {
accel->succeedAllocation();
return 0;
}
accel->failAllocation();
}
return avcodec_default_get_buffer2(codecCtx, frame, flags);
}
HardwareAccel::HardwareAccel(const std::string& name, const AVPixelFormat format)
: name_(name)
, format_(format)
{}
void
HardwareAccel::failAllocation()
{
++allocationFails_;
fail(false);
}
void
HardwareAccel::failExtraction()
{
++extractionFails_;
fail(false);
}
void
HardwareAccel::fail(bool forceFallback)
{
if (allocationFails_ >= MAX_ACCEL_FAILURES || extractionFails_ >= MAX_ACCEL_FAILURES || forceFallback) {
RING_ERR("Hardware acceleration failure");
fallback_ = true;
allocationFails_ = 0;
extractionFails_ = 0;
if (codecCtx_) {
codecCtx_->get_format = avcodec_default_get_format;
codecCtx_->get_buffer2 = avcodec_default_get_buffer2;
int ret;
auto hwType = av_hwdevice_find_type_by_name(accel.name.c_str());
#ifdef HAVE_VAAPI_ACCEL_DRM
// default DRM device may not work on multi GPU computers, so check all possible values
if (accel.name == "vaapi") {
const std::string path = "/dev/dri/";
auto files = ring::fileutils::readDirectory(path);
// renderD* is preferred over card*
std::sort(files.rbegin(), files.rend());
for (auto& entry : files) {
std::string deviceName = path + entry;
if ((ret = av_hwdevice_ctx_create(hardwareDeviceCtx, hwType, deviceName.c_str(), nullptr, 0)) >= 0) {
RING_DBG("Using '%s' hardware acceleration with device '%s'", accel.name.c_str(), deviceName.c_str());
return ret;
}
}
}
#endif
// default device (nullptr) works for most cases
if ((ret = av_hwdevice_ctx_create(hardwareDeviceCtx, hwType, nullptr, nullptr, 0)) >= 0)
RING_DBG("Using '%s' hardware acceleration", accel.name.c_str());
return ret;
}
bool
HardwareAccel::extractData(VideoFrame& input)
const HardwareAccel
setupHardwareDecoding(AVCodecContext* codecCtx)
{
try {
auto inFrame = input.pointer();
if (inFrame->format != format_) {
std::stringstream buf;
buf << "Frame format mismatch: expected " << av_get_pix_fmt_name(format_);
buf << ", got " << av_get_pix_fmt_name((AVPixelFormat)inFrame->format);
throw std::runtime_error(buf.str());
}
// FFmpeg requires a second frame in which to transfer the data
// from the GPU buffer to the main memory
auto output = std::unique_ptr<VideoFrame>(new VideoFrame());
auto outFrame = output->pointer();
outFrame->format = AV_PIX_FMT_YUV420P;
extractData(input, *output);
// move outFrame into inFrame so the caller receives extracted image data
// but we have to delete inFrame first
av_frame_unref(inFrame);
av_frame_move_ref(inFrame, outFrame);
} catch (const std::runtime_error& e) {
failExtraction();
RING_ERR("%s", e.what());
return false;
}
succeedExtraction();
return true;
}
template <class T>
static std::unique_ptr<HardwareAccel>
makeHardwareAccel(const std::string name, const AVPixelFormat format) {
return std::unique_ptr<HardwareAccel>(new T(name, format));
}
std::unique_ptr<HardwareAccel>
makeHardwareAccel(AVCodecContext* codecCtx)
{
enum class AccelID {
NoAccel,
Vdpau,
Vaapi,
VideoToolbox,
};
struct AccelInfo {
AccelID type;
std::string name;
AVPixelFormat format;
std::unique_ptr<HardwareAccel> (*create)(const std::string name, const AVPixelFormat format);
};
/* Each item in this array reprensents a fully implemented hardware acceleration in Ring.
* Each item should be enclosed in an #ifdef to prevent its compilation on an
* unsupported platform (VAAPI for Linux Intel won't compile on a Mac).
* A new item should be added when support for an acceleration has been added to Ring,
* which is also supported by FFmpeg.
* Steps to add an acceleration (after its implementation):
* - Create an AccelID and add it to the switch statement
* - Give it a name (this is used for the daemon logs)
* - Specify its AVPixelFormat (the one used by FFmpeg: check pixfmt.h)
* - Add a function pointer that returns an instance (makeHardwareAccel<> does this already)
* Note: the include of the acceleration's header file must be guarded by the same #ifdef as
* in this array.
/**
* This array represents FFmpeg's hwaccels, along with their pixel format
* and their potentially supported codecs. Each item contains:
* - Name (must match the name used in FFmpeg)
* - Pixel format (tells FFmpeg which hwaccel to use)
* - Array of AVCodecID (potential codecs that can be accelerated by the hwaccel)
* Note: an empty name means the video isn't accelerated
*/
const AccelInfo accels[] = {
#ifdef RING_VAAPI
{ AccelID::Vaapi, "vaapi", AV_PIX_FMT_VAAPI, makeHardwareAccel<VaapiAccel> },
#endif
#ifdef RING_VDPAU
{ AccelID::Vdpau, "vdpau", AV_PIX_FMT_VDPAU, makeHardwareAccel<VdpauAccel> },
#endif
#ifdef RING_VIDEOTOOLBOX
{ AccelID::VideoToolbox, "videotoolbox", AV_PIX_FMT_VIDEOTOOLBOX, makeHardwareAccel<VideoToolboxAccel> },
#endif
{ AccelID::NoAccel, "none", AV_PIX_FMT_NONE, nullptr },
const HardwareAccel accels[] = {
{ "vaapi", AV_PIX_FMT_VAAPI, { AV_CODEC_ID_H264, AV_CODEC_ID_MPEG4, AV_CODEC_ID_H263 } },
{ "vdpau", AV_PIX_FMT_VDPAU, { AV_CODEC_ID_H264, AV_CODEC_ID_MPEG4, AV_CODEC_ID_H263 } },
{ "videotoolbox", AV_PIX_FMT_VIDEOTOOLBOX, { AV_CODEC_ID_H264, AV_CODEC_ID_MPEG4, AV_CODEC_ID_H263 } },
};
std::vector<AccelID> possibleAccels = {};
switch (codecCtx->codec_id) {
case AV_CODEC_ID_H264:
possibleAccels.push_back(AccelID::Vdpau);
possibleAccels.push_back(AccelID::Vaapi);
possibleAccels.push_back(AccelID::VideoToolbox);
break;
case AV_CODEC_ID_MPEG4:
case AV_CODEC_ID_H263P:
possibleAccels.push_back(AccelID::Vdpau);
possibleAccels.push_back(AccelID::Vaapi);
possibleAccels.push_back(AccelID::VideoToolbox);
break;
case AV_CODEC_ID_VP8:
break;
default:
break;
}
for (auto& info : accels) {
for (auto& pa : possibleAccels) {
if (info.type == pa) {
auto accel = info.create(info.name, info.format);
// don't break if the check fails, we want to check every possibility
if (accel->checkAvailability()) {
codecCtx->get_format = getFormatCb;
codecCtx->get_buffer2 = allocateBufferCb;
codecCtx->thread_safe_callbacks = 1;
RING_DBG("Attempting to use '%s' hardware acceleration", accel->name().c_str());
return accel;
}
AVBufferRef* hardwareDeviceCtx = nullptr;
for (auto accel : accels) {
if (std::find(accel.supportedCodecs.begin(), accel.supportedCodecs.end(),
static_cast<AVCodecID>(codecCtx->codec_id)) != accel.supportedCodecs.end()) {
if (openDevice(accel, &hardwareDeviceCtx) >= 0) {
codecCtx->hw_device_ctx = av_buffer_ref(hardwareDeviceCtx);
codecCtx->get_format = getFormatCb;
codecCtx->thread_safe_callbacks = 1;
return accel;
}
}
}
RING_WARN("Not using hardware acceleration");
return nullptr;
RING_WARN("Not using hardware accelerated decoding");
return {};
}
}} // namespace ring::video

52
src/media/video/accel.h
View File

@ -21,57 +21,19 @@
#pragma once
#include "libav_deps.h"
#include "media_buffer.h"
#include "config.h"
#include <string>
#include <memory>
#include <vector>
namespace ring { namespace video {
class HardwareAccel {
public:
HardwareAccel(const std::string& name, const AVPixelFormat format);
virtual ~HardwareAccel() {};
AVPixelFormat format() const { return format_; }
std::string name() const { return name_; }
bool hasFailed() const { return fallback_; }
void setCodecCtx(AVCodecContext* codecCtx) { codecCtx_ = codecCtx; }
void setWidth(int width) { width_ = width; }
void setHeight(int height) { height_ = height; }
void setProfile(int profile) { profile_ = profile; }
void failAllocation();
void failExtraction();
void fail(bool forceFallback);
void succeedAllocation() { allocationFails_ = 0; }
void succeedExtraction() { extractionFails_ = 0; }
// wrapper to take care of boilerplate before calling the derived class's implementation
bool extractData(VideoFrame& input);
public: // must be implemented by derived classes
virtual bool checkAvailability() = 0;
virtual bool init() = 0;
virtual int allocateBuffer(AVFrame* frame, int flags) = 0;
virtual void extractData(VideoFrame& input, VideoFrame& output) = 0;
protected:
AVCodecContext* codecCtx_ = nullptr;
std::string name_;
AVPixelFormat format_;
unsigned allocationFails_ = 0; // how many times in a row allocateBuffer has failed
unsigned extractionFails_ = 0; // how many times in a row extractData has failed
bool fallback_ = false; // set to true when successive failures exceeds MAX_ACCEL_FAILURES
int width_ = -1;
int height_ = -1;
int profile_ = -1;
struct HardwareAccel {
std::string name;
AVPixelFormat format;
std::vector<AVCodecID> supportedCodecs;
};
// HardwareAccel factory
// Checks if codec acceleration is possible
std::unique_ptr<HardwareAccel> makeHardwareAccel(AVCodecContext* codecCtx);
const HardwareAccel setupHardwareDecoding(AVCodecContext* codecCtx);
int transferFrameData(HardwareAccel accel, AVCodecContext* codecCtx, VideoFrame& frame);
}} // namespace ring::video

4
src/media/video/osxvideo/Makefile.am
View File

@ -6,8 +6,4 @@ libosxvideo_la_SOURCES = \
video_device_impl.mm \
video_device_monitor_impl.mm
if RING_ACCEL
libosxvideo_la_SOURCES += videotoolbox.h videotoolbox.mm
endif
AM_OBJCXXFLAGS = -std=c++11

60
src/media/video/osxvideo/videotoolbox.h
View File

@ -1,60 +0,0 @@
/*
* Copyright (C) 2016-2017 Savoir-faire Linux Inc.
*
* Author: Philippe Gorley <philippe.gorley@savoirfairelinux.com>
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
*/
#pragma once
#include "libav_deps.h" // MUST BE INCLUDED FIRST
#include "config.h"
#ifdef RING_VIDEOTOOLBOX
extern "C" {
#include <libavcodec/avcodec.h>
#include <libavutil/hwcontext.h>
#include <libavcodec/videotoolbox.h>
#include <libavutil/imgutils.h>
}
#include "video/accel.h"
#include <memory>
#include <functional>
namespace ring { namespace video {
class VideoToolboxAccel : public HardwareAccel {
public:
VideoToolboxAccel(const std::string name, const AVPixelFormat format);
~VideoToolboxAccel();
bool checkAvailability() override;
bool init() override;
int allocateBuffer(AVFrame* frame, int flags) override;
void extractData(VideoFrame& input, VideoFrame& output) override;
private:
using AVBufferRefPtr = std::unique_ptr<AVBufferRef, std::function<void(AVBufferRef*)>>;
AVBufferRefPtr deviceBufferRef_;
};
}} // namespace ring::video
#endif // RING_VIDEOTOOLBOX

158
src/media/video/osxvideo/videotoolbox.mm
View File

@ -1,158 +0,0 @@
/*
* Copyright (C) 2016-2017 Savoir-faire Linux Inc.
*
* Author: Philippe Gorley <philippe.gorley@savoirfairelinux.com>
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
*/
#include "libav_deps.h" // MUST BE INCLUDED FIRST
#include "config.h"
#ifdef RING_VIDEOTOOLBOX
#include <string>
#include <sstream>
#include <array>
#include "video/osxvideo/videotoolbox.h"
#include "video/accel.h"
#include "logger.h"
namespace ring { namespace video {
static auto avBufferRefDeleter = [](AVBufferRef* buf){ av_buffer_unref(&buf); };
VideoToolboxAccel::VideoToolboxAccel(const std::string name, const AVPixelFormat format)
: HardwareAccel(name, format)
, deviceBufferRef_(nullptr, avBufferRefDeleter)
{
}
VideoToolboxAccel::~VideoToolboxAccel()
{
if (codecCtx_)
av_videotoolbox_default_free(codecCtx_);
}
int
VideoToolboxAccel::allocateBuffer(AVFrame* frame, int flags)
{
// do nothing, as this is done during extractData
(void) frame; // unused
(void) flags; // unused
return 0;
}
void
VideoToolboxAccel::extractData(VideoFrame& input, VideoFrame& output)
{
auto inFrame = input.pointer();
auto outFrame = output.pointer();
auto pixelBuffer = reinterpret_cast<CVPixelBufferRef>(inFrame->data[3]);
auto pixelFormat = CVPixelBufferGetPixelFormatType(pixelBuffer);
switch (pixelFormat) {
case kCVPixelFormatType_420YpCbCr8Planar:
outFrame->format = AV_PIX_FMT_YUV420P;
break;
case kCVPixelFormatType_32BGRA:
outFrame->format = AV_PIX_FMT_BGRA;
break;
case kCVPixelFormatType_422YpCbCr8:
outFrame->format = AV_PIX_FMT_UYVY422;
break;
case kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange: // OS X 10.7+
outFrame->format = AV_PIX_FMT_NV12;
break;
default:
char codecTag[32];
av_get_codec_tag_string(codecTag, sizeof(codecTag), codecCtx_->codec_tag);
std::stringstream buf;
buf << "VideoToolbox (" << codecTag << "): unsupported pixel format (";
buf << av_get_pix_fmt_name(format_) << ")";
throw std::runtime_error(buf.str());
}
outFrame->width = inFrame->width;
outFrame->height = inFrame->height;
// align on 32 bytes
if (av_frame_get_buffer(outFrame, 32) < 0) {
std::stringstream buf;
buf << "Could not allocate a buffer for VideoToolbox";
throw std::runtime_error(buf.str());
}
if (CVPixelBufferLockBaseAddress(pixelBuffer, kCVPixelBufferLock_ReadOnly) != kCVReturnSuccess) {
throw std::runtime_error("Could not lock the pixel buffer");
}
// av_image_copy function takes a 4 element array (according to its signature)
std::array<uint8_t*, 4> buffer = {};
std::array<int, 4> lineSize = {};
if (CVPixelBufferIsPlanar(pixelBuffer)) {
int planeCount = CVPixelBufferGetPlaneCount(pixelBuffer);
for (int i = 0; i < planeCount; i++) {
buffer[i] = static_cast<uint8_t*>(CVPixelBufferGetBaseAddressOfPlane(pixelBuffer, i));
lineSize[i] = CVPixelBufferGetBytesPerRowOfPlane(pixelBuffer, i);
}
} else {
buffer[0] = static_cast<uint8_t*>(CVPixelBufferGetBaseAddress(pixelBuffer));
lineSize[0] = CVPixelBufferGetBytesPerRow(pixelBuffer);
}
av_image_copy(outFrame->data, outFrame->linesize,
const_cast<const uint8_t**>(static_cast<uint8_t**>(buffer.data())),
lineSize.data(), static_cast<AVPixelFormat>(outFrame->format),
inFrame->width, inFrame->height);
if (av_frame_copy_props(outFrame, inFrame) < 0) {
av_frame_unref(outFrame);
}
CVPixelBufferUnlockBaseAddress(pixelBuffer, kCVPixelBufferLock_ReadOnly);
}
bool
VideoToolboxAccel::checkAvailability()
{
AVBufferRef* hardwareDeviceCtx;
if (av_hwdevice_ctx_create(&hardwareDeviceCtx, AV_HWDEVICE_TYPE_VIDEOTOOLBOX, nullptr, nullptr, 0) == 0) {
deviceBufferRef_.reset(hardwareDeviceCtx);
return true;
}
av_buffer_unref(&hardwareDeviceCtx);
return false;
}
bool
VideoToolboxAccel::init()
{
if (av_videotoolbox_default_init(codecCtx_) >= 0) {
RING_DBG("VideoToolbox decoder initialized");
codecCtx_->hw_device_ctx = av_buffer_ref(deviceBufferRef_.get());
return true;
} else {
RING_ERR("Failed to initialize VideoToolbox decoder");
return false;
}
}
}}
#endif // RING_VIDEOTOOLBOX

10
src/media/video/v4l2/Makefile.am
View File

@ -6,15 +6,7 @@ libv4l2_la_SOURCES = \
video_device_impl.cpp \
video_device_monitor_impl.cpp
if RING_VDPAU
libv4l2_la_SOURCES += vdpau.h vdpau.cpp
endif
if RING_VAAPI
libv4l2_la_SOURCES += vaapi.h vaapi.cpp
endif
AM_CXXFLAGS = @LIBAVCODEC_CFLAGS@ @LIBAVFORMAT_CFLAGS@ @LIBAVDEVICE_CFLAGS@ @LIBSWSCALE_CFLAGS@
AM_CXXFLAGS = @LIBAVCODEC_CFLAGS@ @LIBAVFORMAT_CFLAGS@ @LIBAVDEVICE_CFLAGS@ @LIBSWSCALE_CFLAGS@ @LIBAVUTIL_CFLAGS@
AM_CXXFLAGS += @UDEV_CFLAGS@ @VDPAU_CFLAGS@ @LIBVA_CFLAGS@ @LIBVA_DRM_CFLAGS@ @LIBVA_X11_CFLAGS@
libv4l2_la_LIBADD = @LIBAVCODEC_LIBS@ @LIBAVFORMAT_LIBS@ @LIBAVDEVICE_LIBS@ @LIBSWSCALE_LIBS@ @LIBAVUTIL_LIBS@

136
src/media/video/v4l2/vaapi.cpp
View File

@ -1,136 +0,0 @@
/*
* Copyright (C) 2016-2017 Savoir-faire Linux Inc.
*
* Author: Philippe Gorley <philippe.gorley@savoirfairelinux.com>
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
*/
#include "libav_deps.h" // MUST BE INCLUDED FIRST
#include "config.h"
#ifdef RING_VAAPI
#include "video/v4l2/vaapi.h"
#include "video/accel.h"
#include "fileutils.h"
#include <sstream>
#include <stdexcept>
#include <map>
#include <algorithm>
#include <vector>
#include "logger.h"
namespace ring { namespace video {
static auto avBufferRefDeleter = [](AVBufferRef* buf){ av_buffer_unref(&buf); };
VaapiAccel::VaapiAccel(const std::string name, const AVPixelFormat format)
: HardwareAccel(name, format)
, deviceBufferRef_(nullptr, avBufferRefDeleter)
, framesBufferRef_(nullptr, avBufferRefDeleter)
{
}
VaapiAccel::~VaapiAccel()
{
}
int
VaapiAccel::allocateBuffer(AVFrame* frame, int flags)
{
(void) flags; // unused
return av_hwframe_get_buffer(framesBufferRef_.get(), frame, 0);
}
void
VaapiAccel::extractData(VideoFrame& input, VideoFrame& output)
{
auto inFrame = input.pointer();
auto outFrame = output.pointer();
if (av_hwframe_transfer_data(outFrame, inFrame, 0) < 0) {
throw std::runtime_error("Unable to extract data from VAAPI frame");
}
if (av_frame_copy_props(outFrame, inFrame) < 0 ) {
av_frame_unref(outFrame);
}
}
bool
VaapiAccel::checkAvailability()
{
AVBufferRef* hardwareDeviceCtx = nullptr;
#ifdef HAVE_VAAPI_ACCEL_DRM
const std::string path = "/dev/dri/";
auto files = ring::fileutils::readDirectory(path);
// renderD* is preferred over card*
std::sort(files.rbegin(), files.rend());
for (auto& entry : files) {
std::string deviceName = path + entry;
if (av_hwdevice_ctx_create(&hardwareDeviceCtx, AV_HWDEVICE_TYPE_VAAPI, deviceName.c_str(), nullptr, 0) >= 0) {
deviceName_ = deviceName;
break;
}
}
if (hardwareDeviceCtx == nullptr)
return false;
#elif HAVE_VAAPI_ACCEL_X11
if (av_hwdevice_ctx_create(&hardwareDeviceCtx, AV_HWDEVICE_TYPE_VAAPI, nullptr, nullptr, 0) < 0) {
return false;
}
#endif
deviceBufferRef_.reset(hardwareDeviceCtx);
return true;
}
bool
VaapiAccel::init()
{
int numSurfaces = 16; // based on codec instead?
if (codecCtx_->active_thread_type & FF_THREAD_FRAME)
numSurfaces += codecCtx_->thread_count; // need extra surface per thread
framesBufferRef_.reset(av_hwframe_ctx_alloc(deviceBufferRef_.get()));
auto frames = reinterpret_cast<AVHWFramesContext*>(framesBufferRef_->data);
frames->format = format_;
frames->sw_format = AV_PIX_FMT_YUV420P;
frames->width = width_;
frames->height = height_;
frames->initial_pool_size = numSurfaces;
if (av_hwframe_ctx_init(framesBufferRef_.get()) < 0) {
RING_ERR("Failed to initialize VAAPI frame context");
return false;
}
codecCtx_->hw_frames_ctx = av_buffer_ref(framesBufferRef_.get());
if (!deviceName_.empty())
RING_DBG("VAAPI decoder initialized via device: %s", deviceName_.c_str());
else
RING_DBG("VAAPI decoder initialized");
return true;
}
}}
#endif // RING_VAAPI

79
src/media/video/v4l2/vaapi.h
View File

@ -1,79 +0,0 @@
/*
* Copyright (C) 2016-2017 Savoir-faire Linux Inc.
*
* Author: Philippe Gorley <philippe.gorley@savoirfairelinux.com>
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
*/
#pragma once
#include "libav_deps.h" // MUST BE INCLUDED FIRST
#include "config.h"
#ifdef RING_VAAPI
extern "C" {
#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>
#include <unistd.h>
#include <va/va.h>
#ifdef HAVE_VAAPI_ACCEL_DRM
# include <va/va_drm.h>
#endif
#ifdef HAVE_VAAPI_ACCEL_X11
# include <va/va_x11.h>
#endif
#include <libavutil/avconfig.h>
#include <libavutil/buffer.h>
#include <libavutil/frame.h>
#include <libavutil/hwcontext.h>
#include <libavutil/hwcontext_vaapi.h>
#include <libavcodec/vaapi.h>
}
#include "video/accel.h"
#include <memory>
#include <functional>
namespace ring { namespace video {
class VaapiAccel : public HardwareAccel {
public:
VaapiAccel(const std::string name, const AVPixelFormat format);
~VaapiAccel();
bool checkAvailability() override;
bool init() override;
int allocateBuffer(AVFrame* frame, int flags) override;
void extractData(VideoFrame& input, VideoFrame& output) override;
private:
using AVBufferRefPtr = std::unique_ptr<AVBufferRef, std::function<void(AVBufferRef*)>>;
AVBufferRefPtr deviceBufferRef_;
AVBufferRefPtr framesBufferRef_;
std::string deviceName_;
};
}} // namespace ring::video
#endif // RING_VAAPI

120
src/media/video/v4l2/vdpau.cpp
View File

@ -1,120 +0,0 @@
/*
* Copyright (C) 2017 Savoir-faire Linux Inc.
*
* Author: Philippe Gorley <philippe.gorley@savoirfairelinux.com>
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
*/
#include "libav_deps.h" // MUST BE INCLUDED FIRST
#include "config.h"
#ifdef RING_VDPAU
#include "video/v4l2/vdpau.h"
#include "video/accel.h"
#include "fileutils.h"
#include <sstream>
#include <stdexcept>
#include <map>
#include <algorithm>
#include <vector>
#include "logger.h"
namespace ring { namespace video {
static auto avBufferRefDeleter = [](AVBufferRef* buf){ av_buffer_unref(&buf); };
VdpauAccel::VdpauAccel(const std::string name, const AVPixelFormat format)
: HardwareAccel(name, format)
, deviceBufferRef_(nullptr, avBufferRefDeleter)
, framesBufferRef_(nullptr, avBufferRefDeleter)
{
}
VdpauAccel::~VdpauAccel()
{
}
int
VdpauAccel::allocateBuffer(AVFrame* frame, int flags)
{
(void) flags;
return av_hwframe_get_buffer(framesBufferRef_.get(), frame, 0);
}
void
VdpauAccel::extractData(VideoFrame& input, VideoFrame& output)
{
auto inFrame = input.pointer();
auto outFrame = output.pointer();
if (av_hwframe_transfer_data(outFrame, inFrame, 0) < 0) {
throw std::runtime_error("Unable to extract data from VDPAU frame");
}
if (av_frame_copy_props(outFrame, inFrame) < 0 ) {
av_frame_unref(outFrame);
}
}
bool
VdpauAccel::checkAvailability()
{
AVBufferRef* hardwareDeviceCtx;
if (av_hwdevice_ctx_create(&hardwareDeviceCtx, AV_HWDEVICE_TYPE_VDPAU, nullptr, nullptr, 0) == 0) {
deviceBufferRef_.reset(hardwareDeviceCtx);
return true;
}
av_buffer_unref(&hardwareDeviceCtx);
return false;
}
bool
VdpauAccel::init()
{
auto device = reinterpret_cast<AVHWDeviceContext*>(deviceBufferRef_->data);
auto hardwareContext = static_cast<AVVDPAUDeviceContext*>(device->hwctx);
framesBufferRef_.reset(av_hwframe_ctx_alloc(deviceBufferRef_.get()));
auto frames = reinterpret_cast<AVHWFramesContext*>(framesBufferRef_->data);
frames->format = AV_PIX_FMT_VDPAU;
frames->sw_format = AV_PIX_FMT_YUV420P;
frames->width = width_;
frames->height = height_;
if (av_hwframe_ctx_init(framesBufferRef_.get()) < 0) {
RING_ERR("Failed to initialize VDPAU frame context");
return false;
}
if (av_vdpau_bind_context(codecCtx_, hardwareContext->device, hardwareContext->get_proc_address, 0)) {
RING_ERR("Could not bind VDPAU context");
return false;
}
RING_DBG("VDPAU decoder initialized");
return true;
}
}} // namespace ring::video
#endif // RING_VDPAU

64
src/media/video/v4l2/vdpau.h
View File

@ -1,64 +0,0 @@
/*
* Copyright (C) 2017 Savoir-faire Linux Inc.
*
* Author: Philippe Gorley <philippe.gorley@savoirfairelinux.com>
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
*/
#pragma once
#include "libav_deps.h" // MUST BE INCLUDED FIRST
#include "config.h"
#ifdef RING_VDPAU
extern "C" {
#include <stdint.h>
#include <libavcodec/vdpau.h>
#include <libavutil/buffer.h>
#include <libavutil/frame.h>
#include <libavutil/hwcontext.h>
#include <libavutil/hwcontext_vdpau.h>
}
#include "video/accel.h"
#include <memory>
#include <functional>
namespace ring { namespace video {
class VdpauAccel : public HardwareAccel {
public:
VdpauAccel(const std::string name, const AVPixelFormat format);
~VdpauAccel();
bool checkAvailability() override;
bool init() override;
int allocateBuffer(AVFrame* frame, int flags) override;
void extractData(VideoFrame& input, VideoFrame& output) override;
private:
using AVBufferRefPtr = std::unique_ptr<AVBufferRef, std::function<void(AVBufferRef*)>>;
AVBufferRefPtr deviceBufferRef_;
AVBufferRefPtr framesBufferRef_;
};
}} // namespace ring::video
#endif // RING_VDPAU