encoder: add error checking to videoFrameSize

av_image_get_buffer_size can return a negative error code. Prevents
casting such an error to a large positive integer that may cause a
bad_alloc exception on the subsequent buffer allocation.

Change-Id: Ie8a7987dcf16957a21496e791c34957bb2e35bfc
This commit is contained in:
jinboli
2018-11-22 15:30:33 -05:00
committed by philippegorley
parent 7ae0214dd7
commit c499a22be4
3 changed files with 5 additions and 3 deletions

View File

@ -34,7 +34,7 @@ namespace ring {
//=== HELPERS ==================================================================
std::size_t
int
videoFrameSize(int format, int width, int height)
{
return av_image_get_buffer_size((AVPixelFormat)format, width, height, 1);

View File

@ -42,7 +42,7 @@ using AudioFrame = DRing::AudioFrame;
using VideoFrame = DRing::VideoFrame;
// Some helpers
std::size_t videoFrameSize(int format, int width, int height);
int videoFrameSize(int format, int width, int height);
#endif // RING_VIDEO

View File

@ -286,7 +286,9 @@ MediaEncoder::addStream(const SystemCodecInfo& systemCodecInfo, std::string para
const int height = encoderCtx->height;
const int format = encoderCtx->pix_fmt;
scaledFrameBufferSize_ = videoFrameSize(format, width, height);
if (scaledFrameBufferSize_ <= AV_INPUT_BUFFER_MIN_SIZE)
if (scaledFrameBufferSize_ < 0)
throw MediaEncoderException(("Could not compute buffer size: " + libav_utils::getError(scaledFrameBufferSize_)).c_str());
else if (scaledFrameBufferSize_ <= AV_INPUT_BUFFER_MIN_SIZE)
throw MediaEncoderException("buffer too small");
scaledFrameBuffer_.reserve(scaledFrameBufferSize_);