From 417b8529678039e7c8ecbfd954685019e4fc387a Mon Sep 17 00:00:00 2001 From: John Koleszar Date: Tue, 7 Feb 2012 10:26:48 -0800 Subject: Align internal mfqe framebuffer dimensions MFQE postproc crashed with stream dimensions not a multiple of 16. The buffer was memset unconditionally, so if the buffer allocation fails we end up trying to write to NULL. This patch traps an allocation failure with vpx_internal_error(), and aligns the buffer dimensions to what vp8_yv12_alloc_frame_buffer() expects. Change-Id: I3915d597cd66886a24f4ef39752751ebe6425066 --- vp8/common/postproc.c | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) (limited to 'vp8/common') diff --git a/vp8/common/postproc.c b/vp8/common/postproc.c index 0dc14b8bd..87e999772 100644 --- a/vp8/common/postproc.c +++ b/vp8/common/postproc.c @@ -930,10 +930,16 @@ int vp8_post_proc_frame(VP8_COMMON *oci, YV12_BUFFER_CONFIG *dest, vp8_ppflags_t { if ((flags & VP8D_DEBLOCK) || (flags & VP8D_DEMACROBLOCK)) { - if (vp8_yv12_alloc_frame_buffer(&oci->post_proc_buffer_int, oci->Width, oci->Height, VP8BORDERINPIXELS) >= 0) - { - oci->post_proc_buffer_int_used = 1; - } + int width = (oci->Width + 15) & ~15; + int height = (oci->Height + 15) & ~15; + + if (vp8_yv12_alloc_frame_buffer(&oci->post_proc_buffer_int, + width, height, VP8BORDERINPIXELS)) + vpx_internal_error(&oci->error, VPX_CODEC_MEM_ERROR, + "Failed to allocate MFQE framebuffer"); + + oci->post_proc_buffer_int_used = 1; + // insure that postproc is set to all 0's so that post proc // doesn't pull random data in from edge vpx_memset((&oci->post_proc_buffer_int)->buffer_alloc,126,(&oci->post_proc_buffer)->frame_size); -- cgit v1.2.3