diff options
| author | Alexander Voronov <avoronov@graphics.cs.msu.ru> | 2014-10-02 17:14:12 +0400 |
|---|---|---|
| committer | Alexander Voronov <avoronov@graphics.cs.msu.ru> | 2014-10-02 18:57:47 +0400 |
| commit | befc36d4a791447e36699d0390dff5205cc6dd3f (patch) | |
| tree | bc685fdd342a9d291e6ca74b69cf4e7d0901d636 | |
| parent | e9f90513940fc5205c7842cabf63f036de606bce (diff) | |
| download | libvpx-befc36d4a791447e36699d0390dff5205cc6dd3f.tar libvpx-befc36d4a791447e36699d0390dff5205cc6dd3f.tar.gz libvpx-befc36d4a791447e36699d0390dff5205cc6dd3f.tar.bz2 libvpx-befc36d4a791447e36699d0390dff5205cc6dd3f.zip | |
Fix invalid memory access in inter prediction (issue 853).
Change-Id: I5a566d6ade720f212a60c0ad5d6f1ee1d1d37f2e
| -rw-r--r-- | vp9/common/vp9_convolve.c | 12 |
1 files changed, 4 insertions, 8 deletions
diff --git a/vp9/common/vp9_convolve.c b/vp9/common/vp9_convolve.c index ad70e59bf..973d0244c 100644 --- a/vp9/common/vp9_convolve.c +++ b/vp9/common/vp9_convolve.c @@ -130,16 +130,14 @@ static void convolve(const uint8_t *src, ptrdiff_t src_stride, // --Require an additional SUBPEL_TAPS rows for the 8-tap filter tails. // --((64 - 1) * 32 + 15) >> 4 + 8 = 135. uint8_t temp[135 * 64]; - int intermediate_height = (((h - 1) * y_step_q4 + 15) >> 4) + SUBPEL_TAPS; + int intermediate_height = + (((h - 1) * y_step_q4 + y0_q4) >> SUBPEL_BITS) + SUBPEL_TAPS; assert(w <= 64); assert(h <= 64); assert(y_step_q4 <= 32); assert(x_step_q4 <= 32); - if (intermediate_height < h) - intermediate_height = h; - convolve_horiz(src - src_stride * (SUBPEL_TAPS / 2 - 1), src_stride, temp, 64, x_filters, x0_q4, x_step_q4, w, intermediate_height); convolve_vert(temp + 64 * (SUBPEL_TAPS / 2 - 1), 64, dst, dst_stride, @@ -407,16 +405,14 @@ static void high_convolve(const uint8_t *src, ptrdiff_t src_stride, // --Require an additional SUBPEL_TAPS rows for the 8-tap filter tails. // --((64 - 1) * 32 + 15) >> 4 + 8 = 135. uint16_t temp[64 * 135]; - int intermediate_height = (((h - 1) * y_step_q4 + 15) >> 4) + SUBPEL_TAPS; + int intermediate_height = + (((h - 1) * y_step_q4 + y0_q4) >> SUBPEL_BITS) + SUBPEL_TAPS; assert(w <= 64); assert(h <= 64); assert(y_step_q4 <= 32); assert(x_step_q4 <= 32); - if (intermediate_height < h) - intermediate_height = h; - high_convolve_horiz(src - src_stride * (SUBPEL_TAPS / 2 - 1), src_stride, CONVERT_TO_BYTEPTR(temp), 64, x_filters, x0_q4, x_step_q4, w, |