diff options
| author | Alex Converse <aconverse@google.com> | 2015-11-19 16:46:46 -0800 |
|---|---|---|
| committer | Alex Converse <aconverse@google.com> | 2016-04-27 15:07:22 -0700 |
| commit | d3fe3b2abb90caf87f76e46d83a2acfcfcc5b687 (patch) | |
| tree | a8d46d30bf800effb0e1a7fa86c86eb10ff1fad4 | |
| parent | b2ccb9c189069d45d201c988184e9e0796b96270 (diff) | |
| download | libvpx-d3fe3b2abb90caf87f76e46d83a2acfcfcc5b687.tar libvpx-d3fe3b2abb90caf87f76e46d83a2acfcfcc5b687.tar.gz libvpx-d3fe3b2abb90caf87f76e46d83a2acfcfcc5b687.tar.bz2 libvpx-d3fe3b2abb90caf87f76e46d83a2acfcfcc5b687.zip | |
Avoid an unsigned overflow in invert_quant
Change-Id: I16a570b2af66b6580d1cd6f8345a25f079009bf4
| -rw-r--r-- | vp10/encoder/quantize.c | 6 | ||||
| -rw-r--r-- | vp8/encoder/vp8_quantize.c | 6 | ||||
| -rw-r--r-- | vp9/encoder/vp9_quantize.c | 6 |
3 files changed, 9 insertions, 9 deletions
diff --git a/vp10/encoder/quantize.c b/vp10/encoder/quantize.c index 86b324f1a..136efe34a 100644 --- a/vp10/encoder/quantize.c +++ b/vp10/encoder/quantize.c @@ -219,12 +219,12 @@ void vp10_regular_quantize_b_4x4(MACROBLOCK *x, int plane, int block, static void invert_quant(int16_t *quant, int16_t *shift, int d) { unsigned t; - int l; + int l, m; t = d; for (l = 0; t > 1; l++) t >>= 1; - t = 1 + (1 << (16 + l)) / d; - *quant = (int16_t)(t - (1 << 16)); + m = 1 + (1 << (16 + l)) / d; + *quant = (int16_t)(m - (1 << 16)); *shift = 1 << (16 - l); } diff --git a/vp8/encoder/vp8_quantize.c b/vp8/encoder/vp8_quantize.c index ee922c9d6..0d101ba5a 100644 --- a/vp8/encoder/vp8_quantize.c +++ b/vp8/encoder/vp8_quantize.c @@ -227,12 +227,12 @@ static void invert_quant(int improved_quant, short *quant, if(improved_quant) { unsigned t; - int l; + int l, m; t = d; for(l = 0; t > 1; l++) t>>=1; - t = 1 + (1<<(16+l))/d; - *quant = (short)(t - (1<<16)); + m = 1 + (1<<(16+l))/d; + *quant = (short)(m - (1<<16)); *shift = l; /* use multiplication and constant shift by 16 */ *shift = 1 << (16 - *shift); diff --git a/vp9/encoder/vp9_quantize.c b/vp9/encoder/vp9_quantize.c index 91f877ed7..9766c059c 100644 --- a/vp9/encoder/vp9_quantize.c +++ b/vp9/encoder/vp9_quantize.c @@ -219,12 +219,12 @@ void vp9_regular_quantize_b_4x4(MACROBLOCK *x, int plane, int block, static void invert_quant(int16_t *quant, int16_t *shift, int d) { unsigned t; - int l; + int l, m; t = d; for (l = 0; t > 1; l++) t >>= 1; - t = 1 + (1 << (16 + l)) / d; - *quant = (int16_t)(t - (1 << 16)); + m = 1 + (1 << (16 + l)) / d; + *quant = (int16_t)(m - (1 << 16)); *shift = 1 << (16 - l); } |