diff options
| author | James Zern <jzern@google.com> | 2016-09-28 20:37:18 -0700 |
|---|---|---|
| committer | James Zern <jzern@google.com> | 2016-09-28 22:42:03 -0700 |
| commit | 450d89034bf61cc21257867a24ff62355f82aba6 (patch) | |
| tree | 4b718307061459e1b1ee1140386d90dba9b4a097 /vp9 | |
| parent | e094e151de32659fae15b3567e5dead4a5ba310d (diff) | |
| download | libvpx-450d89034bf61cc21257867a24ff62355f82aba6.tar libvpx-450d89034bf61cc21257867a24ff62355f82aba6.tar.gz libvpx-450d89034bf61cc21257867a24ff62355f82aba6.tar.bz2 libvpx-450d89034bf61cc21257867a24ff62355f82aba6.zip | |
vp9_detokenize,decode_coefs: fix signed int overflow
when decoding an invalid bitstream with --enable-vp9-highbitdepth
BUG=webm:1297
Change-Id: I401d87033b4293f2ca595bc51678aad9951ecf15
Diffstat (limited to 'vp9')
| -rw-r--r-- | vp9/decoder/vp9_detokenize.c | 5 |
1 files changed, 5 insertions, 0 deletions
diff --git a/vp9/decoder/vp9_detokenize.c b/vp9/decoder/vp9_detokenize.c index 90ab54f73..a441f3add 100644 --- a/vp9/decoder/vp9_detokenize.c +++ b/vp9/decoder/vp9_detokenize.c @@ -170,7 +170,12 @@ static int decode_coefs(const MACROBLOCKD *xd, PLANE_TYPE type, read_coeff(r, vp9_cat1_prob, 1, &value, &count, &range); } } +#if CONFIG_VP9_HIGHBITDEPTH + // val may use 18-bits + v = (int)(((int64_t)val * dqv) >> dq_shift); +#else v = (val * dqv) >> dq_shift; +#endif } else { if (read_bool(r, p[1], &value, &count, &range)) { token_cache[scan[c]] = 3; |