diff options
| author | James Zern <jzern@google.com> | 2017-02-24 15:36:52 -0800 |
|---|---|---|
| committer | James Zern <jzern@google.com> | 2017-02-24 15:36:52 -0800 |
| commit | 47d6f16a047ea5b5344c7bae0b85d43263f1b26f (patch) | |
| tree | 2987af5efc74fa6e4302b8ceb41ec3fd98af94e4 /vpx_dsp/prob.h | |
| parent | af9002dd16b65db3feb5f4591479a564674129c2 (diff) | |
| download | libvpx-47d6f16a047ea5b5344c7bae0b85d43263f1b26f.tar libvpx-47d6f16a047ea5b5344c7bae0b85d43263f1b26f.tar.gz libvpx-47d6f16a047ea5b5344c7bae0b85d43263f1b26f.tar.bz2 libvpx-47d6f16a047ea5b5344c7bae0b85d43263f1b26f.zip | |
get_prob(): rationalize int types
promote the unsigned int calculation to uint64_t rather than int64_t for
type consistency
Change-Id: Ic34dee1dc707d9faf6a3ae250bfe39b60bef3438
Diffstat (limited to 'vpx_dsp/prob.h')
| -rw-r--r-- | vpx_dsp/prob.h | 2 |
1 files changed, 1 insertions, 1 deletions
diff --git a/vpx_dsp/prob.h b/vpx_dsp/prob.h index 5656ddbab..f1cc0eaa1 100644 --- a/vpx_dsp/prob.h +++ b/vpx_dsp/prob.h @@ -48,7 +48,7 @@ typedef const vpx_tree_index vpx_tree[]; static INLINE vpx_prob get_prob(unsigned int num, unsigned int den) { assert(den != 0); { - const int p = (int)(((int64_t)num * 256 + (den >> 1)) / den); + const int p = (int)(((uint64_t)num * 256 + (den >> 1)) / den); // (p > 255) ? 255 : (p < 1) ? 1 : p; const int clipped_prob = p | ((255 - p) >> 23) | (p == 0); return (vpx_prob)clipped_prob; |