diff options
| author | Ronald S. Bultje <rbultje@google.com> | 2012-02-15 08:30:36 -0800 |
|---|---|---|
| committer | Ronald S. Bultje <rbultje@google.com> | 2012-02-15 15:57:49 -0800 |
| commit | 0930dde2496e911d24c930751b08c71ae722bd89 (patch) | |
| tree | c9e96ad45416f3a9c210278124e44822af33612e /vp8/encoder/rdopt.c | |
| parent | 46f9ad2ca5f161ccbdfc6787da1333427f246263 (diff) | |
| download | libvpx-0930dde2496e911d24c930751b08c71ae722bd89.tar libvpx-0930dde2496e911d24c930751b08c71ae722bd89.tar.gz libvpx-0930dde2496e911d24c930751b08c71ae722bd89.tar.bz2 libvpx-0930dde2496e911d24c930751b08c71ae722bd89.zip | |
Fix overflows in dual prediction mode selection.
Change-Id: I265ad46e01a307bca21e6223725e4055f5e08648
Diffstat (limited to 'vp8/encoder/rdopt.c')
| -rw-r--r-- | vp8/encoder/rdopt.c | 15 |
1 files changed, 12 insertions, 3 deletions
diff --git a/vp8/encoder/rdopt.c b/vp8/encoder/rdopt.c index 69dd9adc7..1d593fbee 100644 --- a/vp8/encoder/rdopt.c +++ b/vp8/encoder/rdopt.c @@ -3142,9 +3142,18 @@ void vp8_rd_pick_inter_mode(VP8_COMP *cpi, MACROBLOCK *x, int recon_yoffset, int rd_update_mvcount(cpi, x, &frame_best_ref_mv[xd->mode_info_context->mbmi.ref_frame]); - *best_single_rd_diff = best_rd - best_single_rd; - *best_dual_rd_diff = best_rd - best_dual_rd; - *best_hybrid_rd_diff = best_rd - best_hybrid_rd; + if (best_single_rd == INT_MAX) + *best_single_rd_diff = INT_MIN; + else + *best_single_rd_diff = best_rd - best_single_rd; + if (best_dual_rd == INT_MAX) + *best_dual_rd_diff = INT_MIN; + else + *best_dual_rd_diff = best_rd - best_dual_rd; + if (best_hybrid_rd == INT_MAX) + *best_hybrid_rd_diff = INT_MIN; + else + *best_hybrid_rd_diff = best_rd - best_hybrid_rd; } void vp8_rd_pick_intra_mode(VP8_COMP *cpi, MACROBLOCK *x, int *rate_) |