diff options
author | Yaowu Xu <yaowu@google.com> | 2016-06-20 10:13:30 -0700 |
---|---|---|
committer | Yaowu Xu <yaowu@google.com> | 2016-06-21 15:37:59 -0700 |
commit | 87bf1a149c2e6f89ed0a6fd804a1f4dc54e346be (patch) | |
tree | 05b200c035fb287b37f615d8505eee1bcd731ca8 /vp9 | |
parent | 40477f1f70556cbd17c7296a332e8014129baa21 (diff) | |
download | libvpx-87bf1a149c2e6f89ed0a6fd804a1f4dc54e346be.tar libvpx-87bf1a149c2e6f89ed0a6fd804a1f4dc54e346be.tar.gz libvpx-87bf1a149c2e6f89ed0a6fd804a1f4dc54e346be.tar.bz2 libvpx-87bf1a149c2e6f89ed0a6fd804a1f4dc54e346be.zip |
Fix ubsan warnings: vp9/encoder/vp9_mcomp.c
This commit fixes a number of ubsan warnings in HBD build.
BUG=webm:1219
Change-Id: I05f0fd0ef50e93db4ba34205005c54af1ed32acc
Diffstat (limited to 'vp9')
-rw-r--r-- | vp9/encoder/vp9_mbgraph.c | 4 | ||||
-rw-r--r-- | vp9/encoder/vp9_mcomp.c | 106 | ||||
-rw-r--r-- | vp9/encoder/vp9_mcomp.h | 4 | ||||
-rw-r--r-- | vp9/encoder/vp9_pickmode.c | 7 | ||||
-rw-r--r-- | vp9/encoder/vp9_rdopt.c | 8 | ||||
-rw-r--r-- | vp9/encoder/vp9_temporal_filter.c | 4 |
6 files changed, 88 insertions, 45 deletions
diff --git a/vp9/encoder/vp9_mbgraph.c b/vp9/encoder/vp9_mbgraph.c index 3ae5b21a9..14a0b162b 100644 --- a/vp9/encoder/vp9_mbgraph.c +++ b/vp9/encoder/vp9_mbgraph.c @@ -59,8 +59,8 @@ static unsigned int do_16x16_motion_iteration(VP9_COMP *cpi, // Try sub-pixel MC // if (bestsme > error_thresh && bestsme < INT_MAX) { - int distortion; - unsigned int sse; + uint32_t distortion; + uint32_t sse; cpi->find_fractional_mv_step( x, dst_mv, ref_mv, cpi->common.allow_high_precision_mv, x->errorperbit, &v_fn_ptr, 0, mv_sf->subpel_iters_per_step, diff --git a/vp9/encoder/vp9_mcomp.c b/vp9/encoder/vp9_mcomp.c index f3ffe35ae..e747277a8 100644 --- a/vp9/encoder/vp9_mcomp.c +++ b/vp9/encoder/vp9_mcomp.c @@ -162,6 +162,33 @@ static INLINE const uint8_t *pre(const uint8_t *buf, int stride, int r, int c) { return &buf[(r >> 3) * stride + (c >> 3)]; } +#if CONFIG_VP9_HIGHBITDEPTH +/* checks if (r, c) has better score than previous best */ +#define CHECK_BETTER(v, r, c) \ + if (c >= minc && c <= maxc && r >= minr && r <= maxr) { \ + int64_t tmpmse; \ + if (second_pred == NULL) { \ + thismse = vfp->svf(pre(y, y_stride, r, c), y_stride, sp(c), \ + sp(r), z, src_stride, &sse); \ + } else { \ + thismse = vfp->svaf(pre(y, y_stride, r, c), y_stride, sp(c), \ + sp(r), z, src_stride, &sse, second_pred); \ + } \ + tmpmse = thismse; \ + tmpmse += MVC(r, c); \ + if (tmpmse >= INT_MAX) { \ + v = INT_MAX; \ + } else if ((v = (uint32_t)tmpmse) < besterr) { \ + besterr = v; \ + br = r; \ + bc = c; \ + *distortion = thismse; \ + *sse1 = sse; \ + } \ + } else { \ + v = INT_MAX; \ + } +#else /* checks if (r, c) has better score than previous best */ #define CHECK_BETTER(v, r, c) \ if (c >= minc && c <= maxc && r >= minr && r <= maxr) { \ @@ -182,6 +209,7 @@ static INLINE const uint8_t *pre(const uint8_t *buf, int stride, int r, int c) { v = INT_MAX; \ } +#endif #define FIRST_LEVEL_CHECKS \ { \ unsigned int left, right, up, down, diag; \ @@ -310,10 +338,10 @@ static unsigned int setup_center_error(const MACROBLOCKD *xd, const uint8_t *second_pred, int w, int h, int offset, int *mvjcost, int *mvcost[2], - unsigned int *sse1, - int *distortion) { - unsigned int besterr; + uint32_t *sse1, + uint32_t *distortion) { #if CONFIG_VP9_HIGHBITDEPTH + uint64_t besterr; if (second_pred != NULL) { if (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) { DECLARE_ALIGNED(16, uint16_t, comp_pred16[64 * 64]); @@ -329,9 +357,13 @@ static unsigned int setup_center_error(const MACROBLOCKD *xd, } else { besterr = vfp->vf(y + offset, y_stride, src, src_stride, sse1); } - *distortion = besterr; + *distortion = (uint32_t)besterr; besterr += mv_err_cost(bestmv, ref_mv, mvjcost, mvcost, error_per_bit); + if (besterr >= UINT32_MAX) + return UINT32_MAX; + return (uint32_t)besterr; #else + uint32_t besterr; (void) xd; if (second_pred != NULL) { DECLARE_ALIGNED(16, uint8_t, comp_pred[64 * 64]); @@ -342,8 +374,8 @@ static unsigned int setup_center_error(const MACROBLOCKD *xd, } *distortion = besterr; besterr += mv_err_cost(bestmv, ref_mv, mvjcost, mvcost, error_per_bit); -#endif // CONFIG_VP9_HIGHBITDEPTH return besterr; +#endif // CONFIG_VP9_HIGHBITDEPTH } static INLINE int divide_and_round(const int n, const int d) { @@ -373,7 +405,7 @@ static void get_cost_surf_min(int *cost_list, int *ir, int *ic, (cost_list[4] - 2 * cost_list[0] + cost_list[2])); } -int vp9_skip_sub_pixel_tree( +uint32_t vp9_skip_sub_pixel_tree( const MACROBLOCK *x, MV *bestmv, const MV *ref_mv, int allow_hp, @@ -383,8 +415,8 @@ int vp9_skip_sub_pixel_tree( int iters_per_step, int *cost_list, int *mvjcost, int *mvcost[2], - int *distortion, - unsigned int *sse1, + uint32_t *distortion, + uint32_t *sse1, const uint8_t *second_pred, int w, int h) { SETUP_SUBPEL_SEARCH; @@ -418,7 +450,7 @@ int vp9_skip_sub_pixel_tree( return besterr; } -int vp9_find_best_sub_pixel_tree_pruned_evenmore( +uint32_t vp9_find_best_sub_pixel_tree_pruned_evenmore( const MACROBLOCK *x, MV *bestmv, const MV *ref_mv, int allow_hp, @@ -428,8 +460,8 @@ int vp9_find_best_sub_pixel_tree_pruned_evenmore( int iters_per_step, int *cost_list, int *mvjcost, int *mvcost[2], - int *distortion, - unsigned int *sse1, + uint32_t *distortion, + uint32_t *sse1, const uint8_t *second_pred, int w, int h) { SETUP_SUBPEL_SEARCH; @@ -498,7 +530,7 @@ int vp9_find_best_sub_pixel_tree_pruned_evenmore( return besterr; } -int vp9_find_best_sub_pixel_tree_pruned_more(const MACROBLOCK *x, +uint32_t vp9_find_best_sub_pixel_tree_pruned_more(const MACROBLOCK *x, MV *bestmv, const MV *ref_mv, int allow_hp, int error_per_bit, @@ -507,8 +539,8 @@ int vp9_find_best_sub_pixel_tree_pruned_more(const MACROBLOCK *x, int iters_per_step, int *cost_list, int *mvjcost, int *mvcost[2], - int *distortion, - unsigned int *sse1, + uint32_t *distortion, + uint32_t *sse1, const uint8_t *second_pred, int w, int h) { SETUP_SUBPEL_SEARCH; @@ -572,7 +604,7 @@ int vp9_find_best_sub_pixel_tree_pruned_more(const MACROBLOCK *x, return besterr; } -int vp9_find_best_sub_pixel_tree_pruned(const MACROBLOCK *x, +uint32_t vp9_find_best_sub_pixel_tree_pruned(const MACROBLOCK *x, MV *bestmv, const MV *ref_mv, int allow_hp, int error_per_bit, @@ -581,8 +613,8 @@ int vp9_find_best_sub_pixel_tree_pruned(const MACROBLOCK *x, int iters_per_step, int *cost_list, int *mvjcost, int *mvcost[2], - int *distortion, - unsigned int *sse1, + uint32_t *distortion, + uint32_t *sse1, const uint8_t *second_pred, int w, int h) { SETUP_SUBPEL_SEARCH; @@ -674,19 +706,19 @@ static const MV search_step_table[12] = { {0, -1}, {0, 1}, {-1, 0}, {1, 0} }; -int vp9_find_best_sub_pixel_tree(const MACROBLOCK *x, - MV *bestmv, const MV *ref_mv, - int allow_hp, - int error_per_bit, - const vp9_variance_fn_ptr_t *vfp, - int forced_stop, - int iters_per_step, - int *cost_list, - int *mvjcost, int *mvcost[2], - int *distortion, - unsigned int *sse1, - const uint8_t *second_pred, - int w, int h) { +uint32_t vp9_find_best_sub_pixel_tree(const MACROBLOCK *x, + MV *bestmv, const MV *ref_mv, + int allow_hp, + int error_per_bit, + const vp9_variance_fn_ptr_t *vfp, + int forced_stop, + int iters_per_step, + int *cost_list, + int *mvjcost, int *mvcost[2], + uint32_t *distortion, + uint32_t *sse1, + const uint8_t *second_pred, + int w, int h) { const uint8_t *const z = x->plane[0].src.buf; const uint8_t *const src_address = z; const int src_stride = x->plane[0].src.stride; @@ -1381,12 +1413,22 @@ int vp9_get_mvpred_var(const MACROBLOCK *x, const struct buf_2d *const what = &x->plane[0].src; const struct buf_2d *const in_what = &xd->plane[0].pre[0]; const MV mv = {best_mv->row * 8, best_mv->col * 8}; - unsigned int unused; - + uint32_t unused; +#if CONFIG_VP9_HIGHBITDEPTH + uint64_t err= vfp->vf(what->buf, what->stride, + get_buf_from_mv(in_what, best_mv), + in_what->stride, &unused); + err += (use_mvcost ? mv_err_cost(&mv, center_mv, x->nmvjointcost, + x->mvcost, x->errorperbit) : 0); + if (err >= INT_MAX) + return INT_MAX; + return (int)err; +#else return vfp->vf(what->buf, what->stride, get_buf_from_mv(in_what, best_mv), in_what->stride, &unused) + (use_mvcost ? mv_err_cost(&mv, center_mv, x->nmvjointcost, x->mvcost, x->errorperbit) : 0); +#endif } int vp9_get_mvpred_av_var(const MACROBLOCK *x, diff --git a/vp9/encoder/vp9_mcomp.h b/vp9/encoder/vp9_mcomp.h index 1b0c860c5..86cd267f3 100644 --- a/vp9/encoder/vp9_mcomp.h +++ b/vp9/encoder/vp9_mcomp.h @@ -74,7 +74,7 @@ unsigned int vp9_int_pro_motion_estimation(const struct VP9_COMP *cpi, BLOCK_SIZE bsize, int mi_row, int mi_col); -typedef int (fractional_mv_step_fp) ( +typedef uint32_t (fractional_mv_step_fp) ( const MACROBLOCK *x, MV *bestmv, const MV *ref_mv, int allow_hp, @@ -84,7 +84,7 @@ typedef int (fractional_mv_step_fp) ( int iters_per_step, int *cost_list, int *mvjcost, int *mvcost[2], - int *distortion, unsigned int *sse1, + uint32_t *distortion, uint32_t *sse1, const uint8_t *second_pred, int w, int h); diff --git a/vp9/encoder/vp9_pickmode.c b/vp9/encoder/vp9_pickmode.c index f027f9f31..c13f24fb0 100644 --- a/vp9/encoder/vp9_pickmode.c +++ b/vp9/encoder/vp9_pickmode.c @@ -157,7 +157,7 @@ static int combined_motion_search(VP9_COMP *cpi, MACROBLOCK *x, const int ref = mi->ref_frame[0]; const MV ref_mv = x->mbmi_ext->ref_mvs[ref][0].as_mv; MV center_mv; - int dis; + uint32_t dis; int rate_mode; const int tmp_col_min = x->mv_col_min; const int tmp_col_max = x->mv_col_max; @@ -1564,7 +1564,8 @@ void vp9_pick_inter_mode(VP9_COMP *cpi, MACROBLOCK *x, !cpi->use_svc && cpi->oxcf.rc_mode == VPX_CBR) { int tmp_sad; - int dis, cost_list[5]; + uint32_t dis; + int cost_list[5]; if (bsize < BLOCK_16X16) continue; @@ -2175,7 +2176,7 @@ void vp9_pick_inter_mode_sub8x8(VP9_COMP *cpi, MACROBLOCK *x, const int tmp_col_max = x->mv_col_max; const int tmp_row_min = x->mv_row_min; const int tmp_row_max = x->mv_row_max; - int dummy_dist; + uint32_t dummy_dist; if (i == 0) { mvp_full.row = b_mv[NEARESTMV].as_mv.row >> 3; diff --git a/vp9/encoder/vp9_rdopt.c b/vp9/encoder/vp9_rdopt.c index a30418286..2e46aa4cd 100644 --- a/vp9/encoder/vp9_rdopt.c +++ b/vp9/encoder/vp9_rdopt.c @@ -1710,8 +1710,8 @@ static void joint_motion_search(VP9_COMP *cpi, MACROBLOCK *x, x->mv_row_max = tmp_row_max; if (bestsme < INT_MAX) { - int dis; /* TODO: use dis in distortion calculation later. */ - unsigned int sse; + uint32_t dis; /* TODO: use dis in distortion calculation later. */ + uint32_t sse; bestsme = cpi->find_fractional_mv_step( x, &tmp_mv, &ref_mv[id].as_mv, @@ -1911,7 +1911,7 @@ static int64_t rd_pick_best_sub8x8_mode(VP9_COMP *cpi, MACROBLOCK *x, INT_MAX, 1); if (bestsme < INT_MAX) { - int distortion; + uint32_t distortion; cpi->find_fractional_mv_step( x, new_mv, @@ -2341,7 +2341,7 @@ static void single_motion_search(VP9_COMP *cpi, MACROBLOCK *x, x->mv_row_max = tmp_row_max; if (bestsme < INT_MAX) { - int dis; /* TODO: use dis in distortion calculation later. */ + uint32_t dis; /* TODO: use dis in distortion calculation later. */ cpi->find_fractional_mv_step(x, &tmp_mv->as_mv, &ref_mv, cm->allow_high_precision_mv, x->errorperbit, diff --git a/vp9/encoder/vp9_temporal_filter.c b/vp9/encoder/vp9_temporal_filter.c index 2ba2750e4..02bcf5a24 100644 --- a/vp9/encoder/vp9_temporal_filter.c +++ b/vp9/encoder/vp9_temporal_filter.c @@ -264,8 +264,8 @@ static int temporal_filter_find_matching_mb_c(VP9_COMP *cpi, int step_param; int sadpb = x->sadperbit16; int bestsme = INT_MAX; - int distortion; - unsigned int sse; + uint32_t distortion; + uint32_t sse; int cost_list[5]; MV best_ref_mv1 = {0, 0}; |