From 5e065cf9d3c622b37eaaebf701d0be6371b890c3 Mon Sep 17 00:00:00 2001 From: James Zern Date: Sat, 21 Mar 2020 16:12:39 -0700 Subject: vp8/{ratectrl,onyx_if}: fix some signed integer overflows in calculations involving bitrate in encode_frame_to_data_rate() and vp8_compute_frame_size_bounds() note this isn't exhaustive, it's just the result of a vpxenc run with: -w 800 -h 480 --cpu-used=8 --rt --target-bitrate=1400000000 Bug: b/151945689 Change-Id: I3a4f878046fcf80e87482761588c977c283ae917 --- vp8/encoder/onyx_if.c | 18 +++++++++--------- vp8/encoder/ratectrl.c | 41 ++++++++++++++++++++++++----------------- 2 files changed, 33 insertions(+), 26 deletions(-) (limited to 'vp8/encoder') diff --git a/vp8/encoder/onyx_if.c b/vp8/encoder/onyx_if.c index 29c8cc66c..3f5b9816d 100644 --- a/vp8/encoder/onyx_if.c +++ b/vp8/encoder/onyx_if.c @@ -4520,15 +4520,15 @@ static void encode_frame_to_data_rate(VP8_COMP *cpi, size_t *size, /* Rolling monitors of whether we are over or underspending used to * help regulate min and Max Q in two pass. */ - cpi->rolling_target_bits = - ((cpi->rolling_target_bits * 3) + cpi->this_frame_target + 2) / 4; - cpi->rolling_actual_bits = - ((cpi->rolling_actual_bits * 3) + cpi->projected_frame_size + 2) / 4; - cpi->long_rolling_target_bits = - ((cpi->long_rolling_target_bits * 31) + cpi->this_frame_target + 16) / 32; - cpi->long_rolling_actual_bits = - ((cpi->long_rolling_actual_bits * 31) + cpi->projected_frame_size + 16) / - 32; + cpi->rolling_target_bits = (int)ROUND64_POWER_OF_TWO( + (int64_t)cpi->rolling_target_bits * 3 + cpi->this_frame_target, 2); + cpi->rolling_actual_bits = (int)ROUND64_POWER_OF_TWO( + (int64_t)cpi->rolling_actual_bits * 3 + cpi->projected_frame_size, 2); + cpi->long_rolling_target_bits = (int)ROUND64_POWER_OF_TWO( + (int64_t)cpi->long_rolling_target_bits * 31 + cpi->this_frame_target, 5); + cpi->long_rolling_actual_bits = (int)ROUND64_POWER_OF_TWO( + (int64_t)cpi->long_rolling_actual_bits * 31 + cpi->projected_frame_size, + 5); /* Actual bits spent */ cpi->total_actual_bits += cpi->projected_frame_size; diff --git a/vp8/encoder/ratectrl.c b/vp8/encoder/ratectrl.c index 1a30c95ae..ba124c359 100644 --- a/vp8/encoder/ratectrl.c +++ b/vp8/encoder/ratectrl.c @@ -1375,14 +1375,17 @@ void vp8_compute_frame_size_bounds(VP8_COMP *cpi, int *frame_under_shoot_limit, *frame_under_shoot_limit = 0; *frame_over_shoot_limit = INT_MAX; } else { + const int64_t this_frame_target = cpi->this_frame_target; + int64_t over_shoot_limit, under_shoot_limit; + if (cpi->common.frame_type == KEY_FRAME) { - *frame_over_shoot_limit = cpi->this_frame_target * 9 / 8; - *frame_under_shoot_limit = cpi->this_frame_target * 7 / 8; + over_shoot_limit = this_frame_target * 9 / 8; + under_shoot_limit = this_frame_target * 7 / 8; } else { if (cpi->oxcf.number_of_layers > 1 || cpi->common.refresh_alt_ref_frame || cpi->common.refresh_golden_frame) { - *frame_over_shoot_limit = cpi->this_frame_target * 9 / 8; - *frame_under_shoot_limit = cpi->this_frame_target * 7 / 8; + over_shoot_limit = this_frame_target * 9 / 8; + under_shoot_limit = this_frame_target * 7 / 8; } else { /* For CBR take buffer fullness into account */ if (cpi->oxcf.end_usage == USAGE_STREAM_FROM_SERVER) { @@ -1392,18 +1395,18 @@ void vp8_compute_frame_size_bounds(VP8_COMP *cpi, int *frame_under_shoot_limit, /* Buffer is too full so relax overshoot and tighten * undershoot */ - *frame_over_shoot_limit = cpi->this_frame_target * 12 / 8; - *frame_under_shoot_limit = cpi->this_frame_target * 6 / 8; + over_shoot_limit = this_frame_target * 12 / 8; + under_shoot_limit = this_frame_target * 6 / 8; } else if (cpi->buffer_level <= (cpi->oxcf.optimal_buffer_level >> 1)) { /* Buffer is too low so relax undershoot and tighten * overshoot */ - *frame_over_shoot_limit = cpi->this_frame_target * 10 / 8; - *frame_under_shoot_limit = cpi->this_frame_target * 4 / 8; + over_shoot_limit = this_frame_target * 10 / 8; + under_shoot_limit = this_frame_target * 4 / 8; } else { - *frame_over_shoot_limit = cpi->this_frame_target * 11 / 8; - *frame_under_shoot_limit = cpi->this_frame_target * 5 / 8; + over_shoot_limit = this_frame_target * 11 / 8; + under_shoot_limit = this_frame_target * 5 / 8; } } /* VBR and CQ mode */ @@ -1413,11 +1416,11 @@ void vp8_compute_frame_size_bounds(VP8_COMP *cpi, int *frame_under_shoot_limit, else { /* Stron overshoot limit for constrained quality */ if (cpi->oxcf.end_usage == USAGE_CONSTRAINED_QUALITY) { - *frame_over_shoot_limit = cpi->this_frame_target * 11 / 8; - *frame_under_shoot_limit = cpi->this_frame_target * 2 / 8; + over_shoot_limit = this_frame_target * 11 / 8; + under_shoot_limit = this_frame_target * 2 / 8; } else { - *frame_over_shoot_limit = cpi->this_frame_target * 11 / 8; - *frame_under_shoot_limit = cpi->this_frame_target * 5 / 8; + over_shoot_limit = this_frame_target * 11 / 8; + under_shoot_limit = this_frame_target * 5 / 8; } } } @@ -1427,9 +1430,13 @@ void vp8_compute_frame_size_bounds(VP8_COMP *cpi, int *frame_under_shoot_limit, * (eg * 7/8) may be tiny make sure there is at least a minimum * range. */ - *frame_over_shoot_limit += 200; - *frame_under_shoot_limit -= 200; - if (*frame_under_shoot_limit < 0) *frame_under_shoot_limit = 0; + over_shoot_limit += 200; + under_shoot_limit -= 200; + if (under_shoot_limit < 0) under_shoot_limit = 0; + if (under_shoot_limit > INT_MAX) under_shoot_limit = INT_MAX; + if (over_shoot_limit > INT_MAX) over_shoot_limit = INT_MAX; + *frame_under_shoot_limit = (int)under_shoot_limit; + *frame_over_shoot_limit = (int)over_shoot_limit; } } -- cgit v1.2.3