vp8/{ratectrl,onyx_if}: fix some signed integer overflows

in calculations involving bitrate in encode_frame_to_data_rate() and
vp8_compute_frame_size_bounds()

note this isn't exhaustive, it's just the result of a vpxenc run with:
-w 800 -h 480 --cpu-used=8 --rt --target-bitrate=1400000000

Bug: b/151945689
Change-Id: I3a4f878046fcf80e87482761588c977c283ae917

2 files changed, 33 insertions, 26 deletions

diff --git a/vp8/encoder/onyx_if.c b/vp8/encoder/onyx_if.c
index 29c8cc66c..3f5b9816d 100644
--- a/vp8/encoder/onyx_if.c
+++ b/vp8/encoder/onyx_if.c
@@ -4520,15 +4520,15 @@ static void encode_frame_to_data_rate(VP8_COMP *cpi, size_t *size,
   /* Rolling monitors of whether we are over or underspending used to
    * help regulate min and Max Q in two pass.
    */
-  cpi->rolling_target_bits =
-      ((cpi->rolling_target_bits * 3) + cpi->this_frame_target + 2) / 4;
-  cpi->rolling_actual_bits =
-      ((cpi->rolling_actual_bits * 3) + cpi->projected_frame_size + 2) / 4;
-  cpi->long_rolling_target_bits =
-      ((cpi->long_rolling_target_bits * 31) + cpi->this_frame_target + 16) / 32;
-  cpi->long_rolling_actual_bits =
-      ((cpi->long_rolling_actual_bits * 31) + cpi->projected_frame_size + 16) /
-      32;
+  cpi->rolling_target_bits = (int)ROUND64_POWER_OF_TWO(
+      (int64_t)cpi->rolling_target_bits * 3 + cpi->this_frame_target, 2);
+  cpi->rolling_actual_bits = (int)ROUND64_POWER_OF_TWO(
+      (int64_t)cpi->rolling_actual_bits * 3 + cpi->projected_frame_size, 2);
+  cpi->long_rolling_target_bits = (int)ROUND64_POWER_OF_TWO(
+      (int64_t)cpi->long_rolling_target_bits * 31 + cpi->this_frame_target, 5);
+  cpi->long_rolling_actual_bits = (int)ROUND64_POWER_OF_TWO(
+      (int64_t)cpi->long_rolling_actual_bits * 31 + cpi->projected_frame_size,
+      5);
 
   /* Actual bits spent */
   cpi->total_actual_bits += cpi->projected_frame_size;
diff --git a/vp8/encoder/ratectrl.c b/vp8/encoder/ratectrl.c
index 1a30c95ae..ba124c359 100644
--- a/vp8/encoder/ratectrl.c
+++ b/vp8/encoder/ratectrl.c
@@ -1375,14 +1375,17 @@ void vp8_compute_frame_size_bounds(VP8_COMP *cpi, int *frame_under_shoot_limit,
     *frame_under_shoot_limit = 0;
     *frame_over_shoot_limit = INT_MAX;
   } else {
+    const int64_t this_frame_target = cpi->this_frame_target;
+    int64_t over_shoot_limit, under_shoot_limit;
+
     if (cpi->common.frame_type == KEY_FRAME) {
-      *frame_over_shoot_limit = cpi->this_frame_target * 9 / 8;
-      *frame_under_shoot_limit = cpi->this_frame_target * 7 / 8;
+      over_shoot_limit = this_frame_target * 9 / 8;
+      under_shoot_limit = this_frame_target * 7 / 8;
     } else {
       if (cpi->oxcf.number_of_layers > 1 || cpi->common.refresh_alt_ref_frame ||
           cpi->common.refresh_golden_frame) {
-        *frame_over_shoot_limit = cpi->this_frame_target * 9 / 8;
-        *frame_under_shoot_limit = cpi->this_frame_target * 7 / 8;
+        over_shoot_limit = this_frame_target * 9 / 8;
+        under_shoot_limit = this_frame_target * 7 / 8;
       } else {
         /* For CBR take buffer fullness into account */
         if (cpi->oxcf.end_usage == USAGE_STREAM_FROM_SERVER) {
@@ -1392,18 +1395,18 @@ void vp8_compute_frame_size_bounds(VP8_COMP *cpi, int *frame_under_shoot_limit,
             /* Buffer is too full so relax overshoot and tighten
              * undershoot
              */
-            *frame_over_shoot_limit = cpi->this_frame_target * 12 / 8;
-            *frame_under_shoot_limit = cpi->this_frame_target * 6 / 8;
+            over_shoot_limit = this_frame_target * 12 / 8;
+            under_shoot_limit = this_frame_target * 6 / 8;
           } else if (cpi->buffer_level <=
                      (cpi->oxcf.optimal_buffer_level >> 1)) {
             /* Buffer is too low so relax undershoot and tighten
              * overshoot
              */
-            *frame_over_shoot_limit = cpi->this_frame_target * 10 / 8;
-            *frame_under_shoot_limit = cpi->this_frame_target * 4 / 8;
+            over_shoot_limit = this_frame_target * 10 / 8;
+            under_shoot_limit = this_frame_target * 4 / 8;
           } else {
-            *frame_over_shoot_limit = cpi->this_frame_target * 11 / 8;
-            *frame_under_shoot_limit = cpi->this_frame_target * 5 / 8;
+            over_shoot_limit = this_frame_target * 11 / 8;
+            under_shoot_limit = this_frame_target * 5 / 8;
           }
         }
         /* VBR and CQ mode */
@@ -1413,11 +1416,11 @@ void vp8_compute_frame_size_bounds(VP8_COMP *cpi, int *frame_under_shoot_limit,
         else {
           /* Stron overshoot limit for constrained quality */
           if (cpi->oxcf.end_usage == USAGE_CONSTRAINED_QUALITY) {
-            *frame_over_shoot_limit = cpi->this_frame_target * 11 / 8;
-            *frame_under_shoot_limit = cpi->this_frame_target * 2 / 8;
+            over_shoot_limit = this_frame_target * 11 / 8;
+            under_shoot_limit = this_frame_target * 2 / 8;
           } else {
-            *frame_over_shoot_limit = cpi->this_frame_target * 11 / 8;
-            *frame_under_shoot_limit = cpi->this_frame_target * 5 / 8;
+            over_shoot_limit = this_frame_target * 11 / 8;
+            under_shoot_limit = this_frame_target * 5 / 8;
           }
         }
       }
@@ -1427,9 +1430,13 @@ void vp8_compute_frame_size_bounds(VP8_COMP *cpi, int *frame_under_shoot_limit,
      * (eg * 7/8) may be tiny make sure there is at least a minimum
      * range.
      */
-    *frame_over_shoot_limit += 200;
-    *frame_under_shoot_limit -= 200;
-    if (*frame_under_shoot_limit < 0) *frame_under_shoot_limit = 0;
+    over_shoot_limit += 200;
+    under_shoot_limit -= 200;
+    if (under_shoot_limit < 0) under_shoot_limit = 0;
+    if (under_shoot_limit > INT_MAX) under_shoot_limit = INT_MAX;
+    if (over_shoot_limit > INT_MAX) over_shoot_limit = INT_MAX;
+    *frame_under_shoot_limit = (int)under_shoot_limit;
+    *frame_over_shoot_limit = (int)over_shoot_limit;
   }
 }