Fix a bug in motion search code

The MV's range is 256. Since the new motion search uses a different
starting MV than the center ref MV, a MV range checking needs to
be done to avoid corruption.

Change-Id: I8ae0721d1bd203639e13891e2e54a2e87276f306

4 files changed, 88 insertions, 53 deletions

diff --git a/vp8/encoder/mcomp.c b/vp8/encoder/mcomp.c
index b9d86a20e..c5209ff85 100644
--- a/vp8/encoder/mcomp.c
+++ b/vp8/encoder/mcomp.c
@@ -941,6 +941,18 @@ int vp8_diamond_search_sad
     unsigned char *check_here;
     int thissad;
 
+    int search_range = 128>>search_param;
+
+    *num00 = 0;
+
+    // Trap uncodable vectors
+    if (((abs(ref_mv->col - center_mv->col) + (search_range<<4)) > MAX_POSSIBLE_MV) || ((abs(ref_mv->row - center_mv->row) + (search_range<<4)) > MAX_POSSIBLE_MV))
+    {
+        best_mv->row = ref_row;
+        best_mv->col = ref_col;
+        return INT_MAX;
+    }
+
     // Work out the start point for the search
     in_what = (unsigned char *)(*(d->base_pre) + d->pre + (ref_row * (d->pre_stride)) + ref_col);
     best_address = in_what;
@@ -962,8 +974,6 @@ int vp8_diamond_search_sad
     best_mv->row = ref_row;
     best_mv->col = ref_col;
 
-    *num00 = 0;
-
     for (step = 0; step < tot_steps ; step++)
     {
         for (j = 0 ; j < x->searches_per_step ; j++)
@@ -1057,6 +1067,18 @@ int vp8_diamond_search_sadx4
     unsigned char *check_here;
     unsigned int thissad;
 
+    int search_range = 128>>search_param;
+
+    *num00 = 0;
+
+    // Trap uncodable vectors
+    if (((abs(ref_mv->col - center_mv->col) + (search_range<<4)) > MAX_POSSIBLE_MV) || ((abs(ref_mv->row - center_mv->row) + (search_range<<4)) > MAX_POSSIBLE_MV))
+    {
+        best_mv->row = ref_row;
+        best_mv->col = ref_col;
+        return INT_MAX;
+    }
+
     // Work out the start point for the search
     in_what = (unsigned char *)(*(d->base_pre) + d->pre + (ref_row * (d->pre_stride)) + ref_col);
     best_address = in_what;
@@ -1078,8 +1100,6 @@ int vp8_diamond_search_sadx4
     best_mv->row = ref_row;
     best_mv->col = ref_col;
 
-    *num00 = 0;
-
     for (step = 0; step < tot_steps ; step++)
     {
         int all_in = 1, t;
@@ -1194,10 +1214,24 @@ int vp8_full_search_sad(MACROBLOCK *x, BLOCK *b, BLOCKD *d, MV *ref_mv, int erro
     int ref_row = ref_mv->row >> 3;
     int ref_col = ref_mv->col >> 3;
 
-    int row_min = ref_row - distance;
-    int row_max = ref_row + distance;
-    int col_min = ref_col - distance;
-    int col_max = ref_col + distance;
+    int row_min, row_max, col_min, col_max;
+
+    int drow = abs(ref_mv->row - center_mv->row);
+    int dcol = abs(ref_mv->col - center_mv->col);
+
+    // reduce search distance and make sure MV obtained is in range.
+    if (((dcol + (distance<<3)) > MAX_POSSIBLE_MV) || (( drow + (distance<<3)) > MAX_POSSIBLE_MV))
+    {
+        if(dcol > drow)
+            distance = (MAX_POSSIBLE_MV - dcol)>>3;
+        else
+            distance = (MAX_POSSIBLE_MV - drow)>>3;
+    }
+
+    row_min = ref_row - distance;
+    row_max = ref_row + distance;
+    col_min = ref_col - distance;
+    col_max = ref_col + distance;
 
     // Work out the mid point for the search
     in_what = *(d->base_pre) + d->pre;
@@ -1284,12 +1318,24 @@ int vp8_full_search_sadx3(MACROBLOCK *x, BLOCK *b, BLOCKD *d, MV *ref_mv, int er
     int ref_row = ref_mv->row >> 3;
     int ref_col = ref_mv->col >> 3;
 
-    int row_min = ref_row - distance;
-    int row_max = ref_row + distance;
-    int col_min = ref_col - distance;
-    int col_max = ref_col + distance;
-
+    int row_min, row_max, col_min, col_max;
     unsigned int sad_array[3];
+    int drow = abs(ref_mv->row - center_mv->row);
+    int dcol = abs(ref_mv->col - center_mv->col);
+
+    // reduce search distance and make sure MV obtained is in range.
+    if (((dcol + (distance<<3)) > MAX_POSSIBLE_MV) || (( drow + (distance<<3)) > MAX_POSSIBLE_MV))
+    {
+        if(dcol > drow)
+            distance = (MAX_POSSIBLE_MV - dcol)>>3;
+        else
+            distance = (MAX_POSSIBLE_MV - drow)>>3;
+    }
+
+    row_min = ref_row - distance;
+    row_max = ref_row + distance;
+    col_min = ref_col - distance;
+    col_max = ref_col + distance;
 
     // Work out the mid point for the search
     in_what = *(d->base_pre) + d->pre;
@@ -1409,13 +1455,25 @@ int vp8_full_search_sadx8(MACROBLOCK *x, BLOCK *b, BLOCKD *d, MV *ref_mv, int er
     int ref_row = ref_mv->row >> 3;
     int ref_col = ref_mv->col >> 3;
 
-    int row_min = ref_row - distance;
-    int row_max = ref_row + distance;
-    int col_min = ref_col - distance;
-    int col_max = ref_col + distance;
-
+    int row_min, row_max, col_min, col_max;
     unsigned short sad_array8[8];
     unsigned int sad_array[3];
+    int drow = abs(ref_mv->row - center_mv->row);
+    int dcol = abs(ref_mv->col - center_mv->col);
+
+    // reduce search distance and make sure MV obtained is in range.
+    if (((dcol + (distance<<3)) > MAX_POSSIBLE_MV) || (( drow + (distance<<3)) > MAX_POSSIBLE_MV))
+    {
+        if(dcol > drow)
+            distance = (MAX_POSSIBLE_MV - dcol)>>3;
+        else
+            distance = (MAX_POSSIBLE_MV - drow)>>3;
+    }
+
+    row_min = ref_row - distance;
+    row_max = ref_row + distance;
+    col_min = ref_col - distance;
+    col_max = ref_col + distance;
 
     // Work out the mid point for the search
     in_what = *(d->base_pre) + d->pre;
diff --git a/vp8/encoder/mcomp.h b/vp8/encoder/mcomp.h
index b54fe7377..122debcae 100644
--- a/vp8/encoder/mcomp.h
+++ b/vp8/encoder/mcomp.h
@@ -24,7 +24,7 @@ extern void accum_mv_refs(MB_PREDICTION_MODE, const int near_mv_ref_cts[4]);
 #define MAX_MVSEARCH_STEPS 8                                    // The maximum number of steps in a step search given the largest allowed initial step
 #define MAX_FULL_PEL_VAL ((1 << (MAX_MVSEARCH_STEPS+3)) - 8)    // Max full pel mv specified in 1/8 pel units
 #define MAX_FIRST_STEP (1 << (MAX_MVSEARCH_STEPS-1))            // Maximum size of the first step in full pel units
-
+#define MAX_POSSIBLE_MV (1 << 11)                               // Maximum MV in 1/8 pel units
 
 extern void print_mode_context(void);
 extern int vp8_mv_bit_cost(MV *mv, MV *ref, int *mvcost[2], int Weight);
diff --git a/vp8/encoder/onyx_if.c b/vp8/encoder/onyx_if.c
index 508190c03..3ccffb749 100644
--- a/vp8/encoder/onyx_if.c
+++ b/vp8/encoder/onyx_if.c
@@ -4221,8 +4221,7 @@ static void encode_frame_to_data_rate
     // Update the GF useage maps.
     // This is done after completing the compression of a frame when all modes etc. are finalized but before loop filter
     vp8_update_gf_useage_maps(cpi, cm, &cpi->mb);
-////////////////////////////////
-////////////////////////////////
+
     // This frame's MVs are saved and will be used in next frame's MV prediction.
     if(cm->show_frame)   //do not save for altref frame
     {
@@ -4231,21 +4230,6 @@ static void encode_frame_to_data_rate
       MODE_INFO *tmp = cm->mip; //point to beginning of allocated MODE_INFO arrays.
       //static int last_video_frame = 0;
 
-      /*
-      if (cm->current_video_frame == 0)   //first frame: set to 0
-      {
-        for (mb_row = 0; mb_row < cm->mb_rows+1; mb_row ++)
-        {
-            for (mb_col = 0; mb_col < cm->mb_cols+1; mb_col ++)
-            {
-                cpi->lfmv[mb_col + mb_row*(cm->mode_info_stride)].as_int = 0;
-                cpi->lf_ref_frame_sign_bias[mb_col + mb_row*(cm->mode_info_stride)] = 0;
-                cpi->lf_ref_frame[mb_col + mb_row*(cm->mode_info_stride)] = 0;
-            }
-        }
-      }else
-      */
-
       if(cm->frame_type != KEY_FRAME)
       {
         for (mb_row = 0; mb_row < cm->mb_rows+1; mb_row ++)
@@ -4257,20 +4241,12 @@ static void encode_frame_to_data_rate
 
               cpi->lf_ref_frame_sign_bias[mb_col + mb_row*(cm->mode_info_stride)] = cm->ref_frame_sign_bias[tmp->mbmi.ref_frame];
               cpi->lf_ref_frame[mb_col + mb_row*(cm->mode_info_stride)] = tmp->mbmi.ref_frame;
-              //printf("[%d, %d]  ", cpi->lfmv[mb_col + mb_row*(cm->mode_info_stride-1)].as_mv.row, cpi->lfmv[mb_col + mb_row*(cm->mode_info_stride-1)].as_mv.col);
               tmp++;
           }
         }
-
-      //last_video_frame = cm->current_video_frame;
       }
     }
 
-//printf("after: %d   %d \n", cm->current_video_frame, cm->show_frame );
-
-
-
-
 
     // Update the GF useage maps.
     // This is done after completing the compression of a frame when all modes etc. are finalized but before loop filter
diff --git a/vp8/encoder/rdopt.c b/vp8/encoder/rdopt.c
index 9b8b3b326..aa17b31e3 100644
--- a/vp8/encoder/rdopt.c
+++ b/vp8/encoder/rdopt.c
@@ -1344,8 +1344,6 @@ static int vp8_rd_pick_best_mbsegmentation(VP8_COMP *cpi, MACROBLOCK *x,
 }
 
 
-
-/////////////////////////
 static void mv_bias(const MODE_INFO *x, int refframe, int_mv *mvp, const int *ref_frame_sign_bias)
 {
     MV xmv;
@@ -1884,6 +1882,16 @@ int vp8_rd_pick_inter_mode(VP8_COMP *cpi, MACROBLOCK *x, int recon_yoffset, int
 
             vp8_mv_pred(cpi, &x->e_mbd, x->e_mbd.mode_info_context, &mvp,
                         x->e_mbd.mode_info_context->mbmi.ref_frame, cpi->common.ref_frame_sign_bias, &sr, &near_sadidx[0]);
+
+            /* adjust mvp to make sure it is within MV range */
+            if(mvp.row > best_ref_mv.row + MAX_POSSIBLE_MV)
+                mvp.row = best_ref_mv.row + MAX_POSSIBLE_MV;
+            if(mvp.row < best_ref_mv.row - MAX_POSSIBLE_MV)
+                mvp.row = best_ref_mv.row - MAX_POSSIBLE_MV;
+            if(mvp.col > best_ref_mv.col + MAX_POSSIBLE_MV)
+                mvp.col = best_ref_mv.col + MAX_POSSIBLE_MV;
+            if(mvp.col < best_ref_mv.col - MAX_POSSIBLE_MV)
+                mvp.col = best_ref_mv.col - MAX_POSSIBLE_MV;
         }
 
         // Estimate the reference frame signaling cost and add it to the rolling cost variable.
@@ -2163,13 +2171,6 @@ int vp8_rd_pick_inter_mode(VP8_COMP *cpi, MACROBLOCK *x, int recon_yoffset, int
                     {
                         int sadpb = x->sadperbit16 >> 2;
                         thissme = cpi->full_search_sad(x, b, d, &full_mvp, sadpb, search_range, &cpi->fn_ptr[BLOCK_16X16], x->mvcost, x->mvsadcost,&best_ref_mv);
-                        /*
-                        MV dia_ref_mv;
-                        dia_ref_mv.row = d->bmi.mv.as_mv.row << 3;
-                        dia_ref_mv.col = d->bmi.mv.as_mv.col << 3;
-                        thissme = cpi->full_search_sad(x, b, d, &dia_ref_mv, sadpb, search_range, &cpi->fn_ptr[BLOCK_16X16], x->mvcost, x->mvsadcost,&best_ref_mv);
-                        */
-
                     }
 
                     // Barrier threshold to initiating full search