changed loop filter for MBs using 8x8 transform

This commit added a set of loop filter functions for macroblocks
using 8x8 transform. First we turned off the regular loop filtering
on 4x4 block boundaries that do not exist in macroblocks using 8x8
transform. Second, we change to use the same loop filter(mask and
7 tap filter) that used for macroblock edge filtering.

Change-Id: I3a00460b7674ced116917d86812ffc32578c1d3a

3 files changed, 74 insertions, 12 deletions

diff --git a/vp8/common/alloccommon.c b/vp8/common/alloccommon.c
index 4457c8821..e643eff09 100644
--- a/vp8/common/alloccommon.c
+++ b/vp8/common/alloccommon.c
@@ -153,10 +153,10 @@ void vp8_setup_version(VP8_COMMON *cm)
         if (!CONFIG_EXPERIMENTAL)
             vpx_internal_error(&cm->error, VPX_CODEC_UNSUP_BITSTREAM,
                                "Bitstream was created by an experimental "
-                               "encoder");        
+                               "encoder");
         cm->experimental = 1;
     }
-    
+
     switch (cm->version & 0x3)
     {
     case 0:
diff --git a/vp8/common/loopfilter.c b/vp8/common/loopfilter.c
index 052d00e0f..677f7c6e4 100644
--- a/vp8/common/loopfilter.c
+++ b/vp8/common/loopfilter.c
@@ -22,6 +22,8 @@ typedef unsigned char uc;
 
 prototype_loopfilter(vp8_loop_filter_horizontal_edge_c);
 prototype_loopfilter(vp8_loop_filter_vertical_edge_c);
+
+
 prototype_loopfilter(vp8_mbloop_filter_horizontal_edge_c);
 prototype_loopfilter(vp8_mbloop_filter_vertical_edge_c);
 
@@ -72,6 +74,16 @@ void vp8_loop_filter_bh_c(unsigned char *y_ptr, unsigned char *u_ptr,
         vp8_loop_filter_horizontal_edge_c(v_ptr + 4 * uv_stride, uv_stride, lfi->blim, lfi->lim, lfi->hev_thr, 1);
 }
 
+#if CONFIG_T8X8
+void vp8_loop_filter_bh8x8_c(unsigned char *y_ptr, unsigned char *u_ptr,
+                          unsigned char *v_ptr, int y_stride, int uv_stride,
+                          loop_filter_info *lfi)
+{
+    vp8_mbloop_filter_horizontal_edge_c(
+        y_ptr + 8 * y_stride, y_stride, lfi->blim, lfi->lim, lfi->hev_thr, 2);
+}
+#endif
+
 void vp8_loop_filter_bhs_c(unsigned char *y_ptr, int y_stride,
                            const unsigned char *blimit)
 {
@@ -96,6 +108,17 @@ void vp8_loop_filter_bv_c(unsigned char *y_ptr, unsigned char *u_ptr,
         vp8_loop_filter_vertical_edge_c(v_ptr + 4, uv_stride, lfi->blim, lfi->lim, lfi->hev_thr, 1);
 }
 
+#if CONFIG_T8X8
+void vp8_loop_filter_bv8x8_c(unsigned char *y_ptr, unsigned char *u_ptr,
+                          unsigned char *v_ptr, int y_stride, int uv_stride,
+                          loop_filter_info *lfi)
+{
+    vp8_mbloop_filter_vertical_edge_c(
+        y_ptr + 8, y_stride, lfi->blim, lfi->lim, lfi->hev_thr, 2);
+}
+
+#endif
+
 void vp8_loop_filter_bvs_c(unsigned char *y_ptr, int y_stride,
                            const unsigned char *blimit)
 {
@@ -328,7 +351,9 @@ void vp8_loop_filter_frame
             const int mode_index = lfi_n->mode_lf_lut[mode_info_context->mbmi.mode];
             const int seg = mode_info_context->mbmi.segment_id;
             const int ref_frame = mode_info_context->mbmi.ref_frame;
-
+#if CONFIG_T8X8
+            int tx_type = mode_info_context->mbmi.txfm_size;
+#endif
             filter_level = lfi_n->lvl[seg][ref_frame][mode_index];
 
             if (filter_level)
@@ -350,8 +375,17 @@ void vp8_loop_filter_frame
                         (y_ptr, u_ptr, v_ptr, post->y_stride, post->uv_stride, &lfi);
 #endif
                     if (!skip_lf)
-                        LF_INVOKE(&cm->rtcd.loopfilter, normal_b_v)
-                        (y_ptr, u_ptr, v_ptr, post->y_stride, post->uv_stride, &lfi);
+                    {
+#if CONFIG_T8X8
+                        if(tx_type == TX_8X8)
+                            vp8_loop_filter_bv8x8_c
+                            (y_ptr, u_ptr, v_ptr, post->y_stride, post->uv_stride, &lfi);
+                        else
+#endif
+                            LF_INVOKE(&cm->rtcd.loopfilter, normal_b_v)
+                            (y_ptr, u_ptr, v_ptr, post->y_stride, post->uv_stride, &lfi);
+
+                    }
 
                     /* don't apply across umv border */
                     if (mb_row > 0)
@@ -362,9 +396,18 @@ void vp8_loop_filter_frame
                         LF_INVOKE(&cm->rtcd.loopfilter, normal_mb_h)
                         (y_ptr, u_ptr, v_ptr, post->y_stride, post->uv_stride, &lfi);
 #endif
+
                     if (!skip_lf)
-                        LF_INVOKE(&cm->rtcd.loopfilter, normal_b_h)
-                        (y_ptr, u_ptr, v_ptr, post->y_stride, post->uv_stride, &lfi);
+                    {
+#if CONFIG_T8X8
+                        if(tx_type == TX_8X8)
+                            vp8_loop_filter_bh8x8_c
+                            (y_ptr, u_ptr, v_ptr, post->y_stride, post->uv_stride, &lfi);
+                        else
+#endif
+                            LF_INVOKE(&cm->rtcd.loopfilter, normal_b_h)
+                            (y_ptr, u_ptr, v_ptr, post->y_stride, post->uv_stride, &lfi);
+                    }
                 }
                 else
                 {
@@ -448,6 +491,9 @@ void vp8_loop_filter_frame_yonly
             const int mode_index = lfi_n->mode_lf_lut[mode_info_context->mbmi.mode];
             const int seg = mode_info_context->mbmi.segment_id;
             const int ref_frame = mode_info_context->mbmi.ref_frame;
+#if CONFIG_T8X8
+            int tx_type = mode_info_context->mbmi.txfm_size;
+#endif
 
             filter_level = lfi_n->lvl[seg][ref_frame][mode_index];
 
@@ -469,10 +515,17 @@ void vp8_loop_filter_frame_yonly
                         LF_INVOKE(&cm->rtcd.loopfilter, normal_mb_v)
                         (y_ptr, 0, 0, post->y_stride, 0, &lfi);
 #endif
-
                     if (!skip_lf)
-                        LF_INVOKE(&cm->rtcd.loopfilter, normal_b_v)
-                        (y_ptr, 0, 0, post->y_stride, 0, &lfi);
+                    {
+#if CONFIG_T8X8
+                        if(tx_type == TX_8X8)
+                            vp8_loop_filter_bv8x8_c
+                            (y_ptr, 0, 0, post->y_stride, 0, &lfi);
+                        else
+#endif
+                            LF_INVOKE(&cm->rtcd.loopfilter, normal_b_v)
+                            (y_ptr, 0, 0, post->y_stride, 0, &lfi);
+                    }
 
                     /* don't apply across umv border */
                     if (mb_row > 0)
@@ -484,8 +537,16 @@ void vp8_loop_filter_frame_yonly
                         (y_ptr, 0, 0, post->y_stride, 0, &lfi);
 #endif
                     if (!skip_lf)
-                        LF_INVOKE(&cm->rtcd.loopfilter, normal_b_h)
-                        (y_ptr, 0, 0, post->y_stride, 0, &lfi);
+                    {
+#if CONFIG_T8X8
+                        if(tx_type == TX_8X8)
+                            vp8_loop_filter_bh8x8_c
+                            (y_ptr, 0, 0, post->y_stride, 0, &lfi);
+                        else
+#endif
+                            LF_INVOKE(&cm->rtcd.loopfilter, normal_b_h)
+                            (y_ptr, 0, 0, post->y_stride, 0, &lfi);
+                    }
                 }
                 else
                 {
diff --git a/vp8/common/loopfilter_filters.c b/vp8/common/loopfilter_filters.c
index 72ba31eaa..afd0fde28 100644
--- a/vp8/common/loopfilter_filters.c
+++ b/vp8/common/loopfilter_filters.c
@@ -10,6 +10,7 @@
 
 
 #include <stdlib.h>
+#include "vpx_config.h"
 #include "loopfilter.h"
 #include "onyxc_int.h"