new loop filter functions for macroblock boundaries

The commit adds a new set of loop filter for macroblock edge filtering.
The new loop filter has a mask to detect so-called "flat" regions. The
detection checks 5 pixels of each side of an edge. If the all pixels
have value with +/-1 from the edge pixel on the same side, the region
is treated as a "flat" region. For such case, a 7 tap filter is used
to change 3 pixel values on each side. The 7 taps are:
               [1, 1, 1, 2, 1, 1, 1]/8
The furthest away pixels used as input are +/-5 away from edge. For
non-flat region, we fall back to old filtering. It should be noted
here that the thresholds and filter taps may require more optimization
for best possible results.

Tests on a set of hd clips showed consistent gains:
http://www.corp.google.com/~yaowu/no_crawl/mblpf_hd.html
(avg psnr: .83% glb psnr: .77% ssim: .82%)

Tests on derf set also showed consistent gains:
http://www.corp.google.com/~yaowu/no_crawl/mblpf_derf.html
(avg psnr: .24% glb psnr: .22% ssim: .48%)

Change-Id: I0855b1ff48e79e1175c20b81967137e18b2af352

3 files changed, 161 insertions, 18 deletions

diff --git a/configure b/configure
index 6e96899c9..a955d1466 100755
--- a/configure
+++ b/configure
@@ -226,6 +226,7 @@ EXPERIMENT_LIST="
     qimode
     uvintra
     newnear
+    newlpf
 "
 CONFIG_LIST="
     external_build
diff --git a/vp8/common/loopfilter.c b/vp8/common/loopfilter.c
index aa205f10e..052d00e0f 100644
--- a/vp8/common/loopfilter.c
+++ b/vp8/common/loopfilter.c
@@ -342,18 +342,26 @@ void vp8_loop_filter_frame
                     lfi.hev_thr = lfi_n->hev_thr[hev_index];
 
                     if (mb_col > 0)
+#if CONFIG_NEWLPF
+                        vp8_loop_filter_mbv_c
+                        (y_ptr, u_ptr, v_ptr, post->y_stride, post->uv_stride, &lfi);
+#else
                         LF_INVOKE(&cm->rtcd.loopfilter, normal_mb_v)
                         (y_ptr, u_ptr, v_ptr, post->y_stride, post->uv_stride, &lfi);
-
+#endif
                     if (!skip_lf)
                         LF_INVOKE(&cm->rtcd.loopfilter, normal_b_v)
                         (y_ptr, u_ptr, v_ptr, post->y_stride, post->uv_stride, &lfi);
 
                     /* don't apply across umv border */
                     if (mb_row > 0)
+#if CONFIG_NEWLPF
+                        vp8_loop_filter_mbh_c
+                        (y_ptr, u_ptr, v_ptr, post->y_stride, post->uv_stride, &lfi);
+#else
                         LF_INVOKE(&cm->rtcd.loopfilter, normal_mb_h)
                         (y_ptr, u_ptr, v_ptr, post->y_stride, post->uv_stride, &lfi);
-
+#endif
                     if (!skip_lf)
                         LF_INVOKE(&cm->rtcd.loopfilter, normal_b_h)
                         (y_ptr, u_ptr, v_ptr, post->y_stride, post->uv_stride, &lfi);
@@ -454,8 +462,13 @@ void vp8_loop_filter_frame_yonly
                     lfi.hev_thr = lfi_n->hev_thr[hev_index];
 
                     if (mb_col > 0)
+#if CONFIG_NEWLPF
+                        vp8_loop_filter_mbv_c
+                        (y_ptr, 0, 0, post->y_stride, 0, &lfi);
+#else
                         LF_INVOKE(&cm->rtcd.loopfilter, normal_mb_v)
                         (y_ptr, 0, 0, post->y_stride, 0, &lfi);
+#endif
 
                     if (!skip_lf)
                         LF_INVOKE(&cm->rtcd.loopfilter, normal_b_v)
@@ -463,9 +476,13 @@ void vp8_loop_filter_frame_yonly
 
                     /* don't apply across umv border */
                     if (mb_row > 0)
+#if CONFIG_NEWLPF
+                        vp8_loop_filter_mbh_c
+                        (y_ptr, 0, 0, post->y_stride, 0, &lfi);
+#else
                         LF_INVOKE(&cm->rtcd.loopfilter, normal_mb_h)
                         (y_ptr, 0, 0, post->y_stride, 0, &lfi);
-
+#endif
                     if (!skip_lf)
                         LF_INVOKE(&cm->rtcd.loopfilter, normal_b_h)
                         (y_ptr, 0, 0, post->y_stride, 0, &lfi);
diff --git a/vp8/common/loopfilter_filters.c b/vp8/common/loopfilter_filters.c
index 10228ae09..72ba31eaa 100644
--- a/vp8/common/loopfilter_filters.c
+++ b/vp8/common/loopfilter_filters.c
@@ -148,7 +148,8 @@ void vp8_loop_filter_vertical_edge_c
     do
     {
         mask = vp8_filter_mask(limit[0], blimit[0],
-                               s[-4], s[-3], s[-2], s[-1], s[0], s[1], s[2], s[3]);
+                               s[-4], s[-3], s[-2], s[-1],
+                               s[0], s[1], s[2], s[3]);
 
         hev = vp8_hevmask(thresh[0], s[-2], s[-1], s[0], s[1]);
 
@@ -158,9 +159,100 @@ void vp8_loop_filter_vertical_edge_c
     }
     while (++i < count * 8);
 }
+#if CONFIG_NEWLPF
+static __inline signed char vp8_flatmask(uc thresh,
+                                         uc p4, uc p3, uc p2, uc p1, uc p0,
+                                         uc q0, uc q1, uc q2, uc q3, uc q4)
+{
+    signed char flat = 0;
+    flat |= (abs(p1 - p0) > 1) * -1;
+    flat |= (abs(q1 - q0) > 1) * -1;
+    flat |= (abs(p0 - p2) > 1) * -1;
+    flat |= (abs(q0 - q2) > 1) * -1;
+    flat |= (abs(p3 - p0) > 1) * -1;
+    flat |= (abs(q3 - q0) > 1) * -1;
+    flat |= (abs(p4 - p0) > 1) * -1;
+    flat |= (abs(q4 - q0) > 1) * -1;
+    flat = ~flat;
+    return flat;
+}
 
+static __inline void vp8_mbfilter(signed char mask, uc hev, uc flat,
+                                  uc *op4, uc *op3, uc *op2, uc *op1, uc *op0,
+                                  uc *oq0, uc *oq1, uc *oq2, uc *oq3, uc *oq4)
+{
+    /* use a 7 tap filter [1, 1, 1, 2, 1, 1, 1] for flat line */
+    if(flat && mask)
+    {
+        unsigned char p0, q0;
+        unsigned char p1, q1;
+        unsigned char p2, q2;
+        unsigned char p3, q3;
+        unsigned char p4, q4;
+
+        p4 = *op4;
+        p3 = *op3;
+        p2 = *op2;
+        p1 = *op1;
+        p0 = *op0;
+        q0 = *oq0;
+        q1 = *oq1;
+        q2 = *oq2;
+        q3 = *oq3;
+        q4 = *oq4;
+
+        *op2 = ( p4 + p4 + p3 + p2 + p2 + p1 + p0 + q0 + 4)>>3;
+        *op1 = ( p4 + p3 + p2 + p1 + p1 + p0 + q0 + q1 + 4)>>3;
+        *op0 = ( p3 + p2 + p1 + p0 + p0 + q0 + q1 + q2 + 4)>>3;
+        *oq0 = ( p2 + p1 + p0 + q0 + q0 + q1 + q2 + q3 + 4)>>3;
+        *oq1 = ( p1 + p0 + q0 + q1 + q1 + q2 + q3 + q4 + 4)>>3;
+        *oq2 = ( p0 + q0 + q1 + q2 + q2 + q3 + q4 + q4 + 4)>>3;
+    }
+    else
+    {
+        signed char ps0, qs0;
+        signed char ps1, qs1;
+        signed char vp8_filter, Filter1, Filter2;
+        signed char u;
+
+        ps1 = (signed char) * op1 ^ 0x80;
+        ps0 = (signed char) * op0 ^ 0x80;
+        qs0 = (signed char) * oq0 ^ 0x80;
+        qs1 = (signed char) * oq1 ^ 0x80;
+
+        /* add outer taps if we have high edge variance */
+        vp8_filter = vp8_signed_char_clamp(ps1 - qs1);
+        vp8_filter &= hev;
+
+        /* inner taps */
+        vp8_filter = vp8_signed_char_clamp(vp8_filter + 3 * (qs0 - ps0));
+        vp8_filter &= mask;
+
+        Filter1 = vp8_signed_char_clamp(vp8_filter + 4);
+        Filter2 = vp8_signed_char_clamp(vp8_filter + 3);
+        Filter1 >>= 3;
+        Filter2 >>= 3;
+        u = vp8_signed_char_clamp(qs0 - Filter1);
+        *oq0 = u ^ 0x80;
+        u = vp8_signed_char_clamp(ps0 + Filter2);
+        *op0 = u ^ 0x80;
+        vp8_filter = Filter1;
+
+        /* outer tap adjustments */
+        vp8_filter += 1;
+        vp8_filter >>= 1;
+        vp8_filter &= ~hev;
+
+        u = vp8_signed_char_clamp(qs1 - vp8_filter);
+        *oq1 = u ^ 0x80;
+        u = vp8_signed_char_clamp(ps1 + vp8_filter);
+        *op1 = u ^ 0x80;
+    }
+}
+#else
 static __inline void vp8_mbfilter(signed char mask, uc hev,
-                           uc *op2, uc *op1, uc *op0, uc *oq0, uc *oq1, uc *oq2)
+                                  uc *op2, uc *op1, uc *op0,
+                                  uc *oq0, uc *oq1, uc *oq2)
 {
     signed char s, u;
     signed char vp8_filter, Filter1, Filter2;
@@ -213,7 +305,7 @@ static __inline void vp8_mbfilter(signed char mask, uc hev,
     s = vp8_signed_char_clamp(ps2 + u);
     *op2 = s ^ 0x80;
 }
-
+#endif
 void vp8_mbloop_filter_horizontal_edge_c
 (
     unsigned char *s,
@@ -226,6 +318,9 @@ void vp8_mbloop_filter_horizontal_edge_c
 {
     signed char hev = 0; /* high edge variance */
     signed char mask = 0;
+#if CONFIG_NEWLPF
+    signed char flat = 0;
+#endif
     int i = 0;
 
     /* loop filter designed to work using chars so that we can make maximum use
@@ -236,12 +331,21 @@ void vp8_mbloop_filter_horizontal_edge_c
 
         mask = vp8_filter_mask(limit[0], blimit[0],
                                s[-4*p], s[-3*p], s[-2*p], s[-1*p],
-                               s[0*p], s[1*p], s[2*p], s[3*p]);
+                               s[ 0*p], s[ 1*p], s[ 2*p], s[ 3*p]);
 
         hev = vp8_hevmask(thresh[0], s[-2*p], s[-1*p], s[0*p], s[1*p]);
-
-        vp8_mbfilter(mask, hev, s - 3 * p, s - 2 * p, s - 1 * p, s, s + 1 * p, s + 2 * p);
-
+#if CONFIG_NEWLPF
+        flat = vp8_flatmask(thresh[0],
+                            s[-5*p], s[-4*p], s[-3*p], s[-2*p], s[-1*p],
+                            s[ 0*p], s[ 1*p], s[ 2*p], s[ 3*p], s[ 4*p]);
+        vp8_mbfilter(mask, hev, flat,
+                     s - 5*p, s - 4*p, s- 3*p, s - 2*p, s - 1*p,
+                     s,       s + 1*p, s+ 2*p, s + 3*p, s + 4*p );
+#else
+        vp8_mbfilter(mask, hev,
+            s - 3*p, s - 2*p, s - 1*p,
+            s,       s + 1*p, s + 2*p);
+#endif
         ++s;
     }
     while (++i < count * 8);
@@ -261,18 +365,31 @@ void vp8_mbloop_filter_vertical_edge_c
 {
     signed char hev = 0; /* high edge variance */
     signed char mask = 0;
+#if CONFIG_NEWLPF
+    signed char flat = 0;
+#endif
     int i = 0;
 
     do
     {
 
         mask = vp8_filter_mask(limit[0], blimit[0],
-                               s[-4], s[-3], s[-2], s[-1], s[0], s[1], s[2], s[3]);
+                               s[-4], s[-3], s[-2], s[-1],
+                               s[0], s[1], s[2], s[3]);
 
         hev = vp8_hevmask(thresh[0], s[-2], s[-1], s[0], s[1]);
-
-        vp8_mbfilter(mask, hev, s - 3, s - 2, s - 1, s, s + 1, s + 2);
-
+#if CONFIG_NEWLPF
+        flat = vp8_flatmask(thresh[0],
+                            s[-5],s[-4],s[-3],s[-2],s[-1],
+                            s[ 0],s[ 1],s[ 2],s[ 3],s[ 4]);
+        vp8_mbfilter(mask, hev, flat,
+                            s - 5, s - 4, s - 3, s - 2, s - 1,
+                            s,     s + 1, s + 2, s + 3, s + 4);
+#else
+        vp8_mbfilter(mask, hev,
+                     s - 3, s - 2, s - 1,
+                     s,     s + 1, s + 2);
+#endif
         s += p;
     }
     while (++i < count * 8);
@@ -280,7 +397,9 @@ void vp8_mbloop_filter_vertical_edge_c
 }
 
 /* should we apply any filter at all ( 11111111 yes, 00000000 no) */
-static __inline signed char vp8_simple_filter_mask(uc blimit, uc p1, uc p0, uc q0, uc q1)
+static __inline signed char vp8_simple_filter_mask(uc blimit,
+                                                   uc p1, uc p0,
+                                                   uc q0, uc q1)
 {
 /* Why does this cause problems for win32?
  * error C2143: syntax error : missing ';' before 'type'
@@ -290,7 +409,9 @@ static __inline signed char vp8_simple_filter_mask(uc blimit, uc p1, uc p0, uc q
     return mask;
 }
 
-static __inline void vp8_simple_filter(signed char mask, uc *op1, uc *op0, uc *oq0, uc *oq1)
+static __inline void vp8_simple_filter(signed char mask,
+                                       uc *op1, uc *op0,
+                                       uc *oq0, uc *oq1)
 {
     signed char vp8_filter, Filter1, Filter2;
     signed char p1 = (signed char) * op1 ^ 0x80;
@@ -327,8 +448,12 @@ void vp8_loop_filter_simple_horizontal_edge_c
 
     do
     {
-        mask = vp8_simple_filter_mask(blimit[0], s[-2*p], s[-1*p], s[0*p], s[1*p]);
-        vp8_simple_filter(mask, s - 2 * p, s - 1 * p, s, s + 1 * p);
+        mask = vp8_simple_filter_mask(blimit[0],
+                                      s[-2*p], s[-1*p],
+                                      s[0*p], s[1*p]);
+        vp8_simple_filter(mask,
+                          s - 2 * p, s - 1 * p,
+                          s, s + 1 * p);
         ++s;
     }
     while (++i < 16);