diff options
author | Deb Mukherjee <debargha@google.com> | 2012-10-05 05:08:20 -0700 |
---|---|---|
committer | Deb Mukherjee <debargha@google.com> | 2012-10-05 05:30:00 -0700 |
commit | 105746c8fec14874babf97aee0d3660130e2ae08 (patch) | |
tree | fe1c5e9fd52b17e8912cd625d110bdf47a185674 /vp8/encoder | |
parent | 3f5d60b38442e7f0ba37883c4b636150305dfae0 (diff) | |
download | libvpx-105746c8fec14874babf97aee0d3660130e2ae08.tar libvpx-105746c8fec14874babf97aee0d3660130e2ae08.tar.gz libvpx-105746c8fec14874babf97aee0d3660130e2ae08.tar.bz2 libvpx-105746c8fec14874babf97aee0d3660130e2ae08.zip |
Mismatch bug-fix when transform expts are off
A variety of issues related to tangling of various hybrid transform
experiments.
Change-Id: I84dac65ff513f8b024407760df55fffa95e198de
Diffstat (limited to 'vp8/encoder')
-rw-r--r-- | vp8/encoder/tokenize.c | 44 |
1 files changed, 28 insertions, 16 deletions
diff --git a/vp8/encoder/tokenize.c b/vp8/encoder/tokenize.c index 91fafd088..4f4bd5010 100644 --- a/vp8/encoder/tokenize.c +++ b/vp8/encoder/tokenize.c @@ -252,6 +252,7 @@ static void tokenize2nd_order_b_8x8 t->context_tree = cpi->common.fc.coef_probs_8x8 [type] [band] [pt]; t->skip_eob_node = pt == 0 && ((band > 0 && type > 0) || (band > 1 && type == 0)); + assert(vp8_coef_encodings[t->Token].Len - t->skip_eob_node > 0); if (!dry_run) ++cpi->coef_counts_8x8 [type] [band] [pt] [x]; @@ -302,6 +303,7 @@ static void tokenize2nd_order_b(MACROBLOCKD *xd, t->context_tree = cpi->common.fc.coef_probs [1] [band] [pt]; t->skip_eob_node = ((pt == 0) && (band > 0)); + assert(vp8_coef_encodings[t->Token].Len - t->skip_eob_node > 0); if (!dry_run) ++cpi->coef_counts [1] [band] [pt] [token]; @@ -316,6 +318,7 @@ static void tokenize2nd_order_b(MACROBLOCKD *xd, t->context_tree = cpi->common.fc.coef_probs [1] [band] [pt]; t->skip_eob_node = ((pt == 0) && (band > 0)); + assert(vp8_coef_encodings[t->Token].Len - t->skip_eob_node > 0); if (!dry_run) ++cpi->coef_counts [1] [band] [pt] [DCT_EOB_TOKEN]; @@ -346,7 +349,8 @@ static void tokenize1st_order_b_8x8 TOKENEXTRA *t = *tp; /* store tokens starting here */ const short *qcoeff_ptr = b->qcoeff; #if CONFIG_HYBRIDTRANSFORM8X8 - TX_TYPE tx_type = type == 3 ? get_tx_type(xd, b) : DCT_DCT; + TX_TYPE tx_type = xd->mode_info_context->mbmi.mode == I8X8_PRED ? + get_tx_type(xd, b) : DCT_DCT; #endif int seg_eob = 64; @@ -377,6 +381,7 @@ static void tokenize1st_order_b_8x8 t->context_tree = cpi->common.fc.coef_probs_8x8[type][band][pt]; t->skip_eob_node = pt == 0 && ((band > 0 && type > 0) || (band > 1 && type == 0)); + assert(vp8_coef_encodings[t->Token].Len - t->skip_eob_node > 0); if (!dry_run) { #if CONFIG_HYBRIDTRANSFORM8X8 @@ -403,6 +408,11 @@ static void tokenize1st_order_b_8x8 t->context_tree = cpi->common.fc.coef_probs_8x8 [type] [band] [pt]; t->skip_eob_node = pt == 0 && ((band > 0 && type > 0) || (band > 1 && type == 0)); + if (vp8_coef_encodings[t->Token].Len - t->skip_eob_node <= 0) { + printf("type %d, seg-eob %d, eob %d, pt %d, c %d band %d\n", type, seg_eob, b->eob, pt, c, band); + fflush(stdout); + } + assert(vp8_coef_encodings[t->Token].Len - t->skip_eob_node > 0); if (!dry_run) { #if CONFIG_HYBRIDTRANSFORM8X8 @@ -458,7 +468,6 @@ static void tokenize1st_order_ht( MACROBLOCKD *xd, b_mode = b->bmi.as_mode.first; tx_type = get_tx_type(xd, b); } - //if (!dry_run) printf("4: %d\n", tx_type); // assign scanning order for luma components coded in intra4x4 mode if( (xd->mode_info_context->mbmi.mode == B_PRED) && @@ -508,6 +517,7 @@ static void tokenize1st_order_ht( MACROBLOCKD *xd, t->skip_eob_node = pt == 0 && ((band > 0 && type > 0) || (band > 1 && type == 0)); + assert(vp8_coef_encodings[t->Token].Len - t->skip_eob_node > 0); if (!dry_run) { if (tx_type != DCT_DCT) @@ -530,6 +540,7 @@ static void tokenize1st_order_ht( MACROBLOCKD *xd, t->skip_eob_node = pt == 0 && ((band > 0 && type > 0) || (band > 1 && type == 0)); + assert(vp8_coef_encodings[t->Token].Len - t->skip_eob_node > 0); if (!dry_run) { if (tx_type != DCT_DCT) ++cpi->hybrid_coef_counts[type] [band] [pt] [DCT_EOB_TOKEN]; @@ -570,6 +581,7 @@ static void tokenize1st_order_ht( MACROBLOCKD *xd, t->context_tree = cpi->common.fc.coef_probs [2] [band] [pt]; t->skip_eob_node = ((pt == 0) && (band > 0)); + assert(vp8_coef_encodings[t->Token].Len - t->skip_eob_node > 0); if (!dry_run) ++cpi->coef_counts [2] [band] [pt] [token]; @@ -584,6 +596,7 @@ static void tokenize1st_order_ht( MACROBLOCKD *xd, t->context_tree = cpi->common.fc.coef_probs [2] [band] [pt]; t->skip_eob_node = ((pt == 0) && (band > 0)); + assert(vp8_coef_encodings[t->Token].Len - t->skip_eob_node > 0); if (!dry_run) ++cpi->coef_counts [2] [band] [pt] [DCT_EOB_TOKEN]; t++; @@ -597,7 +610,6 @@ static void tokenize1st_order_ht( MACROBLOCKD *xd, #endif -#if CONFIG_HYBRIDTRANSFORM8X8 static void tokenize1st_order_chroma ( MACROBLOCKD *xd, @@ -649,6 +661,7 @@ static void tokenize1st_order_chroma t->context_tree = cpi->common.fc.coef_probs [2] [band] [pt]; t->skip_eob_node = ((pt == 0) && (band > 0)); + assert(vp8_coef_encodings[t->Token].Len - t->skip_eob_node > 0); if (!dry_run) ++cpi->coef_counts [2] [band] [pt] [token]; @@ -663,6 +676,7 @@ static void tokenize1st_order_chroma t->context_tree = cpi->common.fc.coef_probs [2] [band] [pt]; t->skip_eob_node = ((pt == 0) && (band > 0)); + assert(vp8_coef_encodings[t->Token].Len - t->skip_eob_node > 0); if (!dry_run) ++cpi->coef_counts [2] [band] [pt] [DCT_EOB_TOKEN]; @@ -674,7 +688,6 @@ static void tokenize1st_order_chroma *a = *l = pt; } } -#endif static void tokenize1st_order_b ( @@ -708,6 +721,8 @@ static void tokenize1st_order_b c = type ? 0 : 1; + assert(b->eob <= 16); + for (; c < b->eob; c++) { rc = vp8_default_zig_zag1d[c]; band = vp8_coef_bands[c]; @@ -721,6 +736,7 @@ static void tokenize1st_order_b t->skip_eob_node = pt == 0 && ((band > 0 && type > 0) || (band > 1 && type == 0)); + assert(vp8_coef_encodings[t->Token].Len - t->skip_eob_node > 0); if (!dry_run) ++cpi->coef_counts [type] [band] [pt] [token]; @@ -735,6 +751,7 @@ static void tokenize1st_order_b t->skip_eob_node = pt == 0 && ((band > 0 && type > 0) || (band > 1 && type == 0)); + assert(vp8_coef_encodings[t->Token].Len - t->skip_eob_node > 0); if (!dry_run) ++cpi->coef_counts [type] [band] [pt] [DCT_EOB_TOKEN]; @@ -753,6 +770,7 @@ static void tokenize1st_order_b VP8_COMBINEENTROPYCONTEXTS(pt, *a, *l); + assert(b->eob <= 16); for (c = 0; c < b->eob; c++) { rc = vp8_default_zig_zag1d[c]; band = vp8_coef_bands[c]; @@ -765,6 +783,7 @@ static void tokenize1st_order_b t->context_tree = cpi->common.fc.coef_probs [2] [band] [pt]; t->skip_eob_node = ((pt == 0) && (band > 0)); + assert(vp8_coef_encodings[t->Token].Len - t->skip_eob_node > 0); if (!dry_run) ++cpi->coef_counts [2] [band] [pt] [token]; @@ -779,6 +798,7 @@ static void tokenize1st_order_b t->context_tree = cpi->common.fc.coef_probs [2] [band] [pt]; t->skip_eob_node = ((pt == 0) && (band > 0)); + assert(vp8_coef_encodings[t->Token].Len - t->skip_eob_node > 0); if (!dry_run) ++cpi->coef_counts [2] [band] [pt] [DCT_EOB_TOKEN]; @@ -908,12 +928,10 @@ void vp8_tokenize_mb(VP8_COMP *cpi, break; #endif case TX_8X8: -#if CONFIG_HYBRIDTRANSFORM8X8 if (xd->mode_info_context->mbmi.mode == I8X8_PRED) xd->mode_info_context->mbmi.mb_skip_coeff = mb_is_skippable_8x8_4x4uv(xd, 0); else -#endif - xd->mode_info_context->mbmi.mb_skip_coeff = mb_is_skippable_8x8(xd, has_y2_block); + xd->mode_info_context->mbmi.mb_skip_coeff = mb_is_skippable_8x8(xd, has_y2_block); break; default: @@ -931,11 +949,9 @@ void vp8_tokenize_mb(VP8_COMP *cpi, else #endif if (tx_size == TX_8X8) { -#if CONFIG_HYBRIDTRANSFORM8X8 if (xd->mode_info_context->mbmi.mode == I8X8_PRED) vp8_stuff_mb_8x8_4x4uv(cpi, xd, t, dry_run); else -#endif vp8_stuff_mb_8x8(cpi, xd, t, dry_run); } else vp8_stuff_mb(cpi, xd, t, dry_run); @@ -992,11 +1008,9 @@ void vp8_tokenize_mb(VP8_COMP *cpi, if (tx_size == TX_8X8) { ENTROPY_CONTEXT *A = (ENTROPY_CONTEXT *)xd->above_context; ENTROPY_CONTEXT *L = (ENTROPY_CONTEXT *)xd->left_context; -#if CONFIG_HYBRIDTRANSFORM8X8 if (xd->mode_info_context->mbmi.mode == I8X8_PRED) { plane_type = PLANE_TYPE_Y_WITH_DC; } -#endif for (b = 0; b < 16; b += 4) { tokenize1st_order_b_8x8(xd, xd->block + b, @@ -1007,12 +1021,9 @@ void vp8_tokenize_mb(VP8_COMP *cpi, *(A + vp8_block2above_8x8[b] + 1) = *(A + vp8_block2above_8x8[b]); *(L + vp8_block2left_8x8[b] + 1) = *(L + vp8_block2left_8x8[b]); } -#if CONFIG_HYBRIDTRANSFORM8X8 if (xd->mode_info_context->mbmi.mode == I8X8_PRED) { tokenize1st_order_chroma(xd, t, PLANE_TYPE_UV, cpi, dry_run); - } else -#endif - { + } else { for (b = 16; b < 24; b += 4) { tokenize1st_order_b_8x8(xd, xd->block + b, t, 2, xd->frame_type, @@ -1343,7 +1354,8 @@ static __inline void stuff1st_order_b_8x8 int pt; /* near block/prev token context index */ TOKENEXTRA *t = *tp; /* store tokens starting here */ #if CONFIG_HYBRIDTRANSFORM8X8 - TX_TYPE tx_type = type == 3 ? get_tx_type(xd, b) : DCT_DCT; + TX_TYPE tx_type = xd->mode_info_context->mbmi.mode == I8X8_PRED ? + get_tx_type(xd, b) : DCT_DCT; #endif VP8_COMBINEENTROPYCONTEXTS(pt, *a, *l); |