diff options
-rwxr-xr-x | configure | 3 | ||||
-rw-r--r-- | examples.mk | 5 | ||||
-rw-r--r-- | examples/decode_with_partial_drops.txt | 232 | ||||
-rw-r--r-- | examples/decoder_tmpl.c | 2 | ||||
-rw-r--r-- | vp8/common/alloccommon.c | 21 | ||||
-rw-r--r-- | vp8/common/onyxc_int.h | 2 | ||||
-rw-r--r-- | vp8/common/onyxd.h | 1 | ||||
-rw-r--r-- | vp8/decoder/decodemv.c | 23 | ||||
-rw-r--r-- | vp8/decoder/decodframe.c | 219 | ||||
-rw-r--r-- | vp8/decoder/ec_types.h | 49 | ||||
-rw-r--r-- | vp8/decoder/error_concealment.c | 613 | ||||
-rw-r--r-- | vp8/decoder/error_concealment.h | 41 | ||||
-rw-r--r-- | vp8/decoder/onyxd_if.c | 49 | ||||
-rw-r--r-- | vp8/decoder/onyxd_int.h | 10 | ||||
-rw-r--r-- | vp8/vp8_dx_iface.c | 8 | ||||
-rw-r--r-- | vp8/vp8dx.mk | 3 | ||||
-rw-r--r-- | vpx/src/vpx_decoder.c | 3 | ||||
-rw-r--r-- | vpx/vpx_decoder.h | 4 | ||||
-rw-r--r-- | vpxdec.c | 14 |
19 files changed, 1239 insertions, 63 deletions
@@ -37,6 +37,7 @@ Advanced options: ${toggle_multithread} multithreaded encoding and decoding. ${toggle_spatial_resampling} spatial sampling (scaling) support ${toggle_realtime_only} enable this option while building for real-time encoding + ${toggle_error_concealment} enable this option to get a decoder which is able to conceal losses ${toggle_runtime_cpu_detect} runtime cpu detection ${toggle_shared} shared library support ${toggle_small} favor smaller size over speed @@ -249,6 +250,7 @@ CONFIG_LIST=" static_msvcrt spatial_resampling realtime_only + error_concealment shared small postproc_visualizer @@ -289,6 +291,7 @@ CMDLINE_SELECT=" mem_tracker spatial_resampling realtime_only + error_concealment shared small postproc_visualizer diff --git a/examples.mk b/examples.mk index 3e34b5697..ad56ff040 100644 --- a/examples.mk +++ b/examples.mk @@ -77,6 +77,11 @@ GEN_EXAMPLES-$(CONFIG_ENCODERS) += decode_with_drops.c endif decode_with_drops.GUID = CE5C53C4-8DDA-438A-86ED-0DDD3CDB8D26 decode_with_drops.DESCRIPTION = Drops frames while decoding +ifeq ($(CONFIG_DECODERS),yes) +GEN_EXAMPLES-$(CONFIG_ERROR_CONCEALMENT) += decode_with_partial_drops.c +endif +decode_with_partial_drops.GUID = 61C2D026-5754-46AC-916F-1343ECC5537E +decode_with_partial_drops.DESCRIPTION = Drops parts of frames while decoding GEN_EXAMPLES-$(CONFIG_ENCODERS) += error_resilient.c error_resilient.GUID = DF5837B9-4145-4F92-A031-44E4F832E00C error_resilient.DESCRIPTION = Error Resiliency Feature diff --git a/examples/decode_with_partial_drops.txt b/examples/decode_with_partial_drops.txt new file mode 100644 index 000000000..30854d669 --- /dev/null +++ b/examples/decode_with_partial_drops.txt @@ -0,0 +1,232 @@ +@TEMPLATE decoder_tmpl.c +Decode With Partial Drops Example +========================= +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ INTRODUCTION +This is an example utility which drops a series of frames (or parts of frames), +as specified on the command line. This is useful for observing the error +recovery features of the codec. +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ INTRODUCTION + +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ EXTRA_INCLUDES +#include <time.h> +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ EXTRA_INCLUDES + +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ HELPERS +struct parsed_header +{ + char key_frame; + int version; + char show_frame; + int first_part_size; +}; + +int next_packet(struct parsed_header* hdr, int pos, int length, int mtu) +{ + int size = 0; + int remaining = length - pos; + /* Uncompressed part is 3 bytes for P frames and 10 bytes for I frames */ + int uncomp_part_size = (hdr->key_frame ? 10 : 3); + /* number of bytes yet to send from header and the first partition */ + int remainFirst = uncomp_part_size + hdr->first_part_size - pos; + if (remainFirst > 0) + { + if (remainFirst <= mtu) + { + size = remainFirst; + } + else + { + size = mtu; + } + + return size; + } + + /* second partition; just slot it up according to MTU */ + if (remaining <= mtu) + { + size = remaining; + return size; + } + return mtu; +} + +void throw_packets(unsigned char* frame, int* size, int loss_rate, + int* thrown, int* kept) +{ + unsigned char loss_frame[256*1024]; + int pkg_size = 1; + int pos = 0; + int loss_pos = 0; + struct parsed_header hdr; + unsigned int tmp; + int mtu = 1500; + + if (*size < 3) + { + return; + } + putc('|', stdout); + /* parse uncompressed 3 bytes */ + tmp = (frame[2] << 16) | (frame[1] << 8) | frame[0]; + hdr.key_frame = !(tmp & 0x1); /* inverse logic */ + hdr.version = (tmp >> 1) & 0x7; + hdr.show_frame = (tmp >> 4) & 0x1; + hdr.first_part_size = (tmp >> 5) & 0x7FFFF; + + /* don't drop key frames */ + if (hdr.key_frame) + { + int i; + *kept = *size/mtu + ((*size % mtu > 0) ? 1 : 0); /* approximate */ + for (i=0; i < *kept; i++) + putc('.', stdout); + return; + } + + while ((pkg_size = next_packet(&hdr, pos, *size, mtu)) > 0) + { + int loss_event = ((rand() + 1.0)/(RAND_MAX + 1.0) < loss_rate/100.0); + if (*thrown == 0 && !loss_event) + { + memcpy(loss_frame + loss_pos, frame + pos, pkg_size); + loss_pos += pkg_size; + (*kept)++; + putc('.', stdout); + } + else + { + (*thrown)++; + putc('X', stdout); + } + pos += pkg_size; + } + memcpy(frame, loss_frame, loss_pos); + memset(frame + loss_pos, 0, *size - loss_pos); + *size = loss_pos; +} +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ HELPERS + +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ DEC_INIT +/* Initialize codec */ +flags = VPX_CODEC_USE_ERROR_CONCEALMENT; +res = vpx_codec_dec_init(&codec, interface, NULL, flags); +if(res) + die_codec(&codec, "Failed to initialize decoder"); + +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ DEC_INIT + +Usage +----- +This example adds a single argument to the `simple_decoder` example, +which specifies the range or pattern of frames to drop. The parameter is +parsed as follows: + +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ USAGE +if(argc!=4 && argc != 5) + die("Usage: %s <infile> <outfile> <N-M|N/M|L,S>\n", argv[0]); +{ + char *nptr; + n = strtol(argv[3], &nptr, 0); + mode = (*nptr == '\0' || *nptr == ',') ? 2 : (*nptr == '-') ? 1 : 0; + + m = strtol(nptr+1, NULL, 0); + if((!n && !m) || (*nptr != '-' && *nptr != '/' && + *nptr != '\0' && *nptr != ',')) + die("Couldn't parse pattern %s\n", argv[3]); +} +seed = (m > 0) ? m : (unsigned int)time(NULL); +srand(seed);thrown_frame = 0; +printf("Seed: %u\n", seed); +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ USAGE + + +Dropping A Range Of Frames +-------------------------- +To drop a range of frames, specify the starting frame and the ending +frame to drop, separated by a dash. The following command will drop +frames 5 through 10 (base 1). + + $ ./decode_with_partial_drops in.ivf out.i420 5-10 + + +Dropping A Pattern Of Frames +---------------------------- +To drop a pattern of frames, specify the number of frames to drop and +the number of frames after which to repeat the pattern, separated by +a forward-slash. The following command will drop 3 of 7 frames. +Specifically, it will decode 4 frames, then drop 3 frames, and then +repeat. + + $ ./decode_with_partial_drops in.ivf out.i420 3/7 + +Dropping Random Parts Of Frames +------------------------------- +A third argument tuple is available to split the frame into 1500 bytes pieces +and randomly drop pieces rather than frames. The frame will be split at +partition boundaries where possible. The following example will seed the RNG +with the seed 123 and drop approximately 5% of the pieces. Pieces which +are depending on an already dropped piece will also be dropped. + + $ ./decode_with_partial_drops in.ivf out.i420 5,123 + + +Extra Variables +--------------- +This example maintains the pattern passed on the command line in the +`n`, `m`, and `is_range` variables: + +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ EXTRA_VARS +int n, m, mode; +unsigned int seed; +int thrown=0, kept=0; +int thrown_frame=0, kept_frame=0; +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ EXTRA_VARS + + +Making The Drop Decision +------------------------ +The example decides whether to drop the frame based on the current +frame number, immediately before decoding the frame. + +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ PRE_DECODE +/* Decide whether to throw parts of the frame or the whole frame + depending on the drop mode */ +thrown_frame = 0; +kept_frame = 0; +switch (mode) +{ +case 0: + if (m - (frame_cnt-1)%m <= n) + { + frame_sz = 0; + } + break; +case 1: + if (frame_cnt >= n && frame_cnt <= m) + { + frame_sz = 0; + } + break; +case 2: + throw_packets(frame, &frame_sz, n, &thrown_frame, &kept_frame); + break; +default: break; +} +if (mode < 2) +{ + if (frame_sz == 0) + { + putc('X', stdout); + thrown_frame++; + } + else + { + putc('.', stdout); + kept_frame++; + } +} +thrown += thrown_frame; +kept += kept_frame; +fflush(stdout); +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ PRE_DECODE diff --git a/examples/decoder_tmpl.c b/examples/decoder_tmpl.c index c70681b1a..8194f0ade 100644 --- a/examples/decoder_tmpl.c +++ b/examples/decoder_tmpl.c @@ -42,6 +42,8 @@ static void die(const char *fmt, ...) { @DIE_CODEC +@HELPERS + int main(int argc, char **argv) { FILE *infile, *outfile; vpx_codec_ctx_t codec; diff --git a/vp8/common/alloccommon.c b/vp8/common/alloccommon.c index b0c7363a7..c2e5bfddf 100644 --- a/vp8/common/alloccommon.c +++ b/vp8/common/alloccommon.c @@ -27,6 +27,9 @@ static void update_mode_info_border(MODE_INFO *mi, int rows, int cols) for (i = 0; i < rows; i++) { + /* TODO(holmer): Bug? This updates the last element of each row + * rather than the border element! + */ vpx_memset(&mi[i*cols-1], 0, sizeof(MODE_INFO)); } } @@ -43,9 +46,11 @@ void vp8_de_alloc_frame_buffers(VP8_COMMON *oci) vpx_free(oci->above_context); vpx_free(oci->mip); + vpx_free(oci->prev_mip); oci->above_context = 0; oci->mip = 0; + oci->prev_mip = 0; } @@ -110,6 +115,21 @@ int vp8_alloc_frame_buffers(VP8_COMMON *oci, int width, int height) oci->mi = oci->mip + oci->mode_info_stride + 1; + /* allocate memory for last frame MODE_INFO array */ +#if CONFIG_ERROR_CONCEALMENT + oci->prev_mip = vpx_calloc((oci->mb_cols + 1) * (oci->mb_rows + 1), sizeof(MODE_INFO)); + + if (!oci->prev_mip) + { + vp8_de_alloc_frame_buffers(oci); + return 1; + } + + oci->prev_mi = oci->prev_mip + oci->mode_info_stride + 1; +#else + oci->prev_mip = NULL; + oci->prev_mi = NULL; +#endif oci->above_context = vpx_calloc(sizeof(ENTROPY_CONTEXT_PLANES) * oci->mb_cols, 1); @@ -120,6 +140,7 @@ int vp8_alloc_frame_buffers(VP8_COMMON *oci, int width, int height) } update_mode_info_border(oci->mi, oci->mb_rows, oci->mb_cols); + update_mode_info_border(oci->prev_mi, oci->mb_rows, oci->mb_cols); return 0; } diff --git a/vp8/common/onyxc_int.h b/vp8/common/onyxc_int.h index cf29d03df..862307ebb 100644 --- a/vp8/common/onyxc_int.h +++ b/vp8/common/onyxc_int.h @@ -139,6 +139,8 @@ typedef struct VP8Common MODE_INFO *mip; /* Base of allocated array */ MODE_INFO *mi; /* Corresponds to upper left visible macroblock */ + MODE_INFO *prev_mip; /* MODE_INFO array 'mip' from last decoded frame */ + MODE_INFO *prev_mi; /* 'mi' from last frame (points into prev_mip) */ INTERPOLATIONFILTERTYPE mcomp_filter_type; diff --git a/vp8/common/onyxd.h b/vp8/common/onyxd.h index e53bc3138..140dc5728 100644 --- a/vp8/common/onyxd.h +++ b/vp8/common/onyxd.h @@ -31,6 +31,7 @@ extern "C" int Version; int postprocess; int max_threads; + int error_concealment; } VP8D_CONFIG; typedef enum { diff --git a/vp8/decoder/decodemv.c b/vp8/decoder/decodemv.c index 4de28688b..cd67536bc 100644 --- a/vp8/decoder/decodemv.c +++ b/vp8/decoder/decodemv.c @@ -244,6 +244,12 @@ static void mb_mode_mv_init(VP8D_COMP *pbi) vp8_reader *const bc = & pbi->bc; MV_CONTEXT *const mvc = pbi->common.fc.mvc; +#if CONFIG_ERROR_CONCEALMENT + /* Default is that no macroblock is corrupt, therefore we initialize + * mvs_corrupt_from_mb to something very big, which we can be sure is + * outside the frame. */ + pbi->mvs_corrupt_from_mb = UINT_MAX; +#endif pbi->prob_skip_false = 0; if (pbi->common.mb_no_coeff_skip) pbi->prob_skip_false = (vp8_prob)vp8_read_literal(bc, 8); @@ -280,6 +286,7 @@ static void mb_mode_mv_init(VP8D_COMP *pbi) } } + static void read_mb_modes_mv(VP8D_COMP *pbi, MODE_INFO *mi, MB_MODE_INFO *mbmi, int mb_row, int mb_col) { @@ -403,7 +410,7 @@ static void read_mb_modes_mv(VP8D_COMP *pbi, MODE_INFO *mi, MB_MODE_INFO *mbmi, do { mi->bmi[ *fill_offset] = bmi; - fill_offset++; + fill_offset++; }while (--fill_count); } @@ -524,12 +531,26 @@ void vp8_decode_mode_mvs(VP8D_COMP *pbi) while (++mb_col < pbi->common.mb_cols) { + int mb_num = mb_row * pbi->common.mb_cols + mb_col; /*read_mb_modes_mv(pbi, xd->mode_info_context, &xd->mode_info_context->mbmi, mb_row, mb_col);*/ if(pbi->common.frame_type == KEY_FRAME) vp8_kfread_modes(pbi, mi, mb_row, mb_col); else read_mb_modes_mv(pbi, mi, &mi->mbmi, mb_row, mb_col); +#if CONFIG_ERROR_CONCEALMENT + /* look for corruption. set mvs_corrupt_from_mb to the current + * mb_num if the frame is corrupt from this macroblock. */ + if (vp8dx_bool_error(&pbi->bc) && mb_num < pbi->mvs_corrupt_from_mb) + { + pbi->mvs_corrupt_from_mb = mb_num; + /* no need to continue since the partition is corrupt from + * here on. + */ + return; + } +#endif + mi++; /* next macroblock */ } diff --git a/vp8/decoder/decodframe.c b/vp8/decoder/decodframe.c index a585f774c..f8e04a7d0 100644 --- a/vp8/decoder/decodframe.c +++ b/vp8/decoder/decodframe.c @@ -27,6 +27,9 @@ #include "decodemv.h" #include "vp8/common/extend.h" +#if CONFIG_ERROR_CONCEALMENT +#include "error_concealment.h" +#endif #include "vpx_mem/vpx_mem.h" #include "vp8/common/idct.h" #include "dequantize.h" @@ -176,7 +179,8 @@ void clamp_mvs(MACROBLOCKD *xd) } -static void decode_macroblock(VP8D_COMP *pbi, MACROBLOCKD *xd) +static void decode_macroblock(VP8D_COMP *pbi, MACROBLOCKD *xd, + unsigned int mb_idx) { int eobtotal = 0; int i, do_clamp = xd->mode_info_context->mbmi.need_to_clamp_mvs; @@ -233,6 +237,19 @@ static void decode_macroblock(VP8D_COMP *pbi, MACROBLOCKD *xd) vp8_build_inter_predictors_mb(xd); } +#if CONFIG_ERROR_CONCEALMENT + if (pbi->ec_enabled && + (mb_idx > pbi->mvs_corrupt_from_mb || + vp8dx_bool_error(xd->current_bc))) + { + /* MB with corrupt residuals or corrupt mode/motion vectors. + * Better to use the predictor as reconstruction. + */ + vp8_conceal_corrupt_mb(xd); + return; + } +#endif + /* dequantization and idct */ if (xd->mode_info_context->mbmi.mode != B_PRED && xd->mode_info_context->mbmi.mode != SPLITMV) { @@ -355,6 +372,32 @@ decode_mb_row(VP8D_COMP *pbi, VP8_COMMON *pc, int mb_row, MACROBLOCKD *xd) for (mb_col = 0; mb_col < pc->mb_cols; mb_col++) { + /* Distance of Mb to the various image edges. + * These are specified to 8th pel as they are always compared to values + * that are in 1/8th pel units + */ + xd->mb_to_left_edge = -((mb_col * 16) << 3); + xd->mb_to_right_edge = ((pc->mb_cols - 1 - mb_col) * 16) << 3; + +#if CONFIG_ERROR_CONCEALMENT + if (pbi->ec_enabled && + xd->mode_info_context->mbmi.ref_frame == INTRA_FRAME && + vp8dx_bool_error(xd->current_bc)) + { + /* We have an intra block with corrupt coefficients, better to + * conceal with an inter block. Interpolate MVs from neighboring MBs + * + * Note that for the first mb with corrupt residual in a frame, + * we might not discover that before decoding the residual. That + * happens after this check, and therefore no inter concealment will + * be done. + */ + vp8_interpolate_motion(xd, + mb_row, mb_col, + pc->mb_rows, pc->mb_cols, + pc->mode_info_stride); + } +#endif if (xd->mode_info_context->mbmi.mode == SPLITMV || xd->mode_info_context->mbmi.mode == B_PRED) { @@ -365,12 +408,6 @@ decode_mb_row(VP8D_COMP *pbi, VP8_COMMON *pc, int mb_row, MACROBLOCKD *xd) } } - /* Distance of Mb to the various image edges. - * These are specified to 8th pel as they are always compared to values that are in 1/8th pel units - */ - xd->mb_to_left_edge = -((mb_col * 16) << 3); - xd->mb_to_right_edge = ((pc->mb_cols - 1 - mb_col) * 16) << 3; - xd->dst.y_buffer = pc->yv12_fb[dst_fb_idx].y_buffer + recon_yoffset; xd->dst.u_buffer = pc->yv12_fb[dst_fb_idx].u_buffer + recon_uvoffset; xd->dst.v_buffer = pc->yv12_fb[dst_fb_idx].v_buffer + recon_uvoffset; @@ -403,7 +440,7 @@ decode_mb_row(VP8D_COMP *pbi, VP8_COMMON *pc, int mb_row, MACROBLOCKD *xd) else pbi->debugoutput =0; */ - decode_macroblock(pbi, xd); + decode_macroblock(pbi, xd, mb_row * pc->mb_cols + mb_col); /* check if the boolean decoder has suffered an error */ xd->corrupted |= vp8dx_bool_error(xd->current_bc); @@ -477,8 +514,8 @@ static void setup_token_decoder(VP8D_COMP *pbi, partition_size = user_data_end - partition; } - if (partition + partition_size > user_data_end - || partition + partition_size < partition) + if (!pbi->ec_enabled && (partition + partition_size > user_data_end + || partition + partition_size < partition)) vpx_internal_error(&pc->error, VPX_CODEC_CORRUPT_FRAME, "Truncated packet or corrupt partition " "%d length", i + 1); @@ -593,63 +630,105 @@ int vp8_decode_frame(VP8D_COMP *pbi) pc->yv12_fb[pc->new_fb_idx].corrupted = 0; if (data_end - data < 3) - vpx_internal_error(&pc->error, VPX_CODEC_CORRUPT_FRAME, - "Truncated packet"); - pc->frame_type = (FRAME_TYPE)(data[0] & 1); - pc->version = (data[0] >> 1) & 7; - pc->show_frame = (data[0] >> 4) & 1; - first_partition_length_in_bytes = - (data[0] | (data[1] << 8) | (data[2] << 16)) >> 5; - data += 3; - - if (data + first_partition_length_in_bytes > data_end - || data + first_partition_length_in_bytes < data) - vpx_internal_error(&pc->error, VPX_CODEC_CORRUPT_FRAME, - "Truncated packet or corrupt partition 0 length"); - vp8_setup_version(pc); - - if (pc->frame_type == KEY_FRAME) { - const int Width = pc->Width; - const int Height = pc->Height; - - /* vet via sync code */ - if (data[0] != 0x9d || data[1] != 0x01 || data[2] != 0x2a) - vpx_internal_error(&pc->error, VPX_CODEC_UNSUP_BITSTREAM, - "Invalid frame sync code"); - - pc->Width = (data[3] | (data[4] << 8)) & 0x3fff; - pc->horiz_scale = data[4] >> 6; - pc->Height = (data[5] | (data[6] << 8)) & 0x3fff; - pc->vert_scale = data[6] >> 6; - data += 7; - - if (Width != pc->Width || Height != pc->Height) + if (pbi->ec_enabled) { - int prev_mb_rows = pc->mb_rows; + /* Declare the missing frame as an inter frame since it will + be handled as an inter frame when we have estimated its + motion vectors. */ + pc->frame_type = INTER_FRAME; + pc->version = 0; + pc->show_frame = 1; + first_partition_length_in_bytes = 0; + } + else + { + vpx_internal_error(&pc->error, VPX_CODEC_CORRUPT_FRAME, + "Truncated packet"); + } + } + else + { + pc->frame_type = (FRAME_TYPE)(data[0] & 1); + pc->version = (data[0] >> 1) & 7; + pc->show_frame = (data[0] >> 4) & 1; + first_partition_length_in_bytes = + (data[0] | (data[1] << 8) | (data[2] << 16)) >> 5; + data += 3; + + if (!pbi->ec_enabled && (data + first_partition_length_in_bytes > data_end + || data + first_partition_length_in_bytes < data)) + vpx_internal_error(&pc->error, VPX_CODEC_CORRUPT_FRAME, + "Truncated packet or corrupt partition 0 length"); + vp8_setup_version(pc); - if (pc->Width <= 0) + if (pc->frame_type == KEY_FRAME) + { + const int Width = pc->Width; + const int Height = pc->Height; + + /* vet via sync code */ + /* When error concealment is enabled we should only check the sync + * code if we have enough bits available + */ + if (!pbi->ec_enabled || data + 3 < data_end) { - pc->Width = Width; - vpx_internal_error(&pc->error, VPX_CODEC_CORRUPT_FRAME, - "Invalid frame width"); + if (data[0] != 0x9d || data[1] != 0x01 || data[2] != 0x2a) + vpx_internal_error(&pc->error, VPX_CODEC_UNSUP_BITSTREAM, + "Invalid frame sync code"); } - if (pc->Height <= 0) + /* If error concealment is enabled we should only parse the new size + * if we have enough data. Otherwise we will end up with the wrong + * size. + */ + if (!pbi->ec_enabled || data + 6 < data_end) { - pc->Height = Height; - vpx_internal_error(&pc->error, VPX_CODEC_CORRUPT_FRAME, - "Invalid frame height"); + pc->Width = (data[3] | (data[4] << 8)) & 0x3fff; + pc->horiz_scale = data[4] >> 6; + pc->Height = (data[5] | (data[6] << 8)) & 0x3fff; + pc->vert_scale = data[6] >> 6; } + data += 7; + + if (Width != pc->Width || Height != pc->Height) + { + int prev_mb_rows = pc->mb_rows; + + if (pc->Width <= 0) + { + pc->Width = Width; + vpx_internal_error(&pc->error, VPX_CODEC_CORRUPT_FRAME, + "Invalid frame width"); + } + + if (pc->Height <= 0) + { + pc->Height = Height; + vpx_internal_error(&pc->error, VPX_CODEC_CORRUPT_FRAME, + "Invalid frame height"); + } - if (vp8_alloc_frame_buffers(pc, pc->Width, pc->Height)) - vpx_internal_error(&pc->error, VPX_CODEC_MEM_ERROR, - "Failed to allocate frame buffers"); + if (vp8_alloc_frame_buffers(pc, pc->Width, pc->Height)) + vpx_internal_error(&pc->error, VPX_CODEC_MEM_ERROR, + "Failed to allocate frame buffers"); + +#if CONFIG_ERROR_CONCEALMENT + pbi->overlaps = NULL; + if (pbi->ec_enabled) + { + if (vp8_alloc_overlap_lists(pbi)) + vpx_internal_error(&pc->error, VPX_CODEC_MEM_ERROR, + "Failed to allocate overlap lists " + "for error concealment"); + } +#endif #if CONFIG_MULTITHREAD - if (pbi->b_multithreaded_rd) - vp8mt_alloc_temp_buffers(pbi, pc->Width, prev_mb_rows); + if (pbi->b_multithreaded_rd) + vp8mt_alloc_temp_buffers(pbi, pc->Width, prev_mb_rows); #endif + } } } @@ -792,7 +871,20 @@ int vp8_decode_frame(VP8D_COMP *pbi) { /* Should the GF or ARF be updated from the current frame */ pc->refresh_golden_frame = vp8_read_bit(bc); +#if CONFIG_ERROR_CONCEALMENT + /* Assume we shouldn't refresh golden if the bit is missing */ + xd->corrupted |= vp8dx_bool_error(bc); + if (pbi->ec_enabled && xd->corrupted) + pc->refresh_golden_frame = 0; +#endif + pc->refresh_alt_ref_frame = vp8_read_bit(bc); +#if CONFIG_ERROR_CONCEALMENT + /* Assume we shouldn't refresh altref if the bit is missing */ + xd->corrupted |= vp8dx_bool_error(bc); + if (pbi->ec_enabled && xd->corrupted) + pc->refresh_alt_ref_frame = 0; +#endif /* Buffer to buffer copy flags. */ pc->copy_buffer_to_gf = 0; @@ -817,6 +909,13 @@ int vp8_decode_frame(VP8D_COMP *pbi) pc->refresh_last_frame = pc->frame_type == KEY_FRAME || vp8_read_bit(bc); +#if CONFIG_ERROR_CONCEALMENT + /* Assume we should refresh the last frame if the bit is missing */ + xd->corrupted |= vp8dx_bool_error(bc); + if (pbi->ec_enabled && xd->corrupted) + pc->refresh_last_frame = 1; +#endif + if (0) { FILE *z = fopen("decodestats.stt", "a"); @@ -872,6 +971,16 @@ int vp8_decode_frame(VP8D_COMP *pbi) vp8_decode_mode_mvs(pbi); +#if CONFIG_ERROR_CONCEALMENT + if (pbi->ec_enabled && + pbi->mvs_corrupt_from_mb < (unsigned int)pc->mb_cols * pc->mb_rows) + { + /* Motion vectors are missing in this frame. We will try to estimate + * them and then continue decoding the frame as usual */ + vp8_estimate_missing_mvs(pbi); + } +#endif + vpx_memset(pc->above_context, 0, sizeof(ENTROPY_CONTEXT_PLANES) * pc->mb_cols); vpx_memcpy(&xd->block[0].bmi, &xd->mode_info_context->bmi[0], sizeof(B_MODE_INFO)); diff --git a/vp8/decoder/ec_types.h b/vp8/decoder/ec_types.h new file mode 100644 index 000000000..a4f8c78b3 --- /dev/null +++ b/vp8/decoder/ec_types.h @@ -0,0 +1,49 @@ +/* + * Copyright (c) 2011 The WebM project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef VP8_DEC_EC_TYPES_H +#define VP8_DEC_EC_TYPES_H + +#define MAX_OVERLAPS 16 + +/* The area (pixel area in Q6) the block pointed to by bmi overlaps + * another block with. + */ +typedef struct +{ + int overlap; + B_MODE_INFO *bmi; +} OVERLAP_NODE; + +/* Structure to keep track of overlapping blocks on a block level. */ +typedef struct +{ + /* TODO(holmer): This array should be exchanged for a linked list */ + OVERLAP_NODE overlaps[MAX_OVERLAPS]; +} B_OVERLAP; + +/* Structure used to hold all the overlaps of a macroblock. The overlaps of a + * macroblock is further divided into block overlaps. + */ +typedef struct +{ + B_OVERLAP overlaps[16]; +} MB_OVERLAP; + +/* Structure for keeping track of motion vectors and which reference frame they + * refer to. Used for motion vector interpolation. + */ +typedef struct +{ + MV mv; + MV_REFERENCE_FRAME ref_frame; +} EC_BLOCK; + +#endif /* VP8_DEC_EC_TYPES_H */ diff --git a/vp8/decoder/error_concealment.c b/vp8/decoder/error_concealment.c new file mode 100644 index 000000000..dcb5c86a5 --- /dev/null +++ b/vp8/decoder/error_concealment.c @@ -0,0 +1,613 @@ +/* + * Copyright (c) 2011 The WebM project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "error_concealment.h" +#include "onyxd_int.h" +#include "decodemv.h" +#include "vpx_mem/vpx_mem.h" +#include "vp8/common/recon.h" +#include "vp8/common/findnearmv.h" + +#include <assert.h> + +#define MIN(x,y) (((x)<(y))?(x):(y)) +#define MAX(x,y) (((x)>(y))?(x):(y)) + +#define FLOOR(x,q) ((x) & -(1 << (q))) + +#define NUM_NEIGHBORS 20 + +typedef struct ec_position +{ + int row; + int col; +} EC_POS; + +/* + * Regenerate the table in Matlab with: + * x = meshgrid((1:4), (1:4)); + * y = meshgrid((1:4), (1:4))'; + * W = round((1./(sqrt(x.^2 + y.^2))*2^7)); + * W(1,1) = 0; + */ +static const int weights_q7[5][5] = { + { 0, 128, 64, 43, 32 }, + {128, 91, 57, 40, 31 }, + { 64, 57, 45, 36, 29 }, + { 43, 40, 36, 30, 26 }, + { 32, 31, 29, 26, 23 } +}; + +int vp8_alloc_overlap_lists(VP8D_COMP *pbi) +{ + if (pbi->overlaps != NULL) + { + vpx_free(pbi->overlaps); + pbi->overlaps = NULL; + } + pbi->overlaps = vpx_calloc(pbi->common.mb_rows * pbi->common.mb_cols, + sizeof(MB_OVERLAP)); + if (pbi->overlaps == NULL) + return -1; + vpx_memset(pbi->overlaps, 0, + sizeof(MB_OVERLAP) * pbi->common.mb_rows * pbi->common.mb_cols); + return 0; +} + +void vp8_de_alloc_overlap_lists(VP8D_COMP *pbi) +{ + vpx_free(pbi->overlaps); + pbi->overlaps = NULL; +} + +/* Inserts a new overlap area value to the list of overlaps of a block */ +static void assign_overlap(OVERLAP_NODE* overlaps, + B_MODE_INFO *bmi, + int overlap) +{ + int i; + if (overlap <= 0) + return; + /* Find and assign to the next empty overlap node in the list of overlaps. + * Empty is defined as bmi == NULL */ + for (i = 0; i < MAX_OVERLAPS; i++) + { + if (overlaps[i].bmi == NULL) + { + overlaps[i].bmi = bmi; + overlaps[i].overlap = overlap; + break; + } + } +} + +/* Calculates the overlap area between two 4x4 squares, where the first + * square has its upper-left corner at (b1_row, b1_col) and the second + * square has its upper-left corner at (b2_row, b2_col). Doesn't + * properly handle squares which do not overlap. + */ +static int block_overlap(int b1_row, int b1_col, int b2_row, int b2_col) +{ + const int int_top = MAX(b1_row, b2_row); // top + const int int_left = MAX(b1_col, b2_col); // left + /* Since each block is 4x4 pixels, adding 4 (Q3) to the left/top edge + * gives us the right/bottom edge. + */ + const int int_right = MIN(b1_col + (4<<3), b2_col + (4<<3)); // right + const int int_bottom = MIN(b1_row + (4<<3), b2_row + (4<<3)); // bottom + return (int_bottom - int_top) * (int_right - int_left); +} + +/* Calculates the overlap area for all blocks in a macroblock at position + * (mb_row, mb_col) in macroblocks, which are being overlapped by a given + * overlapping block at position (new_row, new_col) (in pixels, Q3). The + * first block being overlapped in the macroblock has position (first_blk_row, + * first_blk_col) in blocks relative the upper-left corner of the image. + */ +static void calculate_overlaps_mb(B_OVERLAP *b_overlaps, B_MODE_INFO *bmi, + int new_row, int new_col, + int mb_row, int mb_col, + int first_blk_row, int first_blk_col) +{ + /* Find the blocks within this MB (defined by mb_row, mb_col) which are + * overlapped by bmi and calculate and assign overlap for each of those + * blocks. */ + + /* Block coordinates relative the upper-left block */ + const int rel_ol_blk_row = first_blk_row - mb_row * 4; + const int rel_ol_blk_col = first_blk_col - mb_col * 4; + /* If the block partly overlaps any previous MB, these coordinates + * can be < 0. We don't want to access blocks in previous MBs. + */ + const int blk_idx = MAX(rel_ol_blk_row,0) * 4 + MAX(rel_ol_blk_col,0); + /* Upper left overlapping block */ + B_OVERLAP *b_ol_ul = &(b_overlaps[blk_idx]); + + /* Calculate and assign overlaps for all blocks in this MB + * which the motion compensated block overlaps + */ + /* Avoid calculating overlaps for blocks in later MBs */ + int end_row = MIN(4 + mb_row * 4 - first_blk_row, 2); + int end_col = MIN(4 + mb_col * 4 - first_blk_col, 2); + int row, col; + + /* Check if new_row and new_col are evenly divisible by 4 (Q3), + * and if so we shouldn't check neighboring blocks + */ + if (new_row >= 0 && (new_row & 0x1F) == 0) + end_row = 1; + if (new_col >= 0 && (new_col & 0x1F) == 0) + end_col = 1; + + /* Check if the overlapping block partly overlaps a previous MB + * and if so, we're overlapping fewer blocks in this MB. + */ + if (new_row < (mb_row*16)<<3) + end_row = 1; + if (new_col < (mb_col*16)<<3) + end_col = 1; + + for (row = 0; row < end_row; ++row) + { + for (col = 0; col < end_col; ++col) + { + /* input in Q3, result in Q6 */ + const int overlap = block_overlap(new_row, new_col, + (((first_blk_row + row) * + 4) << 3), + (((first_blk_col + col) * + 4) << 3)); + assign_overlap(b_ol_ul[row * 4 + col].overlaps, bmi, overlap); + } + } +} + +void vp8_calculate_overlaps(MB_OVERLAP *overlap_ul, + int mb_rows, int mb_cols, + B_MODE_INFO *bmi, + int b_row, int b_col) +{ + MB_OVERLAP *mb_overlap; + int row, col, rel_row, rel_col; + int new_row, new_col; + int end_row, end_col; + int overlap_b_row, overlap_b_col; + int overlap_mb_row, overlap_mb_col; + + /* mb subpixel position */ + row = (4 * b_row) << 3; /* Q3 */ + col = (4 * b_col) << 3; /* Q3 */ + + /* reverse compensate for motion */ + new_row = row - bmi->mv.as_mv.row; + new_col = col - bmi->mv.as_mv.col; + + if (new_row >= ((16*mb_rows) << 3) || new_col >= ((16*mb_cols) << 3)) + { + /* the new block ended up outside the frame */ + return; + } + + if (new_row <= (-4 << 3) || new_col <= (-4 << 3)) + { + /* outside the frame */ + return; + } + /* overlapping block's position in blocks */ + overlap_b_row = FLOOR(new_row / 4, 3) >> 3; + overlap_b_col = FLOOR(new_col / 4, 3) >> 3; + + /* overlapping block's MB position in MBs + * operations are done in Q3 + */ + overlap_mb_row = FLOOR((overlap_b_row << 3) / 4, 3) >> 3; + overlap_mb_col = FLOOR((overlap_b_col << 3) / 4, 3) >> 3; + + end_row = MIN(mb_rows - overlap_mb_row, 2); + end_col = MIN(mb_cols - overlap_mb_col, 2); + + /* Don't calculate overlap for MBs we don't overlap */ + /* Check if the new block row starts at the last block row of the MB */ + if (abs(new_row - ((16*overlap_mb_row) << 3)) < ((3*4) << 3)) + end_row = 1; + /* Check if the new block col starts at the last block col of the MB */ + if (abs(new_col - ((16*overlap_mb_col) << 3)) < ((3*4) << 3)) + end_col = 1; + + /* find the MB(s) this block is overlapping */ + for (rel_row = 0; rel_row < end_row; ++rel_row) + { + for (rel_col = 0; rel_col < end_col; ++rel_col) + { + if (overlap_mb_row + rel_row < 0 || + overlap_mb_col + rel_col < 0) + continue; + mb_overlap = overlap_ul + (overlap_mb_row + rel_row) * mb_cols + + overlap_mb_col + rel_col; + + calculate_overlaps_mb(mb_overlap->overlaps, bmi, + new_row, new_col, + overlap_mb_row + rel_row, + overlap_mb_col + rel_col, + overlap_b_row + rel_row, + overlap_b_col + rel_col); + } + } +} + +/* Estimates a motion vector given the overlapping blocks' motion vectors. + * Filters out all overlapping blocks which do not refer to the correct + * reference frame type. + */ +static void estimate_mv(const OVERLAP_NODE *overlaps, B_MODE_INFO *bmi) +{ + int i; + int overlap_sum = 0; + int row_acc = 0; + int col_acc = 0; + + bmi->mv.as_int = 0; + for (i=0; i < MAX_OVERLAPS; ++i) + { + if (overlaps[i].bmi == NULL) + break; + col_acc += overlaps[i].overlap * overlaps[i].bmi->mv.as_mv.col; + row_acc += overlaps[i].overlap * overlaps[i].bmi->mv.as_mv.row; + overlap_sum += overlaps[i].overlap; + } + if (overlap_sum > 0) + { + /* Q9 / Q6 = Q3 */ + bmi->mv.as_mv.col = col_acc / overlap_sum; + bmi->mv.as_mv.row = row_acc / overlap_sum; + bmi->mode = NEW4X4; + } + else + { + bmi->mv.as_mv.col = 0; + bmi->mv.as_mv.row = 0; + bmi->mode = NEW4X4; + } +} + +/* Estimates all motion vectors for a macroblock given the lists of + * overlaps for each block. Decides whether or not the MVs must be clamped. + */ +static void estimate_mb_mvs(const B_OVERLAP *block_overlaps, + MODE_INFO *mi, + int mb_to_left_edge, + int mb_to_right_edge, + int mb_to_top_edge, + int mb_to_bottom_edge) +{ + int i; + int non_zero_count = 0; + MV * const filtered_mv = &(mi->mbmi.mv.as_mv); + B_MODE_INFO * const bmi = mi->bmi; + filtered_mv->col = 0; + filtered_mv->row = 0; + for (i = 0; i < 16; ++i) + { + /* Estimate vectors for all blocks which are overlapped by this type */ + /* Interpolate/extrapolate the rest of the block's MVs */ + estimate_mv(block_overlaps[i].overlaps, &(bmi[i])); + mi->mbmi.need_to_clamp_mvs = vp8_check_mv_bounds(&bmi[i].mv, + mb_to_left_edge, + mb_to_right_edge, + mb_to_top_edge, + mb_to_bottom_edge); + if (bmi[i].mv.as_int != 0) + { + ++non_zero_count; + filtered_mv->col += bmi[i].mv.as_mv.col; + filtered_mv->row += bmi[i].mv.as_mv.row; + } + } + if (non_zero_count > 0) + { + filtered_mv->col /= non_zero_count; + filtered_mv->row /= non_zero_count; + } +} + +static void calc_prev_mb_overlaps(MB_OVERLAP *overlaps, MODE_INFO *prev_mi, + int mb_row, int mb_col, + int mb_rows, int mb_cols) +{ + int sub_row; + int sub_col; + for (sub_row = 0; sub_row < 4; ++sub_row) + { + for (sub_col = 0; sub_col < 4; ++sub_col) + { + vp8_calculate_overlaps( + overlaps, mb_rows, mb_cols, + &(prev_mi->bmi[sub_row * 4 + sub_col]), + 4 * mb_row + sub_row, + 4 * mb_col + sub_col); + } + } +} + +/* Estimate all missing motion vectors. This function does the same as the one + * above, but has different input arguments. */ +static void estimate_missing_mvs(MB_OVERLAP *overlaps, + MODE_INFO *mi, MODE_INFO *prev_mi, + int mb_rows, int mb_cols, + unsigned int first_corrupt) +{ + int mb_row, mb_col; + vpx_memset(overlaps, 0, sizeof(MB_OVERLAP) * mb_rows * mb_cols); + /* First calculate the overlaps for all blocks */ + for (mb_row = 0; mb_row < mb_rows; ++mb_row) + { + for (mb_col = 0; mb_col < mb_cols; ++mb_col) + { + /* We're only able to use blocks referring to the last frame + * when extrapolating new vectors. + */ + if (prev_mi->mbmi.ref_frame == LAST_FRAME) + { + calc_prev_mb_overlaps(overlaps, prev_mi, + mb_row, mb_col, + mb_rows, mb_cols); + } + ++prev_mi; + } + ++prev_mi; + } + + mb_row = first_corrupt / mb_cols; + mb_col = first_corrupt - mb_row * mb_cols; + mi += mb_row*(mb_cols + 1) + mb_col; + /* Go through all macroblocks in the current image with missing MVs + * and calculate new MVs using the overlaps. + */ + for (; mb_row < mb_rows; ++mb_row) + { + int mb_to_top_edge = -((mb_row * 16)) << 3; + int mb_to_bottom_edge = ((mb_rows - 1 - mb_row) * 16) << 3; + for (; mb_col < mb_cols; ++mb_col) + { + int mb_to_left_edge = -((mb_col * 16) << 3); + int mb_to_right_edge = ((mb_cols - 1 - mb_col) * 16) << 3; + const B_OVERLAP *block_overlaps = + overlaps[mb_row*mb_cols + mb_col].overlaps; + mi->mbmi.ref_frame = LAST_FRAME; + mi->mbmi.mode = SPLITMV; + mi->mbmi.uv_mode = DC_PRED; + mi->mbmi.partitioning = 3; + estimate_mb_mvs(block_overlaps, + mi, + mb_to_left_edge, + mb_to_right_edge, + mb_to_top_edge, + mb_to_bottom_edge); + ++mi; + } + mb_col = 0; + ++mi; + } +} + +void vp8_estimate_missing_mvs(VP8D_COMP *pbi) +{ + VP8_COMMON * const pc = &pbi->common; + estimate_missing_mvs(pbi->overlaps, + pc->mi, pc->prev_mi, + pc->mb_rows, pc->mb_cols, + pbi->mvs_corrupt_from_mb); +} + +static void assign_neighbor(EC_BLOCK *neighbor, MODE_INFO *mi, int block_idx) +{ + assert(mi->mbmi.ref_frame < MAX_REF_FRAMES); + neighbor->ref_frame = mi->mbmi.ref_frame; + neighbor->mv = mi->bmi[block_idx].mv.as_mv; +} + +/* Finds the neighboring blocks of a macroblocks. In the general case + * 20 blocks are found. If a fewer number of blocks are found due to + * image boundaries, those positions in the EC_BLOCK array are left "empty". + * The neighbors are enumerated with the upper-left neighbor as the first + * element, the second element refers to the neighbor to right of the previous + * neighbor, and so on. The last element refers to the neighbor below the first + * neighbor. + */ +static void find_neighboring_blocks(MODE_INFO *mi, + EC_BLOCK *neighbors, + int mb_row, int mb_col, + int mb_rows, int mb_cols, + int mi_stride) +{ + int i = 0; + int j; + if (mb_row > 0) + { + /* upper left */ + if (mb_col > 0) + assign_neighbor(&neighbors[i], mi - mi_stride - 1, 15); + ++i; + /* above */ + for (j = 12; j < 16; ++j, ++i) + assign_neighbor(&neighbors[i], mi - mi_stride, j); + } + else + i += 5; + if (mb_col < mb_cols - 1) + { + /* upper right */ + if (mb_row > 0) + assign_neighbor(&neighbors[i], mi - mi_stride + 1, 12); + ++i; + /* right */ + for (j = 0; j <= 12; j += 4, ++i) + assign_neighbor(&neighbors[i], mi + 1, j); + } + else + i += 5; + if (mb_row < mb_rows - 1) + { + /* lower right */ + if (mb_col < mb_cols - 1) + assign_neighbor(&neighbors[i], mi + mi_stride + 1, 0); + ++i; + /* below */ + for (j = 0; j < 4; ++j, ++i) + assign_neighbor(&neighbors[i], mi + mi_stride, j); + } + else + i += 5; + if (mb_col > 0) + { + /* lower left */ + if (mb_row < mb_rows - 1) + assign_neighbor(&neighbors[i], mi + mi_stride - 1, 4); + ++i; + /* left */ + for (j = 3; j < 16; j += 4, ++i) + { + assign_neighbor(&neighbors[i], mi - 1, j); + } + } + else + i += 5; + assert(i == 20); +} + +/* Calculates which reference frame type is dominating among the neighbors */ +static MV_REFERENCE_FRAME dominant_ref_frame(EC_BLOCK *neighbors) +{ + /* Default to referring to "skip" */ + MV_REFERENCE_FRAME dom_ref_frame = LAST_FRAME; + int max_ref_frame_cnt = 0; + int ref_frame_cnt[MAX_REF_FRAMES] = {0}; + int i; + /* Count neighboring reference frames */ + for (i = 0; i < NUM_NEIGHBORS; ++i) + { + if (neighbors[i].ref_frame < MAX_REF_FRAMES && + neighbors[i].ref_frame != INTRA_FRAME) + ++ref_frame_cnt[neighbors[i].ref_frame]; + } + /* Find maximum */ + for (i = 0; i < MAX_REF_FRAMES; ++i) + { + if (ref_frame_cnt[i] > max_ref_frame_cnt) + { + dom_ref_frame = i; + max_ref_frame_cnt = ref_frame_cnt[i]; + } + } + return dom_ref_frame; +} + +/* Interpolates all motion vectors for a macroblock from the neighboring blocks' + * motion vectors. + */ +static void interpolate_mvs(MACROBLOCKD *mb, + EC_BLOCK *neighbors, + MV_REFERENCE_FRAME dom_ref_frame) +{ + int row, col, i; + MODE_INFO * const mi = mb->mode_info_context; + /* Table with the position of the neighboring blocks relative the position + * of the upper left block of the current MB. Starting with the upper left + * neighbor and going to the right. + */ + const EC_POS neigh_pos[NUM_NEIGHBORS] = { + {-1,-1}, {-1,0}, {-1,1}, {-1,2}, {-1,3}, + {-1,4}, {0,4}, {1,4}, {2,4}, {3,4}, + {4,4}, {4,3}, {4,2}, {4,1}, {4,0}, + {4,-1}, {3,-1}, {2,-1}, {1,-1}, {0,-1} + }; + for (row = 0; row < 4; ++row) + { + for (col = 0; col < 4; ++col) + { + int w_sum = 0; + int mv_row_sum = 0; + int mv_col_sum = 0; + int_mv * const mv = &(mi->bmi[row*4 + col].mv); + for (i = 0; i < NUM_NEIGHBORS; ++i) + { + /* Calculate the weighted sum of neighboring MVs referring + * to the dominant frame type. + */ + const int w = weights_q7[abs(row - neigh_pos[i].row)] + [abs(col - neigh_pos[i].col)]; + if (neighbors[i].ref_frame != dom_ref_frame) + continue; + w_sum += w; + /* Q7 * Q3 = Q10 */ + mv_row_sum += w*neighbors[i].mv.row; + mv_col_sum += w*neighbors[i].mv.col; + } + if (w_sum > 0) + { + /* Avoid division by zero. + * Normalize with the sum of the coefficients + * Q3 = Q10 / Q7 + */ + mv->as_mv.row = mv_row_sum / w_sum; + mv->as_mv.col = mv_col_sum / w_sum; + mi->bmi[row*4 + col].mode = NEW4X4; + mi->mbmi.need_to_clamp_mvs = vp8_check_mv_bounds(mv, + mb->mb_to_left_edge, + mb->mb_to_right_edge, + mb->mb_to_top_edge, + mb->mb_to_bottom_edge); + } + } + } +} + +void vp8_interpolate_motion(MACROBLOCKD *mb, + int mb_row, int mb_col, + int mb_rows, int mb_cols, + int mi_stride) +{ + /* Find relevant neighboring blocks */ + EC_BLOCK neighbors[NUM_NEIGHBORS]; + MV_REFERENCE_FRAME dom_ref_frame; + int i; + /* Initialize the array. MAX_REF_FRAMES is interpreted as "doesn't exist" */ + for (i = 0; i < NUM_NEIGHBORS; ++i) + { + neighbors[i].ref_frame = MAX_REF_FRAMES; + neighbors[i].mv.row = neighbors[i].mv.col = 0; + } + find_neighboring_blocks(mb->mode_info_context, + neighbors, + mb_row, mb_col, + mb_rows, mb_cols, + mb->mode_info_stride); + /* Determine the dominant block type */ + dom_ref_frame = dominant_ref_frame(neighbors); + /* Interpolate MVs for the missing blocks + * from the dominating MVs */ + interpolate_mvs(mb, neighbors, dom_ref_frame); + + mb->mode_info_context->mbmi.ref_frame = dom_ref_frame; + mb->mode_info_context->mbmi.mode = SPLITMV; + mb->mode_info_context->mbmi.uv_mode = DC_PRED; + mb->mode_info_context->mbmi.partitioning = 3; +} + +void vp8_conceal_corrupt_mb(MACROBLOCKD *xd) +{ + /* This macroblock has corrupt residual, use the motion compensated + image (predictor) for concealment */ + vp8_recon_copy16x16(xd->predictor, 16, xd->dst.y_buffer, xd->dst.y_stride); + vp8_recon_copy8x8(xd->predictor + 256, 8, + xd->dst.u_buffer, xd->dst.uv_stride); + vp8_recon_copy8x8(xd->predictor + 320, 8, + xd->dst.v_buffer, xd->dst.uv_stride); +} diff --git a/vp8/decoder/error_concealment.h b/vp8/decoder/error_concealment.h new file mode 100644 index 000000000..65ae9d9be --- /dev/null +++ b/vp8/decoder/error_concealment.h @@ -0,0 +1,41 @@ +/* + * Copyright (c) 2011 The WebM project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + + +#ifndef ERROR_CONCEALMENT_H +#define ERROR_CONCEALMENT_H + +#include "onyxd_int.h" +#include "ec_types.h" + +/* Allocate memory for the overlap lists */ +int vp8_alloc_overlap_lists(VP8D_COMP *pbi); + +/* Deallocate the overlap lists */ +void vp8_de_alloc_overlap_lists(VP8D_COMP *pbi); + +/* Estimate all missing motion vectors. */ +void vp8_estimate_missing_mvs(VP8D_COMP *pbi); + +/* Functions for spatial MV interpolation */ + +/* Interpolates all motion vectors for a macroblock mb at position + * (mb_row, mb_col). */ +void vp8_interpolate_motion(MACROBLOCKD *mb, + int mb_row, int mb_col, + int mb_rows, int mb_cols, + int mi_stride); + +/* Conceal a macroblock with corrupt residual. + * Copies the prediction signal to the reconstructed image. + */ +void vp8_conceal_corrupt_mb(MACROBLOCKD *xd); + +#endif diff --git a/vp8/decoder/onyxd_if.c b/vp8/decoder/onyxd_if.c index 1e83ab542..705fb334f 100644 --- a/vp8/decoder/onyxd_if.c +++ b/vp8/decoder/onyxd_if.c @@ -30,6 +30,9 @@ #include "vp8/common/systemdependent.h" #include "vpx_ports/vpx_timer.h" #include "detokenize.h" +#if CONFIG_ERROR_CONCEALMENT +#include "error_concealment.h" +#endif #if ARCH_ARM #include "vpx_ports/arm.h" #endif @@ -96,6 +99,13 @@ VP8D_PTR vp8dx_create_decompressor(VP8D_CONFIG *oxcf) } pbi->common.error.setjmp = 0; + +#if CONFIG_ERROR_CONCEALMENT + pbi->ec_enabled = oxcf->error_concealment; +#else + pbi->ec_enabled = 0; +#endif + return (VP8D_PTR) pbi; } @@ -112,6 +122,9 @@ void vp8dx_remove_decompressor(VP8D_PTR ptr) vp8mt_de_alloc_temp_buffers(pbi, pbi->common.mb_rows); vp8_decoder_remove_threads(pbi); #endif +#if CONFIG_ERROR_CONCEALMENT + vp8_de_alloc_overlap_lists(pbi); +#endif vp8_remove_common(&pbi->common); vpx_free(pbi); } @@ -271,11 +284,17 @@ int vp8dx_receive_compressed_data(VP8D_PTR ptr, unsigned long size, const unsign */ cm->yv12_fb[cm->lst_fb_idx].corrupted = 1; - /* Signal that we have no frame to show. */ - cm->show_frame = 0; + /* If error concealment is disabled we won't signal missing frames to + * the decoder. + */ + if (!pbi->ec_enabled) + { + /* Signal that we have no frame to show. */ + cm->show_frame = 0; - /* Nothing more to do. */ - return 0; + /* Nothing more to do. */ + return 0; + } } @@ -388,6 +407,28 @@ int vp8dx_receive_compressed_data(VP8D_PTR ptr, unsigned long size, const unsign vp8_clear_system_state(); +#if CONFIG_ERROR_CONCEALMENT + /* swap the mode infos to storage for future error concealment */ + if (pbi->ec_enabled && pbi->common.prev_mi) + { + const MODE_INFO* tmp = pbi->common.prev_mi; + int row, col; + pbi->common.prev_mi = pbi->common.mi; + pbi->common.mi = tmp; + + /* Propagate the segment_ids to the next frame */ + for (row = 0; row < pbi->common.mb_rows; ++row) + { + for (col = 0; col < pbi->common.mb_cols; ++col) + { + const int i = row*pbi->common.mode_info_stride + col; + pbi->common.mi[i].mbmi.segment_id = + pbi->common.prev_mi[i].mbmi.segment_id; + } + } + } +#endif + /*vp8_print_modes_and_motion_vectors( cm->mi, cm->mb_rows,cm->mb_cols, cm->current_video_frame);*/ if (cm->show_frame) diff --git a/vp8/decoder/onyxd_int.h b/vp8/decoder/onyxd_int.h index 9b9175628..ea6de3dda 100644 --- a/vp8/decoder/onyxd_int.h +++ b/vp8/decoder/onyxd_int.h @@ -17,6 +17,9 @@ #include "vp8/common/onyxc_int.h" #include "vp8/common/threading.h" #include "dequantize.h" +#if CONFIG_ERROR_CONCEALMENT +#include "ec_types.h" +#endif typedef struct { @@ -128,6 +131,13 @@ typedef struct VP8Decompressor vp8_prob prob_gf; vp8_prob prob_skip_false; +#if CONFIG_ERROR_CONCEALMENT + MB_OVERLAP *overlaps; + /* the mb num from which modes and mvs (first partition) are corrupt */ + unsigned int mvs_corrupt_from_mb; +#endif + int ec_enabled; + } VP8D_COMP; int vp8_decode_frame(VP8D_COMP *cpi); diff --git a/vp8/vp8_dx_iface.c b/vp8/vp8_dx_iface.c index cca17c23d..e0f4c0a96 100644 --- a/vp8/vp8_dx_iface.c +++ b/vp8/vp8_dx_iface.c @@ -19,6 +19,8 @@ #include "decoder/onyxd_int.h" #define VP8_CAP_POSTPROC (CONFIG_POSTPROC ? VPX_CODEC_CAP_POSTPROC : 0) +#define VP8_CAP_ERROR_CONCEALMENT (CONFIG_ERROR_CONCEALMENT ? \ + VPX_CODEC_CAP_ERROR_CONCEALMENT : 0) typedef vpx_codec_stream_info_t vp8_stream_info_t; @@ -364,6 +366,8 @@ static vpx_codec_err_t vp8_decode(vpx_codec_alg_priv_t *ctx, oxcf.Version = 9; oxcf.postprocess = 0; oxcf.max_threads = ctx->cfg.threads; + oxcf.error_concealment = + (ctx->base.init_flags & VPX_CODEC_USE_ERROR_CONCEALMENT); optr = vp8dx_create_decompressor(&oxcf); @@ -719,7 +723,7 @@ CODEC_INTERFACE(vpx_codec_vp8_dx) = { "WebM Project VP8 Decoder" VERSION_STRING, VPX_CODEC_INTERNAL_ABI_VERSION, - VPX_CODEC_CAP_DECODER | VP8_CAP_POSTPROC, + VPX_CODEC_CAP_DECODER | VP8_CAP_POSTPROC | VP8_CAP_ERROR_CONCEALMENT, /* vpx_codec_caps_t caps; */ vp8_init, /* vpx_codec_init_fn_t init; */ vp8_destroy, /* vpx_codec_destroy_fn_t destroy; */ @@ -749,7 +753,7 @@ vpx_codec_iface_t vpx_codec_vp8_algo = { "WebM Project VP8 Decoder (Deprecated API)" VERSION_STRING, VPX_CODEC_INTERNAL_ABI_VERSION, - VPX_CODEC_CAP_DECODER | VP8_CAP_POSTPROC, + VPX_CODEC_CAP_DECODER | VP8_CAP_POSTPROC | VP8_CAP_ERROR_CONCEALMENT, /* vpx_codec_caps_t caps; */ vp8_init, /* vpx_codec_init_fn_t init; */ vp8_destroy, /* vpx_codec_destroy_fn_t destroy; */ diff --git a/vp8/vp8dx.mk b/vp8/vp8dx.mk index 564967191..85d6f513e 100644 --- a/vp8/vp8dx.mk +++ b/vp8/vp8dx.mk @@ -53,6 +53,9 @@ VP8_DX_SRCS-yes += decoder/decodemv.c VP8_DX_SRCS-yes += decoder/decodframe.c VP8_DX_SRCS-yes += decoder/dequantize.c VP8_DX_SRCS-yes += decoder/detokenize.c +VP8_DX_SRCS-$(CONFIG_ERROR_CONCEALMENT) += decoder/ec_types.h +VP8_DX_SRCS-$(CONFIG_ERROR_CONCEALMENT) += decoder/error_concealment.h +VP8_DX_SRCS-$(CONFIG_ERROR_CONCEALMENT) += decoder/error_concealment.c VP8_DX_SRCS-yes += decoder/generic/dsystemdependent.c VP8_DX_SRCS-yes += decoder/dboolhuff.h VP8_DX_SRCS-yes += decoder/decodemv.h diff --git a/vpx/src/vpx_decoder.c b/vpx/src/vpx_decoder.c index 4ffb00d32..42aedee7e 100644 --- a/vpx/src/vpx_decoder.c +++ b/vpx/src/vpx_decoder.c @@ -36,6 +36,9 @@ vpx_codec_err_t vpx_codec_dec_init_ver(vpx_codec_ctx_t *ctx, res = VPX_CODEC_INCAPABLE; else if ((flags & VPX_CODEC_USE_POSTPROC) && !(iface->caps & VPX_CODEC_CAP_POSTPROC)) res = VPX_CODEC_INCAPABLE; + else if ((flags & VPX_CODEC_USE_ERROR_CONCEALMENT) && + !(iface->caps & VPX_CODEC_CAP_ERROR_CONCEALMENT)) + res = VPX_CODEC_INCAPABLE; else if (!(iface->caps & VPX_CODEC_CAP_DECODER)) res = VPX_CODEC_INCAPABLE; else diff --git a/vpx/vpx_decoder.h b/vpx/vpx_decoder.h index 4c5740945..e50fd765f 100644 --- a/vpx/vpx_decoder.h +++ b/vpx/vpx_decoder.h @@ -53,6 +53,8 @@ extern "C" { #define VPX_CODEC_CAP_PUT_SLICE 0x10000 /**< Will issue put_slice callbacks */ #define VPX_CODEC_CAP_PUT_FRAME 0x20000 /**< Will issue put_frame callbacks */ #define VPX_CODEC_CAP_POSTPROC 0x40000 /**< Can postprocess decoded frame */ +#define VPX_CODEC_CAP_ERROR_CONCEALMENT 0x80000 /**< Can conceal errors due to + packet loss */ /*! \brief Initialization-time Feature Enabling * @@ -62,6 +64,8 @@ extern "C" { * The available flags are specified by VPX_CODEC_USE_* defines. */ #define VPX_CODEC_USE_POSTPROC 0x10000 /**< Postprocess decoded frame */ +#define VPX_CODEC_USE_ERROR_CONCEALMENT 0x20000 /**< Conceal errors in decoded + frames */ /*!\brief Stream properties * @@ -87,6 +87,9 @@ static const arg_def_t threadsarg = ARG_DEF("t", "threads", 1, "Max threads to use"); static const arg_def_t verbosearg = ARG_DEF("v", "verbose", 0, "Show version string"); +static const arg_def_t error_concealment = ARG_DEF(NULL, "error-concealment", 0, + "Enable decoder error-concealment"); + #if CONFIG_MD5 static const arg_def_t md5arg = ARG_DEF(NULL, "md5", 0, @@ -100,6 +103,7 @@ static const arg_def_t *all_args[] = #if CONFIG_MD5 &md5arg, #endif + &error_concealment, NULL }; @@ -700,6 +704,7 @@ int main(int argc, const char **argv_) FILE *infile; int frame_in = 0, frame_out = 0, flipuv = 0, noblit = 0, do_md5 = 0, progress = 0; int stop_after = 0, postproc = 0, summary = 0, quiet = 1; + int ec_enabled = 0; vpx_codec_iface_t *iface = NULL; unsigned int fourcc; unsigned long dx_time = 0; @@ -724,6 +729,7 @@ int main(int argc, const char **argv_) #endif struct input_ctx input = {0}; int frames_corrupted = 0; + int dec_flags = 0; /* Parse command line */ exec_name = argv_[0]; @@ -843,6 +849,10 @@ int main(int argc, const char **argv_) vp8_dbg_display_mv = flags; } } + else if (arg_match(&arg, &error_concealment, argi)) + { + ec_enabled = 1; + } #endif else @@ -963,8 +973,10 @@ int main(int argc, const char **argv_) break; } + dec_flags = (postproc ? VPX_CODEC_USE_POSTPROC : 0) | + (ec_enabled ? VPX_CODEC_USE_ERROR_CONCEALMENT : 0); if (vpx_codec_dec_init(&decoder, iface ? iface : ifaces[0].iface, &cfg, - postproc ? VPX_CODEC_USE_POSTPROC : 0)) + dec_flags)) { fprintf(stderr, "Failed to initialize decoder: %s\n", vpx_codec_error(&decoder)); return EXIT_FAILURE; |