From 9ca907b53e6653866f985831b8580925f4dd4ba4 Mon Sep 17 00:00:00 2001 From: Frank Galligan Date: Fri, 18 Jan 2013 09:44:23 -0800 Subject: libvpx: Fix some warnings. Change-Id: If8be8b9d28a29631f29c46daea8a226ab3580610 --- vp9/encoder/vp9_lookahead.c | 2 +- vp9/encoder/vp9_rdopt.c | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) (limited to 'vp9/encoder') diff --git a/vp9/encoder/vp9_lookahead.c b/vp9/encoder/vp9_lookahead.c index a51c78656..1bca9d267 100644 --- a/vp9/encoder/vp9_lookahead.c +++ b/vp9/encoder/vp9_lookahead.c @@ -174,7 +174,7 @@ vp9_lookahead_peek(struct lookahead_ctx *ctx, int index) { struct lookahead_entry *buf = NULL; - assert(index < ctx->max_sz); + assert(index < (int)ctx->max_sz); if (index < (int)ctx->sz) { index += ctx->read_idx; if (index >= (int)ctx->max_sz) diff --git a/vp9/encoder/vp9_rdopt.c b/vp9/encoder/vp9_rdopt.c index 5e76d9372..e8d0cc68e 100644 --- a/vp9/encoder/vp9_rdopt.c +++ b/vp9/encoder/vp9_rdopt.c @@ -847,7 +847,7 @@ static int vp9_sb_block_error_c(int16_t *coeff, int16_t *dqcoeff, error += this_diff * this_diff; } - return error > INT_MAX ? INT_MAX : error; + return error > INT_MAX ? INT_MAX : (int)error; } #define DEBUG_ERROR 0 @@ -3123,9 +3123,9 @@ static void store_coding_context(MACROBLOCK *x, PICK_MODE_CONTEXT *ctx, ctx->best_ref_mv.as_int = ref_mv->as_int; ctx->second_best_ref_mv.as_int = second_ref_mv->as_int; - ctx->single_pred_diff = comp_pred_diff[SINGLE_PREDICTION_ONLY]; - ctx->comp_pred_diff = comp_pred_diff[COMP_PREDICTION_ONLY]; - ctx->hybrid_pred_diff = comp_pred_diff[HYBRID_PREDICTION]; + ctx->single_pred_diff = (int)comp_pred_diff[SINGLE_PREDICTION_ONLY]; + ctx->comp_pred_diff = (int)comp_pred_diff[COMP_PREDICTION_ONLY]; + ctx->hybrid_pred_diff = (int)comp_pred_diff[HYBRID_PREDICTION]; memcpy(ctx->txfm_rd_diff, txfm_size_diff, sizeof(ctx->txfm_rd_diff)); } -- cgit v1.2.3 From f67d740b342f3a0d7d0605bfb0a13db0987e57e7 Mon Sep 17 00:00:00 2001 From: Frank Galligan Date: Thu, 31 Jan 2013 15:36:55 -0800 Subject: Add support for x64 and win64 yasm flags. Some projects must define only win64 for Windows 64bit builds using yasm. Change-Id: I1d09590d66a7bfc8b4412e1cc8685978ac60b748 --- vp9/encoder/x86/vp9_dct_sse2.asm | 4 ++-- vp9/encoder/x86/vp9_quantize_sse2.asm | 12 ++++++------ vp9/encoder/x86/vp9_quantize_sse4.asm | 6 +++--- vp9/encoder/x86/vp9_quantize_ssse3.asm | 6 +++--- vp9/encoder/x86/vp9_sad_sse3.asm | 8 ++++---- 5 files changed, 18 insertions(+), 18 deletions(-) (limited to 'vp9/encoder') diff --git a/vp9/encoder/x86/vp9_dct_sse2.asm b/vp9/encoder/x86/vp9_dct_sse2.asm index 57b81a566..bbd6086da 100644 --- a/vp9/encoder/x86/vp9_dct_sse2.asm +++ b/vp9/encoder/x86/vp9_dct_sse2.asm @@ -29,7 +29,7 @@ movsxd rax, dword ptr arg(2) lea rcx, [rsi + rax*2] %else - %ifidn __OUTPUT_FORMAT__,x64 + %if LIBVPX_YASM_WIN64 %define input rcx %define output rdx %define pitch r8 @@ -53,7 +53,7 @@ RESTORE_GOT pop rbp %else - %ifidn __OUTPUT_FORMAT__,x64 + %if LIBVPX_YASM_WIN64 RESTORE_XMM %endif %endif diff --git a/vp9/encoder/x86/vp9_quantize_sse2.asm b/vp9/encoder/x86/vp9_quantize_sse2.asm index 060acc2ac..2a686f5a8 100644 --- a/vp9/encoder/x86/vp9_quantize_sse2.asm +++ b/vp9/encoder/x86/vp9_quantize_sse2.asm @@ -27,7 +27,7 @@ sym(vp9_regular_quantize_b_sse2): push rdi push rsi %else - %ifidn __OUTPUT_FORMAT__,x64 + %if LIBVPX_YASM_WIN64 push rdi push rsi %endif @@ -46,7 +46,7 @@ sym(vp9_regular_quantize_b_sse2): mov rdi, arg(0) ; BLOCK *b mov rsi, arg(1) ; BLOCKD *d %else - %ifidn __OUTPUT_FORMAT__,x64 + %if LIBVPX_YASM_WIN64 mov rdi, rcx ; BLOCK *b mov rsi, rdx ; BLOCKD *d %else @@ -223,7 +223,7 @@ ZIGZAG_LOOP 15 pop rsi pop rdi %else - %ifidn __OUTPUT_FORMAT__,x64 + %if LIBVPX_YASM_WIN64 pop rsi pop rdi %endif @@ -247,7 +247,7 @@ sym(vp9_fast_quantize_b_sse2): push rdi push rsi %else - %ifidn __OUTPUT_FORMAT__,x64 + %if LIBVPX_YASM_WIN64 push rdi push rsi %else @@ -261,7 +261,7 @@ sym(vp9_fast_quantize_b_sse2): mov rdi, arg(0) ; BLOCK *b mov rsi, arg(1) ; BLOCKD *d %else - %ifidn __OUTPUT_FORMAT__,x64 + %if LIBVPX_YASM_WIN64 mov rdi, rcx ; BLOCK *b mov rsi, rdx ; BLOCKD *d %else @@ -361,7 +361,7 @@ sym(vp9_fast_quantize_b_sse2): pop rsi pop rdi %else - %ifidn __OUTPUT_FORMAT__,x64 + %if LIBVPX_YASM_WIN64 pop rsi pop rdi %endif diff --git a/vp9/encoder/x86/vp9_quantize_sse4.asm b/vp9/encoder/x86/vp9_quantize_sse4.asm index 1d43ce958..d7779bd0d 100644 --- a/vp9/encoder/x86/vp9_quantize_sse4.asm +++ b/vp9/encoder/x86/vp9_quantize_sse4.asm @@ -31,7 +31,7 @@ sym(vp9_regular_quantize_b_sse4): %define stack_size 32 sub rsp, stack_size %else - %ifidn __OUTPUT_FORMAT__,x64 + %if LIBVPX_YASM_WIN64 SAVE_XMM 8, u push rdi push rsi @@ -43,7 +43,7 @@ sym(vp9_regular_quantize_b_sse4): mov rdi, arg(0) ; BLOCK *b mov rsi, arg(1) ; BLOCKD *d %else - %ifidn __OUTPUT_FORMAT__,x64 + %if LIBVPX_YASM_WIN64 mov rdi, rcx ; BLOCK *b mov rsi, rdx ; BLOCKD *d %else @@ -238,7 +238,7 @@ ZIGZAG_LOOP 15, 7, xmm3, xmm7, xmm8 pop rbp %else %undef xmm5 - %ifidn __OUTPUT_FORMAT__,x64 + %if LIBVPX_YASM_WIN64 pop rsi pop rdi RESTORE_XMM diff --git a/vp9/encoder/x86/vp9_quantize_ssse3.asm b/vp9/encoder/x86/vp9_quantize_ssse3.asm index 41edbc13e..e082af1f5 100644 --- a/vp9/encoder/x86/vp9_quantize_ssse3.asm +++ b/vp9/encoder/x86/vp9_quantize_ssse3.asm @@ -27,7 +27,7 @@ sym(vp9_fast_quantize_b_ssse3): push rdi push rsi %else - %ifidn __OUTPUT_FORMAT__,x64 + %if LIBVPX_YASM_WIN64 push rdi push rsi %endif @@ -38,7 +38,7 @@ sym(vp9_fast_quantize_b_ssse3): mov rdi, arg(0) ; BLOCK *b mov rsi, arg(1) ; BLOCKD *d %else - %ifidn __OUTPUT_FORMAT__,x64 + %if LIBVPX_YASM_WIN64 mov rdi, rcx ; BLOCK *b mov rsi, rdx ; BLOCKD *d %else @@ -122,7 +122,7 @@ sym(vp9_fast_quantize_b_ssse3): pop rsi pop rdi %else - %ifidn __OUTPUT_FORMAT__,x64 + %if LIBVPX_YASM_WIN64 pop rsi pop rdi %endif diff --git a/vp9/encoder/x86/vp9_sad_sse3.asm b/vp9/encoder/x86/vp9_sad_sse3.asm index 2c409cbe5..1c39a08f8 100644 --- a/vp9/encoder/x86/vp9_sad_sse3.asm +++ b/vp9/encoder/x86/vp9_sad_sse3.asm @@ -33,7 +33,7 @@ movsxd rax, dword ptr arg(1) ; src_stride movsxd rdx, dword ptr arg(3) ; ref_stride %else - %ifidn __OUTPUT_FORMAT__,x64 + %if LIBVPX_YASM_WIN64 SAVE_XMM 7, u %define src_ptr rcx %define src_stride rdx @@ -76,7 +76,7 @@ pop rsi pop rbp %else - %ifidn __OUTPUT_FORMAT__,x64 + %if LIBVPX_YASM_WIN64 RESTORE_XMM %endif %endif @@ -111,7 +111,7 @@ xchg rbx, rax %else - %ifidn __OUTPUT_FORMAT__,x64 + %if LIBVPX_YASM_WIN64 SAVE_XMM 7, u %define src_ptr rcx %define src_stride rdx @@ -156,7 +156,7 @@ pop rsi pop rbp %else - %ifidn __OUTPUT_FORMAT__,x64 + %if LIBVPX_YASM_WIN64 pop rsi RESTORE_XMM %endif -- cgit v1.2.3 From 936aa281b589e312aeaca2e7db0ec9dc87a2f99a Mon Sep 17 00:00:00 2001 From: Jingning Han Date: Wed, 20 Feb 2013 15:32:17 -0800 Subject: Fixed the buffer overflow issue The issue that potentially broke the encoding process was due to the fact that the length of token link is calculated from the total number of tokens coded, while it is possible, in high bit-rate setting, this length is greater than the buffer length initially assigned to the cpi->tok. This patch increases the initially allocated buffer length assigned to cpi->tok from (mb_rows * mb_cols * 24 * 16) to (mb_rows * mb_cols * (1 + 24 * 16)). It resolves the buffer overflow problem. Change-Id: I8661a8d39ea0a3c24303e3f71a170787a1d5b1df --- vp9/encoder/vp9_onyx_if.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'vp9/encoder') diff --git a/vp9/encoder/vp9_onyx_if.c b/vp9/encoder/vp9_onyx_if.c index fbdc2d2cc..27e0e48a3 100644 --- a/vp9/encoder/vp9_onyx_if.c +++ b/vp9/encoder/vp9_onyx_if.c @@ -1148,7 +1148,7 @@ void vp9_alloc_compressor_data(VP9_COMP *cpi) { vpx_free(cpi->tok); { - unsigned int tokens = cm->mb_rows * cm->mb_cols * 24 * 16; + unsigned int tokens = cm->mb_rows * cm->mb_cols * (24 * 16 + 1); CHECK_MEM_ERROR(cpi->tok, vpx_calloc(tokens, sizeof(*cpi->tok))); } -- cgit v1.2.3 From fba9772dd2e1c2eb0b0ef506128e3723a3f069dc Mon Sep 17 00:00:00 2001 From: James Zern Date: Fri, 22 Feb 2013 11:42:09 -0800 Subject: vp8_first_pass(): avoid floating point div by 0 Change-Id: Id1e6a12db6b0c1d3f64ead8fd8834aadc30fbed2 --- vp9/encoder/vp9_firstpass.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'vp9/encoder') diff --git a/vp9/encoder/vp9_firstpass.c b/vp9/encoder/vp9_firstpass.c index db981754e..ed944a7c8 100644 --- a/vp9/encoder/vp9_firstpass.c +++ b/vp9/encoder/vp9_firstpass.c @@ -749,7 +749,8 @@ void vp9_first_pass(VP9_COMP *cpi) { ((cm->current_video_frame > 0) && (cpi->twopass.this_frame_stats->pcnt_inter > 0.20) && ((cpi->twopass.this_frame_stats->intra_error / - cpi->twopass.this_frame_stats->coded_error) > 2.0))) { + DOUBLE_DIVIDE_CHECK(cpi->twopass.this_frame_stats->coded_error)) > + 2.0))) { vp8_yv12_copy_frame(lst_yv12, gld_yv12); cpi->twopass.sr_update_lag = 1; } else -- cgit v1.2.3 From 4e00060d296d426814fc01d9ff874b1364691664 Mon Sep 17 00:00:00 2001 From: James Zern Date: Fri, 22 Feb 2013 11:46:34 -0800 Subject: vp8: make gf_group_bits 64-bit avoids signed integer overflow; matches kf_group_bits Change-Id: I193145cdc4fa53e70fba0a1731a03eb1a574931d --- vp9/encoder/vp9_firstpass.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) (limited to 'vp9/encoder') diff --git a/vp9/encoder/vp9_firstpass.c b/vp9/encoder/vp9_firstpass.c index db981754e..f38e5397b 100644 --- a/vp9/encoder/vp9_firstpass.c +++ b/vp9/encoder/vp9_firstpass.c @@ -1707,8 +1707,9 @@ static void define_gf_group(VP9_COMP *cpi, FIRSTPASS_STATS *this_frame) { // Clip cpi->twopass.gf_group_bits based on user supplied data rate // variability limit (cpi->oxcf.two_pass_vbrmax_section) - if (cpi->twopass.gf_group_bits > max_bits * cpi->baseline_gf_interval) - cpi->twopass.gf_group_bits = max_bits * cpi->baseline_gf_interval; + if (cpi->twopass.gf_group_bits > + (int64_t)max_bits * cpi->baseline_gf_interval) + cpi->twopass.gf_group_bits = (int64_t)max_bits * cpi->baseline_gf_interval; // Reset the file position reset_fpf_position(cpi, start_pos); -- cgit v1.2.3 From e5fb6321a12c02035eb0453c9aa0a5721fd5e7f7 Mon Sep 17 00:00:00 2001 From: James Zern Date: Fri, 22 Feb 2013 16:06:48 -0800 Subject: give vp9 variance struct a unique name variance_vtable clashed with vp8/common/variance.h Change-Id: I09c1de44d5519f1bd13f58c01144c0de4706de6f --- vp9/encoder/vp9_variance.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'vp9/encoder') diff --git a/vp9/encoder/vp9_variance.h b/vp9/encoder/vp9_variance.h index 3f0af0855..675dbb63e 100644 --- a/vp9/encoder/vp9_variance.h +++ b/vp9/encoder/vp9_variance.h @@ -69,7 +69,7 @@ typedef unsigned int (*vp9_get16x16prederror_fn_t)(const uint8_t *src_ptr, const uint8_t *ref_ptr, int ref_stride); -typedef struct variance_vtable { +typedef struct vp9_variance_vtable { vp9_sad_fn_t sdf; vp9_variance_fn_t vf; vp9_subpixvariance_fn_t svf; -- cgit v1.2.3 From 82ed3f9a41ed97ebbbd2c764e397a18273158840 Mon Sep 17 00:00:00 2001 From: Jan Kratochvil Date: Wed, 27 Feb 2013 09:51:59 -0800 Subject: Fix --as=nasm compatibility for new asm code. s/movd/movq/ Change-Id: Id1a56de91551f8dc796f14f1056c565dfc1ba626 --- vp9/encoder/x86/vp9_temporal_filter_apply_sse2.asm | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'vp9/encoder') diff --git a/vp9/encoder/x86/vp9_temporal_filter_apply_sse2.asm b/vp9/encoder/x86/vp9_temporal_filter_apply_sse2.asm index a559d5d5a..d2d13b383 100644 --- a/vp9/encoder/x86/vp9_temporal_filter_apply_sse2.asm +++ b/vp9/encoder/x86/vp9_temporal_filter_apply_sse2.asm @@ -50,7 +50,7 @@ sym(vp9_temporal_filter_apply_sse2): ; 0x8000 >> (16 - strength) mov rdx, 16 sub rdx, arg(4) ; 16 - strength - movd xmm4, rdx ; can't use rdx w/ shift + movq xmm4, rdx ; can't use rdx w/ shift movdqa xmm5, [GLOBAL(_const_top_bit)] psrlw xmm5, xmm4 movdqa [rsp + rounding_bit], xmm5 -- cgit v1.2.3 From a07bed2b2b2ca84fdb145ee8e4f1ca100d39915e Mon Sep 17 00:00:00 2001 From: James Zern Date: Thu, 28 Feb 2013 15:45:29 -0800 Subject: firstpass.c: correct casting around gf_group_bits gf_group_bits is int64_t remove casts to int. Change-Id: I3b4225905041fac9af9fdfcbcb6f1c357ea4b593 --- vp9/encoder/vp9_firstpass.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'vp9/encoder') diff --git a/vp9/encoder/vp9_firstpass.c b/vp9/encoder/vp9_firstpass.c index e8525d471..8df6c20a7 100644 --- a/vp9/encoder/vp9_firstpass.c +++ b/vp9/encoder/vp9_firstpass.c @@ -1695,8 +1695,8 @@ static void define_gf_group(VP9_COMP *cpi, FIRSTPASS_STATS *this_frame) { if ((cpi->twopass.kf_group_bits > 0) && (cpi->twopass.kf_group_error_left > 0)) { cpi->twopass.gf_group_bits = - (int)((double)cpi->twopass.kf_group_bits * - (gf_group_err / cpi->twopass.kf_group_error_left)); + (int64_t)(cpi->twopass.kf_group_bits * + (gf_group_err / cpi->twopass.kf_group_error_left)); } else cpi->twopass.gf_group_bits = 0; -- cgit v1.2.3