From fced63ed300c721c115c00af0bd91907702242eb Mon Sep 17 00:00:00 2001 From: Deb Mukherjee Date: Mon, 6 Oct 2014 14:30:01 -0700 Subject: Resolves some static analysis / undefined warnings Also fixes a case of distortion becoming negative and messing up the RDCOST computation. Change-Id: Id345af9e8dfff31ade622be5756e51f2cdface53 --- vp9/encoder/vp9_rdopt.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'vp9/encoder/vp9_rdopt.c') diff --git a/vp9/encoder/vp9_rdopt.c b/vp9/encoder/vp9_rdopt.c index 20a4b7082..eacbbf7da 100644 --- a/vp9/encoder/vp9_rdopt.c +++ b/vp9/encoder/vp9_rdopt.c @@ -232,7 +232,7 @@ static void model_rd_for_sb(VP9_COMP *cpi, BLOCK_SIZE bsize, // Fast approximate the modelling function. if (cpi->oxcf.speed > 4) { int64_t rate; - int64_t square_error = sse; + const int64_t square_error = sum_sse; int quantizer = (pd->dequant[1] >> 3); #if CONFIG_VP9_HIGHBITDEPTH if (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) { @@ -497,7 +497,7 @@ static void block_rd_txfm(int plane, int block, BLOCK_SIZE plane_bsize, if (tx_size != TX_32X32) dc_correct >>= 2; - args->dist = args->sse - dc_correct; + args->dist = MAX(0, args->sse - dc_correct); } } else { // skip forward transform @@ -2448,7 +2448,7 @@ static int64_t handle_inter_mode(VP9_COMP *cpi, MACROBLOCK *x, #if CONFIG_VP9_HIGHBITDEPTH DECLARE_ALIGNED_ARRAY(16, uint16_t, tmp_buf16, MAX_MB_PLANE * 64 * 64); DECLARE_ALIGNED_ARRAY(16, uint8_t, tmp_buf8, MAX_MB_PLANE * 64 * 64); - uint8_t *tmp_buf = tmp_buf8; + uint8_t *tmp_buf; #else DECLARE_ALIGNED_ARRAY(16, uint8_t, tmp_buf, MAX_MB_PLANE * 64 * 64); #endif // CONFIG_VP9_HIGHBITDEPTH -- cgit v1.2.3