diff options
| author | Johann <johannkoenig@google.com> | 2012-02-09 12:38:31 -0800 |
|---|---|---|
| committer | Johann <johannkoenig@google.com> | 2012-02-09 12:38:31 -0800 |
| commit | fea3556e20f63ed1a03daa45804f724e70705aa0 (patch) | |
| tree | 6f77544f3b6063f61876dc22bb0436d6affafa8d /vp8/encoder/arm/armv6 | |
| parent | 2e0d55314c28c546d98595e4b3799ca451a5a777 (diff) | |
| download | libvpx-fea3556e20f63ed1a03daa45804f724e70705aa0.tar libvpx-fea3556e20f63ed1a03daa45804f724e70705aa0.tar.gz libvpx-fea3556e20f63ed1a03daa45804f724e70705aa0.tar.bz2 libvpx-fea3556e20f63ed1a03daa45804f724e70705aa0.zip | |
Fix variance overflow
In the variance calculations the difference is summed and later squared.
When the sum exceeds sqrt(2^31) the value is treated as a negative when
it is shifted which gives incorrect results.
To fix this we cast the result of the multiplication as unsigned.
The alternative fix is to shift sum down by 4 before multiplying.
However that will reduce precision.
For 16x16 blocks the maximum sum is 65280 and sqrt(2^31) is 46340 (and
change).
PPC change is untested.
Change-Id: I1bad27ea0720067def6d71a6da5f789508cec265
Diffstat (limited to 'vp8/encoder/arm/armv6')
4 files changed, 4 insertions, 4 deletions
diff --git a/vp8/encoder/arm/armv6/vp8_variance16x16_armv6.asm b/vp8/encoder/arm/armv6/vp8_variance16x16_armv6.asm index 5feaa8bc2..dc84c30da 100644 --- a/vp8/encoder/arm/armv6/vp8_variance16x16_armv6.asm +++ b/vp8/encoder/arm/armv6/vp8_variance16x16_armv6.asm @@ -144,7 +144,7 @@ loop ldr r6, [sp, #40] ; get address of sse mul r0, r8, r8 ; sum * sum str r11, [r6] ; store sse - sub r0, r11, r0, asr #8 ; return (sse - ((sum * sum) >> 8)) + sub r0, r11, r0, lsr #8 ; return (sse - ((sum * sum) >> 8)) ldmfd sp!, {r4-r12, pc} diff --git a/vp8/encoder/arm/armv6/vp8_variance_halfpixvar16x16_h_armv6.asm b/vp8/encoder/arm/armv6/vp8_variance_halfpixvar16x16_h_armv6.asm index 1b5489795..dd2ce685c 100644 --- a/vp8/encoder/arm/armv6/vp8_variance_halfpixvar16x16_h_armv6.asm +++ b/vp8/encoder/arm/armv6/vp8_variance_halfpixvar16x16_h_armv6.asm @@ -169,7 +169,7 @@ loop ldr r6, [sp, #40] ; get address of sse mul r0, r8, r8 ; sum * sum str r11, [r6] ; store sse - sub r0, r11, r0, asr #8 ; return (sse - ((sum * sum) >> 8)) + sub r0, r11, r0, lsr #8 ; return (sse - ((sum * sum) >> 8)) ldmfd sp!, {r4-r12, pc} diff --git a/vp8/encoder/arm/armv6/vp8_variance_halfpixvar16x16_hv_armv6.asm b/vp8/encoder/arm/armv6/vp8_variance_halfpixvar16x16_hv_armv6.asm index 38c55edf8..f972d9b5b 100644 --- a/vp8/encoder/arm/armv6/vp8_variance_halfpixvar16x16_hv_armv6.asm +++ b/vp8/encoder/arm/armv6/vp8_variance_halfpixvar16x16_hv_armv6.asm @@ -210,7 +210,7 @@ loop ldr r6, [sp, #40] ; get address of sse mul r0, r8, r8 ; sum * sum str r11, [r6] ; store sse - sub r0, r11, r0, asr #8 ; return (sse - ((sum * sum) >> 8)) + sub r0, r11, r0, lsr #8 ; return (sse - ((sum * sum) >> 8)) ldmfd sp!, {r4-r12, pc} diff --git a/vp8/encoder/arm/armv6/vp8_variance_halfpixvar16x16_v_armv6.asm b/vp8/encoder/arm/armv6/vp8_variance_halfpixvar16x16_v_armv6.asm index 22a50eb00..f5da9c09e 100644 --- a/vp8/encoder/arm/armv6/vp8_variance_halfpixvar16x16_v_armv6.asm +++ b/vp8/encoder/arm/armv6/vp8_variance_halfpixvar16x16_v_armv6.asm @@ -171,7 +171,7 @@ loop ldr r6, [sp, #40] ; get address of sse mul r0, r8, r8 ; sum * sum str r11, [r6] ; store sse - sub r0, r11, r0, asr #8 ; return (sse - ((sum * sum) >> 8)) + sub r0, r11, r0, lsr #8 ; return (sse - ((sum * sum) >> 8)) ldmfd sp!, {r4-r12, pc} |