diff options
| author | James Zern <jzern@google.com> | 2016-06-08 17:33:34 -0700 |
|---|---|---|
| committer | James Zern <jzern@google.com> | 2016-06-08 17:33:34 -0700 |
| commit | 95d2dc8981f1ec2da22784c011e1d11e726601e5 (patch) | |
| tree | 556a45e938d96750c128c8041f0d9bd8ecbb4f28 /test/fdct8x8_test.cc | |
| parent | 06c6e4cbf6e42db1de8589eadd1566c1ef36aed8 (diff) | |
| download | libvpx-95d2dc8981f1ec2da22784c011e1d11e726601e5.tar libvpx-95d2dc8981f1ec2da22784c011e1d11e726601e5.tar.gz libvpx-95d2dc8981f1ec2da22784c011e1d11e726601e5.tar.bz2 libvpx-95d2dc8981f1ec2da22784c011e1d11e726601e5.zip | |
fdct8x8_test: fix unsigned overflow
the difference between src and dst will be signed, the error will be
unsigned.
quiets -fsanitize=integer:
unsigned integer overflow: 4294967295 * 4294967295
Change-Id: I580813093ee46284fde7954520dfcb1188f79268
Diffstat (limited to 'test/fdct8x8_test.cc')
| -rw-r--r-- | test/fdct8x8_test.cc | 8 |
1 files changed, 4 insertions, 4 deletions
diff --git a/test/fdct8x8_test.cc b/test/fdct8x8_test.cc index 0c081ee1f..29f215817 100644 --- a/test/fdct8x8_test.cc +++ b/test/fdct8x8_test.cc @@ -425,10 +425,10 @@ class FwdTrans8x8TestBase { for (int j = 0; j < kNumCoeffs; ++j) { #if CONFIG_VP9_HIGHBITDEPTH - const uint32_t diff = + const int diff = bit_depth_ == VPX_BITS_8 ? dst[j] - src[j] : dst16[j] - src16[j]; #else - const uint32_t diff = dst[j] - src[j]; + const int diff = dst[j] - src[j]; #endif const uint32_t error = diff * diff; EXPECT_GE(1u << 2 * (bit_depth_ - 8), error) @@ -511,10 +511,10 @@ void CompareInvReference(IdctFunc ref_txfm, int thresh) { for (int j = 0; j < kNumCoeffs; ++j) { #if CONFIG_VP9_HIGHBITDEPTH - const uint32_t diff = + const int diff = bit_depth_ == VPX_BITS_8 ? dst[j] - ref[j] : dst16[j] - ref16[j]; #else - const uint32_t diff = dst[j] - ref[j]; + const int diff = dst[j] - ref[j]; #endif const uint32_t error = diff * diff; EXPECT_EQ(0u, error) |