vpx_mem: add basic size check

set a max allocable size to prevent overflows in 32-bit and extremely
large allocation attempts in 64-bit. this could be amended to allow size
or num parameters to be 64-bits with the correct size being used at each
call site.

BUG=webm:819

Change-Id: Ia81004d6c4279680714c4488b4f6cf287ab396a5

1 files changed, 34 insertions, 11 deletions

diff --git a/vpx_mem/vpx_mem.c b/vpx_mem/vpx_mem.c
index 783a88bd0..4f4e0dddf 100644
--- a/vpx_mem/vpx_mem.c
+++ b/vpx_mem/vpx_mem.c
@@ -9,18 +9,36 @@
  */
 
 #include "vpx_mem.h"
+#include <limits.h>
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
 #include "include/vpx_mem_intrnl.h"
 #include "vpx/vpx_integer.h"
 
+#if SIZE_MAX > (1ULL << 40)
+#define VPX_MAX_ALLOCABLE_MEMORY (1ULL << 40)
+#else
+// For 32-bit targets keep this below INT_MAX to avoid valgrind warnings.
+#define VPX_MAX_ALLOCABLE_MEMORY ((1ULL << 31) - (1 << 16))
+#endif
+
+// Returns 0 in case of overflow of nmemb * size.
+static int check_size_argument_overflow(uint64_t nmemb, uint64_t size) {
+  const uint64_t total_size = nmemb * size;
+  if (nmemb == 0) return 1;
+  if (size > VPX_MAX_ALLOCABLE_MEMORY / nmemb) return 0;
+  if (total_size != (size_t)total_size) return 0;
+
+  return 1;
+}
+
 static INLINE size_t *get_malloc_address_location(void *const mem) {
   return ((size_t *)mem) - 1;
 }
 
-static INLINE size_t get_aligned_malloc_size(size_t size, size_t align) {
-  return size + align - 1 + ADDRESS_STORAGE_SIZE;
+static INLINE uint64_t get_aligned_malloc_size(size_t size, size_t align) {
+  return (uint64_t)size + align - 1 + ADDRESS_STORAGE_SIZE;
 }
 
 static INLINE void set_actual_malloc_address(void *const mem,
@@ -35,9 +53,11 @@ static INLINE void *get_actual_malloc_address(void *const mem) {
 }
 
 void *vpx_memalign(size_t align, size_t size) {
-  void *x = NULL;
-  const size_t aligned_size = get_aligned_malloc_size(size, align);
-  void *const addr = malloc(aligned_size);
+  void *x = NULL, *addr;
+  const uint64_t aligned_size = get_aligned_malloc_size(size, align);
+  if (!check_size_argument_overflow(1, aligned_size)) return NULL;
+
+  addr = malloc((size_t)aligned_size);
   if (addr) {
     x = align_addr((unsigned char *)addr + ADDRESS_STORAGE_SIZE, (int)align);
     set_actual_malloc_address(x, addr);
@@ -48,9 +68,11 @@ void *vpx_memalign(size_t align, size_t size) {
 void *vpx_malloc(size_t size) { return vpx_memalign(DEFAULT_ALIGNMENT, size); }
 
 void *vpx_calloc(size_t num, size_t size) {
-  const size_t total_size = num * size;
-  void *const x = vpx_malloc(total_size);
-  if (x) memset(x, 0, total_size);
+  void *x;
+  if (!check_size_argument_overflow(num, size)) return NULL;
+
+  x = vpx_malloc(num * size);
+  if (x) memset(x, 0, num * size);
   return x;
 }
 
@@ -71,10 +93,11 @@ void *vpx_realloc(void *memblk, size_t size) {
     vpx_free(memblk);
   else {
     void *addr = get_actual_malloc_address(memblk);
-    const size_t aligned_size =
+    const uint64_t aligned_size =
         get_aligned_malloc_size(size, DEFAULT_ALIGNMENT);
-    memblk = NULL;
-    addr = realloc(addr, aligned_size);
+    if (!check_size_argument_overflow(1, aligned_size)) return NULL;
+
+    addr = realloc(addr, (size_t)aligned_size);
     if (addr) {
       new_addr = align_addr((unsigned char *)addr + ADDRESS_STORAGE_SIZE,
                             DEFAULT_ALIGNMENT);