1 files changed, 56 insertions, 25 deletions

diff --git a/vpx_mem/vpx_mem.c b/vpx_mem/vpx_mem.c
index cfced2f7d..4f4e0dddf 100644
--- a/vpx_mem/vpx_mem.c
+++ b/vpx_mem/vpx_mem.c
@@ -9,23 +9,59 @@
  */
 
 #include "vpx_mem.h"
+#include <limits.h>
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
 #include "include/vpx_mem_intrnl.h"
 #include "vpx/vpx_integer.h"
 
-void *vpx_memalign(size_t align, size_t size) {
-  void *addr, *x = NULL;
+#if SIZE_MAX > (1ULL << 40)
+#define VPX_MAX_ALLOCABLE_MEMORY (1ULL << 40)
+#else
+// For 32-bit targets keep this below INT_MAX to avoid valgrind warnings.
+#define VPX_MAX_ALLOCABLE_MEMORY ((1ULL << 31) - (1 << 16))
+#endif
+
+// Returns 0 in case of overflow of nmemb * size.
+static int check_size_argument_overflow(uint64_t nmemb, uint64_t size) {
+  const uint64_t total_size = nmemb * size;
+  if (nmemb == 0) return 1;
+  if (size > VPX_MAX_ALLOCABLE_MEMORY / nmemb) return 0;
+  if (total_size != (size_t)total_size) return 0;
+
+  return 1;
+}
+
+static INLINE size_t *get_malloc_address_location(void *const mem) {
+  return ((size_t *)mem) - 1;
+}
+
+static INLINE uint64_t get_aligned_malloc_size(size_t size, size_t align) {
+  return (uint64_t)size + align - 1 + ADDRESS_STORAGE_SIZE;
+}
+
+static INLINE void set_actual_malloc_address(void *const mem,
+                                             const void *const malloc_addr) {
+  size_t *const malloc_addr_location = get_malloc_address_location(mem);
+  *malloc_addr_location = (size_t)malloc_addr;
+}
+
+static INLINE void *get_actual_malloc_address(void *const mem) {
+  size_t *const malloc_addr_location = get_malloc_address_location(mem);
+  return (void *)(*malloc_addr_location);
+}
 
-  addr = malloc(size + align - 1 + ADDRESS_STORAGE_SIZE);
+void *vpx_memalign(size_t align, size_t size) {
+  void *x = NULL, *addr;
+  const uint64_t aligned_size = get_aligned_malloc_size(size, align);
+  if (!check_size_argument_overflow(1, aligned_size)) return NULL;
 
+  addr = malloc((size_t)aligned_size);
   if (addr) {
     x = align_addr((unsigned char *)addr + ADDRESS_STORAGE_SIZE, (int)align);
-    /* save the actual malloc address */
-    ((size_t *)x)[-1] = (size_t)addr;
+    set_actual_malloc_address(x, addr);
   }
-
   return x;
 }
 
@@ -33,17 +69,15 @@ void *vpx_malloc(size_t size) { return vpx_memalign(DEFAULT_ALIGNMENT, size); }
 
 void *vpx_calloc(size_t num, size_t size) {
   void *x;
+  if (!check_size_argument_overflow(num, size)) return NULL;
 
-  x = vpx_memalign(DEFAULT_ALIGNMENT, num * size);
-
+  x = vpx_malloc(num * size);
   if (x) memset(x, 0, num * size);
-
   return x;
 }
 
 void *vpx_realloc(void *memblk, size_t size) {
-  void *addr, *new_addr = NULL;
-  int align = DEFAULT_ALIGNMENT;
+  void *new_addr = NULL;
 
   /*
   The realloc() function changes the size of the object pointed to by
@@ -58,19 +92,16 @@ void *vpx_realloc(void *memblk, size_t size) {
   else if (!size)
     vpx_free(memblk);
   else {
-    addr = (void *)(((size_t *)memblk)[-1]);
-    memblk = NULL;
-
-    new_addr = realloc(addr, size + align + ADDRESS_STORAGE_SIZE);
-
-    if (new_addr) {
-      addr = new_addr;
-      new_addr =
-          (void *)(((size_t)((unsigned char *)new_addr + ADDRESS_STORAGE_SIZE) +
-                    (align - 1)) &
-                   (size_t)-align);
-      /* save the actual malloc address */
-      ((size_t *)new_addr)[-1] = (size_t)addr;
+    void *addr = get_actual_malloc_address(memblk);
+    const uint64_t aligned_size =
+        get_aligned_malloc_size(size, DEFAULT_ALIGNMENT);
+    if (!check_size_argument_overflow(1, aligned_size)) return NULL;
+
+    addr = realloc(addr, (size_t)aligned_size);
+    if (addr) {
+      new_addr = align_addr((unsigned char *)addr + ADDRESS_STORAGE_SIZE,
+                            DEFAULT_ALIGNMENT);
+      set_actual_malloc_address(new_addr, addr);
     }
   }
 
@@ -79,7 +110,7 @@ void *vpx_realloc(void *memblk, size_t size) {
 
 void vpx_free(void *memblk) {
   if (memblk) {
-    void *addr = (void *)(((size_t *)memblk)[-1]);
+    void *addr = get_actual_malloc_address(memblk);
     free(addr);
   }
 }