Fix BZ #15089: malloc_trim always trim for large padding.

3 files changed, 50 insertions, 43 deletions

diff --git a/ChangeLog b/ChangeLog
index 7c8164ac7c..c03e0567a8 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,8 @@
+2013-12-06  Fernando J. V. da Silva  <fernandojvdasilva@gmail.com>
+
+	[BZ #15089]
+	* malloc/malloc.c: Exit systrim() if pad is bigger than heap top size.
+
 2013-12-06  Adhemerval Zanella  <azanella@linux.vnet.ibm.com>
 
 	* NEWS: Mention ppc32/power4+ STT_GNU_IFUNC support.
diff --git a/NEWS b/NEWS
index 2e80cbedbf..59ae1c236c 100644
--- a/NEWS
+++ b/NEWS
@@ -12,16 +12,16 @@ Version 2.19
   156, 387, 431, 832, 926, 2801, 4772, 6786, 6787, 6807, 6810, 7003, 9954,
   10253, 10278, 11087, 11157, 11214, 12486, 13028, 13982, 13985, 14029,
   14032, 14143, 14155, 14547, 14699, 14752, 14876, 14910, 15004, 15048,
-  15218, 15268, 15277, 15308, 15362, 15374, 15400, 15425, 15427, 15483,
-  15522, 15531, 15532, 15601, 15608, 15609, 15610, 15632, 15640, 15670,
-  15672, 15680, 15681, 15723, 15734, 15735, 15736, 15748, 15749, 15754,
-  15760, 15763, 15764, 15797, 15799, 15825, 15844, 15847, 15849, 15855,
-  15856, 15857, 15859, 15867, 15886, 15887, 15890, 15892, 15893, 15895,
-  15897, 15901, 15905, 15909, 15915, 15917, 15919, 15921, 15923, 15939,
-  15941, 15948, 15963, 15966, 15985, 15988, 15997, 16032, 16034, 16036,
-  16037, 16038, 16041, 16055, 16071, 16072, 16074, 16077, 16078, 16103,
-  16112, 16143, 16144, 16146, 16150, 16151, 16153, 16167, 16172, 16195,
-  16214, 16245, 16271, 16274, 16283, 16289.
+  15089, 15218, 15268, 15277, 15308, 15362, 15374, 15400, 15425, 15427,
+  15483, 15522, 15531, 15532, 15601, 15608, 15609, 15610, 15632, 15640,
+  15670, 15672, 15680, 15681, 15723, 15734, 15735, 15736, 15748, 15749,
+  15754, 15760, 15763, 15764, 15797, 15799, 15825, 15844, 15847, 15849,
+  15855, 15856, 15857, 15859, 15867, 15886, 15887, 15890, 15892, 15893,
+  15895, 15897, 15901, 15905, 15909, 15915, 15917, 15919, 15921, 15923,
+  15939, 15941, 15948, 15963, 15966, 15985, 15988, 15997, 16032, 16034,
+  16036, 16037, 16038, 16041, 16055, 16071, 16072, 16074, 16077, 16078,
+  16103, 16112, 16143, 16144, 16146, 16150, 16151, 16153, 16167, 16172,
+  16195, 16214, 16245, 16271, 16274, 16283, 16289.
 
 * The public headers no longer use __unused nor __block.  This change is to
   support compiling programs that are derived from BSD sources and use
diff --git a/malloc/malloc.c b/malloc/malloc.c
index 8977687f8c..b3353bd0a3 100644
--- a/malloc/malloc.c
+++ b/malloc/malloc.c
@@ -2709,52 +2709,54 @@ static int systrim(size_t pad, mstate av)
   char* current_brk;     /* address returned by pre-check sbrk call */
   char* new_brk;         /* address returned by post-check sbrk call */
   size_t pagesz;
+  long  top_area;
 
   pagesz = GLRO(dl_pagesize);
   top_size = chunksize(av->top);
 
+  top_area = top_size - MINSIZE - 1;
+  if (top_area <= pad)
+    return 0;
+
   /* Release in pagesize units, keeping at least one page */
-  extra = (top_size - pad - MINSIZE - 1) & ~(pagesz - 1);
+  extra = (top_area - pad) & ~(pagesz - 1);
 
-  if (extra > 0) {
+  /*
+    Only proceed if end of memory is where we last set it.
+    This avoids problems if there were foreign sbrk calls.
+  */
+  current_brk = (char*)(MORECORE(0));
+  if (current_brk == (char*)(av->top) + top_size) {
 
     /*
-      Only proceed if end of memory is where we last set it.
-      This avoids problems if there were foreign sbrk calls.
+      Attempt to release memory. We ignore MORECORE return value,
+      and instead call again to find out where new end of memory is.
+      This avoids problems if first call releases less than we asked,
+      of if failure somehow altered brk value. (We could still
+      encounter problems if it altered brk in some very bad way,
+      but the only thing we can do is adjust anyway, which will cause
+      some downstream failure.)
     */
-    current_brk = (char*)(MORECORE(0));
-    if (current_brk == (char*)(av->top) + top_size) {
-
-      /*
-	Attempt to release memory. We ignore MORECORE return value,
-	and instead call again to find out where new end of memory is.
-	This avoids problems if first call releases less than we asked,
-	of if failure somehow altered brk value. (We could still
-	encounter problems if it altered brk in some very bad way,
-	but the only thing we can do is adjust anyway, which will cause
-	some downstream failure.)
-      */
 
-      MORECORE(-extra);
-      /* Call the `morecore' hook if necessary.  */
-      void (*hook) (void) = force_reg (__after_morecore_hook);
-      if (__builtin_expect (hook != NULL, 0))
-	(*hook) ();
-      new_brk = (char*)(MORECORE(0));
+    MORECORE(-extra);
+    /* Call the `morecore' hook if necessary.  */
+    void (*hook) (void) = force_reg (__after_morecore_hook);
+    if (__builtin_expect (hook != NULL, 0))
+      (*hook) ();
+    new_brk = (char*)(MORECORE(0));
 
-      LIBC_PROBE (memory_sbrk_less, 2, new_brk, extra);
+    LIBC_PROBE (memory_sbrk_less, 2, new_brk, extra);
 
-      if (new_brk != (char*)MORECORE_FAILURE) {
-	released = (long)(current_brk - new_brk);
+    if (new_brk != (char*)MORECORE_FAILURE) {
+      released = (long)(current_brk - new_brk);
 
-	if (released != 0) {
-	  /* Success. Adjust top. */
-	  av->system_mem -= released;
-	  set_head(av->top, (top_size - released) | PREV_INUSE);
-	  check_malloc_state(av);
-	  return 1;
-	}
-      }
+      if (released != 0) {
+	/* Success. Adjust top. */
+	av->system_mem -= released;
+	set_head(av->top, (top_size - released) | PREV_INUSE);
+	check_malloc_state(av);
+	return 1;
+       }
     }
   }
   return 0;