vfprintf: Fix memory with large width and precision [BZ #19931]

Free a previously allocated work buffer if it is not large enough.

3 files changed, 121 insertions, 5 deletions

diff --git a/stdio-common/Makefile b/stdio-common/Makefile
index cc79d347bc..6c597c1926 100644
--- a/stdio-common/Makefile
+++ b/stdio-common/Makefile
@@ -58,16 +58,18 @@ tests := tstscanf test_rdwr test-popen tstgetln test-fseek \
 	 scanf16 scanf17 tst-setvbuf1 tst-grouping bug23 bug24 \
 	 bug-vfprintf-nargs tst-long-dbl-fphex tst-fphex-wide tst-sprintf3 \
 	 bug25 tst-printf-round bug23-2 bug23-3 bug23-4 bug26 tst-fmemopen3 \
-	 tst-printf-bz18872
+	 tst-printf-bz18872 tst-vfprintf-width-prec
 
 test-srcs = tst-unbputc tst-printf
 
 ifeq ($(run-built-tests),yes)
 tests-special += $(objpfx)tst-unbputc.out $(objpfx)tst-printf.out \
 		 $(objpfx)tst-printf-bz18872-mem.out \
-		 $(objpfx)tst-setvbuf1-cmp.out
+		 $(objpfx)tst-setvbuf1-cmp.out \
+		 $(objpfx)tst-vfprintf-width-prec-mem.out
 generated += tst-printf-bz18872.c tst-printf-bz18872.mtrace \
-	     tst-printf-bz18872-mem.out
+	     tst-printf-bz18872-mem.out \
+	     tst-vfprintf-width-prec.mtrace tst-vfprintf-width-prec-mem.out
 endif
 
 include ../Rules
@@ -86,6 +88,8 @@ $(objpfx)tst-swprintf.out: $(gen-locales)
 endif
 
 tst-printf-bz18872-ENV = MALLOC_TRACE=$(objpfx)tst-printf-bz18872.mtrace
+tst-vfprintf-width-prec-ENV = \
+  MALLOC_TRACE=$(objpfx)tst-vfprintf-width-prec.mtrace
 
 $(objpfx)tst-unbputc.out: tst-unbputc.sh $(objpfx)tst-unbputc
 	$(SHELL) $< $(common-objpfx) '$(test-program-prefix)'; \
@@ -100,8 +104,8 @@ $(objpfx)tst-printf.out: tst-printf.sh $(objpfx)tst-printf
 $(objpfx)tst-printf-bz18872.c: tst-printf-bz18872.sh
 	rm -f $@ && $(BASH) $^ > $@.new && mv $@.new $@
 
-$(objpfx)tst-printf-bz18872-mem.out: $(objpfx)tst-printf-bz18872.out
-	$(common-objpfx)malloc/mtrace $(objpfx)tst-printf-bz18872.mtrace > $@; \
+$(objpfx)tst-%-mem.out: $(objpfx)tst-%.out
+	$(common-objpfx)malloc/mtrace $(objpfx)tst-$*.mtrace > $@; \
 	$(evaluate-test)
 
 CFLAGS-vfprintf.c = -Wno-uninitialized
diff --git a/stdio-common/tst-vfprintf-width-prec.c b/stdio-common/tst-vfprintf-width-prec.c
new file mode 100644
index 0000000000..28927419ee
--- /dev/null
+++ b/stdio-common/tst-vfprintf-width-prec.c
@@ -0,0 +1,107 @@
+/* Test for memory leak with large width and precision.
+   Copyright (C) 1991-2016 Free Software Foundation, Inc.
+   This file is part of the GNU C Library.
+
+   The GNU C Library is free software; you can redistribute it and/or
+   modify it under the terms of the GNU Lesser General Public
+   License as published by the Free Software Foundation; either
+   version 2.1 of the License, or (at your option) any later version.
+
+   The GNU C Library is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+   Lesser General Public License for more details.
+
+   You should have received a copy of the GNU Lesser General Public
+   License along with the GNU C Library; if not, see
+   <http://www.gnu.org/licenses/>.  */
+
+#include <mcheck.h>
+#include <stdio.h>
+#include <sys/resource.h>
+#include <wchar.h>
+
+static int
+do_test (void)
+{
+  mtrace ();
+
+  int ret;
+  {
+    char *result;
+    ret = asprintf (&result, "%133000.133001x", 17);
+    if (ret < 0)
+      {
+        printf ("error: asprintf: %m\n");
+        return 1;
+      }
+    free (result);
+  }
+  {
+    wchar_t *result = calloc (ret + 1, sizeof (wchar_t));
+    if (result == NULL)
+      {
+        printf ("error: calloc (%d, %zu): %m", ret + 1, sizeof (wchar_t));
+        return 1;
+      }
+
+    ret = swprintf (result, ret + 1, L"%133000.133001x", 17);
+    if (ret < 0)
+      {
+        printf ("error: swprintf: %d (%m)\n", ret);
+        return 1;
+      }
+    free (result);
+  }
+
+  /* Limit the size of the process, so that the second allocation will
+     fail.  */
+  {
+    struct rlimit limit;
+    if (getrlimit (RLIMIT_AS, &limit) != 0)
+      {
+        printf ("getrlimit (RLIMIT_AS) failed: %m\n");
+        return 1;
+      }
+    long target = 200 * 1024 * 1024;
+    if (limit.rlim_cur == RLIM_INFINITY || limit.rlim_cur > target)
+      {
+        limit.rlim_cur = target;
+        if (setrlimit (RLIMIT_AS, &limit) != 0)
+          {
+            printf ("setrlimit (RLIMIT_AS) failed: %m\n");
+            return 1;
+          }
+      }
+  }
+
+  {
+    char *result;
+    ret = asprintf (&result, "%133000.999999999x", 17);
+    if (ret >= 0)
+      {
+        printf ("error: asprintf: incorrect result %d\n", ret);
+        return 1;
+      }
+  }
+  {
+    wchar_t result[100];
+    if (result == NULL)
+      {
+        printf ("error: calloc (%d, %zu): %m", ret + 1, sizeof (wchar_t));
+        return 1;
+      }
+
+    ret = swprintf (result, 100, L"%133000.999999999x", 17);
+    if (ret >= 0)
+      {
+        printf ("error: swprintf: incorrect result %d\n", ret);
+        return 1;
+      }
+  }
+
+  return 0;
+}
+
+#define TEST_FUNCTION do_test ()
+#include "../test-skeleton.c"
diff --git a/stdio-common/vfprintf.c b/stdio-common/vfprintf.c
index 6829d4dc8e..f24020a585 100644
--- a/stdio-common/vfprintf.c
+++ b/stdio-common/vfprintf.c
@@ -1564,6 +1564,11 @@ vfprintf (FILE *s, const CHAR_T *format, va_list ap)
 	prec = 0;
       if (prec > width && prec > WORK_BUFFER_SIZE - 32)
 	{
+	  /* Deallocate any previously allocated buffer because it is
+	     too small.  */
+	  if (__glibc_unlikely (workstart != NULL))
+	    free (workstart);
+	  workstart = NULL;
 	  if (__glibc_unlikely (prec >= INT_MAX / sizeof (CHAR_T) - 32))
 	    {
 	      __set_errno (EOVERFLOW);