Fix memory leak in regexp compiler (BZ #17069)

(cherry picked from commit 4d43ef1e7434d7d419afbcd754931cb0c794763c)

Conflicts:
	posix/Makefile

4 files changed, 59 insertions, 5 deletions

diff --git a/ChangeLog b/ChangeLog
index a69a26953a..8dd7e30c5e 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,15 @@
+2014-06-19  Andreas Schwab  <schwab@linux-m68k.org>
+
+	[BZ #17069]
+	* posix/regcomp.c (parse_expression): Deallocate partially
+	constructed tree before returning error.
+	* posix/Makefile.c (tests): Add bug-regex36.
+	(generated): Add bug-regex36.mtrace.
+	(tests-special): Add $(objpfx)bug-regex36-mem.out
+	(bug-regex36-ENV): New variable.
+	($(objpfx)bug-regex36-mem.out): New rule.
+	* posix/bug-regex36.c: New file.
+
 2014-06-03  Andreas Schwab  <schwab@suse.de>
 
 	[BZ #15946]
diff --git a/posix/Makefile b/posix/Makefile
index 6709900cb2..9dd5fa47ff 100644
--- a/posix/Makefile
+++ b/posix/Makefile
@@ -86,7 +86,7 @@ tests		:= tstgetopt testfnm runtests runptests	     \
 		   tst-getaddrinfo3 tst-fnmatch2 tst-cpucount tst-cpuset \
 		   bug-getopt1 bug-getopt2 bug-getopt3 bug-getopt4 \
 		   bug-getopt5 tst-getopt_long1 bug-regex34 bug-regex35 \
-		   tst-pathconf tst-getaddrinfo4
+		   tst-pathconf tst-getaddrinfo4 bug-regex36
 xtests		:= bug-ga2
 ifeq (yes,$(build-shared))
 test-srcs	:= globtest
@@ -110,7 +110,7 @@ generated := $(addprefix wordexp-test-result, 1 2 3 4 5 6 7 8 9 10) \
 	     tst-pcre-mem tst-pcre.mtrace tst-boost-mem tst-boost.mtrace \
 	     bug-ga2.mtrace bug-ga2-mem bug-glob2.mtrace bug-glob2-mem \
 	     tst-vfork3-mem tst-vfork3.mtrace getconf.speclist \
-	     tst-fnmatch-mem tst-fnmatch.mtrace
+	     tst-fnmatch-mem tst-fnmatch.mtrace bug-regex36.mtrace
 
 include ../Rules
 
@@ -260,6 +260,12 @@ bug-regex31-ENV = MALLOC_TRACE=$(objpfx)bug-regex31.mtrace
 $(objpfx)bug-regex31-mem: $(objpfx)bug-regex31.out
 	$(common-objpfx)malloc/mtrace $(objpfx)bug-regex31.mtrace > $@
 
+bug-regex36-ENV = MALLOC_TRACE=$(objpfx)bug-regex36.mtrace
+
+$(objpfx)bug-regex36-mem.out: $(objpfx)bug-regex36.out
+	$(common-objpfx)malloc/mtrace $(objpfx)bug-regex36.mtrace > $@; \
+	$(evaluate-test)
+
 tst-vfork3-ENV = MALLOC_TRACE=$(objpfx)tst-vfork3.mtrace
 
 $(objpfx)tst-vfork3-mem: $(objpfx)tst-vfork3.out
diff --git a/posix/bug-regex36.c b/posix/bug-regex36.c
new file mode 100644
index 0000000000..3dda026585
--- /dev/null
+++ b/posix/bug-regex36.c
@@ -0,0 +1,29 @@
+/* Test regcomp not leaking memory on invalid repetition operator
+   Copyright (C) 2014 Free Software Foundation, Inc.
+   This file is part of the GNU C Library.
+
+   The GNU C Library is free software; you can redistribute it and/or
+   modify it under the terms of the GNU Lesser General Public
+   License as published by the Free Software Foundation; either
+   version 2.1 of the License, or (at your option) any later version.
+
+   The GNU C Library is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+   Lesser General Public License for more details.
+
+   You should have received a copy of the GNU Lesser General Public
+   License along with the GNU C Library; if not, see
+   <http://www.gnu.org/licenses/>.  */
+
+#include <mcheck.h>
+#include <regex.h>
+
+int
+main (int argc, char **argv)
+{
+  regex_t r;
+  mtrace ();
+  regcomp (&r, "[a]\\{-2,}", 0);
+  regfree (&r);
+}
diff --git a/posix/regcomp.c b/posix/regcomp.c
index 921d0f49a3..a5020be192 100644
--- a/posix/regcomp.c
+++ b/posix/regcomp.c
@@ -2415,14 +2415,21 @@ parse_expression (re_string_t *regexp, regex_t *preg, re_token_t *token,
   while (token->type == OP_DUP_ASTERISK || token->type == OP_DUP_PLUS
 	 || token->type == OP_DUP_QUESTION || token->type == OP_OPEN_DUP_NUM)
     {
-      tree = parse_dup_op (tree, regexp, dfa, token, syntax, err);
-      if (BE (*err != REG_NOERROR && tree == NULL, 0))
-	return NULL;
+      bin_tree_t *dup_tree = parse_dup_op (tree, regexp, dfa, token, syntax, err);
+      if (BE (*err != REG_NOERROR && dup_tree == NULL, 0))
+	{
+	  if (tree != NULL)
+	    postorder (tree, free_tree, NULL);
+	  return NULL;
+	}
+      tree = dup_tree;
       /* In BRE consecutive duplications are not allowed.  */
       if ((syntax & RE_CONTEXT_INVALID_DUP)
 	  && (token->type == OP_DUP_ASTERISK
 	      || token->type == OP_OPEN_DUP_NUM))
 	{
+	  if (tree != NULL)
+	    postorder (tree, free_tree, NULL);
 	  *err = REG_BADRPT;
 	  return NULL;
 	}