Handle overflow in __hcreate_r

Hi,

As in bugzilla entry there is overflow in hsearch when looking for prime
number as SIZE_MAX - 1 is divisible by 5. We fix that by rejecting large
inputs before looking for prime.

	* misc/hsearch_r.c (__hcreate_r): Handle overflow.

2 files changed, 18 insertions, 1 deletions

diff --git a/ChangeLog b/ChangeLog
index 0dfa3b3c8e..73645c3773 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,13 @@
+2015-08-25  Ondřej Bílka  <neleai@seznam.cz>
+
+	* debug/strcpy_chk.c: Improve performance.
+	* debug/stpcpy_chk.c: Likewise.
+	* sysdeps/x86_64/strcpy_chk.S: Remove.
+	* sysdeps/x86_64/stpcpy_chk.S: Remove.
+
+	[BZ #18240]
+	* misc/hsearch_r.c (__hcreate_r): Handle overflow.
+
 2015-08-25  H.J. Lu  <hongjiu.lu@intel.com>
 
 	[BZ #15128]
diff --git a/misc/hsearch_r.c b/misc/hsearch_r.c
index 9f55e845cf..559df29cf7 100644
--- a/misc/hsearch_r.c
+++ b/misc/hsearch_r.c
@@ -19,7 +19,7 @@
 #include <errno.h>
 #include <malloc.h>
 #include <string.h>
-
+#include <stdint.h>
 #include <search.h>
 
 /* [Aho,Sethi,Ullman] Compilers: Principles, Techniques and Tools, 1986
@@ -73,6 +73,13 @@ __hcreate_r (nel, htab)
       return 0;
     }
 
+  if (nel >= SIZE_MAX / sizeof (_ENTRY))
+    {
+      __set_errno (ENOMEM);
+      return 0;
+    }
+
+
   /* There is still another table active. Return with error. */
   if (htab->table != NULL)
     return 0;