Handle overflow in __hcreate_r

Hi,

As in bugzilla entry there is overflow in hsearch when looking for prime
number as SIZE_MAX - 1 is divisible by 5. We fix that by rejecting large
inputs before looking for prime.

	* misc/hsearch_r.c (__hcreate_r): Handle overflow.

(cherry picked from commit 2f5c1750558fe64bac361f52d6827ab1bcfe52bc)

3 files changed, 20 insertions, 3 deletions

diff --git a/ChangeLog b/ChangeLog
index c0b0a3677c..64dea2e8f1 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,13 @@
+2016-05-24  Ondřej Bílka  <neleai@seznam.cz>
+
+	* debug/strcpy_chk.c: Improve performance.
+	* debug/stpcpy_chk.c: Likewise.
+	* sysdeps/x86_64/strcpy_chk.S: Remove.
+	* sysdeps/x86_64/stpcpy_chk.S: Remove.
+
+	[BZ #18240]
+	* misc/hsearch_r.c (__hcreate_r): Handle overflow.
+
 2016-05-24  Florian Weimer  <fweimer@redhat.com>
 
 	[BZ #19779]
diff --git a/NEWS b/NEWS
index 92e0316113..d60193833b 100644
--- a/NEWS
+++ b/NEWS
@@ -10,8 +10,8 @@ Version 2.20.1
 * The following bugs are resolved with this release:
 
   16009, 16617, 16618, 17266, 17269, 17370, 17371, 17460, 17485, 17555,
-  17625, 17630, 17801, 18032, 18080, 18508, 18665, 18694, 18928, 19018,
-  19682.
+  17625, 17630, 17801, 18032, 18080, 18240, 18508, 18665, 18694, 18928,
+  19018, 19682.
 
 * The glob function suffered from a stack-based buffer overflow when it was
   called with the GLOB_ALTDIRFUNC flag and encountered a long file name.
diff --git a/misc/hsearch_r.c b/misc/hsearch_r.c
index 81c27d800c..4414a25cba 100644
--- a/misc/hsearch_r.c
+++ b/misc/hsearch_r.c
@@ -19,7 +19,7 @@
 #include <errno.h>
 #include <malloc.h>
 #include <string.h>
-
+#include <stdint.h>
 #include <search.h>
 
 /* [Aho,Sethi,Ullman] Compilers: Principles, Techniques and Tools, 1986
@@ -73,6 +73,13 @@ hcreate_r (nel, htab)
       return 0;
     }
 
+  if (nel >= SIZE_MAX / sizeof (_ENTRY))
+    {
+      __set_errno (ENOMEM);
+      return 0;
+    }
+
+
   /* There is still another table active. Return with error. */
   if (htab->table != NULL)
     return 0;