Fix ldbl-128/ldbl-128ibm lgamma spurious "invalid", incorrect signgam (bug 18952).

The ldbl-128 / ldbl-128ibm implementation of lgammal converts (the
floor of minus) non-integer negative arguments to int to determine the
value of signgam.  When those values are outside the range of int,
this produces spurious "invalid" exceptions and incorrect values of
signgam.  This patch fixes this by instead determining signgam through
comparing half the integer in question to floor of half the integer.

Tested for mips64, x86_64 and x86.

	[BZ #18952]
	* sysdeps/ieee754/ldbl-128/e_lgammal_r.c (__ieee754_lgammal_r): Do
	not convert non-integer negative arguments to int to determine the
	value of signgam.
	* math/auto-libm-test-in: Add more tests of lgamma.
	* math/auto-libm-test-out: Regenerated.

5 files changed, 55 insertions, 3 deletions

diff --git a/ChangeLog b/ChangeLog
index 61a4b539fe..9e23da11b4 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,5 +1,12 @@
 2015-09-11  Joseph Myers  <joseph@codesourcery.com>
 
+	[BZ #18952]
+	* sysdeps/ieee754/ldbl-128/e_lgammal_r.c (__ieee754_lgammal_r): Do
+	not convert non-integer negative arguments to int to determine the
+	value of signgam.
+	* math/auto-libm-test-in: Add more tests of lgamma.
+	* math/auto-libm-test-out: Regenerated.
+
 	* math/auto-libm-test-in: Add more tests of acosh, atanh, cbrt,
 	cosh, csqrt, erfc, expm1 and lgamma.
 	* math/auto-libm-test-out: Regenerated.
diff --git a/NEWS b/NEWS
index 3e78630caf..65e706c9da 100644
--- a/NEWS
+++ b/NEWS
@@ -13,7 +13,7 @@ Version 2.23
   16973, 17787, 17905, 18084, 18086, 18240, 18265, 18370, 18421, 18480,
   18525, 18610, 18618, 18647, 18661, 18674, 18675, 18681, 18757, 18778,
   18781, 18787, 18789, 18790, 18795, 18796, 18820, 18823, 18824, 18863,
-  18870, 18873, 18887, 18921.
+  18870, 18873, 18887, 18921, 18952.
 
 * The obsolete header <regexp.h> has been removed.  Programs that require
   this header must be updated to use <regex.h> instead.
diff --git a/math/auto-libm-test-in b/math/auto-libm-test-in
index 84b3df0bee..83eb4cf8c3 100644
--- a/math/auto-libm-test-in
+++ b/math/auto-libm-test-in
@@ -2047,6 +2047,9 @@ lgamma -0xffffffffffffffffp-1
 lgamma -0x3ffffffffffffffffffffffffffp-1
 lgamma -0x1ffffffffffffffffffffffffffffp-1
 
+lgamma -0x100000000.8p0
+lgamma -0x100000001.8p0
+
 lgamma -0.25
 lgamma -0.5
 lgamma -0.75
diff --git a/math/auto-libm-test-out b/math/auto-libm-test-out
index 14caa1d842..747815b1c9 100644
--- a/math/auto-libm-test-out
+++ b/math/auto-libm-test-out
@@ -153351,6 +153351,48 @@ lgamma -0x1ffffffffffffffffffffffffffffp-1
 = lgamma tonearest ldbl-128 -0xf.fffffffffffffffffffffffffff8p+108L : -0x4.ca1ea7c6bcac53b28539e9281ba4p+116L 1 : inexact-ok
 = lgamma towardzero ldbl-128 -0xf.fffffffffffffffffffffffffff8p+108L : -0x4.ca1ea7c6bcac53b28539e9281ba4p+116L 1 : inexact-ok
 = lgamma upward ldbl-128 -0xf.fffffffffffffffffffffffffff8p+108L : -0x4.ca1ea7c6bcac53b28539e9281ba4p+116L 1 : inexact-ok
+lgamma -0x100000000.8p0
+= lgamma downward dbl-64 -0x1.000000008p+32 : -0x1.52e42ff102e65p+36 -1 : inexact-ok
+= lgamma tonearest dbl-64 -0x1.000000008p+32 : -0x1.52e42ff102e65p+36 -1 : inexact-ok
+= lgamma towardzero dbl-64 -0x1.000000008p+32 : -0x1.52e42ff102e64p+36 -1 : inexact-ok
+= lgamma upward dbl-64 -0x1.000000008p+32 : -0x1.52e42ff102e64p+36 -1 : inexact-ok
+= lgamma downward ldbl-96-intel -0x1.000000008p+32L : -0x1.52e42ff102e64be4p+36L -1 : inexact-ok
+= lgamma tonearest ldbl-96-intel -0x1.000000008p+32L : -0x1.52e42ff102e64be2p+36L -1 : inexact-ok
+= lgamma towardzero ldbl-96-intel -0x1.000000008p+32L : -0x1.52e42ff102e64be2p+36L -1 : inexact-ok
+= lgamma upward ldbl-96-intel -0x1.000000008p+32L : -0x1.52e42ff102e64be2p+36L -1 : inexact-ok
+= lgamma downward ldbl-96-m68k -0x1.000000008p+32L : -0x1.52e42ff102e64be4p+36L -1 : inexact-ok
+= lgamma tonearest ldbl-96-m68k -0x1.000000008p+32L : -0x1.52e42ff102e64be2p+36L -1 : inexact-ok
+= lgamma towardzero ldbl-96-m68k -0x1.000000008p+32L : -0x1.52e42ff102e64be2p+36L -1 : inexact-ok
+= lgamma upward ldbl-96-m68k -0x1.000000008p+32L : -0x1.52e42ff102e64be2p+36L -1 : inexact-ok
+= lgamma downward ldbl-128 -0x1.000000008p+32L : -0x1.52e42ff102e64be289794d246256p+36L -1 : inexact-ok
+= lgamma tonearest ldbl-128 -0x1.000000008p+32L : -0x1.52e42ff102e64be289794d246255p+36L -1 : inexact-ok
+= lgamma towardzero ldbl-128 -0x1.000000008p+32L : -0x1.52e42ff102e64be289794d246255p+36L -1 : inexact-ok
+= lgamma upward ldbl-128 -0x1.000000008p+32L : -0x1.52e42ff102e64be289794d246255p+36L -1 : inexact-ok
+= lgamma downward ldbl-128ibm -0x1.000000008p+32L : -0x1.52e42ff102e64be289794d24628p+36L -1 : inexact-ok
+= lgamma tonearest ldbl-128ibm -0x1.000000008p+32L : -0x1.52e42ff102e64be289794d24628p+36L -1 : inexact-ok
+= lgamma towardzero ldbl-128ibm -0x1.000000008p+32L : -0x1.52e42ff102e64be289794d2462p+36L -1 : inexact-ok
+= lgamma upward ldbl-128ibm -0x1.000000008p+32L : -0x1.52e42ff102e64be289794d2462p+36L -1 : inexact-ok
+lgamma -0x100000001.8p0
+= lgamma downward dbl-64 -0x1.000000018p+32 : -0x1.52e42ff265ca8p+36 1 : inexact-ok
+= lgamma tonearest dbl-64 -0x1.000000018p+32 : -0x1.52e42ff265ca8p+36 1 : inexact-ok
+= lgamma towardzero dbl-64 -0x1.000000018p+32 : -0x1.52e42ff265ca7p+36 1 : inexact-ok
+= lgamma upward dbl-64 -0x1.000000018p+32 : -0x1.52e42ff265ca7p+36 1 : inexact-ok
+= lgamma downward ldbl-96-intel -0x1.000000018p+32L : -0x1.52e42ff265ca7bd4p+36L 1 : inexact-ok
+= lgamma tonearest ldbl-96-intel -0x1.000000018p+32L : -0x1.52e42ff265ca7bd2p+36L 1 : inexact-ok
+= lgamma towardzero ldbl-96-intel -0x1.000000018p+32L : -0x1.52e42ff265ca7bd2p+36L 1 : inexact-ok
+= lgamma upward ldbl-96-intel -0x1.000000018p+32L : -0x1.52e42ff265ca7bd2p+36L 1 : inexact-ok
+= lgamma downward ldbl-96-m68k -0x1.000000018p+32L : -0x1.52e42ff265ca7bd4p+36L 1 : inexact-ok
+= lgamma tonearest ldbl-96-m68k -0x1.000000018p+32L : -0x1.52e42ff265ca7bd2p+36L 1 : inexact-ok
+= lgamma towardzero ldbl-96-m68k -0x1.000000018p+32L : -0x1.52e42ff265ca7bd2p+36L 1 : inexact-ok
+= lgamma upward ldbl-96-m68k -0x1.000000018p+32L : -0x1.52e42ff265ca7bd2p+36L 1 : inexact-ok
+= lgamma downward ldbl-128 -0x1.000000018p+32L : -0x1.52e42ff265ca7bd24518407be41dp+36L 1 : inexact-ok
+= lgamma tonearest ldbl-128 -0x1.000000018p+32L : -0x1.52e42ff265ca7bd24518407be41dp+36L 1 : inexact-ok
+= lgamma towardzero ldbl-128 -0x1.000000018p+32L : -0x1.52e42ff265ca7bd24518407be41cp+36L 1 : inexact-ok
+= lgamma upward ldbl-128 -0x1.000000018p+32L : -0x1.52e42ff265ca7bd24518407be41cp+36L 1 : inexact-ok
+= lgamma downward ldbl-128ibm -0x1.000000018p+32L : -0x1.52e42ff265ca7bd24518407be48p+36L 1 : inexact-ok
+= lgamma tonearest ldbl-128ibm -0x1.000000018p+32L : -0x1.52e42ff265ca7bd24518407be4p+36L 1 : inexact-ok
+= lgamma towardzero ldbl-128ibm -0x1.000000018p+32L : -0x1.52e42ff265ca7bd24518407be4p+36L 1 : inexact-ok
+= lgamma upward ldbl-128ibm -0x1.000000018p+32L : -0x1.52e42ff265ca7bd24518407be4p+36L 1 : inexact-ok
 lgamma -0.25
 = lgamma downward flt-32 -0x4p-4f : 0x1.96ee68p+0f -1 : inexact-ok
 = lgamma tonearest flt-32 -0x4p-4f : 0x1.96ee68p+0f -1 : inexact-ok
diff --git a/sysdeps/ieee754/ldbl-128/e_lgammal_r.c b/sysdeps/ieee754/ldbl-128/e_lgammal_r.c
index abf0f15995..500aacc9c4 100644
--- a/sysdeps/ieee754/ldbl-128/e_lgammal_r.c
+++ b/sysdeps/ieee754/ldbl-128/e_lgammal_r.c
@@ -787,8 +787,8 @@ __ieee754_lgammal_r (long double x, int *signgamp)
       p = __floorl (q);
       if (p == q)
 	return (one / (p - p));
-      i = p;
-      if ((i & 1) == 0)
+      long double halfp = p * 0.5L;
+      if (halfp == __floorl (halfp))
 	*signgamp = -1;
       else
 	*signgamp = 1;