Use rounds{s,d} for x86 rint, ceil, floor

15 files changed, 339 insertions, 10 deletions

diff --git a/sysdeps/x86_64/dl-machine.h b/sysdeps/x86_64/dl-machine.h
index 6d66ff69c7..1068af6bb9 100644
--- a/sysdeps/x86_64/dl-machine.h
+++ b/sysdeps/x86_64/dl-machine.h
@@ -207,6 +207,10 @@ _dl_start_user:\n\
 /* A reloc type used for ld.so cmdline arg lookups to reject PLT entries.  */
 #define ELF_MACHINE_JMP_SLOT	R_X86_64_JUMP_SLOT
 
+/* The relative ifunc relocation.  */
+// XXX This is a work-around for a broken linker.  Remove!
+#define ELF_MACHINE_IRELATIVE	R_X86_64_IRELATIVE
+
 /* The x86-64 never uses Elf64_Rel relocations.  */
 #define ELF_MACHINE_NO_REL 1
 
diff --git a/sysdeps/x86_64/fpu/bits/mathinline.h b/sysdeps/x86_64/fpu/bits/mathinline.h
index 5bdf47ecbf..c3e03e8ca1 100644
--- a/sysdeps/x86_64/fpu/bits/mathinline.h
+++ b/sysdeps/x86_64/fpu/bits/mathinline.h
@@ -30,32 +30,34 @@
 #endif
 
 
-#if defined __USE_ISOC99 && defined __GNUC__ && __GNUC__ >= 2
+#if defined __GNUC__ && __GNUC__ >= 2
+# ifdef __USE_ISOC99
+__BEGIN_NAMESPACE_C99
 
 /* Test for negative number.  Used in the signbit() macro.  */
 __MATH_INLINE int
 __NTH (__signbitf (float __x))
 {
-# if __WORDSIZE == 32
+#  if __WORDSIZE == 32
   __extension__ union { float __f; int __i; } __u = { __f: __x };
   return __u.__i < 0;
-# else
+#  else
   int __m;
   __asm ("pmovmskb %1, %0" : "=r" (__m) : "x" (__x));
   return __m & 0x8;
-# endif
+#  endif
 }
 __MATH_INLINE int
 __NTH (__signbit (double __x))
 {
-# if __WORDSIZE == 32
+#  if __WORDSIZE == 32
   __extension__ union { double __d; int __i[2]; } __u = { __d: __x };
   return __u.__i[1] < 0;
-# else
+#  else
   int __m;
   __asm ("pmovmskb %1, %0" : "=r" (__m) : "x" (__x));
   return __m & 0x80;
-# endif
+#  endif
 }
 __MATH_INLINE int
 __NTH (__signbitl (long double __x))
@@ -64,9 +66,6 @@ __NTH (__signbitl (long double __x))
   return (__u.__i[2] & 0x8000) != 0;
 }
 
-#ifdef __USE_ISOC99
-__BEGIN_NAMESPACE_C99
-
 /* Round to nearest integer.  */
 #  if __WORDSIZE == 64 || defined __SSE_MATH__
 __MATH_INLINE long int
@@ -101,10 +100,14 @@ __NTH (llrint (double __x))
   __asm ("cvtsd2si %1, %0" : "=r" (__res) : "xm" (__x));
   return __res;
 }
+
+__END_NAMESPACE_C99
 #  endif
 
 #  if defined __FINITE_MATH_ONLY__ && __FINITE_MATH_ONLY__ > 0 \
       && (__WORDSIZE == 64 || defined __SSE2_MATH__)
+__BEGIN_NAMESPACE_C99
+
 /* Determine maximum of two values.  */
 __MATH_INLINE float
 __NTH (fmaxf (float __x, float __y))
@@ -137,4 +140,70 @@ __NTH (fmin (double __x, double __y))
 __END_NAMESPACE_C99
 # endif
 
+# if defined __SSE4_1__ && (__WORDSIZE == 64 || defined __SSE2_MATH__)
+#  if defined __USE_MISC || defined __USE_XOPEN_EXTENDED || defined __USE_ISOC99
+__BEGIN_NAMESPACE_C99
+
+/* Round to nearest integer.  */
+__MATH_INLINE double
+__NTH (rint (double __x))
+{
+  double __res;
+  __asm ("roundsd $4, %1, %0" : "=x" (__res) : "x" (__x));
+  return __res;
+}
+__MATH_INLINE float
+__NTH (rintf (float __x))
+{
+  float __res;
+  __asm ("roundss $4, %1, %0" : "=x" (__res) : "x" (__x));
+  return __res;
+}
+
+__END_NAMESPACE_C99
+#  endif
+
+__BEGIN_NAMESPACE_STD
+/* Smallest integral value not less than X.  */
+__MATH_INLINE double
+__NTH (ceil (double __x))
+{
+  double __res;
+  __asm ("roundsd $2, %1, %0" : "=x" (__res) : "x" (__x));
+  return __res;
+}
+__END_NAMESPACE_STD
+
+__BEGIN_NAMESPACE_C99
+__MATH_INLINE float
+__NTH (ceilf (float __x))
+{
+  float __res;
+  __asm ("roundss $2, %1, %0" : "=x" (__res) : "x" (__x));
+  return __res;
+}
+__END_NAMESPACE_C99
+
+__BEGIN_NAMESPACE_STD
+/* Largest integer not greater than X.  */
+__MATH_INLINE double
+__NTH (ceil (double __x))
+{
+  double __res;
+  __asm ("roundsd $1, %1, %0" : "=x" (__res) : "x" (__x));
+  return __res;
+}
+__END_NAMESPACE_STD
+
+__BEGIN_NAMESPACE_C99
+__MATH_INLINE float
+__NTH (ceilf (float __x))
+{
+  float __res;
+  __asm ("roundss $1, %1, %0" : "=x" (__res) : "x" (__x));
+  return __res;
+}
+__END_NAMESPACE_C99
+# endif
+
 #endif
diff --git a/sysdeps/x86_64/fpu/multiarch/Makefile b/sysdeps/x86_64/fpu/multiarch/Makefile
new file mode 100644
index 0000000000..b29feedd57
--- /dev/null
+++ b/sysdeps/x86_64/fpu/multiarch/Makefile
@@ -0,0 +1,4 @@
+ifeq ($(subdir),math)
+libm-sysdep_routines += s_floor-c s_ceil-c s_floorf-c s_ceilf-c \
+			s_rint-c s_rintf-c
+endif
diff --git a/sysdeps/x86_64/fpu/multiarch/s_ceil-c.c b/sysdeps/x86_64/fpu/multiarch/s_ceil-c.c
new file mode 100644
index 0000000000..6a5ea3ff27
--- /dev/null
+++ b/sysdeps/x86_64/fpu/multiarch/s_ceil-c.c
@@ -0,0 +1,2 @@
+#define __ceil __ceil_c
+#include <sysdeps/ieee754/dbl-64/wordsize-64/s_ceil.c>
diff --git a/sysdeps/x86_64/fpu/multiarch/s_ceil.S b/sysdeps/x86_64/fpu/multiarch/s_ceil.S
new file mode 100644
index 0000000000..d0f8da3443
--- /dev/null
+++ b/sysdeps/x86_64/fpu/multiarch/s_ceil.S
@@ -0,0 +1,40 @@
+/* Copyright (C) 2011 Free Software Foundation, Inc.
+   This file is part of the GNU C Library.
+   Contributed by Ulrich Drepper <drepper@gmail.come>, 2011.
+
+   The GNU C Library is free software; you can redistribute it and/or
+   modify it under the terms of the GNU Lesser General Public
+   License as published by the Free Software Foundation; either
+   version 2.1 of the License, or (at your option) any later version.
+
+   The GNU C Library is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+   Lesser General Public License for more details.
+
+   You should have received a copy of the GNU Lesser General Public
+   License along with the GNU C Library; if not, write to the Free
+   Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA
+   02111-1307 USA.  */
+
+#include <machine/asm.h>
+#include <init-arch.h>
+
+
+ENTRY(__ceil)
+	.type	__ceil, @gnu_indirect_function
+	call	__get_cpu_features@plt
+	movq	%rax, %rdx
+	leaq	__ceil_sse41(%rip), %rax
+	testl	$bit_SSE4_1, CPUID_OFFSET+index_SSE4_1(%rdx)
+	jnz	2f
+	leaq	__ceil_c(%rip), %rax
+2:	ret
+END(__ceil)
+weak_alias (__ceil, ceil)
+
+
+ENTRY(__ceil_sse41)
+	roundsd	$2, %xmm0, %xmm0
+	ret
+END(__ceil_sse41)
diff --git a/sysdeps/x86_64/fpu/multiarch/s_ceilf-c.c b/sysdeps/x86_64/fpu/multiarch/s_ceilf-c.c
new file mode 100644
index 0000000000..229a6273b2
--- /dev/null
+++ b/sysdeps/x86_64/fpu/multiarch/s_ceilf-c.c
@@ -0,0 +1,2 @@
+#define __ceilf __ceilf_c
+#include <sysdeps/ieee754/flt-32/s_ceilf.c>
diff --git a/sysdeps/x86_64/fpu/multiarch/s_ceilf.S b/sysdeps/x86_64/fpu/multiarch/s_ceilf.S
new file mode 100644
index 0000000000..65ce252f6f
--- /dev/null
+++ b/sysdeps/x86_64/fpu/multiarch/s_ceilf.S
@@ -0,0 +1,40 @@
+/* Copyright (C) 2011 Free Software Foundation, Inc.
+   This file is part of the GNU C Library.
+   Contributed by Ulrich Drepper <drepper@gmail.come>, 2011.
+
+   The GNU C Library is free software; you can redistribute it and/or
+   modify it under the terms of the GNU Lesser General Public
+   License as published by the Free Software Foundation; either
+   version 2.1 of the License, or (at your option) any later version.
+
+   The GNU C Library is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+   Lesser General Public License for more details.
+
+   You should have received a copy of the GNU Lesser General Public
+   License along with the GNU C Library; if not, write to the Free
+   Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA
+   02111-1307 USA.  */
+
+#include <machine/asm.h>
+#include <init-arch.h>
+
+
+ENTRY(__ceilf)
+	.type	__ceilf, @gnu_indirect_function
+	call	__get_cpu_features@plt
+	movq	%rax, %rdx
+	leaq	__ceilf_sse41(%rip), %rax
+	testl	$bit_SSE4_1, CPUID_OFFSET+index_SSE4_1(%rdx)
+	jnz	2f
+	leaq	__ceilf_c(%rip), %rax
+2:	ret
+END(__ceilf)
+weak_alias (__ceilf, ceilf)
+
+
+ENTRY(__ceilf_sse41)
+	roundss	$2, %xmm0, %xmm0
+	ret
+END(__ceilf_sse41)
diff --git a/sysdeps/x86_64/fpu/multiarch/s_floor-c.c b/sysdeps/x86_64/fpu/multiarch/s_floor-c.c
new file mode 100644
index 0000000000..8b8c31d945
--- /dev/null
+++ b/sysdeps/x86_64/fpu/multiarch/s_floor-c.c
@@ -0,0 +1,2 @@
+#define __floor __floor_c
+#include <sysdeps/ieee754/dbl-64/wordsize-64/s_floor.c>
diff --git a/sysdeps/x86_64/fpu/multiarch/s_floor.S b/sysdeps/x86_64/fpu/multiarch/s_floor.S
new file mode 100644
index 0000000000..514ea95411
--- /dev/null
+++ b/sysdeps/x86_64/fpu/multiarch/s_floor.S
@@ -0,0 +1,40 @@
+/* Copyright (C) 2011 Free Software Foundation, Inc.
+   This file is part of the GNU C Library.
+   Contributed by Ulrich Drepper <drepper@gmail.come>, 2011.
+
+   The GNU C Library is free software; you can redistribute it and/or
+   modify it under the terms of the GNU Lesser General Public
+   License as published by the Free Software Foundation; either
+   version 2.1 of the License, or (at your option) any later version.
+
+   The GNU C Library is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+   Lesser General Public License for more details.
+
+   You should have received a copy of the GNU Lesser General Public
+   License along with the GNU C Library; if not, write to the Free
+   Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA
+   02111-1307 USA.  */
+
+#include <machine/asm.h>
+#include <init-arch.h>
+
+
+ENTRY(__floor)
+	.type	__floor, @gnu_indirect_function
+	call	__get_cpu_features@plt
+	movq	%rax, %rdx
+	leaq	__floor_sse41(%rip), %rax
+	testl	$bit_SSE4_1, CPUID_OFFSET+index_SSE4_1(%rdx)
+	jnz	2f
+	leaq	__floor_c(%rip), %rax
+2:	ret
+END(__floor)
+weak_alias (__floor, floor)
+
+
+ENTRY(__floor_sse41)
+	roundsd	$1, %xmm0, %xmm0
+	ret
+END(__floor_sse41)
diff --git a/sysdeps/x86_64/fpu/multiarch/s_floorf-c.c b/sysdeps/x86_64/fpu/multiarch/s_floorf-c.c
new file mode 100644
index 0000000000..3f367863a7
--- /dev/null
+++ b/sysdeps/x86_64/fpu/multiarch/s_floorf-c.c
@@ -0,0 +1,2 @@
+#define __floorf __floorf_c
+#include <sysdeps/ieee754/flt-32/s_floorf.c>
diff --git a/sysdeps/x86_64/fpu/multiarch/s_floorf.S b/sysdeps/x86_64/fpu/multiarch/s_floorf.S
new file mode 100644
index 0000000000..d8cd56e496
--- /dev/null
+++ b/sysdeps/x86_64/fpu/multiarch/s_floorf.S
@@ -0,0 +1,40 @@
+/* Copyright (C) 2011 Free Software Foundation, Inc.
+   This file is part of the GNU C Library.
+   Contributed by Ulrich Drepper <drepper@gmail.come>, 2011.
+
+   The GNU C Library is free software; you can redistribute it and/or
+   modify it under the terms of the GNU Lesser General Public
+   License as published by the Free Software Foundation; either
+   version 2.1 of the License, or (at your option) any later version.
+
+   The GNU C Library is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+   Lesser General Public License for more details.
+
+   You should have received a copy of the GNU Lesser General Public
+   License along with the GNU C Library; if not, write to the Free
+   Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA
+   02111-1307 USA.  */
+
+#include <machine/asm.h>
+#include <init-arch.h>
+
+
+ENTRY(__floorf)
+	.type	__floorf, @gnu_indirect_function
+	call	__get_cpu_features@plt
+	movq	%rax, %rdx
+	leaq	__floorf_sse41(%rip), %rax
+	testl	$bit_SSE4_1, CPUID_OFFSET+index_SSE4_1(%rdx)
+	jnz	2f
+	leaq	__floorf_c(%rip), %rax
+2:	ret
+END(__floorf)
+weak_alias (__floorf, floorf)
+
+
+ENTRY(__floorf_sse41)
+	roundss	$1, %xmm0, %xmm0
+	ret
+END(__floorf_sse41)
diff --git a/sysdeps/x86_64/fpu/multiarch/s_rint-c.c b/sysdeps/x86_64/fpu/multiarch/s_rint-c.c
new file mode 100644
index 0000000000..f29f45b062
--- /dev/null
+++ b/sysdeps/x86_64/fpu/multiarch/s_rint-c.c
@@ -0,0 +1,2 @@
+#define __rint __rint_c
+#include <sysdeps/ieee754/dbl-64/wordsize-64/s_rint.c>
diff --git a/sysdeps/x86_64/fpu/multiarch/s_rint.S b/sysdeps/x86_64/fpu/multiarch/s_rint.S
new file mode 100644
index 0000000000..75beffad98
--- /dev/null
+++ b/sysdeps/x86_64/fpu/multiarch/s_rint.S
@@ -0,0 +1,40 @@
+/* Copyright (C) 2011 Free Software Foundation, Inc.
+   This file is part of the GNU C Library.
+   Contributed by Ulrich Drepper <drepper@gmail.come>, 2011.
+
+   The GNU C Library is free software; you can redistribute it and/or
+   modify it under the terms of the GNU Lesser General Public
+   License as published by the Free Software Foundation; either
+   version 2.1 of the License, or (at your option) any later version.
+
+   The GNU C Library is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+   Lesser General Public License for more details.
+
+   You should have received a copy of the GNU Lesser General Public
+   License along with the GNU C Library; if not, write to the Free
+   Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA
+   02111-1307 USA.  */
+
+#include <machine/asm.h>
+#include <init-arch.h>
+
+
+ENTRY(__rint)
+	.type	__rint, @gnu_indirect_function
+	call	__get_cpu_features@plt
+	movq	%rax, %rdx
+	leaq	__rint_sse41(%rip), %rax
+	testl	$bit_SSE4_1, CPUID_OFFSET+index_SSE4_1(%rdx)
+	jnz	2f
+	leaq	__rint_c(%rip), %rax
+2:	ret
+END(__rint)
+weak_alias (__rint, rint)
+
+
+ENTRY(__rint_sse41)
+	roundsd	$4, %xmm0, %xmm0
+	ret
+END(__rint_sse41)
diff --git a/sysdeps/x86_64/fpu/multiarch/s_rintf-c.c b/sysdeps/x86_64/fpu/multiarch/s_rintf-c.c
new file mode 100644
index 0000000000..30ed42a656
--- /dev/null
+++ b/sysdeps/x86_64/fpu/multiarch/s_rintf-c.c
@@ -0,0 +1,2 @@
+#define __rintf __rintf_c
+#include <sysdeps/ieee754/flt-32/s_rintf.c>
diff --git a/sysdeps/x86_64/fpu/multiarch/s_rintf.S b/sysdeps/x86_64/fpu/multiarch/s_rintf.S
new file mode 100644
index 0000000000..512d28cacf
--- /dev/null
+++ b/sysdeps/x86_64/fpu/multiarch/s_rintf.S
@@ -0,0 +1,40 @@
+/* Copyright (C) 2011 Free Software Foundation, Inc.
+   This file is part of the GNU C Library.
+   Contributed by Ulrich Drepper <drepper@gmail.come>, 2011.
+
+   The GNU C Library is free software; you can redistribute it and/or
+   modify it under the terms of the GNU Lesser General Public
+   License as published by the Free Software Foundation; either
+   version 2.1 of the License, or (at your option) any later version.
+
+   The GNU C Library is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+   Lesser General Public License for more details.
+
+   You should have received a copy of the GNU Lesser General Public
+   License along with the GNU C Library; if not, write to the Free
+   Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA
+   02111-1307 USA.  */
+
+#include <machine/asm.h>
+#include <init-arch.h>
+
+
+ENTRY(__rintf)
+	.type	__rintf, @gnu_indirect_function
+	call	__get_cpu_features@plt
+	movq	%rax, %rdx
+	leaq	__rintf_sse41(%rip), %rax
+	testl	$bit_SSE4_1, CPUID_OFFSET+index_SSE4_1(%rdx)
+	jnz	2f
+	leaq	__rintf_c(%rip), %rax
+2:	ret
+END(__rintf)
+weak_alias (__rintf, rintf)
+
+
+ENTRY(__rintf_sse41)
+	roundss	$4, %xmm0, %xmm0
+	ret
+END(__rintf_sse41)