* init.c (__pthread_initialize_minimal_internal): Check whether

	private futexes are available.
	* allocatestack.c (allocate_stack): Copy private_futex field from
	current thread into the new stack.
	* sysdeps/unix/sysv/linux/x86_64/libc-lowlevellock.S: Use private
	futexes if they are available.
	* sysdeps/unix/sysv/linux/i386/i486/libc-lowlevellock.S: Likewise
	* sysdeps/unix/sysv/linux/x86_64/lowlevellock.S: Adjust so that change
	in libc-lowlevellock.S allow using private futexes.
	* sysdeps/unix/sysv/linux/i386/i486/lowlevellock.S: Likewise.
	* sysdeps/unix/sysv/linux/x86_64/lowlevellock.h: Define
	FUTEX_PRIVATE_FLAG.
	* sysdeps/unix/sysv/linux/i386/lowlevellock.h: Likewise.
	* sysdeps/unix/sysv/linux/x86_64/pthread_once.S: Use private futexes
	if they are available.
	* sysdeps/unix/sysv/linux/i386/pthread_once.S: Likewise.
	* sysdeps/x86_64/tcb-offsets.sym: Add PRIVATE_FUTEX.
	* sysdeps/i386/tcb-offsets.sym: Likewise.
	* sysdeps/x86_64/tls.h (tcbhead_t): Add private_futex field.
	* sysdeps/i386/tls.h (tcbhead_t): Likewise.

2 files changed, 38 insertions, 7 deletions

diff --git a/nptl/sysdeps/unix/sysv/linux/i386/i486/libc-lowlevellock.S b/nptl/sysdeps/unix/sysv/linux/i386/i486/libc-lowlevellock.S
index 88885b735d..f246711489 100644
--- a/nptl/sysdeps/unix/sysv/linux/i386/i486/libc-lowlevellock.S
+++ b/nptl/sysdeps/unix/sysv/linux/i386/i486/libc-lowlevellock.S
@@ -1,4 +1,4 @@
-/* Copyright (C) 2002, 2003 Free Software Foundation, Inc.
+/* Copyright (C) 2002, 2003, 2007 Free Software Foundation, Inc.
    This file is part of the GNU C Library.
    Contributed by Ulrich Drepper <drepper@redhat.com>, 2002.
 
@@ -17,6 +17,8 @@
    Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA
    02111-1307 USA.  */
 
+#include <kernel-features.h>
+
 /* In libc.so we do not unconditionally use the lock prefix.  Only if
    the application is using threads.  */
 #ifndef UP
@@ -27,4 +29,17 @@
 0:
 #endif
 
+/* All locks in libc are private.  Use the kernel feature if possible.  */
+#define FUTEX_PRIVATE_FLAG	128
+#ifdef __ASSUME_PRIVATE_FUTEX
+# define FUTEX_WAIT		(0 | FUTEX_PRIVATE_FLAG)
+# define FUTEX_WAKE		(1 | FUTEX_PRIVATE_FLAG)
+#else
+# define LOAD_FUTEX_WAIT(reg) \
+	movl	%gs:PRIVATE_FUTEX, reg
+# define LOAD_FUTEX_WAKE(reg) \
+	movl	%gs:PRIVATE_FUTEX, reg ; \
+	orl	$FUTEX_WAKE, reg
+#endif
+
 #include "lowlevellock.S"
diff --git a/nptl/sysdeps/unix/sysv/linux/i386/i486/lowlevellock.S b/nptl/sysdeps/unix/sysv/linux/i386/i486/lowlevellock.S
index e2da5b04cf..1c47b37bb0 100644
--- a/nptl/sysdeps/unix/sysv/linux/i386/i486/lowlevellock.S
+++ b/nptl/sysdeps/unix/sysv/linux/i386/i486/lowlevellock.S
@@ -1,4 +1,4 @@
-/* Copyright (C) 2002, 2003, 2004, 2006 Free Software Foundation, Inc.
+/* Copyright (C) 2002, 2003, 2004, 2006, 2007 Free Software Foundation, Inc.
    This file is part of the GNU C Library.
    Contributed by Ulrich Drepper <drepper@redhat.com>, 2002.
 
@@ -32,8 +32,22 @@
 
 #define SYS_gettimeofday	__NR_gettimeofday
 #define SYS_futex		240
-#define FUTEX_WAIT		0
-#define FUTEX_WAKE		1
+#ifndef FUTEX_WAIT
+# define FUTEX_WAIT		0
+# define FUTEX_WAKE		1
+#endif
+
+#ifndef LOAD_FUTEX_WAIT
+# if FUTEX_WAIT == 0
+#  define LOAD_FUTEX_WAIT(reg) \
+	xorl	reg, reg
+# else
+#  define LOAD_FUTEX_WAIT(reg) \
+	movl	$FUTEX_WAIT, reg
+# endif
+# define LOAD_FUTEX_WAKE(reg) \
+	movl	$FUTEX_WAKE, reg
+#endif
 
 
 	.globl	__lll_mutex_lock_wait
@@ -55,7 +69,7 @@ __lll_mutex_lock_wait:
 	movl	$2, %edx
 	movl	%ecx, %ebx
 	xorl	%esi, %esi	/* No timeout.  */
-	xorl	%ecx, %ecx	/* movl $FUTEX_WAIT, %ecx */
+	LOAD_FUTEX_WAIT (%ecx)
 
 	cmpl	%edx, %eax	/* NB:	 %edx == 2 */
 	jne 2f
@@ -151,7 +165,7 @@ __lll_mutex_timedlock_wait:
 
 	/* Futex call.  */
 	movl	%esp, %esi
-	xorl	%ecx, %ecx	/* movl $FUTEX_WAIT, %ecx */
+	LOAD_FUTEX_WAIT (%ecx)
 	movl	$SYS_futex, %eax
 	ENTER_KERNEL
 	movl	%eax, %ecx
@@ -252,7 +266,7 @@ __lll_mutex_unlock_wake:
 
 	movl	%eax, %ebx
 	movl	$0, (%eax)
-	movl	$FUTEX_WAKE, %ecx
+	LOAD_FUTEX_WAKE (%ecx)
 	movl	$1, %edx	/* Wake one thread.  */
 	movl	$SYS_futex, %eax
 	ENTER_KERNEL
@@ -314,6 +328,8 @@ __lll_timedwait_tid:
 	jz	4f
 
 	movl	%esp, %esi
+	/* XXX The kernel so far uses global futex for the wakeup at
+	   all times.  */
 	xorl	%ecx, %ecx	/* movl $FUTEX_WAIT, %ecx */
 	movl	%ebp, %ebx
 	movl	$SYS_futex, %eax