* sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_rdlock.S: Likewise.

	* sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_timedrdlock.S:
	Likewise.
	* sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_timedwrlock.S:
	Likewise.
	* sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_unlock.S: Likewise.
	* sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_wrlock.S: Likewise.

10 files changed, 117 insertions, 29 deletions

diff --git a/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_rdlock.S b/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_rdlock.S
index db0639d21c..763c14b08f 100644
--- a/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_rdlock.S
+++ b/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_rdlock.S
@@ -1,4 +1,4 @@
-/* Copyright (C) 2002, 2003 Free Software Foundation, Inc.
+/* Copyright (C) 2002, 2003, 2007 Free Software Foundation, Inc.
    This file is part of the GNU C Library.
    Contributed by Ulrich Drepper <drepper@redhat.com>, 2002.
 
@@ -77,8 +77,20 @@ __pthread_rwlock_rdlock:
 #endif
 	jne	10f
 
-11:	addl	$READERS_WAKEUP, %ebx
-	movl	%esi, %ecx	/* movl $FUTEX_WAIT, %ecx */
+11:
+#if __ASSUME_PRIVATE_FUTEX
+	movl	$FUTEX_PRIVATE_FLAG|FUTEX_WAIT, %ecx
+	xorl	PSHARED(%ebx), %ecx
+#else
+# if FUTEX_WAIT == 0
+	movl	PSHARED(%ebx), %ecx
+# else
+	movl	$FUTEX_WAIT, %ecx
+	orl	PSHARED(%ebx), %ecx
+# endif
+	xorl	%gs:PRIVATE_FUTEX, %ecx
+#endif
+	addl	$READERS_WAKEUP, %ebx
 	movl	$SYS_futex, %eax
 	ENTER_KERNEL
 
diff --git a/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_timedrdlock.S b/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_timedrdlock.S
index eb5665b432..a6d3db5302 100644
--- a/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_timedrdlock.S
+++ b/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_timedrdlock.S
@@ -1,4 +1,4 @@
-/* Copyright (C) 2002, 2003 Free Software Foundation, Inc.
+/* Copyright (C) 2002, 2003, 2007 Free Software Foundation, Inc.
    This file is part of the GNU C Library.
    Contributed by Ulrich Drepper <drepper@redhat.com>, 2002.
 
@@ -108,8 +108,20 @@ pthread_rwlock_timedrdlock:
 	/* Futex call.  */
 	movl	%ecx, (%esp)	/* Store relative timeout.  */
 	movl	%edx, 4(%esp)
+
 	movl	%esi, %edx
-	xorl	%ecx, %ecx	/* movl $FUTEX_WAIT, %ecx */
+#if __ASSUME_PRIVATE_FUTEX
+	movl	$FUTEX_PRIVATE_FLAG|FUTEX_WAIT, %ecx
+	xorl	PSHARED(%ebp), %ecx
+#else
+# if FUTEX_WAIT == 0
+	movl	PSHARED(%ebp), %ecx
+# else
+	movl	$FUTEX_WAIT, %ecx
+	orl	PSHARED(%ebp), %ecx
+# endif
+	xorl	%gs:PRIVATE_FUTEX, %ecx
+#endif
 	movl	%esp, %esi
 	leal	READERS_WAKEUP(%ebp), %ebx
 	movl	$SYS_futex, %eax
diff --git a/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_timedwrlock.S b/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_timedwrlock.S
index d9db77ba05..6c6e2faf50 100644
--- a/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_timedwrlock.S
+++ b/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_timedwrlock.S
@@ -1,4 +1,4 @@
-/* Copyright (C) 2002, 2003 Free Software Foundation, Inc.
+/* Copyright (C) 2002, 2003, 2007 Free Software Foundation, Inc.
    This file is part of the GNU C Library.
    Contributed by Ulrich Drepper <drepper@redhat.com>, 2002.
 
@@ -106,8 +106,20 @@ pthread_rwlock_timedwrlock:
 	/* Futex call.  */
 	movl	%ecx, (%esp)	/* Store relative timeout.  */
 	movl	%edx, 4(%esp)
+
 	movl	%esi, %edx
-	xorl	%ecx, %ecx	/* movl $FUTEX_WAIT, %ecx */
+#if __ASSUME_PRIVATE_FUTEX
+	movl	$FUTEX_PRIVATE_FLAG|FUTEX_WAIT, %ecx
+	xorl	PSHARED(%ebp), %ecx
+#else
+# if FUTEX_WAIT == 0
+	movl	PSHARED(%ebp), %ecx
+# else
+	movl	$FUTEX_WAIT, %ecx
+	orl	PSHARED(%ebp), %ecx
+# endif
+	xorl	%gs:PRIVATE_FUTEX, %ecx
+#endif
 	movl	%esp, %esi
 	leal	WRITERS_WAKEUP(%ebp), %ebx
 	movl	$SYS_futex, %eax
diff --git a/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_unlock.S b/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_unlock.S
index 64aac3255a..122ac7646e 100644
--- a/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_unlock.S
+++ b/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_unlock.S
@@ -1,4 +1,4 @@
-/* Copyright (C) 2002, 2003 Free Software Foundation, Inc.
+/* Copyright (C) 2002, 2003, 2007 Free Software Foundation, Inc.
    This file is part of the GNU C Library.
    Contributed by Ulrich Drepper <drepper@redhat.com>, 2002.
 
@@ -61,9 +61,8 @@ __pthread_rwlock_unlock:
 
 5:	movl	$0, WRITER(%edi)
 
-	movl	$1, %ecx
+	movl	$1, %edx
 	leal	WRITERS_WAKEUP(%edi), %ebx
-	movl	%ecx, %edx
 	cmpl	$0, WRITERS_QUEUED(%edi)
 	jne	0f
 
@@ -83,7 +82,16 @@ __pthread_rwlock_unlock:
 #endif
 	jne	7f
 
-8:	movl	$SYS_futex, %eax
+8:
+#if __ASSUME_PRIVATE_FUTEX
+	movl	$FUTEX_PRIVATE_FLAG|FUTEX_WAKE, %ecx
+	xorl	PSHARED(%edi), %ecx
+#else
+	movl	$FUTEX_WAKE, %ecx
+	orl	PSHARED(%edi), %ecx
+	xorl	%gs:PRIVATE_FUTEX, %ecx
+#endif
+	movl	$SYS_futex, %eax
 	ENTER_KERNEL
 
 	xorl	%eax, %eax
diff --git a/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_wrlock.S b/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_wrlock.S
index ea9cc170db..9083419a4b 100644
--- a/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_wrlock.S
+++ b/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_wrlock.S
@@ -1,4 +1,4 @@
-/* Copyright (C) 2002, 2003 Free Software Foundation, Inc.
+/* Copyright (C) 2002, 2003, 2007 Free Software Foundation, Inc.
    This file is part of the GNU C Library.
    Contributed by Ulrich Drepper <drepper@redhat.com>, 2002.
 
@@ -75,8 +75,20 @@ __pthread_rwlock_wrlock:
 #endif
 	jne	10f
 
-11:	addl	$WRITERS_WAKEUP, %ebx
-	movl	%esi, %ecx	/* movl $FUTEX_WAIT, %ecx */
+11:
+#if __ASSUME_PRIVATE_FUTEX
+	movl	$FUTEX_PRIVATE_FLAG|FUTEX_WAIT, %ecx
+	xorl	PSHARED(%ebx), %ecx
+#else
+# if FUTEX_WAIT == 0
+	movl	PSHARED(%ebx), %ecx
+# else
+	movl	$FUTEX_WAIT, %ecx
+	orl	PSHARED(%ebx), %ecx
+# endif
+	xorl	%gs:PRIVATE_FUTEX, %ecx
+#endif
+	addl	$WRITERS_WAKEUP, %ebx
 	movl	$SYS_futex, %eax
 	ENTER_KERNEL
 
diff --git a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_rdlock.S b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_rdlock.S
index 5e9d8fb1d6..da98ab3d49 100644
--- a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_rdlock.S
+++ b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_rdlock.S
@@ -1,4 +1,4 @@
-/* Copyright (C) 2002, 2003, 2005 Free Software Foundation, Inc.
+/* Copyright (C) 2002, 2003, 2005, 2007 Free Software Foundation, Inc.
    This file is part of the GNU C Library.
    Contributed by Ulrich Drepper <drepper@redhat.com>, 2002.
 
@@ -20,11 +20,13 @@
 #include <sysdep.h>
 #include <lowlevelrwlock.h>
 #include <pthread-errnos.h>
+#include <kernel-features.h>
 
 
 #define SYS_futex		202
 #define FUTEX_WAIT		0
 #define FUTEX_WAKE		1
+#define FUTEX_PRIVATE_FLAG	128
 
 #ifndef UP
 # define LOCK lock
@@ -73,12 +75,20 @@ __pthread_rwlock_rdlock:
 #endif
 	jne	10f
 
-11:	addq	$READERS_WAKEUP, %rdi
-#if FUTEX_WAIT == 0
-	xorl	%esi, %esi
+11:
+#if __ASSUME_PRIVATE_FUTEX
+	movl	$FUTEX_PRIVATE_FLAG|FUTEX_WAIT, %esi
+	xorl	PSHARED(%rdi), %esi
 #else
+# if FUTEX_WAIT == 0
+	movl	PSHARED(%rdi), %esi
+# else
 	movl	$FUTEX_WAIT, %esi
+	orl	PSHARED(%rdi), %esi
+# endif
+	xorl	%fs:PRIVATE_FUTEX, %esi
 #endif
+	addq	$READERS_WAKEUP, %rdi
 	movl	$SYS_futex, %eax
 	syscall
 
diff --git a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_timedrdlock.S b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_timedrdlock.S
index b44660418a..19ad27aab8 100644
--- a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_timedrdlock.S
+++ b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_timedrdlock.S
@@ -1,4 +1,4 @@
-/* Copyright (C) 2002, 2003, 2004, 2005 Free Software Foundation, Inc.
+/* Copyright (C) 2002, 2003, 2004, 2005, 2007 Free Software Foundation, Inc.
    This file is part of the GNU C Library.
    Contributed by Ulrich Drepper <drepper@redhat.com>, 2002.
 
@@ -112,10 +112,17 @@ pthread_rwlock_timedrdlock:
 	movq	%rcx, (%rsp)	/* Store relative timeout.  */
 	movq	%rdi, 8(%rsp)
 
-#if FUTEX_WAIT == 0
-	xorl	%esi, %esi
+#if __ASSUME_PRIVATE_FUTEX
+	movl	$FUTEX_PRIVATE_FLAG|FUTEX_WAIT, %esi
+	xorl	PSHARED(%rdi), %esi
 #else
+# if FUTEX_WAIT == 0
+	movl	PSHARED(%r12), %esi
+# else
 	movl	$FUTEX_WAIT, %esi
+	orl	PSHARED(%r12), %esi
+# endif
+	xorl	%fs:PRIVATE_FUTEX, %esi
 #endif
 	movq	%rsp, %r10
 	movl	%r14d, %edx
diff --git a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_timedwrlock.S b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_timedwrlock.S
index 525e5b6b93..4c3e7413a8 100644
--- a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_timedwrlock.S
+++ b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_timedwrlock.S
@@ -1,4 +1,4 @@
-/* Copyright (C) 2002, 2003, 2005 Free Software Foundation, Inc.
+/* Copyright (C) 2002, 2003, 2005, 2007 Free Software Foundation, Inc.
    This file is part of the GNU C Library.
    Contributed by Ulrich Drepper <drepper@redhat.com>, 2002.
 
@@ -108,10 +108,17 @@ pthread_rwlock_timedwrlock:
 	movq	%rcx, (%rsp)	/* Store relative timeout.  */
 	movq	%rdi, 8(%rsp)
 
-#if FUTEX_WAIT == 0
-	xorl	%esi, %esi
+#if __ASSUME_PRIVATE_FUTEX
+	movl	$FUTEX_PRIVATE_FLAG|FUTEX_WAIT, %esi
+	xorl	PSHARED(%rdi), %esi
 #else
+# if FUTEX_WAIT == 0
+	movl	PSHARED(%r12), %esi
+# else
 	movl	$FUTEX_WAIT, %esi
+	orl	PSHARED(%r12), %esi
+# endif
+	xorl	%fs:PRIVATE_FUTEX, %esi
 #endif
 	movq	%rsp, %r10
 	movl	%r14d, %edx
diff --git a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_unlock.S b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_unlock.S
index 4ac9ae62ea..bf688252c8 100644
--- a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_unlock.S
+++ b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_unlock.S
@@ -56,8 +56,8 @@ __pthread_rwlock_unlock:
 
 5:	movl	$0, WRITER(%rdi)
 
+	movl	$1, %edx
 	leaq	WRITERS_WAKEUP(%rdi), %r10
-	movq	%rsi, %rdx
 	cmpl	$0, WRITERS_QUEUED(%rdi)
 	jne	0f
 
diff --git a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_wrlock.S b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_wrlock.S
index 0e82f890aa..46d97e033e 100644
--- a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_wrlock.S
+++ b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_wrlock.S
@@ -1,4 +1,4 @@
-/* Copyright (C) 2002, 2003, 2005 Free Software Foundation, Inc.
+/* Copyright (C) 2002, 2003, 2005, 2007 Free Software Foundation, Inc.
    This file is part of the GNU C Library.
    Contributed by Ulrich Drepper <drepper@redhat.com>, 2002.
 
@@ -71,12 +71,20 @@ __pthread_rwlock_wrlock:
 #endif
 	jne	10f
 
-11:	addq	$WRITERS_WAKEUP, %rdi
-#if FUTEX_WAIT == 0
-	xorl	%esi, %esi
+11:
+#if __ASSUME_PRIVATE_FUTEX
+	movl	$FUTEX_PRIVATE_FLAG|FUTEX_WAIT, %esi
+	xorl	PSHARED(%rdi), %esi
 #else
+# if FUTEX_WAIT == 0
+	movl	PSHARED(%rdi), %esi
+# else
 	movl	$FUTEX_WAIT, %esi
+	orl	PSHARED(%rdi), %esi
+# endif
+	xorl	%fs:PRIVATE_FUTEX, %esi
 #endif
+	addq	$WRITERS_WAKEUP, %rdi
 	movl	$SYS_futex, %eax
 	syscall