* init.c: Make it compile with older kernel headers.

	* tst-initializers1.c: Show through exit code which test failed.

	* pthread_rwlock_init.c: Also initialize __shared field.
	* sysdeps/unix/sysv/linux/i386/bits/pthreadtypes.h: Split __flags
	element in rwlock structure into four byte elements.  One of them is
	the new __shared element.
	* sysdeps/unix/sysv/linux/x86_64/bits/pthreadtypes.h [__WORDSIZE=32]:
	Likewise.
	[__WORDSIZE=64]: Renamed __pad1 element int rwlock structire to
	__shared, adjust names of other padding elements.
	* sysdeps/unix/sysv/linux/powerpc/bits/pthreadtypes.h: Likewise.
	* sysdeps/pthread/pthread.h: Adjust rwlock initializers.
	* sysdeps/unix/sysv/linux/lowlevelrwlock.sym: Add PSHARED.
	* sysdeps/unix/sysv/linux/powerpc/lowlevellock.h: Define
	FUTEX_PRIVATE_FLAG.
	* sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_rdlock.S: Change main
	futex to use private operations if possible.
	* sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_timedrdlock.S:
	Likewise.
	* sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_timedwrlock.S:
	Likewise.
	* sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_unlock.S: Likewise.
	* sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_wrlock.S: Likewise.

12 files changed, 120 insertions, 38 deletions

diff --git a/nptl/ChangeLog b/nptl/ChangeLog
index fc79f9ad87..9f57cc4de1 100644
--- a/nptl/ChangeLog
+++ b/nptl/ChangeLog
@@ -1,3 +1,31 @@
+2007-05-27  Ulrich Drepper  <drepper@redhat.com>
+
+	* init.c: Make it compile with older kernel headers.
+
+	* tst-initializers1.c: Show through exit code which test failed.
+
+	* pthread_rwlock_init.c: Also initialize __shared field.
+	* sysdeps/unix/sysv/linux/i386/bits/pthreadtypes.h: Split __flags
+	element in rwlock structure into four byte elements.  One of them is
+	the new __shared element.
+	* sysdeps/unix/sysv/linux/x86_64/bits/pthreadtypes.h [__WORDSIZE=32]:
+	Likewise.
+	[__WORDSIZE=64]: Renamed __pad1 element int rwlock structire to
+	__shared, adjust names of other padding elements.
+	* sysdeps/unix/sysv/linux/powerpc/bits/pthreadtypes.h: Likewise.
+	* sysdeps/pthread/pthread.h: Adjust rwlock initializers.
+	* sysdeps/unix/sysv/linux/lowlevelrwlock.sym: Add PSHARED.
+	* sysdeps/unix/sysv/linux/powerpc/lowlevellock.h: Define
+	FUTEX_PRIVATE_FLAG.
+	* sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_rdlock.S: Change main
+	futex to use private operations if possible.
+	* sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_timedrdlock.S:
+	Likewise.
+	* sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_timedwrlock.S:
+	Likewise.
+	* sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_unlock.S: Likewise.
+	* sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_wrlock.S: Likewise.
+
 2007-05-26  Ulrich Drepper  <drepper@redhat.com>
 
 	* pthreadP.h (PTHREAD_RWLOCK_PREFER_READER_P): Define.
diff --git a/nptl/allocatestack.c b/nptl/allocatestack.c
index aad157a9f9..c8c221f4fd 100644
--- a/nptl/allocatestack.c
+++ b/nptl/allocatestack.c
@@ -516,10 +516,9 @@ allocate_stack (const struct pthread_attr *attr, struct pthread **pdp,
 	  __pthread_multiple_threads = *__libc_multiple_threads_ptr = 1;
 #endif
 
-#ifdef THREAD_COPY_PRIVATE_FUTEX
 	  /* The thread must know when private futexes are supported.  */
-	  THREAD_COPY_PRIVATE_FUTEX (pd);
-#endif
+	  pd->header.private_futex = THREAD_GETMEM (THREAD_SELF,
+                                                    header.private_futex);
 
 #ifdef NEED_DL_SYSINFO
 	  /* Copy the sysinfo value from the parent.  */
diff --git a/nptl/init.c b/nptl/init.c
index 212b835a0d..82adeae6a6 100644
--- a/nptl/init.c
+++ b/nptl/init.c
@@ -276,15 +276,15 @@ __pthread_initialize_minimal_internal (void)
 #endif
     set_robust_list_not_avail ();
 
-#ifdef THREAD_SET_PRIVATE_FUTEX
+#ifndef __ASSUME_PRIVATE_FUTEX
   /* Private futexes are always used (at least internally) so that
      doing the test once this early is beneficial.  */
   {
     int word;
-    res = INTERNAL_SYSCALL (futex, err, 3, &word,
+    word = INTERNAL_SYSCALL (futex, err, 3, &word,
 			    FUTEX_WAKE | FUTEX_PRIVATE_FLAG, 1);
-    if (!INTERNAL_SYSCALL_ERROR_P (res, err))
-      pd->header.private_futex = FUTEX_PRIVATE_FLAG;
+    if (!INTERNAL_SYSCALL_ERROR_P (word, err))
+      THREAD_SETMEM (pd, header.private_futex, FUTEX_PRIVATE_FLAG);
   }
 #endif
 
diff --git a/nptl/pthread_rwlock_init.c b/nptl/pthread_rwlock_init.c
index f664dd8104..27f25ac2ab 100644
--- a/nptl/pthread_rwlock_init.c
+++ b/nptl/pthread_rwlock_init.c
@@ -1,4 +1,4 @@
-/* Copyright (C) 2002 Free Software Foundation, Inc.
+/* Copyright (C) 2002, 2007 Free Software Foundation, Inc.
    This file is part of the GNU C Library.
    Contributed by Ulrich Drepper <drepper@redhat.com>, 2002.
 
@@ -18,6 +18,7 @@
    02111-1307 USA.  */
 
 #include "pthreadP.h"
+#include <kernel-features.h>
 
 
 static const struct pthread_rwlockattr default_attr =
@@ -37,14 +38,44 @@ __pthread_rwlock_init (rwlock, attr)
   iattr = ((const struct pthread_rwlockattr *) attr) ?: &default_attr;
 
   rwlock->__data.__lock = 0;
-  rwlock->__data.__flags
-    = iattr->lockkind == PTHREAD_RWLOCK_PREFER_WRITER_NONRECURSIVE_NP;
   rwlock->__data.__nr_readers = 0;
-  rwlock->__data.__writer = 0;
   rwlock->__data.__readers_wakeup = 0;
   rwlock->__data.__writer_wakeup = 0;
   rwlock->__data.__nr_readers_queued = 0;
   rwlock->__data.__nr_writers_queued = 0;
+  rwlock->__data.__writer = 0;
+
+  rwlock->__data.__flags
+    = iattr->lockkind == PTHREAD_RWLOCK_PREFER_WRITER_NONRECURSIVE_NP;
+
+  /* The __SHARED field is computed to minimize the work that needs to
+     be done while handling the futex.  There are two inputs: the
+     availability of private futexes and whether the rwlock is shared
+     or private.  Unfortunately the value of a private rwlock is
+     fixed: it must be zero.  The PRIVATE_FUTEX flag has the value
+     0x80 in case private futexes are available and zero otherwise.
+     This leads to the following table:
+
+                 |     pshared     |     result
+                 | shared  private | shared  private |
+     ------------+-----------------+-----------------+
+     !avail 0    |     0       0   |     0       0   |
+      avail 0x80 |  0x80       0   |     0    0x80   |
+
+     If the pshared value is in locking functions XORed with avail
+     we get the expected result.  */
+#ifdef __ASSUME_PRIVATE_FUTEX
+  rwlock->__data.__shared = (iattr->pshared == PTHREAD_PROCESS_PRIVATE
+			     ? 0 : FUTEX_PRIVATE_FLAG);
+#else
+  rwlock->__data.__shared = (iattr->pshared == PTHREAD_PROCESS_PRIVATE
+			     ? 0
+			     : THREAD_GETMEM (THREAD_SELF,
+					      header.private_futex));
+#endif
+
+  rwlock->__data.__pad1 = 0;
+  rwlock->__data.__pad2 = 0;
 
   return 0;
 }
diff --git a/nptl/sysdeps/pthread/pthread.h b/nptl/sysdeps/pthread/pthread.h
index 4dc0b2272b..d175f312ee 100644
--- a/nptl/sysdeps/pthread/pthread.h
+++ b/nptl/sysdeps/pthread/pthread.h
@@ -21,6 +21,7 @@
 #define _PTHREAD_H	1
 
 #include <features.h>
+#include <endian.h>
 #include <sched.h>
 #include <time.h>
 
@@ -120,21 +121,23 @@ enum
 };
 
 /* Read-write lock initializers.  */
-# if __WORDSIZE == 64
-#  define PTHREAD_RWLOCK_INITIALIZER \
+# define PTHREAD_RWLOCK_INITIALIZER \
   { { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 } }
-# else
-#  define PTHREAD_RWLOCK_INITIALIZER \
-  { { 0, 0, 0, 0, 0, 0, 0, 0 } }
-# endif
 # ifdef __USE_GNU
 #  if __WORDSIZE == 64
 #   define PTHREAD_RWLOCK_WRITER_NONRECURSIVE_INITIALIZER_NP \
   { { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,					      \
       PTHREAD_RWLOCK_PREFER_WRITER_NONRECURSIVE_NP } }
 #  else
-#   define PTHREAD_RWLOCK_WRITER_NONRECURSIVE_INITIALIZER_NP \
-  { { 0, 0, 0, 0, 0, 0, PTHREAD_RWLOCK_PREFER_WRITER_NONRECURSIVE_NP, 0 } }
+#   if __BYTE_ORDER == __LITTLE_ENDIAN
+#    define PTHREAD_RWLOCK_WRITER_NONRECURSIVE_INITIALIZER_NP \
+  { { 0, 0, 0, 0, 0, 0, PTHREAD_RWLOCK_PREFER_WRITER_NONRECURSIVE_NP, \
+      0, 0, 0, 0 } }
+#   else
+#    define PTHREAD_RWLOCK_WRITER_NONRECURSIVE_INITIALIZER_NP \
+  { { 0, 0, 0, 0, 0, 0, 0, 0, 0, PTHREAD_RWLOCK_PREFER_WRITER_NONRECURSIVE_NP,\
+      0 } }
+#   endif
 #  endif
 # endif
 #endif  /* Unix98 or XOpen2K */
diff --git a/nptl/sysdeps/unix/sysv/linux/i386/bits/pthreadtypes.h b/nptl/sysdeps/unix/sysv/linux/i386/bits/pthreadtypes.h
index f53d0e5a72..9e3e016fb8 100644
--- a/nptl/sysdeps/unix/sysv/linux/i386/bits/pthreadtypes.h
+++ b/nptl/sysdeps/unix/sysv/linux/i386/bits/pthreadtypes.h
@@ -1,4 +1,4 @@
-/* Copyright (C) 2002, 2003, 2004, 2005, 2006 Free Software Foundation, Inc.
+/* Copyright (C) 2002,2003,2004,2005,2006,2007 Free Software Foundation, Inc.
    This file is part of the GNU C Library.
 
    The GNU C Library is free software; you can redistribute it and/or
@@ -128,7 +128,10 @@ typedef union
     unsigned int __nr_writers_queued;
     /* FLAGS must stay at this position in the structure to maintain
        binary compatibility.  */
-    unsigned int __flags;
+    unsigned char __flags;
+    unsigned char __shared;
+    unsigned char __pad1;
+    unsigned char __pad2;
     int __writer;
   } __data;
   char __size[__SIZEOF_PTHREAD_RWLOCK_T];
diff --git a/nptl/sysdeps/unix/sysv/linux/lowlevelrwlock.sym b/nptl/sysdeps/unix/sysv/linux/lowlevelrwlock.sym
index e82c878d39..f50b25bfb8 100644
--- a/nptl/sysdeps/unix/sysv/linux/lowlevelrwlock.sym
+++ b/nptl/sysdeps/unix/sysv/linux/lowlevelrwlock.sym
@@ -1,6 +1,7 @@
 #include <stddef.h>
 #include <stdio.h>
 #include <bits/pthreadtypes.h>
+#include <bits/wordsize.h>
 
 --
 
@@ -12,3 +13,4 @@ READERS_QUEUED	offsetof (pthread_rwlock_t, __data.__nr_readers_queued)
 WRITERS_QUEUED	offsetof (pthread_rwlock_t, __data.__nr_writers_queued)
 FLAGS		offsetof (pthread_rwlock_t, __data.__flags)
 WRITER		offsetof (pthread_rwlock_t, __data.__writer)
+PSHARED		offsetof (pthread_rwlock_t, __data.__shared)
diff --git a/nptl/sysdeps/unix/sysv/linux/powerpc/bits/pthreadtypes.h b/nptl/sysdeps/unix/sysv/linux/powerpc/bits/pthreadtypes.h
index a7150f6aef..c0b59c336f 100644
--- a/nptl/sysdeps/unix/sysv/linux/powerpc/bits/pthreadtypes.h
+++ b/nptl/sysdeps/unix/sysv/linux/powerpc/bits/pthreadtypes.h
@@ -1,5 +1,5 @@
 /* Machine-specific pthread type layouts.  PowerPC version.
-   Copyright (C) 2003, 2004, 2005, 2006 Free Software Foundation, Inc.
+   Copyright (C) 2003, 2004, 2005, 2006, 2007 Free Software Foundation, Inc.
    This file is part of the GNU C Library.
    Contributed by Paul Mackerras <paulus@au.ibm.com>, 2003.
 
@@ -160,9 +160,9 @@ typedef union
     unsigned int __nr_readers_queued;
     unsigned int __nr_writers_queued;
     int __writer;
-    int __pad1;
+    int __shared;
+    unsigned long int __pad1;
     unsigned long int __pad2;
-    unsigned long int __pad3;
     /* FLAGS must stay at this position in the structure to maintain
        binary compatibility.  */
     unsigned int __flags;
@@ -176,9 +176,12 @@ typedef union
     unsigned int __writer_wakeup;
     unsigned int __nr_readers_queued;
     unsigned int __nr_writers_queued;
+    unsigned char __pad1;
+    unsigned char __pad2;
+    unsigned char __shared;
     /* FLAGS must stay at this position in the structure to maintain
        binary compatibility.  */
-    unsigned int __flags;
+    unsigned char __flags;
     int __writer;
   } __data;
 # endif
diff --git a/nptl/sysdeps/unix/sysv/linux/powerpc/lowlevellock.h b/nptl/sysdeps/unix/sysv/linux/powerpc/lowlevellock.h
index 239f4ddf55..71fede8384 100644
--- a/nptl/sysdeps/unix/sysv/linux/powerpc/lowlevellock.h
+++ b/nptl/sysdeps/unix/sysv/linux/powerpc/lowlevellock.h
@@ -37,6 +37,8 @@
 #define FUTEX_LOCK_PI		6
 #define FUTEX_UNLOCK_PI		7
 #define FUTEX_TRYLOCK_PI	8
+#define FUTEX_PRIVATE_FLAG	128
+
 
 /* Initializer for compatibility lock.	*/
 #define LLL_MUTEX_LOCK_INITIALIZER (0)
diff --git a/nptl/sysdeps/unix/sysv/linux/x86_64/bits/pthreadtypes.h b/nptl/sysdeps/unix/sysv/linux/x86_64/bits/pthreadtypes.h
index 693387a266..7a09c81194 100644
--- a/nptl/sysdeps/unix/sysv/linux/x86_64/bits/pthreadtypes.h
+++ b/nptl/sysdeps/unix/sysv/linux/x86_64/bits/pthreadtypes.h
@@ -1,4 +1,4 @@
-/* Copyright (C) 2002, 2003, 2004, 2005, 2006 Free Software Foundation, Inc.
+/* Copyright (C) 2002,2003,2004,2005,2006,2007 Free Software Foundation, Inc.
    This file is part of the GNU C Library.
    Contributed by Ulrich Drepper <drepper@redhat.com>, 2002.
 
@@ -159,9 +159,9 @@ typedef union
     unsigned int __nr_readers_queued;
     unsigned int __nr_writers_queued;
     int __writer;
-    int __pad1;
+    int __shared;
+    unsigned long int __pad1;
     unsigned long int __pad2;
-    unsigned long int __pad3;
     /* FLAGS must stay at this position in the structure to maintain
        binary compatibility.  */
     unsigned int __flags;
@@ -177,7 +177,10 @@ typedef union
     unsigned int __nr_writers_queued;
     /* FLAGS must stay at this position in the structure to maintain
        binary compatibility.  */
-    unsigned int __flags;
+    unsigned char __flags;
+    unsigned char __shared;
+    unsigned char __pad1;
+    unsigned char __pad2;
     int __writer;
   } __data;
 # endif
diff --git a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_unlock.S b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_unlock.S
index 3a6b9f0bad..4ac9ae62ea 100644
--- a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_unlock.S
+++ b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_unlock.S
@@ -1,4 +1,4 @@
-/* Copyright (C) 2002, 2003, 2005 Free Software Foundation, Inc.
+/* Copyright (C) 2002, 2003, 2005, 2007 Free Software Foundation, Inc.
    This file is part of the GNU C Library.
    Contributed by Ulrich Drepper <drepper@redhat.com>, 2002.
 
@@ -56,7 +56,6 @@ __pthread_rwlock_unlock:
 
 5:	movl	$0, WRITER(%rdi)
 
-	movl	$1, %esi
 	leaq	WRITERS_WAKEUP(%rdi), %r10
 	movq	%rsi, %rdx
 	cmpl	$0, WRITERS_QUEUED(%rdi)
@@ -78,7 +77,16 @@ __pthread_rwlock_unlock:
 #endif
 	jne	7f
 
-8:	movl	$SYS_futex, %eax
+8:
+#if __ASSUME_PRIVATE_FUTEX
+	movl	$FUTEX_PRIVATE_FLAG|FUTEX_WAKE, %esi
+	xorl	PSHARED(%rdi), %esi
+#else
+	movl	$FUTEX_WAKE, %esi
+	orl	PSHARED(%rdi), %esi
+	xorl	%fs:PRIVATE_FUTEX, %esi
+#endif
+	movl	$SYS_futex, %eax
 	movq	%r10, %rdi
 	syscall
 
diff --git a/nptl/tst-initializers1.c b/nptl/tst-initializers1.c
index eacee35c61..9036e6eac3 100644
--- a/nptl/tst-initializers1.c
+++ b/nptl/tst-initializers1.c
@@ -1,4 +1,4 @@
-/* Copyright (C) 2005, 2006 Free Software Foundation, Inc.
+/* Copyright (C) 2005, 2006, 2007 Free Software Foundation, Inc.
    This file is part of the GNU C Library.
    Contributed by Jakub Jelinek <jakub@redhat.com>, 2005.
 
@@ -37,15 +37,15 @@ main (void)
   if (mtx_normal.__data.__kind != PTHREAD_MUTEX_TIMED_NP)
     return 1;
   if (mtx_recursive.__data.__kind != PTHREAD_MUTEX_RECURSIVE_NP)
-    return 1;
+    return 2;
   if (mtx_errorchk.__data.__kind != PTHREAD_MUTEX_ERRORCHECK_NP)
-    return 1;
+    return 3;
   if (mtx_adaptive.__data.__kind != PTHREAD_MUTEX_ADAPTIVE_NP)
-    return 1;
+    return 4;
   if (rwl_normal.__data.__flags != PTHREAD_RWLOCK_PREFER_READER_NP)
-    return 1;
+    return 5;
   if (rwl_writer.__data.__flags
       != PTHREAD_RWLOCK_PREFER_WRITER_NONRECURSIVE_NP)
-    return 1;
+    return 6;
   return 0;
 }