aboutsummaryrefslogtreecommitdiff
path: root/nptl
diff options
context:
space:
mode:
authorUlrich Drepper <drepper@redhat.com>2004-06-03 16:04:11 +0000
committerUlrich Drepper <drepper@redhat.com>2004-06-03 16:04:11 +0000
commit75fcceded2cfc65e4879521fff4db6a620a96363 (patch)
tree6d0763c5a2e4b59ece4d57a87232d435d15a5f7f /nptl
parent322861e8b62dbca030a66f9ab37e6688b223c65f (diff)
downloadglibc-75fcceded2cfc65e4879521fff4db6a620a96363.tar
glibc-75fcceded2cfc65e4879521fff4db6a620a96363.tar.gz
glibc-75fcceded2cfc65e4879521fff4db6a620a96363.tar.bz2
glibc-75fcceded2cfc65e4879521fff4db6a620a96363.zip
Update.
2004-06-03 Ulrich Drepper <drepper@redhat.com> * sysdeps/i386/i486/bits/atomic.h: Optimize a bit.
Diffstat (limited to 'nptl')
-rw-r--r--nptl/ChangeLog92
-rw-r--r--nptl/Makefile2
-rw-r--r--nptl/pthread_cond_init.c1
-rw-r--r--nptl/sysdeps/pthread/pthread_cond_broadcast.c22
-rw-r--r--nptl/sysdeps/pthread/pthread_cond_signal.c15
-rw-r--r--nptl/sysdeps/pthread/pthread_cond_timedwait.c26
-rw-r--r--nptl/sysdeps/pthread/pthread_cond_wait.c28
-rw-r--r--nptl/sysdeps/unix/sysv/linux/alpha/bits/pthreadtypes.h3
-rw-r--r--nptl/sysdeps/unix/sysv/linux/alpha/lowlevellock.h12
-rw-r--r--nptl/sysdeps/unix/sysv/linux/i386/bits/pthreadtypes.h3
-rw-r--r--nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_broadcast.S53
-rw-r--r--nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_signal.S11
-rw-r--r--nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_wait.S29
-rw-r--r--nptl/sysdeps/unix/sysv/linux/ia64/bits/pthreadtypes.h3
-rw-r--r--nptl/sysdeps/unix/sysv/linux/ia64/lowlevellock.h11
-rw-r--r--nptl/sysdeps/unix/sysv/linux/lowlevelcond.sym1
-rw-r--r--nptl/sysdeps/unix/sysv/linux/powerpc/bits/pthreadtypes.h3
-rw-r--r--nptl/sysdeps/unix/sysv/linux/powerpc/lowlevellock.h12
-rw-r--r--nptl/sysdeps/unix/sysv/linux/s390/bits/pthreadtypes.h3
-rw-r--r--nptl/sysdeps/unix/sysv/linux/s390/lowlevellock.h24
-rw-r--r--nptl/sysdeps/unix/sysv/linux/sh/lowlevellock.h3
-rw-r--r--nptl/sysdeps/unix/sysv/linux/sparc/bits/pthreadtypes.h3
-rw-r--r--nptl/sysdeps/unix/sysv/linux/sparc/lowlevellock.h12
-rw-r--r--nptl/sysdeps/unix/sysv/linux/x86_64/bits/pthreadtypes.h3
-rw-r--r--nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_broadcast.S38
-rw-r--r--nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_signal.S13
-rw-r--r--nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S16
-rw-r--r--nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S26
28 files changed, 289 insertions, 179 deletions
diff --git a/nptl/ChangeLog b/nptl/ChangeLog
index 071e7048cc..1ff058e21b 100644
--- a/nptl/ChangeLog
+++ b/nptl/ChangeLog
@@ -1,3 +1,95 @@
+2004-05-21 Jakub Jelinek <jakub@redhat.com>
+
+ * Makefile (tests): Add tst-cond16.
+ * sysdeps/unix/sysv/linux/lowlevelcond.sym (cond_futex): Add.
+ * pthread_cond_init.c (__pthread_cond_init): Clear __data.__futex.
+ * sysdeps/unix/sysv/linux/i386/bits/pthreadtypes.h (pthread_cond_t):
+ Add __data.__futex field, reshuffle __data.__clock.
+ * sysdeps/unix/sysv/linux/i386/pthread_cond_signal.S
+ (__pthread_cond_signal): Increment __futex at the same time as
+ __wakeup_seq or __total_seq. Pass address of __futex instead of
+ address of low 32-bits of __wakeup_seq to futex syscall.
+ * sysdeps/unix/sysv/linux/i386/pthread_cond_wait.S
+ (__pthread_cond_wait): Likewise. Pass __futex value from before
+ releasing internal lock to FUTEX_WAIT.
+ * sysdeps/unix/sysv/linux/i386/pthread_cond_timedwait.S
+ (__pthread_cond_timedwait): Likewise.
+ * sysdeps/unix/sysv/linux/i386/pthread_cond_broadcast.S
+ (FUTEX_CMP_REQUEUE): Define.
+ (__pthread_cond_broadcast): Set __futex to 2 * __total_seq.
+ Use FUTEX_CMP_REQUEUE operation instead of FUTEX_REQUEUE.
+ Pass __futex value from before the unlock and __futex address instead
+ of address of low 32-bits of __wakeup_seq to futex syscall.
+ Fallback to FUTEX_WAKE all on any errors.
+ * sysdeps/unix/sysv/linux/alpha/lowlevellock.h (FUTEX_CMP_REQUEUE):
+ Define.
+ (lll_futex_requeue): Add val argument, use FUTEX_CMP_REQUEUE
+ internally. Return non-zero if error, zero if success.
+ * sysdeps/unix/sysv/linux/alpha/bits/pthreadtypes.h (pthread_cond_t):
+ Add __data.__futex field, reshuffle __data.__clock.
+ * sysdeps/unix/sysv/linux/s390/lowlevellock.h (FUTEX_CMP_REQUEUE):
+ Define.
+ (lll_futex_requeue): Add val argument, return 1 unconditionally
+ for the time being.
+ * sysdeps/unix/sysv/linux/s390/bits/pthreadtypes.h (pthread_cond_t):
+ Add __data.__futex field, reshuffle __data.__clock.
+ * sysdeps/unix/sysv/linux/powerpc/lowlevellock.h (FUTEX_CMP_REQUEUE):
+ Define.
+ (lll_futex_requeue): Add val argument, use FUTEX_CMP_REQUEUE
+ internally. Return non-zero if error, zero if success.
+ * sysdeps/unix/sysv/linux/powerpc/bits/pthreadtypes.h
+ (pthread_cond_t): Add __data.__futex field, reshuffle __data.__clock.
+ * sysdeps/unix/sysv/linux/sparc/lowlevellock.h (FUTEX_CMP_REQUEUE):
+ Define.
+ (lll_futex_requeue): Add val argument, use FUTEX_CMP_REQUEUE
+ internally. Return non-zero if error, zero if success.
+ * sysdeps/unix/sysv/linux/sparc/bits/pthreadtypes.h (pthread_cond_t):
+ Add __data.__futex field, reshuffle __data.__clock.
+ * sysdeps/unix/sysv/linux/ia64/lowlevellock.h (FUTEX_CMP_REQUEUE):
+ Define.
+ (lll_futex_requeue): Add val argument, use FUTEX_CMP_REQUEUE
+ internally. Return non-zero if error, zero if success.
+ * sysdeps/unix/sysv/linux/ia64/bits/pthreadtypes.h (pthread_cond_t):
+ Add __data.__futex field, reshuffle __data.__clock.
+ * sysdeps/unix/sysv/linux/x86_64/bits/pthreadtypes.h (pthread_cond_t):
+ Add __data.__futex field, reshuffle __data.__clock.
+ * sysdeps/pthread/pthread_cond_signal.c (__pthread_cond_signal):
+ Increment __futex at the same time as __wakeup_seq or __total_seq.
+ Pass address of __futex instead of address of low 32-bits of
+ __wakeup_seq to futex syscall.
+ * sysdeps/pthread/pthread_cond_wait.c (__pthread_cond_wait): Likewise.
+ Pass __futex value from before releasing internal lock
+ to FUTEX_WAIT.
+ * sysdeps/pthread/pthread_cond_timedwait.c (__pthread_cond_timedwait):
+ Likewise. Avoid unnecessary shadowing of variables.
+ * sysdeps/pthread/pthread_cond_broadcast.c (__pthread_cond_broadcast):
+ Set __futex to 2 * __total_seq. Pass __futex value from before the
+ unlock and __futex address instead of address of low 32-bits of
+ __wakeup_seq to futex_requeue macro, adjust for new return value
+ meaning.
+ * sysdeps/unix/sysv/linux/x86_64/pthread_cond_signal.S
+ (__pthread_cond_signal): Increment __futex at the same time as
+ __wakeup_seq or __total_seq. Pass address of __futex instead of
+ address of low 32-bits of __wakeup_seq to futex syscall.
+ * sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S
+ (__pthread_cond_wait): Likewise. Pass __futex value from before
+ releasing internal lock to FUTEX_WAIT.
+ * sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S
+ (__pthread_cond_timedwait): Likewise.
+ * sysdeps/unix/sysv/linux/x86_64/pthread_cond_broadcast.S
+ (FUTEX_CMP_REQUEUE): Define.
+ (__pthread_cond_broadcast): Set __futex to 2 * __total_seq.
+ Use FUTEX_CMP_REQUEUE operation instead of FUTEX_REQUEUE.
+ Pass __futex value from before the unlock and __futex address instead
+ of address of low 32-bits of __wakeup_seq to futex syscall.
+ Fallback to FUTEX_WAKE all on any errors.
+
+2004-06-03 Kaz Kojima <kkojima@rr.iij4u.or.jp>
+
+ * sysdeps/unix/sysv/linux/sh/lowlevellock.h (lll_mutex_lock):
+ Add nop to align the end of critical section.
+ (lll_mutex_cond_lock, lll_mutex_timedlock): Likewise.
+
2004-06-01 Kaz Kojima <kkojima@rr.iij4u.or.jp>
* sysdeps/unix/sysv/linux/sh/bits/pthreadtypes.h (pthread_cond_t):
diff --git a/nptl/Makefile b/nptl/Makefile
index 687a6a0cfc..fd3ed9f699 100644
--- a/nptl/Makefile
+++ b/nptl/Makefile
@@ -194,7 +194,7 @@ tests = tst-attr1 tst-attr2 tst-attr3 \
tst-spin1 tst-spin2 tst-spin3 \
tst-cond1 tst-cond2 tst-cond3 tst-cond4 tst-cond5 tst-cond6 tst-cond7 \
tst-cond8 tst-cond9 tst-cond10 tst-cond11 tst-cond12 tst-cond13 \
- tst-cond14 tst-cond15 tst-cond17 tst-cond18 \
+ tst-cond14 tst-cond15 tst-cond16 tst-cond17 tst-cond18 \
tst-rwlock1 tst-rwlock2 tst-rwlock3 tst-rwlock4 tst-rwlock5 \
tst-rwlock6 tst-rwlock7 tst-rwlock8 tst-rwlock9 tst-rwlock10 \
tst-rwlock11 tst-rwlock12 tst-rwlock13 \
diff --git a/nptl/pthread_cond_init.c b/nptl/pthread_cond_init.c
index d059da2315..f5fbd64e96 100644
--- a/nptl/pthread_cond_init.c
+++ b/nptl/pthread_cond_init.c
@@ -31,6 +31,7 @@ __pthread_cond_init (cond, cond_attr)
struct pthread_condattr *icond_attr = (struct pthread_condattr *) cond_attr;
cond->__data.__lock = LLL_MUTEX_LOCK_INITIALIZER;
+ cond->__data.__futex = 0;
cond->__data.__clock = (icond_attr == NULL
? CLOCK_REALTIME : (icond_attr->value & 0xfe) >> 1);
cond->__data.__total_seq = 0;
diff --git a/nptl/sysdeps/pthread/pthread_cond_broadcast.c b/nptl/sysdeps/pthread/pthread_cond_broadcast.c
index a42c579658..1eac8ecf83 100644
--- a/nptl/sysdeps/pthread/pthread_cond_broadcast.c
+++ b/nptl/sysdeps/pthread/pthread_cond_broadcast.c
@@ -41,35 +41,29 @@ __pthread_cond_broadcast (cond)
/* Yes. Mark them all as woken. */
cond->__data.__wakeup_seq = cond->__data.__total_seq;
cond->__data.__woken_seq = cond->__data.__total_seq;
+ cond->__data.__futex = (unsigned int) cond->__data.__total_seq * 2;
+ int futex_val = cond->__data.__futex;
/* Signal that a broadcast happened. */
++cond->__data.__broadcast_seq;
/* We are done. */
lll_mutex_unlock (cond->__data.__lock);
- /* The futex syscall operates on a 32-bit word. That is fine,
- we just use the low 32 bits of the sequence counter. */
-#if BYTE_ORDER == LITTLE_ENDIAN
- int *futex = ((int *) (&cond->__data.__wakeup_seq));
-#elif BYTE_ORDER == BIG_ENDIAN
- int *futex = ((int *) (&cond->__data.__wakeup_seq)) + 1;
-#else
-# error "No valid byte order"
-#endif
-
/* Do not use requeue for pshared condvars. */
if (cond->__data.__mutex == (void *) ~0l)
goto wake_all;
/* Wake everybody. */
pthread_mutex_t *mut = (pthread_mutex_t *) cond->__data.__mutex;
- if (__builtin_expect (lll_futex_requeue (futex, 1, INT_MAX,
- &mut->__data.__lock) == -EINVAL,
- 0))
+ /* lll_futex_requeue returns 0 for success and non-zero
+ for errors. */
+ if (__builtin_expect (lll_futex_requeue (&cond->__data.__futex, 1,
+ INT_MAX, &mut->__data.__lock,
+ futex_val), 0))
{
/* The requeue functionality is not available. */
wake_all:
- lll_futex_wake (futex, INT_MAX);
+ lll_futex_wake (&cond->__data.__futex, INT_MAX);
}
/* That's all. */
diff --git a/nptl/sysdeps/pthread/pthread_cond_signal.c b/nptl/sysdeps/pthread/pthread_cond_signal.c
index 76203ac2cc..f5623480f8 100644
--- a/nptl/sysdeps/pthread/pthread_cond_signal.c
+++ b/nptl/sysdeps/pthread/pthread_cond_signal.c
@@ -1,4 +1,4 @@
-/* Copyright (C) 2003 Free Software Foundation, Inc.
+/* Copyright (C) 2003, 2004 Free Software Foundation, Inc.
This file is part of the GNU C Library.
Contributed by Martin Schwidefsky <schwidefsky@de.ibm.com>, 2003.
@@ -40,19 +40,10 @@ __pthread_cond_signal (cond)
{
/* Yes. Mark one of them as woken. */
++cond->__data.__wakeup_seq;
-
- /* The futex syscall operates on a 32-bit word. That is fine,
- we just use the low 32 bits of the sequence counter. */
-#if BYTE_ORDER == LITTLE_ENDIAN
- int *futex = ((int *) (&cond->__data.__wakeup_seq));
-#elif BYTE_ORDER == BIG_ENDIAN
- int *futex = ((int *) (&cond->__data.__wakeup_seq)) + 1;
-#else
-# error "No valid byte order"
-#endif
+ ++cond->__data.__futex;
/* Wake one. */
- lll_futex_wake (futex, 1);
+ lll_futex_wake (&cond->__data.__futex, 1);
}
/* We are done. */
diff --git a/nptl/sysdeps/pthread/pthread_cond_timedwait.c b/nptl/sysdeps/pthread/pthread_cond_timedwait.c
index 96b1029cf5..940b51b4be 100644
--- a/nptl/sysdeps/pthread/pthread_cond_timedwait.c
+++ b/nptl/sysdeps/pthread/pthread_cond_timedwait.c
@@ -66,6 +66,7 @@ __pthread_cond_timedwait (cond, mutex, abstime)
/* We have one new user of the condvar. */
++cond->__data.__total_seq;
+ ++cond->__data.__futex;
/* Remember the mutex we are using here. If there is already a
different address store this is a bad user bug. Do not store
@@ -89,27 +90,17 @@ __pthread_cond_timedwait (cond, mutex, abstime)
/* Remember the broadcast counter. */
cbuffer.bc_seq = cond->__data.__broadcast_seq;
- /* The futex syscall operates on a 32-bit word. That is fine, we
- just use the low 32 bits of the sequence counter. */
-#if BYTE_ORDER == LITTLE_ENDIAN
- int *futex = ((int *) (&cond->__data.__wakeup_seq));
-#elif BYTE_ORDER == BIG_ENDIAN
- int *futex = ((int *) (&cond->__data.__wakeup_seq)) + 1;
-#else
-# error "No valid byte order"
-#endif
-
while (1)
{
struct timespec rt;
{
#ifdef __NR_clock_gettime
INTERNAL_SYSCALL_DECL (err);
- int val;
- val = INTERNAL_SYSCALL (clock_gettime, err, 2,
+ int ret;
+ ret = INTERNAL_SYSCALL (clock_gettime, err, 2,
cond->__data.__clock, &rt);
# ifndef __ASSUME_POSIX_TIMERS
- if (__builtin_expect (INTERNAL_SYSCALL_ERROR_P (val, err), 0))
+ if (__builtin_expect (INTERNAL_SYSCALL_ERROR_P (ret, err), 0))
{
struct timeval tv;
(void) gettimeofday (&tv, NULL);
@@ -149,15 +140,17 @@ __pthread_cond_timedwait (cond, mutex, abstime)
goto timeout;
}
+ unsigned int futex_val = cond->__data.__futex;
+
/* Prepare to wait. Release the condvar futex. */
lll_mutex_unlock (cond->__data.__lock);
/* Enable asynchronous cancellation. Required by the standard. */
cbuffer.oldtype = __pthread_enable_asynccancel ();
- /* Wait until woken by signal or broadcast. Note that we
- truncate the 'val' value to 32 bits. */
- err = lll_futex_timed_wait (futex, (unsigned int) val, &rt);
+ /* Wait until woken by signal or broadcast. */
+ err = lll_futex_timed_wait (&cond->__data.__futex,
+ futex_val, &rt);
/* Disable asynchronous cancellation. */
__pthread_disable_asynccancel (cbuffer.oldtype);
@@ -180,6 +173,7 @@ __pthread_cond_timedwait (cond, mutex, abstime)
timeout:
/* Yep. Adjust the counters. */
++cond->__data.__wakeup_seq;
+ ++cond->__data.__futex;
/* The error value. */
result = ETIMEDOUT;
diff --git a/nptl/sysdeps/pthread/pthread_cond_wait.c b/nptl/sysdeps/pthread/pthread_cond_wait.c
index 128f5a2d99..45187b5240 100644
--- a/nptl/sysdeps/pthread/pthread_cond_wait.c
+++ b/nptl/sysdeps/pthread/pthread_cond_wait.c
@@ -52,20 +52,14 @@ __condvar_cleanup (void *arg)
appropriately. */
++cbuffer->cond->__data.__wakeup_seq;
++cbuffer->cond->__data.__woken_seq;
+ ++cbuffer->cond->__data.__futex;
}
/* We are done. */
lll_mutex_unlock (cbuffer->cond->__data.__lock);
/* Wake everybody to make sure no condvar signal gets lost. */
-#if BYTE_ORDER == LITTLE_ENDIAN
- int *futex = ((int *) (&cbuffer->cond->__data.__wakeup_seq));
-#elif BYTE_ORDER == BIG_ENDIAN
- int *futex = ((int *) (&cbuffer->cond->__data.__wakeup_seq)) + 1;
-#else
-# error "No valid byte order"
-#endif
- lll_futex_wake (futex, INT_MAX);
+ lll_futex_wake (&cbuffer->cond->__data.__futex, INT_MAX);
/* Get the mutex before returning unless asynchronous cancellation
is in effect. */
@@ -95,6 +89,7 @@ __pthread_cond_wait (cond, mutex)
/* We have one new user of the condvar. */
++cond->__data.__total_seq;
+ ++cond->__data.__futex;
/* Remember the mutex we are using here. If there is already a
different address store this is a bad user bug. Do not store
@@ -118,27 +113,18 @@ __pthread_cond_wait (cond, mutex)
/* Remember the broadcast counter. */
cbuffer.bc_seq = cond->__data.__broadcast_seq;
- /* The futex syscall operates on a 32-bit word. That is fine, we
- just use the low 32 bits of the sequence counter. */
-#if BYTE_ORDER == LITTLE_ENDIAN
- int *futex = ((int *) (&cond->__data.__wakeup_seq));
-#elif BYTE_ORDER == BIG_ENDIAN
- int *futex = ((int *) (&cond->__data.__wakeup_seq)) + 1;
-#else
-# error "No valid byte order"
-#endif
-
do
{
+ unsigned int futex_val = cond->__data.__futex;
+
/* Prepare to wait. Release the condvar futex. */
lll_mutex_unlock (cond->__data.__lock);
/* Enable asynchronous cancellation. Required by the standard. */
cbuffer.oldtype = __pthread_enable_asynccancel ();
- /* Wait until woken by signal or broadcast. Note that we
- truncate the 'val' value to 32 bits. */
- lll_futex_wait (futex, (unsigned int) val);
+ /* Wait until woken by signal or broadcast. */
+ lll_futex_wait (&cond->__data.__futex, futex_val);
/* Disable asynchronous cancellation. */
__pthread_disable_asynccancel (cbuffer.oldtype);
diff --git a/nptl/sysdeps/unix/sysv/linux/alpha/bits/pthreadtypes.h b/nptl/sysdeps/unix/sysv/linux/alpha/bits/pthreadtypes.h
index 1bb2968120..62c853cee9 100644
--- a/nptl/sysdeps/unix/sysv/linux/alpha/bits/pthreadtypes.h
+++ b/nptl/sysdeps/unix/sysv/linux/alpha/bits/pthreadtypes.h
@@ -76,11 +76,12 @@ typedef union
struct
{
int __lock;
- int __clock;
+ unsigned int __futex;
unsigned long long int __total_seq;
unsigned long long int __wakeup_seq;
unsigned long long int __woken_seq;
void *__mutex;
+ int __clock;
unsigned int __broadcast_seq;
} __data;
char __size[__SIZEOF_PTHREAD_COND_T];
diff --git a/nptl/sysdeps/unix/sysv/linux/alpha/lowlevellock.h b/nptl/sysdeps/unix/sysv/linux/alpha/lowlevellock.h
index a7079a8490..fd4a7ca4bb 100644
--- a/nptl/sysdeps/unix/sysv/linux/alpha/lowlevellock.h
+++ b/nptl/sysdeps/unix/sysv/linux/alpha/lowlevellock.h
@@ -30,6 +30,7 @@
#define FUTEX_WAIT 0
#define FUTEX_WAKE 1
#define FUTEX_REQUEUE 3
+#define FUTEX_CMP_REQUEUE 4
/* Initializer for compatibility lock. */
#define LLL_MUTEX_LOCK_INITIALIZER (0)
@@ -61,14 +62,15 @@
INTERNAL_SYSCALL_ERROR_P (__ret, __err)? -__ret : __ret; \
})
-#define lll_futex_requeue(futexp, nr_wake, nr_move, mutex) \
+/* Returns non-zero if error happened, zero if success. */
+#define lll_futex_requeue(futexp, nr_wake, nr_move, mutex, val) \
({ \
INTERNAL_SYSCALL_DECL (__err); \
long int __ret; \
- __ret = INTERNAL_SYSCALL (futex, __err, 5, \
- (futexp), FUTEX_REQUEUE, (nr_wake), (nr_move), \
- (mutex)); \
- INTERNAL_SYSCALL_ERROR_P (__ret, __err)? -__ret : __ret; \
+ __ret = INTERNAL_SYSCALL (futex, __err, 6, \
+ (futexp), FUTEX_CMP_REQUEUE, (nr_wake), \
+ (nr_move), (mutex), (val)); \
+ INTERNAL_SYSCALL_ERROR_P (__ret, __err); \
})
diff --git a/nptl/sysdeps/unix/sysv/linux/i386/bits/pthreadtypes.h b/nptl/sysdeps/unix/sysv/linux/i386/bits/pthreadtypes.h
index 9da84c66d6..8d7858a071 100644
--- a/nptl/sysdeps/unix/sysv/linux/i386/bits/pthreadtypes.h
+++ b/nptl/sysdeps/unix/sysv/linux/i386/bits/pthreadtypes.h
@@ -76,11 +76,12 @@ typedef union
struct
{
int __lock;
- int __clock;
+ unsigned int __futex;
unsigned long long int __total_seq;
unsigned long long int __wakeup_seq;
unsigned long long int __woken_seq;
void *__mutex;
+ int __clock;
unsigned int __broadcast_seq;
} __data;
char __size[__SIZEOF_PTHREAD_COND_T];
diff --git a/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_broadcast.S b/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_broadcast.S
index 456f3dbfb1..5471c1c927 100644
--- a/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_broadcast.S
+++ b/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_broadcast.S
@@ -32,6 +32,7 @@
#define FUTEX_WAIT 0
#define FUTEX_WAKE 1
#define FUTEX_REQUEUE 3
+#define FUTEX_CMP_REQUEUE 4
#define EINVAL 22
@@ -47,8 +48,9 @@ __pthread_cond_broadcast:
pushl %ebx
pushl %esi
pushl %edi
+ pushl %ebp
- movl 16(%esp), %ebx
+ movl 20(%esp), %ebx
/* Get internal lock. */
movl $1, %edx
@@ -61,29 +63,31 @@ __pthread_cond_broadcast:
#endif
jnz 1f
-2: addl $wakeup_seq, %ebx
- movl total_seq+4-wakeup_seq(%ebx), %eax
- movl total_seq-wakeup_seq(%ebx), %ecx
- cmpl 4(%ebx), %eax
+2: addl $cond_futex, %ebx
+ movl total_seq+4-cond_futex(%ebx), %eax
+ movl total_seq-cond_futex(%ebx), %ebp
+ cmpl wakeup_seq+4-cond_futex(%ebx), %eax
ja 3f
jb 4f
- cmpl (%ebx), %ecx
+ cmpl wakeup_seq-cond_futex(%ebx), %ebp
jna 4f
/* Cause all currently waiting threads to recognize they are
woken up. */
-3: movl %ecx, (%ebx)
- movl %eax, 4(%ebx)
- movl %ecx, woken_seq-wakeup_seq(%ebx)
- movl %eax, woken_seq-wakeup_seq+4(%ebx)
- addl $1, broadcast_seq-wakeup_seq(%ebx)
+3: movl %ebp, wakeup_seq-cond_futex(%ebx)
+ movl %eax, wakeup_seq-cond_futex+4(%ebx)
+ movl %ebp, woken_seq-cond_futex(%ebx)
+ movl %eax, woken_seq-cond_futex+4(%ebx)
+ addl %ebp, %ebp
+ addl $1, broadcast_seq-cond_futex(%ebx)
+ movl %ebp, (%ebx)
/* Get the address of the mutex used. */
- movl dep_mutex-wakeup_seq(%ebx), %edi
+ movl dep_mutex-cond_futex(%ebx), %edi
/* Unlock. */
LOCK
- subl $1, cond_lock-wakeup_seq(%ebx)
+ subl $1, cond_lock-cond_futex(%ebx)
jne 7f
/* Don't use requeue for pshared condvars. */
@@ -91,7 +95,7 @@ __pthread_cond_broadcast:
je 9f
/* Wake up all threads. */
- movl $FUTEX_REQUEUE, %ecx
+ movl $FUTEX_CMP_REQUEUE, %ecx
movl $SYS_futex, %eax
movl $0x7fffffff, %esi
movl $1, %edx
@@ -99,14 +103,18 @@ __pthread_cond_broadcast:
# if MUTEX_FUTEX != 0
addl $MUTEX_FUTEX, %edi
# endif
- ENTER_KERNEL
+/* FIXME: Until Ingo fixes 4G/4G vDSO, 6 arg syscalls are broken for sysenter.
+ ENTER_KERNEL */
+ int $0x80
-#ifndef __ASSUME_FUTEX_REQUEUE
- cmpl $-EINVAL, %eax
- je 9f
-#endif
+ /* For any kind of error, which mainly is EAGAIN, we try again
+ with WAKE. The general test also covers running on old
+ kernels. */
+ cmpl $0xfffff001, %eax
+ jae 9f
10: xorl %eax, %eax
+ popl %ebp
popl %edi
popl %esi
popl %ebx
@@ -115,10 +123,11 @@ __pthread_cond_broadcast:
.align 16
/* Unlock. */
4: LOCK
- subl $1, cond_lock-wakeup_seq(%ebx)
+ subl $1, cond_lock-cond_futex(%ebx)
jne 5f
6: xorl %eax, %eax
+ popl %ebp
popl %edi
popl %esi
popl %ebx
@@ -135,12 +144,12 @@ __pthread_cond_broadcast:
jmp 2b
/* Unlock in loop requires waekup. */
-5: leal cond_lock-wakeup_seq(%ebx), %eax
+5: leal cond_lock-cond_futex(%ebx), %eax
call __lll_mutex_unlock_wake
jmp 6b
/* Unlock in loop requires waekup. */
-7: leal cond_lock-wakeup_seq(%ebx), %eax
+7: leal cond_lock-cond_futex(%ebx), %eax
call __lll_mutex_unlock_wake
jmp 8b
diff --git a/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_signal.S b/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_signal.S
index 4722b4c0e0..3c5a1db59c 100644
--- a/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_signal.S
+++ b/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_signal.S
@@ -60,22 +60,23 @@ __pthread_cond_signal:
#endif
jnz 1f
-2: leal wakeup_seq(%edi), %ebx
+2: leal cond_futex(%edi), %ebx
movl total_seq+4(%edi), %eax
movl total_seq(%edi), %ecx
- cmpl 4(%ebx), %eax
+ cmpl wakeup_seq+4(%edi), %eax
#if cond_lock != 0
/* Must use leal to preserve the flags. */
leal cond_lock(%edi), %edi
#endif
ja 3f
jb 4f
- cmpl (%ebx), %ecx
+ cmpl wakeup_seq-cond_futex(%ebx), %ecx
jbe 4f
/* Bump the wakeup number. */
-3: addl $1, (%ebx)
- adcl $0, 4(%ebx)
+3: addl $1, wakeup_seq-cond_futex(%ebx)
+ adcl $0, wakeup_seq-cond_futex+4(%ebx)
+ addl $1, (%ebx)
/* Wake up one thread. */
movl $FUTEX_WAKE, %ecx
diff --git a/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_wait.S b/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_wait.S
index 74e3172ab0..3fe7f8c17a 100644
--- a/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_wait.S
+++ b/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_wait.S
@@ -79,6 +79,7 @@ __pthread_cond_wait:
addl $1, total_seq(%ebx)
adcl $0, total_seq+4(%ebx)
+ addl $1, cond_futex(%ebx)
#define FRAME_SIZE 16
subl $FRAME_SIZE, %esp
@@ -92,8 +93,10 @@ __pthread_cond_wait:
movl %edx, 8(%esp)
movl %eax, 12(%esp)
+8: movl cond_futex(%ebx), %edi
+
/* Unlock. */
-8: LOCK
+ LOCK
#if cond_lock == 0
subl $1, (%ebx)
#else
@@ -107,12 +110,12 @@ __pthread_cond_wait:
movl %esi, %ecx /* movl $FUTEX_WAIT, %ecx */
movl %edi, %edx
- addl $wakeup_seq, %ebx
-.Ladd_wakeup:
+ addl $cond_futex, %ebx
+.Ladd_cond_futex:
movl $SYS_futex, %eax
ENTER_KERNEL
- subl $wakeup_seq, %ebx
-.Lsub_wakeup:
+ subl $cond_futex, %ebx
+.Lsub_cond_futex:
movl (%esp), %eax
call __pthread_disable_asynccancel
@@ -246,7 +249,7 @@ versioned_symbol (libpthread, __pthread_cond_wait, pthread_cond_wait,
.type __condvar_w_cleanup2, @function
__condvar_w_cleanup2:
- subl $wakeup_seq, %ebx
+ subl $cond_futex, %ebx
.size __condvar_w_cleanup2, .-__condvar_w_cleanup2
.LSbl4:
.type __condvar_w_cleanup, @function
@@ -278,6 +281,8 @@ __condvar_w_cleanup:
addl $1, wakeup_seq(%ebx)
adcl $0, wakeup_seq+4(%ebx)
+ addl $1, cond_futex(%ebx)
+
addl $1, woken_seq(%ebx)
adcl $0, woken_seq+4(%ebx)
@@ -297,7 +302,7 @@ __condvar_w_cleanup:
call __lll_mutex_unlock_wake
/* Wake up all waiters to make sure no signal gets lost. */
-2: addl $wakeup_seq, %ebx
+2: addl $cond_futex, %ebx
movl $FUTEX_WAKE, %ecx
movl $SYS_futex, %eax
movl $0x7fffffff, %edx
@@ -323,15 +328,15 @@ __condvar_w_cleanup:
.uleb128 .Lcstend-.Lcstbegin
.Lcstbegin:
.long .LcleanupSTART-.LSTARTCODE
- .long .Ladd_wakeup-.LcleanupSTART
+ .long .Ladd_cond_futex-.LcleanupSTART
.long __condvar_w_cleanup-.LSTARTCODE
.uleb128 0
- .long .Ladd_wakeup-.LSTARTCODE
- .long .Lsub_wakeup-.Ladd_wakeup
+ .long .Ladd_cond_futex-.LSTARTCODE
+ .long .Lsub_cond_futex-.Ladd_cond_futex
.long __condvar_w_cleanup2-.LSTARTCODE
.uleb128 0
- .long .Lsub_wakeup-.LSTARTCODE
- .long .LcleanupEND-.Lsub_wakeup
+ .long .Lsub_cond_futex-.LSTARTCODE
+ .long .LcleanupEND-.Lsub_cond_futex
.long __condvar_w_cleanup-.LSTARTCODE
.uleb128 0
.long .LcallUR-.LSTARTCODE
diff --git a/nptl/sysdeps/unix/sysv/linux/ia64/bits/pthreadtypes.h b/nptl/sysdeps/unix/sysv/linux/ia64/bits/pthreadtypes.h
index bdf3ee741b..958af47162 100644
--- a/nptl/sysdeps/unix/sysv/linux/ia64/bits/pthreadtypes.h
+++ b/nptl/sysdeps/unix/sysv/linux/ia64/bits/pthreadtypes.h
@@ -76,11 +76,12 @@ typedef union
struct
{
int __lock;
- int __clock;
+ unsigned int __futex;
unsigned long long int __total_seq;
unsigned long long int __wakeup_seq;
unsigned long long int __woken_seq;
void *__mutex;
+ int __clock;
unsigned int __broadcast_seq;
} __data;
char __size[__SIZEOF_PTHREAD_COND_T];
diff --git a/nptl/sysdeps/unix/sysv/linux/ia64/lowlevellock.h b/nptl/sysdeps/unix/sysv/linux/ia64/lowlevellock.h
index 6ed21fdc14..e13358ffef 100644
--- a/nptl/sysdeps/unix/sysv/linux/ia64/lowlevellock.h
+++ b/nptl/sysdeps/unix/sysv/linux/ia64/lowlevellock.h
@@ -30,6 +30,7 @@
#define FUTEX_WAIT 0
#define FUTEX_WAKE 1
#define FUTEX_REQUEUE 3
+#define FUTEX_CMP_REQUEUE 4
/* Delay in spinlock loop. */
#define BUSY_WAIT_NOP asm ("hint @pause")
@@ -52,11 +53,13 @@
_r10 == -1 ? -_retval : _retval; \
})
-#define lll_futex_requeue(ftx, nr_wake, nr_move, mutex) \
+/* Returns non-zero if error happened, zero if success. */
+#define lll_futex_requeue(ftx, nr_wake, nr_move, mutex, val) \
({ \
- DO_INLINE_SYSCALL(futex, 5, (long) (ftx), FUTEX_REQUEUE, (int) (nr_wake), \
- (int) (nr_move), (long) (mutex)); \
- _r10 == -1 ? -_retval : _retval; \
+ DO_INLINE_SYSCALL(futex, 6, (long) (ftx), FUTEX_CMP_REQUEUE, \
+ (int) (nr_wake), (int) (nr_move), (long) (mutex), \
+ (int) val); \
+ _r10 == -1; \
})
diff --git a/nptl/sysdeps/unix/sysv/linux/lowlevelcond.sym b/nptl/sysdeps/unix/sysv/linux/lowlevelcond.sym
index d0559da86f..2e193e6d2e 100644
--- a/nptl/sysdeps/unix/sysv/linux/lowlevelcond.sym
+++ b/nptl/sysdeps/unix/sysv/linux/lowlevelcond.sym
@@ -4,6 +4,7 @@
--
cond_lock offsetof (pthread_cond_t, __data.__lock)
+cond_futex offsetof (pthread_cond_t, __data.__futex)
cond_clock offsetof (pthread_cond_t, __data.__clock)
total_seq offsetof (pthread_cond_t, __data.__total_seq)
wakeup_seq offsetof (pthread_cond_t, __data.__wakeup_seq)
diff --git a/nptl/sysdeps/unix/sysv/linux/powerpc/bits/pthreadtypes.h b/nptl/sysdeps/unix/sysv/linux/powerpc/bits/pthreadtypes.h
index 794d888a13..5a000d30af 100644
--- a/nptl/sysdeps/unix/sysv/linux/powerpc/bits/pthreadtypes.h
+++ b/nptl/sysdeps/unix/sysv/linux/powerpc/bits/pthreadtypes.h
@@ -96,11 +96,12 @@ typedef union
struct
{
int __lock;
- int __clock;
+ unsigned int __futex;
unsigned long long int __total_seq;
unsigned long long int __wakeup_seq;
unsigned long long int __woken_seq;
void *__mutex;
+ int __clock;
unsigned int __broadcast_seq;
} __data;
char __size[__SIZEOF_PTHREAD_COND_T];
diff --git a/nptl/sysdeps/unix/sysv/linux/powerpc/lowlevellock.h b/nptl/sysdeps/unix/sysv/linux/powerpc/lowlevellock.h
index 8e3f46d534..ead89ba931 100644
--- a/nptl/sysdeps/unix/sysv/linux/powerpc/lowlevellock.h
+++ b/nptl/sysdeps/unix/sysv/linux/powerpc/lowlevellock.h
@@ -30,6 +30,7 @@
#define FUTEX_WAIT 0
#define FUTEX_WAKE 1
#define FUTEX_REQUEUE 3
+#define FUTEX_CMP_REQUEUE 4
/* Initializer for compatibility lock. */
#define LLL_MUTEX_LOCK_INITIALIZER (0)
@@ -64,15 +65,16 @@
INTERNAL_SYSCALL_ERROR_P (__ret, __err) ? -__ret : __ret; \
})
-#define lll_futex_requeue(futexp, nr_wake, nr_move, mutex) \
+/* Returns non-zero if error happened, zero if success. */
+#define lll_futex_requeue(futexp, nr_wake, nr_move, mutex, val) \
({ \
INTERNAL_SYSCALL_DECL (__err); \
long int __ret; \
\
- __ret = INTERNAL_SYSCALL (futex, __err, 5, \
- (futexp), FUTEX_REQUEUE, (nr_wake), (nr_move), \
- (mutex)); \
- INTERNAL_SYSCALL_ERROR_P (__ret, __err) ? -__ret : __ret; \
+ __ret = INTERNAL_SYSCALL (futex, __err, 6, \
+ (futexp), FUTEX_CMP_REQUEUE, (nr_wake), \
+ (nr_move), (mutex), (val)); \
+ INTERNAL_SYSCALL_ERROR_P (__ret, __err); \
})
#ifdef UP
diff --git a/nptl/sysdeps/unix/sysv/linux/s390/bits/pthreadtypes.h b/nptl/sysdeps/unix/sysv/linux/s390/bits/pthreadtypes.h
index 02b061b05c..142286451c 100644
--- a/nptl/sysdeps/unix/sysv/linux/s390/bits/pthreadtypes.h
+++ b/nptl/sysdeps/unix/sysv/linux/s390/bits/pthreadtypes.h
@@ -95,11 +95,12 @@ typedef union
struct
{
int __lock;
- int __clock;
+ unsigned int __futex;
unsigned long long int __total_seq;
unsigned long long int __wakeup_seq;
unsigned long long int __woken_seq;
void *__mutex;
+ int __clock;
unsigned int __broadcast_seq;
} __data;
char __size[__SIZEOF_PTHREAD_COND_T];
diff --git a/nptl/sysdeps/unix/sysv/linux/s390/lowlevellock.h b/nptl/sysdeps/unix/sysv/linux/s390/lowlevellock.h
index 8b460eb98f..60679cca86 100644
--- a/nptl/sysdeps/unix/sysv/linux/s390/lowlevellock.h
+++ b/nptl/sysdeps/unix/sysv/linux/s390/lowlevellock.h
@@ -29,6 +29,7 @@
#define FUTEX_WAIT 0
#define FUTEX_WAKE 1
#define FUTEX_REQUEUE 3
+#define FUTEX_CMP_REQUEUE 4
/* Initializer for compatibility lock. */
#define LLL_MUTEX_LOCK_INITIALIZER (0)
@@ -82,22 +83,29 @@
})
-#define lll_futex_requeue(futex, nr_wake, nr_move, mutex) \
+/* Returns non-zero if error happened, zero if success. */
+#if 0
+/* FIXME: s390 only supports up to 5 argument syscalls. Once FUTEX_CMP_REQUEUE
+ kernel interface for s390 is finalized, adjust this. */
+#define lll_futex_requeue(futex, nr_wake, nr_move, mutex, val) \
({ \
- register unsigned long int __r2 asm ("2") = (unsigned long int) (futex); \
- register unsigned long int __r3 asm ("3") = FUTEX_REQUEUE; \
- register unsigned long int __r4 asm ("4") = (long int) (nr_wake); \
- register unsigned long int __r5 asm ("5") = (long int) (nr_move); \
- register unsigned long int __r6 asm ("6") = (unsigned long int) (mutex); \
- register unsigned long __result asm ("2"); \
+ register unsigned long int __r2 asm ("2") = (unsigned long int) (futex); \
+ register unsigned long int __r3 asm ("3") = FUTEX_CMP_REQUEUE; \
+ register unsigned long int __r4 asm ("4") = (long int) (nr_wake); \
+ register unsigned long int __r5 asm ("5") = (long int) (nr_move); \
+ register unsigned long int __r6 asm ("6") = (unsigned long int) (mutex); \
+ register unsigned long __result asm ("2"); \
\
__asm __volatile ("svc %b1" \
: "=d" (__result) \
: "i" (SYS_futex), "0" (__r2), "d" (__r3), \
"d" (__r4), "d" (__r5), "d" (__r6) \
: "cc", "memory" ); \
- __result; \
+ __result > -4096UL; \
})
+#else
+#define lll_futex_requeue(futex, nr_wake, nr_move, mutex, val) 1
+#endif
#define lll_compare_and_swap(futex, oldval, newval, operation) \
diff --git a/nptl/sysdeps/unix/sysv/linux/sh/lowlevellock.h b/nptl/sysdeps/unix/sysv/linux/sh/lowlevellock.h
index 00ff1b20fb..d9376d45a0 100644
--- a/nptl/sysdeps/unix/sysv/linux/sh/lowlevellock.h
+++ b/nptl/sysdeps/unix/sysv/linux/sh/lowlevellock.h
@@ -89,6 +89,7 @@ extern int __lll_mutex_unlock_wake (int *__futex) attribute_hidden;
__asm __volatile ("\
.align 2\n\
mova 1f,r0\n\
+ nop\n\
mov r15,r1\n\
mov #-8,r15\n\
0: mov.l @%2,%0\n\
@@ -108,6 +109,7 @@ extern int __lll_mutex_unlock_wake (int *__futex) attribute_hidden;
__asm __volatile ("\
.align 2\n\
mova 1f,r0\n\
+ nop\n\
mov r15,r1\n\
mov #-8,r15\n\
0: mov.l @%2,%0\n\
@@ -125,6 +127,7 @@ extern int __lll_mutex_unlock_wake (int *__futex) attribute_hidden;
__asm __volatile ("\
.align 2\n\
mova 1f,r0\n\
+ nop\n\
mov r15,r1\n\
mov #-8,r15\n\
0: mov.l @%2,%0\n\
diff --git a/nptl/sysdeps/unix/sysv/linux/sparc/bits/pthreadtypes.h b/nptl/sysdeps/unix/sysv/linux/sparc/bits/pthreadtypes.h
index b0c1372bba..e86dd000a1 100644
--- a/nptl/sysdeps/unix/sysv/linux/sparc/bits/pthreadtypes.h
+++ b/nptl/sysdeps/unix/sysv/linux/sparc/bits/pthreadtypes.h
@@ -96,11 +96,12 @@ typedef union
struct
{
int __lock;
- int __clock;
+ unsigned int __futex;
unsigned long long int __total_seq;
unsigned long long int __wakeup_seq;
unsigned long long int __woken_seq;
void *__mutex;
+ int __clock;
unsigned int __broadcast_seq;
} __data;
char __size[__SIZEOF_PTHREAD_COND_T];
diff --git a/nptl/sysdeps/unix/sysv/linux/sparc/lowlevellock.h b/nptl/sysdeps/unix/sysv/linux/sparc/lowlevellock.h
index 05bc5a2974..4626aec524 100644
--- a/nptl/sysdeps/unix/sysv/linux/sparc/lowlevellock.h
+++ b/nptl/sysdeps/unix/sysv/linux/sparc/lowlevellock.h
@@ -29,6 +29,7 @@
#define FUTEX_WAIT 0
#define FUTEX_WAKE 1
#define FUTEX_REQUEUE 3
+#define FUTEX_CMP_REQUEUE 4
/* Initializer for compatibility lock. */
#define LLL_MUTEX_LOCK_INITIALIZER (0)
@@ -63,15 +64,16 @@
__ret; \
})
-#define lll_futex_requeue(futexp, nr_wake, nr_move, mutex) \
+/* Returns non-zero if error happened, zero if success. */
+#define lll_futex_requeue(futexp, nr_wake, nr_move, mutex, val) \
({ \
INTERNAL_SYSCALL_DECL (__err); \
long int __ret; \
\
- __ret = INTERNAL_SYSCALL (futex, __err, 5, \
- (futexp), FUTEX_REQUEUE, (nr_wake), (nr_move), \
- (mutex)); \
- __ret; \
+ __ret = INTERNAL_SYSCALL (futex, __err, 6, \
+ (futexp), FUTEX_CMP_REQUEUE, (nr_wake), \
+ (nr_move), (mutex), (val)); \
+ INTERNAL_SYSCALL_ERROR_P (__ret, __err); \
})
#ifdef __sparc32_atomic_do_lock
diff --git a/nptl/sysdeps/unix/sysv/linux/x86_64/bits/pthreadtypes.h b/nptl/sysdeps/unix/sysv/linux/x86_64/bits/pthreadtypes.h
index e29c77529e..661e476aa7 100644
--- a/nptl/sysdeps/unix/sysv/linux/x86_64/bits/pthreadtypes.h
+++ b/nptl/sysdeps/unix/sysv/linux/x86_64/bits/pthreadtypes.h
@@ -76,11 +76,12 @@ typedef union
struct
{
int __lock;
- int __clock;
+ unsigned int __futex;
unsigned long long int __total_seq;
unsigned long long int __wakeup_seq;
unsigned long long int __woken_seq;
void *__mutex;
+ int __clock;
unsigned int __broadcast_seq;
} __data;
char __size[__SIZEOF_PTHREAD_COND_T];
diff --git a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_broadcast.S b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_broadcast.S
index 6d29d761fa..e8d7bd9bb6 100644
--- a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_broadcast.S
+++ b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_broadcast.S
@@ -32,6 +32,7 @@
#define FUTEX_WAIT 0
#define FUTEX_WAKE 1
#define FUTEX_REQUEUE 3
+#define FUTEX_CMP_REQUEUE 4
#define EINVAL 22
@@ -55,39 +56,42 @@ __pthread_cond_broadcast:
#endif
jnz 1f
-2: addq $wakeup_seq, %rdi
- movq total_seq-wakeup_seq(%rdi), %rcx
- cmpq (%rdi), %rcx
+2: addq $cond_futex, %rdi
+ movq total_seq-cond_futex(%rdi), %r9
+ cmpq wakeup_seq-cond_futex(%rdi), %r9
jna 4f
/* Cause all currently waiting threads to recognize they are
woken up. */
- movq %rcx, (%rdi)
- movq %rcx, woken_seq-wakeup_seq(%rdi)
- incl broadcast_seq-wakeup_seq(%rdi)
+ movq %r9, wakeup_seq-cond_futex(%rdi)
+ movq %r9, woken_seq-cond_futex(%rdi)
+ addq %r9, %r9
+ movl %r9d, (%rdi)
+ incl broadcast_seq-cond_futex(%rdi)
/* Get the address of the mutex used. */
- movq dep_mutex-wakeup_seq(%rdi), %r8
+ movq dep_mutex-cond_futex(%rdi), %r8
/* Unlock. */
LOCK
- decl cond_lock-wakeup_seq(%rdi)
+ decl cond_lock-cond_futex(%rdi)
jne 7f
8: cmpq $-1, %r8
je 9f
/* Wake up all threads. */
- movq $FUTEX_REQUEUE, %rsi
+ movq $FUTEX_CMP_REQUEUE, %rsi
movq $SYS_futex, %rax
movl $1, %edx
movq $0x7fffffff, %r10
syscall
-#ifndef __ASSUME_FUTEX_REQUEUE
- cmpq $-EINVAL, %rax
- je 9f
-#endif
+ /* For any kind of error, which mainly is EAGAIN, we try again
+ with WAKE. The general test also covers running on old
+ kernels. */
+ cmpq $-4095, %rax
+ jae 9f
10: xorl %eax, %eax
retq
@@ -95,7 +99,7 @@ __pthread_cond_broadcast:
.align 16
/* Unlock. */
4: LOCK
- decl cond_lock-wakeup_seq(%rdi)
+ decl cond_lock-cond_futex(%rdi)
jne 5f
6: xorl %eax, %eax
@@ -113,14 +117,14 @@ __pthread_cond_broadcast:
jmp 2b
/* Unlock in loop requires wakeup. */
-5: addq $cond_lock-wakeup_seq, %rdi
+5: addq $cond_lock-cond_futex, %rdi
callq __lll_mutex_unlock_wake
jmp 6b
/* Unlock in loop requires wakeup. */
-7: addq $cond_lock-wakeup_seq, %rdi
+7: addq $cond_lock-cond_futex, %rdi
callq __lll_mutex_unlock_wake
- subq $cond_lock-wakeup_seq, %rdi
+ subq $cond_lock-cond_futex, %rdi
jmp 8b
9: /* The futex requeue functionality is not available. */
diff --git a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_signal.S b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_signal.S
index 92eadfc433..62bb74cc1a 100644
--- a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_signal.S
+++ b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_signal.S
@@ -1,4 +1,4 @@
-/* Copyright (C) 2002, 2003 Free Software Foundation, Inc.
+/* Copyright (C) 2002, 2003, 2004 Free Software Foundation, Inc.
This file is part of the GNU C Library.
Contributed by Ulrich Drepper <drepper@redhat.com>, 2002.
@@ -56,13 +56,14 @@ __pthread_cond_signal:
#endif
jnz 1f
-2: addq $wakeup_seq, %rdi
+2: addq $cond_futex, %rdi
movq total_seq(%r8), %rcx
- cmpq (%rdi), %rcx
+ cmpq wakeup_seq(%r8), %rcx
jbe 4f
/* Bump the wakeup number. */
- addq $1, (%rdi)
+ addq $1, wakeup_seq(%r8)
+ addl $1, (%rdi)
/* Wake up one thread. */
movq $FUTEX_WAKE, %rsi
@@ -95,11 +96,7 @@ __pthread_cond_signal:
/* Unlock in loop requires wakeup. */
5:
-#if cond_lock != 0
- addq $cond_lock-wakeup_seq, %rdi
-#else
movq %r8, %rdi
-#endif
callq __lll_mutex_unlock_wake
jmp 6b
.size __pthread_cond_signal, .-__pthread_cond_signal
diff --git a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S
index 80cbf7e430..e75f05e07f 100644
--- a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S
+++ b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S
@@ -105,6 +105,7 @@ __pthread_cond_timedwait:
movq 8(%rsp), %rdi
incq total_seq(%rdi)
+ incl cond_futex(%rdi)
/* Install cancellation handler. */
#ifdef PIC
@@ -118,9 +119,9 @@ __pthread_cond_timedwait:
/* Get and store current wakeup_seq value. */
movq 8(%rsp), %rdi
- movq wakeup_seq(%rdi), %r12
+ movq wakeup_seq(%rdi), %r9
movl broadcast_seq(%rdi), %edx
- movq %r12, 40(%rsp)
+ movq %r9, 40(%rsp)
movl %edx, 4(%rsp)
/* Get the current time. */
@@ -172,6 +173,8 @@ __pthread_cond_timedwait:
21: movq %rcx, 24(%rsp)
movq %rdx, 32(%rsp)
+ movl cond_futex(%rdi), %r12d
+
/* Unlock. */
LOCK
#if cond_lock == 0
@@ -187,7 +190,7 @@ __pthread_cond_timedwait:
leaq 24(%rsp), %r10
xorq %rsi, %rsi /* movq $FUTEX_WAIT, %rsi */
movq %r12, %rdx
- addq $wakeup_seq-cond_lock, %rdi
+ addq $cond_futex, %rdi
movq $SYS_futex, %rax
syscall
movq %rax, %r14
@@ -211,21 +214,22 @@ __pthread_cond_timedwait:
movq woken_seq(%rdi), %rax
- movq wakeup_seq(%rdi), %r12
+ movq wakeup_seq(%rdi), %r9
cmpl 4(%rsp), %edx
jne 23f
- cmpq 40(%rsp), %r12
+ cmpq 40(%rsp), %r9
jbe 15f
- cmpq %rax, %r12
+ cmpq %rax, %r9
ja 9f
15: cmpq $-ETIMEDOUT, %r14
jne 8b
13: incq wakeup_seq(%rdi)
+ incl cond_futex(%rdi)
movq $ETIMEDOUT, %r14
jmp 14f
diff --git a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S
index 065eb11813..9e7da301d3 100644
--- a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S
+++ b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S
@@ -1,4 +1,4 @@
-/* Copyright (C) 2002, 2003 Free Software Foundation, Inc.
+/* Copyright (C) 2002, 2003, 2004 Free Software Foundation, Inc.
This file is part of the GNU C Library.
Contributed by Ulrich Drepper <drepper@redhat.com>, 2002.
@@ -69,6 +69,8 @@ __condvar_cleanup:
incq woken_seq(%rdi)
+ incl cond_futex(%rdi)
+
3: LOCK
#if cond_lock == 0
decl (%rdi)
@@ -82,7 +84,7 @@ __condvar_cleanup:
callq __lll_mutex_unlock_wake
/* Wake up all waiters to make sure no signal gets lost. */
-2: addq $wakeup_seq, %rdi
+2: addq $cond_futex, %rdi
movq $FUTEX_WAKE, %rsi
movl $0x7fffffff, %edx
movq $SYS_futex, %rax
@@ -154,6 +156,7 @@ __pthread_cond_wait:
movq 8(%rsp), %rdi
incq total_seq(%rdi)
+ incl cond_futex(%rdi)
/* Install cancellation handler. */
#ifdef PIC
@@ -167,13 +170,14 @@ __pthread_cond_wait:
/* Get and store current wakeup_seq value. */
movq 8(%rsp), %rdi
- movq wakeup_seq(%rdi), %r12
+ movq wakeup_seq(%rdi), %r9
movl broadcast_seq(%rdi), %edx
- movq %r12, 24(%rsp)
+ movq %r9, 24(%rsp)
movl %edx, 4(%rsp)
/* Unlock. */
-8: LOCK
+8: movl cond_futex(%rdi), %r12d
+ LOCK
#if cond_lock == 0
decl (%rdi)
#else
@@ -187,7 +191,7 @@ __pthread_cond_wait:
movq 8(%rsp), %rdi
xorq %r10, %r10
movq %r12, %rdx
- addq $wakeup_seq-cond_lock, %rdi
+ addq $cond_futex-cond_lock, %rdi
movq $SYS_futex, %rax
movq %r10, %rsi /* movq $FUTEX_WAIT, %rsi */
syscall
@@ -211,15 +215,15 @@ __pthread_cond_wait:
movq woken_seq(%rdi), %rax
- movq wakeup_seq(%rdi), %r12
+ movq wakeup_seq(%rdi), %r9
cmpl 4(%rsp), %edx
jne 16f
- cmpq 24(%rsp), %r12
+ cmpq 24(%rsp), %r9
jbe 8b
- cmpq %rax, %r12
+ cmpq %rax, %r9
jna 8b
incq woken_seq(%rdi)
@@ -359,8 +363,8 @@ versioned_symbol (libpthread, __pthread_cond_wait, pthread_cond_wait,
.byte 0x40+.Lsubq-.Lpush_r12 # DW_CFA_advance_loc+N
.byte 14 # DW_CFA_def_cfa_offset
.uleb128 16+FRAME_SIZE
- .byte 2 # DW_CFA_advance_loc1
- .byte .Laddq-.Lsubq
+ .byte 3 # DW_CFA_advance_loc2
+ .2byte .Laddq-.Lsubq
.byte 14 # DW_CFA_def_cfa_offset
.uleb128 16
.byte 0x40+.Lpop_r12-.Laddq # DW_CFA_advance_loc+N