aboutsummaryrefslogtreecommitdiff
path: root/nptl
diff options
context:
space:
mode:
authorUlrich Drepper <drepper@redhat.com>2006-02-15 17:20:33 +0000
committerUlrich Drepper <drepper@redhat.com>2006-02-15 17:20:33 +0000
commit683040c3b2c7dc2bc08045202a0a0b46d143b3cb (patch)
treefcd73510f2bcfcdf57078331d800db4217db759b /nptl
parentb007ce7cc6971e4bd4cd91c558efd3d4603d941d (diff)
downloadglibc-683040c3b2c7dc2bc08045202a0a0b46d143b3cb.tar
glibc-683040c3b2c7dc2bc08045202a0a0b46d143b3cb.tar.gz
glibc-683040c3b2c7dc2bc08045202a0a0b46d143b3cb.tar.bz2
glibc-683040c3b2c7dc2bc08045202a0a0b46d143b3cb.zip
* sysdeps/unix/sysv/linux/not-cancel.h (__openat_not_cancel,
__openat64_not_cancel): Remove prototypes. (__openat_nocancel, __openat64_nocancel): New prototypes or defines. (openat_not_cancel, openat_not_cancel_3, openat64_not_cancel, openat64_not_cancel_3): Use them.
Diffstat (limited to 'nptl')
-rw-r--r--nptl/ChangeLog27
-rw-r--r--nptl/Makefile2
-rw-r--r--nptl/descr.h2
-rw-r--r--nptl/pthreadP.h12
-rw-r--r--nptl/pthread_create.c7
-rw-r--r--nptl/pthread_mutex_consistent.c6
-rw-r--r--nptl/pthread_mutex_destroy.c12
-rw-r--r--nptl/pthread_mutex_lock.c113
-rw-r--r--nptl/pthread_mutex_timedlock.c113
-rw-r--r--nptl/pthread_mutex_trylock.c130
-rw-r--r--nptl/pthread_mutex_unlock.c41
-rw-r--r--nptl/sysdeps/unix/sysv/linux/Makefile5
-rw-r--r--nptl/sysdeps/unix/sysv/linux/alpha/bits/pthreadtypes.h10
-rw-r--r--nptl/sysdeps/unix/sysv/linux/i386/i486/lowlevelrobustlock.S186
-rw-r--r--nptl/sysdeps/unix/sysv/linux/i386/i586/lowlevelrobustlock.S20
-rw-r--r--nptl/sysdeps/unix/sysv/linux/i386/i686/lowlevelrobustlock.S20
-rw-r--r--nptl/sysdeps/unix/sysv/linux/i386/lowlevellock.h118
-rw-r--r--nptl/sysdeps/unix/sysv/linux/ia64/bits/pthreadtypes.h10
-rw-r--r--nptl/sysdeps/unix/sysv/linux/lowlevelrobustlock.sym6
-rw-r--r--nptl/sysdeps/unix/sysv/linux/powerpc/bits/pthreadtypes.h19
-rw-r--r--nptl/sysdeps/unix/sysv/linux/pthread_mutex_cond_lock.c5
-rw-r--r--nptl/sysdeps/unix/sysv/linux/s390/bits/pthreadtypes.h19
-rw-r--r--nptl/sysdeps/unix/sysv/linux/sh/bits/pthreadtypes.h8
-rw-r--r--nptl/sysdeps/unix/sysv/linux/sparc/bits/pthreadtypes.h19
-rw-r--r--nptl/sysdeps/unix/sysv/linux/x86_64/lowlevellock.h96
-rw-r--r--nptl/sysdeps/unix/sysv/linux/x86_64/lowlevelrobustlock.S192
26 files changed, 997 insertions, 201 deletions
diff --git a/nptl/ChangeLog b/nptl/ChangeLog
index 073b3649b7..c280d8dbc8 100644
--- a/nptl/ChangeLog
+++ b/nptl/ChangeLog
@@ -1,3 +1,30 @@
+2006-02-15 Ulrich Drepper <drepper@redhat.com>
+
+ * pthreadp.h: Define PTHREAD_MUTEX_INCONSISTENT instead of
+ PTHREAD_MUTEX_OWNERDEAD.
+ (PTHREAD_MUTEX_ROBUST_PRIVATE_NP): Define as 16, not 256.
+ Define FUTEX_WAITERS, FUTEX_OWNER_DIED, FUTEX_TID_MASK.
+ * Makefile (libpthread-routines): Add lowlevelrobustlock.
+ * pthread_create.c (start_thread): Very much simplify robust_list loop.
+ * pthread_mutex_consistent.c: Inconsistent mutex have __owner now set
+ to PTHREAD_MUTEX_INCONSISTENT.
+ * pthread_mutex_destroy.c: Allow destroying of inconsistent mutexes.
+ * pthread_mutex_lock.c: Reimplement robust mutex handling.
+ * pthread_mutex_trylock.c: Likewise.
+ * pthread_mutex_timedlock.c: Likewise.
+ * pthread_mutex_unlock.c: Likewise.
+ * sysdeps/unix/sysv/linux/pthread_mutex_cond_lock.c: Likewise.
+ * sysdeps/unix/sysv/linux/Makefile (gen-as-const-headers): Add
+ lowlevelrobustlock.sym.
+ * sysdeps/unix/sysv/linux/lowlevelrobustlock.sym: New file.
+ * sysdeps/unix/sysv/linux/i386/lowlevellock.h: Add lll_robust_mutex_*
+ definitions.
+ * sysdeps/unix/sysv/linux/x86_64/lowlevellock.h: Likewise.
+ * sysdeps/unix/sysv/linux/i386/i486/lowlevelrobustlock.S: New file.
+ * sysdeps/unix/sysv/linux/i386/i586/lowlevelrobustlock.S: New file.
+ * sysdeps/unix/sysv/linux/i386/i686/lowlevelrobustlock.S: New file.
+ * sysdeps/unix/sysv/linux/x86_64/lowlevelrobustlock.S: New file.
+
2006-02-12 Ulrich Drepper <drepper@redhat.com>
* allocatestack.c (allocate_stack): Initialize robust_list.
diff --git a/nptl/Makefile b/nptl/Makefile
index d7b3aaf1ce..e484c6ab61 100644
--- a/nptl/Makefile
+++ b/nptl/Makefile
@@ -100,7 +100,7 @@ libpthread-routines = init vars events version \
cleanup_defer_compat unwind \
pt-longjmp pt-cleanup\
cancellation \
- lowlevellock \
+ lowlevellock lowlevelrobustlock \
pt-vfork \
ptw-write ptw-read ptw-close ptw-fcntl ptw-accept \
ptw-connect ptw-recv ptw-recvfrom ptw-recvmsg ptw-send \
diff --git a/nptl/descr.h b/nptl/descr.h
index 6984138537..d5491c1355 100644
--- a/nptl/descr.h
+++ b/nptl/descr.h
@@ -166,7 +166,7 @@ struct pthread
do { \
__pthread_slist_t *runp = THREAD_GETMEM (THREAD_SELF, robust_list.__next);\
if (runp == &mutex->__data.__list) \
- THREAD_SETMEM (THREAD_SELF, robust_list, runp->__next); \
+ THREAD_SETMEM (THREAD_SELF, robust_list.__next, runp->__next); \
else \
{ \
while (runp->__next != &mutex->__data.__list) \
diff --git a/nptl/pthreadP.h b/nptl/pthreadP.h
index 61b7176159..77d8f5ad24 100644
--- a/nptl/pthreadP.h
+++ b/nptl/pthreadP.h
@@ -1,4 +1,4 @@
-/* Copyright (C) 2002, 2003, 2004, 2005 Free Software Foundation, Inc.
+/* Copyright (C) 2002, 2003, 2004, 2005, 2006 Free Software Foundation, Inc.
This file is part of the GNU C Library.
Contributed by Ulrich Drepper <drepper@redhat.com>, 2002.
@@ -52,7 +52,7 @@
/* Magic cookie representing robust mutex with dead owner. */
-#define PTHREAD_MUTEX_OWNERDEAD INT_MAX
+#define PTHREAD_MUTEX_INCONSISTENT INT_MAX
/* Magic cookie representing not recoverable robust mutex. */
#define PTHREAD_MUTEX_NOTRECOVERABLE (INT_MAX - 1)
@@ -60,7 +60,7 @@
/* Internal mutex type value. */
enum
{
- PTHREAD_MUTEX_ROBUST_PRIVATE_NP = 256,
+ PTHREAD_MUTEX_ROBUST_PRIVATE_NP = 16,
PTHREAD_MUTEX_ROBUST_PRIVATE_RECURSIVE_NP
= PTHREAD_MUTEX_ROBUST_PRIVATE_NP | PTHREAD_MUTEX_RECURSIVE_NP,
PTHREAD_MUTEX_ROBUST_PRIVATE_ERRORCHECK_NP
@@ -77,6 +77,12 @@ enum
(PTHREAD_MUTEXATTR_FLAG_ROBUST | PTHREAD_MUTEXATTR_FLAG_PSHARED)
+/* Bits used in robust mutex implementation. */
+#define FUTEX_WAITERS 0x80000000
+#define FUTEX_OWNER_DIED 0x40000000
+#define FUTEX_TID_MASK 0x1fffffff
+
+
/* Internal variables. */
diff --git a/nptl/pthread_create.c b/nptl/pthread_create.c
index b1253b2243..f3d90ecebf 100644
--- a/nptl/pthread_create.c
+++ b/nptl/pthread_create.c
@@ -324,17 +324,12 @@ start_thread (void *arg)
((char *) robust - offsetof (struct __pthread_mutex_s, __list));
robust = robust->__next;
- assert (lll_mutex_islocked (this->__lock));
- this->__count = 0;
- --this->__nusers;
- assert (this->__owner != PTHREAD_MUTEX_NOTRECOVERABLE);
- this->__owner = PTHREAD_MUTEX_OWNERDEAD;
this->__list.__next = NULL;
#ifdef __PTHREAD_MUTEX_HAVE_PREV
this->__list.__prev = NULL;
#endif
- lll_mutex_unlock (this->__lock);
+ lll_robust_mutex_dead (this->__lock);
}
while (robust != &pd->robust_list);
diff --git a/nptl/pthread_mutex_consistent.c b/nptl/pthread_mutex_consistent.c
index 2edfe8a5a6..0cfe972da0 100644
--- a/nptl/pthread_mutex_consistent.c
+++ b/nptl/pthread_mutex_consistent.c
@@ -1,4 +1,4 @@
-/* Copyright (C) 2005 Free Software Foundation, Inc.
+/* Copyright (C) 2005, 2006 Free Software Foundation, Inc.
This file is part of the GNU C Library.
Contributed by Ulrich Drepper <drepper@redhat.com>, 2005.
@@ -27,10 +27,10 @@ pthread_mutex_consistent_np (mutex)
{
/* Test whether this is a robust mutex with a dead owner. */
if ((mutex->__data.__kind & PTHREAD_MUTEX_ROBUST_PRIVATE_NP) == 0
- || mutex->__data.__owner != -THREAD_GETMEM (THREAD_SELF, tid))
+ || mutex->__data.__owner != PTHREAD_MUTEX_INCONSISTENT)
return EINVAL;
- mutex->__data.__owner = -mutex->__data.__owner;
+ mutex->__data.__owner = THREAD_GETMEM (THREAD_SELF, tid);
return 0;
}
diff --git a/nptl/pthread_mutex_destroy.c b/nptl/pthread_mutex_destroy.c
index 2bf76a9da3..19a647a846 100644
--- a/nptl/pthread_mutex_destroy.c
+++ b/nptl/pthread_mutex_destroy.c
@@ -1,4 +1,4 @@
-/* Copyright (C) 2002, 2003, 2005 Free Software Foundation, Inc.
+/* Copyright (C) 2002, 2003, 2005, 2006 Free Software Foundation, Inc.
This file is part of the GNU C Library.
Contributed by Ulrich Drepper <drepper@redhat.com>, 2002.
@@ -26,9 +26,17 @@ __pthread_mutex_destroy (mutex)
pthread_mutex_t *mutex;
{
if (mutex->__data.__nusers != 0)
- return EBUSY;
+ {
+ if ((mutex->__data.__kind & PTHREAD_MUTEX_ROBUST_PRIVATE_NP) != 0
+ && (mutex->__data.__lock & FUTEX_OWNER_DIED) != 0
+ && mutex->__data.__nusers == 1)
+ goto dead_robust_mutex;
+
+ return EBUSY;
+ }
/* Set to an invalid value. */
+ dead_robust_mutex:
mutex->__data.__kind = -1;
return 0;
diff --git a/nptl/pthread_mutex_lock.c b/nptl/pthread_mutex_lock.c
index 420711a4d4..dd22567c71 100644
--- a/nptl/pthread_mutex_lock.c
+++ b/nptl/pthread_mutex_lock.c
@@ -1,4 +1,4 @@
-/* Copyright (C) 2002, 2003, 2004, 2005 Free Software Foundation, Inc.
+/* Copyright (C) 2002, 2003, 2004, 2005, 2006 Free Software Foundation, Inc.
This file is part of the GNU C Library.
Contributed by Ulrich Drepper <drepper@redhat.com>, 2002.
@@ -27,6 +27,7 @@
#ifndef LLL_MUTEX_LOCK
# define LLL_MUTEX_LOCK(mutex) lll_mutex_lock (mutex)
# define LLL_MUTEX_TRYLOCK(mutex) lll_mutex_trylock (mutex)
+# define LLL_ROBUST_MUTEX_LOCK(mutex, id) lll_robust_mutex_lock (mutex, id)
#endif
@@ -36,6 +37,7 @@ __pthread_mutex_lock (mutex)
{
assert (sizeof (mutex->__size) >= sizeof (mutex->__data));
+ int oldval;
pid_t id = THREAD_GETMEM (THREAD_SELF, tid);
int retval = 0;
@@ -107,60 +109,83 @@ __pthread_mutex_lock (mutex)
break;
case PTHREAD_MUTEX_ROBUST_PRIVATE_RECURSIVE_NP:
- /* Check whether we already hold the mutex. */
- if (abs (mutex->__data.__owner) == id)
+ case PTHREAD_MUTEX_ROBUST_PRIVATE_ERRORCHECK_NP:
+ case PTHREAD_MUTEX_ROBUST_PRIVATE_NP:
+ case PTHREAD_MUTEX_ROBUST_PRIVATE_ADAPTIVE_NP:
+ oldval = mutex->__data.__lock;
+ do
{
- /* Just bump the counter. */
- if (__builtin_expect (mutex->__data.__count + 1 == 0, 0))
- /* Overflow of the counter. */
- return EAGAIN;
-
- ++mutex->__data.__count;
-
- return 0;
- }
-
- /* We have to get the mutex. */
- LLL_MUTEX_LOCK (mutex->__data.__lock);
+ if ((oldval & FUTEX_OWNER_DIED) != 0)
+ {
+ /* The previous owner died. Try locking the mutex. */
+ int newval;
+ while ((newval
+ = atomic_compare_and_exchange_val_acq (&mutex->__data.__lock,
+ id, oldval))
+ != oldval)
+ {
+ if ((newval & FUTEX_OWNER_DIED) == 0)
+ goto normal;
+ oldval = newval;
+ }
- mutex->__data.__count = 1;
+ /* We got the mutex. */
+ mutex->__data.__count = 1;
+ /* But it is inconsistent unless marked otherwise. */
+ mutex->__data.__owner = PTHREAD_MUTEX_INCONSISTENT;
+
+ ENQUEUE_MUTEX (mutex);
+
+ /* Note that we deliberately exit here. If we fall
+ through to the end of the function __nusers would be
+ incremented which is not correct because the old
+ owner has to be discounted. If we are not supposed
+ to increment __nusers we actually have to decrement
+ it here. */
+#ifdef NO_INCR
+ --mutex->__data.__nusers;
+#endif
- goto robust;
+ return EOWNERDEAD;
+ }
- case PTHREAD_MUTEX_ROBUST_PRIVATE_ERRORCHECK_NP:
- /* Check whether we already hold the mutex. */
- if (__builtin_expect (abs (mutex->__data.__owner) == id, 0))
- return EDEADLK;
+ normal:
+ /* Check whether we already hold the mutex. */
+ if (__builtin_expect ((mutex->__data.__lock & FUTEX_TID_MASK)
+ == id, 0))
+ {
+ if (mutex->__data.__kind
+ == PTHREAD_MUTEX_ROBUST_PRIVATE_ERRORCHECK_NP)
+ return EDEADLK;
- /* FALLTHROUGH */
+ if (mutex->__data.__kind
+ == PTHREAD_MUTEX_ROBUST_PRIVATE_RECURSIVE_NP)
+ {
+ /* Just bump the counter. */
+ if (__builtin_expect (mutex->__data.__count + 1 == 0, 0))
+ /* Overflow of the counter. */
+ return EAGAIN;
- case PTHREAD_MUTEX_ROBUST_PRIVATE_NP:
- case PTHREAD_MUTEX_ROBUST_PRIVATE_ADAPTIVE_NP:
- LLL_MUTEX_LOCK (mutex->__data.__lock);
+ ++mutex->__data.__count;
- robust:
- if (__builtin_expect (mutex->__data.__owner
- == PTHREAD_MUTEX_NOTRECOVERABLE, 0))
- {
- /* This mutex is now not recoverable. */
- mutex->__data.__count = 0;
- lll_mutex_unlock (mutex->__data.__lock);
- return ENOTRECOVERABLE;
- }
+ return 0;
+ }
+ }
- /* This mutex is either healthy or we can try to recover it. */
- assert (mutex->__data.__owner == 0
- || mutex->__data.__owner == PTHREAD_MUTEX_OWNERDEAD);
+ oldval = LLL_ROBUST_MUTEX_LOCK (mutex->__data.__lock, id);
- if (__builtin_expect (mutex->__data.__owner
- == PTHREAD_MUTEX_OWNERDEAD, 0))
- {
- retval = EOWNERDEAD;
- /* We signal ownership of a not yet recovered robust mutex
- by storing the negative thread ID. */
- id = -id;
+ if (__builtin_expect (mutex->__data.__owner
+ == PTHREAD_MUTEX_NOTRECOVERABLE, 0))
+ {
+ /* This mutex is now not recoverable. */
+ mutex->__data.__count = 0;
+ lll_mutex_unlock (mutex->__data.__lock);
+ return ENOTRECOVERABLE;
+ }
}
+ while ((oldval & FUTEX_OWNER_DIED) != 0);
+ mutex->__data.__count = 1;
ENQUEUE_MUTEX (mutex);
break;
diff --git a/nptl/pthread_mutex_timedlock.c b/nptl/pthread_mutex_timedlock.c
index bc4ead765d..b69caedc7c 100644
--- a/nptl/pthread_mutex_timedlock.c
+++ b/nptl/pthread_mutex_timedlock.c
@@ -1,4 +1,4 @@
-/* Copyright (C) 2002, 2003, 2004, 2005 Free Software Foundation, Inc.
+/* Copyright (C) 2002, 2003, 2004, 2005, 2006 Free Software Foundation, Inc.
This file is part of the GNU C Library.
Contributed by Ulrich Drepper <drepper@redhat.com>, 2002.
@@ -28,6 +28,7 @@ pthread_mutex_timedlock (mutex, abstime)
pthread_mutex_t *mutex;
const struct timespec *abstime;
{
+ int oldval;
pid_t id = THREAD_GETMEM (THREAD_SELF, tid);
int result = 0;
@@ -103,67 +104,83 @@ pthread_mutex_timedlock (mutex, abstime)
break;
case PTHREAD_MUTEX_ROBUST_PRIVATE_RECURSIVE_NP:
- /* Check whether we already hold the mutex. */
- if (abs (mutex->__data.__owner) == id)
+ case PTHREAD_MUTEX_ROBUST_PRIVATE_ERRORCHECK_NP:
+ case PTHREAD_MUTEX_ROBUST_PRIVATE_NP:
+ case PTHREAD_MUTEX_ROBUST_PRIVATE_ADAPTIVE_NP:
+ oldval = mutex->__data.__lock;
+ do
{
- /* Just bump the counter. */
- if (__builtin_expect (mutex->__data.__count + 1 == 0, 0))
- /* Overflow of the counter. */
- return EAGAIN;
-
- ++mutex->__data.__count;
+ if ((oldval & FUTEX_OWNER_DIED) != 0)
+ {
+ /* The previous owner died. Try locking the mutex. */
+ int newval;
+ while ((newval
+ = atomic_compare_and_exchange_val_acq (&mutex->__data.__lock,
+ id, oldval))
+ != oldval)
+ {
+ if ((newval & FUTEX_OWNER_DIED) == 0)
+ goto normal;
+ oldval = newval;
+ }
- goto out;
- }
+ /* We got the mutex. */
+ mutex->__data.__count = 1;
+ /* But it is inconsistent unless marked otherwise. */
+ mutex->__data.__owner = PTHREAD_MUTEX_INCONSISTENT;
- /* We have to get the mutex. */
- result = lll_mutex_timedlock (mutex->__data.__lock, abstime);
+ ENQUEUE_MUTEX (mutex);
- if (result != 0)
- goto out;
+ /* Note that we deliberately exist here. If we fall
+ through to the end of the function __nusers would be
+ incremented which is not correct because the old
+ owner has to be discounted. */
+ return EOWNERDEAD;
+ }
- /* Only locked once so far. */
- mutex->__data.__count = 1;
- goto robust;
+ normal:
+ /* Check whether we already hold the mutex. */
+ if (__builtin_expect ((mutex->__data.__lock & FUTEX_TID_MASK)
+ == id, 0))
+ {
+ if (mutex->__data.__kind
+ == PTHREAD_MUTEX_ROBUST_PRIVATE_ERRORCHECK_NP)
+ return EDEADLK;
- case PTHREAD_MUTEX_ROBUST_PRIVATE_ERRORCHECK_NP:
- /* Check whether we already hold the mutex. */
- if (__builtin_expect (abs (mutex->__data.__owner) == id, 0))
- return EDEADLK;
+ if (mutex->__data.__kind
+ == PTHREAD_MUTEX_ROBUST_PRIVATE_RECURSIVE_NP)
+ {
+ /* Just bump the counter. */
+ if (__builtin_expect (mutex->__data.__count + 1 == 0, 0))
+ /* Overflow of the counter. */
+ return EAGAIN;
- /* FALLTHROUGH */
+ ++mutex->__data.__count;
- case PTHREAD_MUTEX_ROBUST_PRIVATE_NP:
- case PTHREAD_MUTEX_ROBUST_PRIVATE_ADAPTIVE_NP:
- result = lll_mutex_timedlock (mutex->__data.__lock, abstime);
+ return 0;
+ }
+ }
- if (result != 0)
- goto out;
+ result = lll_robust_mutex_timedlock (mutex->__data.__lock, abstime,
+ id);
- robust:
- if (__builtin_expect (mutex->__data.__owner
- == PTHREAD_MUTEX_NOTRECOVERABLE, 0))
- {
- /* This mutex is now not recoverable. */
- mutex->__data.__count = 0;
- lll_mutex_unlock (mutex->__data.__lock);
- return ENOTRECOVERABLE;
- }
+ if (__builtin_expect (mutex->__data.__owner
+ == PTHREAD_MUTEX_NOTRECOVERABLE, 0))
+ {
+ /* This mutex is now not recoverable. */
+ mutex->__data.__count = 0;
+ lll_mutex_unlock (mutex->__data.__lock);
+ return ENOTRECOVERABLE;
+ }
- /* This mutex is either healthy or we can try to recover it. */
- assert (mutex->__data.__owner == 0
- || mutex->__data.__owner == PTHREAD_MUTEX_OWNERDEAD);
+ if (result == ETIMEDOUT || result == EINVAL)
+ goto out;
- if (__builtin_expect (mutex->__data.__owner
- == PTHREAD_MUTEX_OWNERDEAD, 0))
- {
- result = EOWNERDEAD;
- /* We signal ownership of a not yet recovered robust mutex
- by storing the negative thread ID. */
- mutex->__data.__owner = -id;
- ++mutex->__data.__nusers;
+ oldval = result;
}
+ while ((oldval & FUTEX_OWNER_DIED) != 0);
+ mutex->__data.__count = 1;
ENQUEUE_MUTEX (mutex);
break;
diff --git a/nptl/pthread_mutex_trylock.c b/nptl/pthread_mutex_trylock.c
index ae73ecc39b..5a13ea6925 100644
--- a/nptl/pthread_mutex_trylock.c
+++ b/nptl/pthread_mutex_trylock.c
@@ -1,4 +1,4 @@
-/* Copyright (C) 2002, 2003, 2005 Free Software Foundation, Inc.
+/* Copyright (C) 2002, 2003, 2005, 2006 Free Software Foundation, Inc.
This file is part of the GNU C Library.
Contributed by Ulrich Drepper <drepper@redhat.com>, 2002.
@@ -28,6 +28,7 @@ int
__pthread_mutex_trylock (mutex)
pthread_mutex_t *mutex;
{
+ int oldval;
pid_t id = THREAD_GETMEM (THREAD_SELF, tid);
switch (__builtin_expect (mutex->__data.__kind, PTHREAD_MUTEX_TIMED_NP))
@@ -77,73 +78,88 @@ __pthread_mutex_trylock (mutex)
case PTHREAD_MUTEX_ROBUST_PRIVATE_RECURSIVE_NP:
- /* Check whether we already hold the mutex. */
- if (abs (mutex->__data.__owner) == id)
- {
- /* Just bump the counter. */
- if (__builtin_expect (mutex->__data.__count + 1 == 0, 0))
- /* Overflow of the counter. */
- return EAGAIN;
-
- ++mutex->__data.__count;
-
- return 0;
- }
-
- /* We have to get the mutex. */
- if (lll_mutex_trylock (mutex->__data.__lock) == 0)
- {
- mutex->__data.__count = 1;
-
- goto robust;
- }
-
- break;
-
case PTHREAD_MUTEX_ROBUST_PRIVATE_ERRORCHECK_NP:
- /* Check whether we already hold the mutex. */
- if (__builtin_expect (abs (mutex->__data.__owner) == id, 0))
- return EDEADLK;
-
- /* FALLTHROUGH */
-
case PTHREAD_MUTEX_ROBUST_PRIVATE_NP:
case PTHREAD_MUTEX_ROBUST_PRIVATE_ADAPTIVE_NP:
- if (lll_mutex_trylock (mutex->__data.__lock) != 0)
- break;
-
- robust:
- if (__builtin_expect (mutex->__data.__owner
- == PTHREAD_MUTEX_NOTRECOVERABLE, 0))
- {
- /* This mutex is now not recoverable. */
- mutex->__data.__count = 0;
- lll_mutex_unlock (mutex->__data.__lock);
- return ENOTRECOVERABLE;
- }
-
- /* This mutex is either healthy or we can try to recover it. */
- assert (mutex->__data.__owner == 0
- || mutex->__data.__owner == PTHREAD_MUTEX_OWNERDEAD);
-
- /* Record the ownership. */
- int retval = 0;
- if (__builtin_expect (mutex->__data.__owner
- == PTHREAD_MUTEX_OWNERDEAD, 0))
+ oldval = mutex->__data.__lock;
+ do
{
- retval = EOWNERDEAD;
- /* We signal ownership of a not yet recovered robust
- mutex by storing the negative thread ID. */
- id = -id;
+ if ((oldval & FUTEX_OWNER_DIED) != 0)
+ {
+ /* The previous owner died. Try locking the mutex. */
+ int newval;
+ while ((newval
+ = atomic_compare_and_exchange_val_acq (&mutex->__data.__lock,
+ id, oldval))
+ != oldval)
+ {
+ if ((newval & FUTEX_OWNER_DIED) == 0)
+ goto normal;
+ oldval = newval;
+ }
+
+ /* We got the mutex. */
+ mutex->__data.__count = 1;
+ /* But it is inconsistent unless marked otherwise. */
+ mutex->__data.__owner = PTHREAD_MUTEX_INCONSISTENT;
+
+ ENQUEUE_MUTEX (mutex);
+
+ /* Note that we deliberately exist here. If we fall
+ through to the end of the function __nusers would be
+ incremented which is not correct because the old
+ owner has to be discounted. */
+ return EOWNERDEAD;
+ }
+
+ normal:
+ /* Check whether we already hold the mutex. */
+ if (__builtin_expect ((mutex->__data.__lock & FUTEX_TID_MASK)
+ == id, 0))
+ {
+ if (mutex->__data.__kind
+ == PTHREAD_MUTEX_ROBUST_PRIVATE_ERRORCHECK_NP)
+ return EDEADLK;
+
+ if (mutex->__data.__kind
+ == PTHREAD_MUTEX_ROBUST_PRIVATE_RECURSIVE_NP)
+ {
+ /* Just bump the counter. */
+ if (__builtin_expect (mutex->__data.__count + 1 == 0, 0))
+ /* Overflow of the counter. */
+ return EAGAIN;
+
+ ++mutex->__data.__count;
+
+ return 0;
+ }
+ }
+
+ oldval = lll_robust_mutex_trylock (mutex->__data.__lock, id);
+ if (oldval != 0 && (oldval & FUTEX_OWNER_DIED) == 0)
+ return EBUSY;
+
+ robust:
+ if (__builtin_expect (mutex->__data.__owner
+ == PTHREAD_MUTEX_NOTRECOVERABLE, 0))
+ {
+ /* This mutex is now not recoverable. */
+ mutex->__data.__count = 0;
+ if (oldval == id)
+ lll_mutex_unlock (mutex->__data.__lock);
+ return ENOTRECOVERABLE;
+ }
}
+ while ((oldval & FUTEX_OWNER_DIED) != 0);
ENQUEUE_MUTEX (mutex);
mutex->__data.__owner = id;
++mutex->__data.__nusers;
+ mutex->__data.__count = 1;
+
+ return 0;
- return retval
-;
default:
/* Correct code cannot set any other type. */
return EINVAL;
diff --git a/nptl/pthread_mutex_unlock.c b/nptl/pthread_mutex_unlock.c
index 4d87381065..d41eefe34c 100644
--- a/nptl/pthread_mutex_unlock.c
+++ b/nptl/pthread_mutex_unlock.c
@@ -41,23 +41,32 @@ __pthread_mutex_unlock_usercnt (mutex, decr)
if (--mutex->__data.__count != 0)
/* We still hold the mutex. */
return 0;
- break;
+ goto normal;
case PTHREAD_MUTEX_ERRORCHECK_NP:
/* Error checking mutex. */
if (mutex->__data.__owner != THREAD_GETMEM (THREAD_SELF, tid)
|| ! lll_mutex_islocked (mutex->__data.__lock))
return EPERM;
- break;
+ /* FALLTHROUGH */
case PTHREAD_MUTEX_TIMED_NP:
case PTHREAD_MUTEX_ADAPTIVE_NP:
- /* Normal mutex. Nothing special to do. */
+ /* Always reset the owner field. */
+ normal:
+ mutex->__data.__owner = 0;
+ if (decr)
+ /* One less user. */
+ --mutex->__data.__nusers;
+
+ /* Unlock. */
+ lll_mutex_unlock (mutex->__data.__lock);
break;
case PTHREAD_MUTEX_ROBUST_PRIVATE_RECURSIVE_NP:
/* Recursive mutex. */
- if (mutex->__data.__owner == -THREAD_GETMEM (THREAD_SELF, tid))
+ if ((mutex->__data.__lock & FUTEX_TID_MASK)
+ == THREAD_GETMEM (THREAD_SELF, tid))
{
if (--mutex->__data.__count != 0)
/* We still hold the mutex. */
@@ -78,7 +87,8 @@ __pthread_mutex_unlock_usercnt (mutex, decr)
case PTHREAD_MUTEX_ROBUST_PRIVATE_ERRORCHECK_NP:
case PTHREAD_MUTEX_ROBUST_PRIVATE_NP:
case PTHREAD_MUTEX_ROBUST_PRIVATE_ADAPTIVE_NP:
- if (abs (mutex->__data.__owner) != THREAD_GETMEM (THREAD_SELF, tid)
+ if ((mutex->__data.__lock & FUTEX_TID_MASK)
+ != THREAD_GETMEM (THREAD_SELF, tid)
|| ! lll_mutex_islocked (mutex->__data.__lock))
return EPERM;
@@ -86,15 +96,21 @@ __pthread_mutex_unlock_usercnt (mutex, decr)
making the state consistent, mark the mutex as unrecoverable
and make all waiters. */
if (__builtin_expect (mutex->__data.__owner
- == -THREAD_GETMEM (THREAD_SELF, tid)
- || (mutex->__data.__owner
- == PTHREAD_MUTEX_NOTRECOVERABLE), 0))
+ == PTHREAD_MUTEX_INCONSISTENT, 0))
notrecoverable:
newowner = PTHREAD_MUTEX_NOTRECOVERABLE;
robust:
/* Remove mutex from the list. */
DEQUEUE_MUTEX (mutex);
+
+ mutex->__data.__owner = newowner;
+ if (decr)
+ /* One less user. */
+ --mutex->__data.__nusers;
+
+ /* Unlock. */
+ lll_robust_mutex_unlock (mutex->__data.__lock);
break;
default:
@@ -102,15 +118,6 @@ __pthread_mutex_unlock_usercnt (mutex, decr)
return EINVAL;
}
- /* Always reset the owner field. */
- mutex->__data.__owner = newowner;
- if (decr)
- /* One less user. */
- --mutex->__data.__nusers;
-
- /* Unlock. */
- lll_mutex_unlock (mutex->__data.__lock);
-
return 0;
}
diff --git a/nptl/sysdeps/unix/sysv/linux/Makefile b/nptl/sysdeps/unix/sysv/linux/Makefile
index 14f513c127..88dce1a886 100644
--- a/nptl/sysdeps/unix/sysv/linux/Makefile
+++ b/nptl/sysdeps/unix/sysv/linux/Makefile
@@ -1,4 +1,4 @@
-# Copyright (C) 2002, 2003, 2004, 2005 Free Software Foundation, Inc.
+# Copyright (C) 2002, 2003, 2004, 2005, 2006 Free Software Foundation, Inc.
# This file is part of the GNU C Library.
# Contributed by Ulrich Drepper <drepper@redhat.com>, 2002.
@@ -24,7 +24,8 @@ sysdep_routines += register-atfork unregister-atfork libc_pthread_init \
libpthread-sysdep_routines += pt-fork pthread_mutex_cond_lock
gen-as-const-headers += lowlevelcond.sym lowlevelrwlock.sym \
- lowlevelbarrier.sym unwindbuf.sym
+ lowlevelbarrier.sym unwindbuf.sym \
+ lowlevelrobustlock.sym
endif
ifeq ($(subdir),posix)
diff --git a/nptl/sysdeps/unix/sysv/linux/alpha/bits/pthreadtypes.h b/nptl/sysdeps/unix/sysv/linux/alpha/bits/pthreadtypes.h
index cb916917e5..41a54d4b0d 100644
--- a/nptl/sysdeps/unix/sysv/linux/alpha/bits/pthreadtypes.h
+++ b/nptl/sysdeps/unix/sysv/linux/alpha/bits/pthreadtypes.h
@@ -43,6 +43,13 @@ typedef union
} pthread_attr_t;
+typedef struct __pthread_internal_list
+{
+ struct __pthread_internal_list *__prev;
+ struct __pthread_internal_list *__next;
+} __pthread_list_t;
+
+
/* Data structures for mutex handling. The structure of the attribute
type is deliberately not exposed. */
typedef union
@@ -57,8 +64,7 @@ typedef union
binary compatibility. */
int __kind;
int __spins;
- struct __pthread_mutex_s *__next;
- struct __pthread_mutex_s *__prev;
+ __pthread_list_t __list;
#define __PTHREAD_MUTEX_HAVE_PREV 1
} __data;
char __size[__SIZEOF_PTHREAD_MUTEX_T];
diff --git a/nptl/sysdeps/unix/sysv/linux/i386/i486/lowlevelrobustlock.S b/nptl/sysdeps/unix/sysv/linux/i386/i486/lowlevelrobustlock.S
new file mode 100644
index 0000000000..1c516c7424
--- /dev/null
+++ b/nptl/sysdeps/unix/sysv/linux/i386/i486/lowlevelrobustlock.S
@@ -0,0 +1,186 @@
+/* Copyright (C) 2002, 2003, 2004, 2006 Free Software Foundation, Inc.
+ This file is part of the GNU C Library.
+ Contributed by Ulrich Drepper <drepper@redhat.com>, 2002.
+
+ The GNU C Library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 2.1 of the License, or (at your option) any later version.
+
+ The GNU C Library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with the GNU C Library; if not, write to the Free
+ Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA
+ 02111-1307 USA. */
+
+#include <sysdep.h>
+#include <pthread-errnos.h>
+#include <lowlevelrobustlock.h>
+
+ .text
+
+#ifndef LOCK
+# ifdef UP
+# define LOCK
+# else
+# define LOCK lock
+# endif
+#endif
+
+#define SYS_gettimeofday __NR_gettimeofday
+#define SYS_futex 240
+#define FUTEX_WAIT 0
+#define FUTEX_WAKE 1
+#define FUTEX_WAITERS 0x80000000
+#define FUTEX_OWNER_DIED 0x40000000
+
+
+ .globl __lll_robust_mutex_lock_wait
+ .type __lll_robust_mutex_lock_wait,@function
+ .hidden __lll_robust_mutex_lock_wait
+ .align 16
+__lll_robust_mutex_lock_wait:
+ pushl %edx
+ pushl %ebx
+ pushl %esi
+
+ movl %ecx, %ebx
+ xorl %esi, %esi /* No timeout. */
+ xorl %ecx, %ecx /* movl $FUTEX_WAIT, %ecx */
+
+4: movl %eax, %edx
+ orl $FUTEX_WAITERS, %edx
+
+ testl $FUTEX_OWNER_DIED, %eax
+ jnz 3f
+
+ cmpl %edx, %eax /* NB: %edx == 2 */
+ je 1f
+
+ LOCK
+ cmpxchgl %edx, (%ebx)
+ jnz 2f
+
+1: movl $SYS_futex, %eax
+ ENTER_KERNEL
+
+ movl (%ebx), %eax
+
+2: test %eax, %eax
+ jne 4b
+
+ movl %gs:TID, %edx
+ LOCK
+ cmpxchgl %edx, (%ebx)
+ jnz 4b
+ /* NB: %eax == 0 */
+
+3: popl %esi
+ popl %ebx
+ popl %edx
+ ret
+ .size __lll_robust_mutex_lock_wait,.-__lll_robust_mutex_lock_wait
+
+
+ .globl __lll_robust_mutex_timedlock_wait
+ .type __lll_robust_mutex_timedlock_wait,@function
+ .hidden __lll_robust_mutex_timedlock_wait
+ .align 16
+__lll_robust_mutex_timedlock_wait:
+ /* Check for a valid timeout value. */
+ cmpl $1000000000, 4(%edx)
+ jae 3f
+
+ pushl %edi
+ pushl %esi
+ pushl %ebx
+ pushl %ebp
+
+ /* Stack frame for the timespec and timeval structs. */
+ subl $12, %esp
+
+ movl %ecx, %ebp
+ movl %edx, %edi
+
+1: movl %eax, 8(%esp)
+
+ /* Get current time. */
+ movl %esp, %ebx
+ xorl %ecx, %ecx
+ movl $SYS_gettimeofday, %eax
+ ENTER_KERNEL
+
+ /* Compute relative timeout. */
+ movl 4(%esp), %eax
+ movl $1000, %edx
+ mul %edx /* Milli seconds to nano seconds. */
+ movl (%edi), %ecx
+ movl 4(%edi), %edx
+ subl (%esp), %ecx
+ subl %eax, %edx
+ jns 4f
+ addl $1000000000, %edx
+ subl $1, %ecx
+4: testl %ecx, %ecx
+ js 8f /* Time is already up. */
+
+ /* Store relative timeout. */
+ movl %ecx, (%esp)
+ movl %edx, 4(%esp)
+
+ movl %ebp, %ebx
+
+ movl 8(%esp), %edx
+ movl %edx, %eax
+ orl $FUTEX_WAITERS, %edx
+
+ testl $FUTEX_OWNER_DIED, %eax
+ jnz 6f
+
+ cmpl %eax, %edx
+ je 2f
+
+ LOCK
+ cmpxchgl %edx, (%ebx)
+ movl $0, %ecx /* Must use mov to avoid changing cc. */
+ jnz 5f
+
+2:
+ /* Futex call. */
+ movl %esp, %esi
+ xorl %ecx, %ecx /* movl $FUTEX_WAIT, %ecx */
+ movl $SYS_futex, %eax
+ ENTER_KERNEL
+ movl %eax, %ecx
+
+ movl (%ebx), %eax
+
+5: testl %eax, %eax
+ jne 7f
+
+ movl %gs:TID, %edx
+ LOCK
+ cmpxchgl %edx, (%ebx)
+ jnz 7f
+
+6: addl $12, %esp
+ popl %ebp
+ popl %ebx
+ popl %esi
+ popl %edi
+ ret
+
+ /* Check whether the time expired. */
+7: cmpl $-ETIMEDOUT, %ecx
+ jne 1b
+
+8: movl $ETIMEDOUT, %eax
+ jmp 6b
+
+3: movl $EINVAL, %eax
+ ret
+ .size __lll_robust_mutex_timedlock_wait,.-__lll_robust_mutex_timedlock_wait
diff --git a/nptl/sysdeps/unix/sysv/linux/i386/i586/lowlevelrobustlock.S b/nptl/sysdeps/unix/sysv/linux/i386/i586/lowlevelrobustlock.S
new file mode 100644
index 0000000000..f768e16a7d
--- /dev/null
+++ b/nptl/sysdeps/unix/sysv/linux/i386/i586/lowlevelrobustlock.S
@@ -0,0 +1,20 @@
+/* Copyright (C) 2002, 2006 Free Software Foundation, Inc.
+ This file is part of the GNU C Library.
+ Contributed by Ulrich Drepper <drepper@redhat.com>, 2002.
+
+ The GNU C Library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 2.1 of the License, or (at your option) any later version.
+
+ The GNU C Library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with the GNU C Library; if not, write to the Free
+ Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA
+ 02111-1307 USA. */
+
+#include "../i486/lowlevelrobustlock.S"
diff --git a/nptl/sysdeps/unix/sysv/linux/i386/i686/lowlevelrobustlock.S b/nptl/sysdeps/unix/sysv/linux/i386/i686/lowlevelrobustlock.S
new file mode 100644
index 0000000000..f768e16a7d
--- /dev/null
+++ b/nptl/sysdeps/unix/sysv/linux/i386/i686/lowlevelrobustlock.S
@@ -0,0 +1,20 @@
+/* Copyright (C) 2002, 2006 Free Software Foundation, Inc.
+ This file is part of the GNU C Library.
+ Contributed by Ulrich Drepper <drepper@redhat.com>, 2002.
+
+ The GNU C Library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 2.1 of the License, or (at your option) any later version.
+
+ The GNU C Library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with the GNU C Library; if not, write to the Free
+ Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA
+ 02111-1307 USA. */
+
+#include "../i486/lowlevelrobustlock.S"
diff --git a/nptl/sysdeps/unix/sysv/linux/i386/lowlevellock.h b/nptl/sysdeps/unix/sysv/linux/i386/lowlevellock.h
index b233d9747f..f3b97143c4 100644
--- a/nptl/sysdeps/unix/sysv/linux/i386/lowlevellock.h
+++ b/nptl/sysdeps/unix/sysv/linux/i386/lowlevellock.h
@@ -139,6 +139,16 @@ extern int __lll_mutex_unlock_wake (int *__futex)
ret; })
+#define lll_robust_mutex_trylock(futex, id) \
+ ({ int ret; \
+ __asm __volatile (LOCK_INSTR "cmpxchgl %2, %1" \
+ : "=a" (ret), "=m" (futex) \
+ : "r" (id), "m" (futex), \
+ "0" (LLL_MUTEX_LOCK_INITIALIZER) \
+ : "memory"); \
+ ret; })
+
+
#define lll_mutex_cond_trylock(futex) \
({ int ret; \
__asm __volatile (LOCK_INSTR "cmpxchgl %2, %1" \
@@ -167,6 +177,25 @@ extern int __lll_mutex_unlock_wake (int *__futex)
: "memory"); })
+#define lll_robust_mutex_lock(futex, id) \
+ ({ int result, ignore; \
+ __asm __volatile (LOCK_INSTR "cmpxchgl %1, %2\n\t" \
+ "jnz _L_mutex_lock_%=\n\t" \
+ ".subsection 1\n\t" \
+ ".type _L_mutex_lock_%=,@function\n" \
+ "_L_mutex_lock_%=:\n\t" \
+ "leal %2, %%ecx\n\t" \
+ "call __lll_robust_mutex_lock_wait\n\t" \
+ "jmp 1f\n\t" \
+ ".size _L_mutex_lock_%=,.-_L_mutex_lock_%=\n" \
+ ".previous\n" \
+ "1:" \
+ : "=a" (result), "=c" (ignore), "=m" (futex) \
+ : "0" (0), "1" (id), "m" (futex) \
+ : "memory"); \
+ result; })
+
+
/* Special version of lll_mutex_lock which causes the unlock function to
always wakeup waiters. */
#define lll_mutex_cond_lock(futex) \
@@ -187,6 +216,25 @@ extern int __lll_mutex_unlock_wake (int *__futex)
: "memory"); })
+#define lll_robust_mutex_cond_lock(futex, id) \
+ ({ int result, ignore; \
+ __asm __volatile (LOCK_INSTR "cmpxchgl %1, %2\n\t" \
+ "jnz _L_mutex_cond_lock_%=\n\t" \
+ ".subsection 1\n\t" \
+ ".type _L_mutex_cond_lock_%=,@function\n" \
+ "_L_mutex_cond_lock_%=:\n\t" \
+ "leal %2, %%ecx\n\t" \
+ "call __lll_robust_mutex_lock_wait\n\t" \
+ "jmp 1f\n\t" \
+ ".size _L_mutex_cond_lock_%=,.-_L_mutex_cond_lock_%=\n"\
+ ".previous\n" \
+ "1:" \
+ : "=a" (result), "=c" (ignore), "=m" (futex) \
+ : "0" (0), "1" (id | FUTEX_WAITERS), "m" (futex) \
+ : "memory"); \
+ result; })
+
+
#define lll_mutex_timedlock(futex, timeout) \
({ int result, ignore1, ignore2; \
__asm __volatile (LOCK_INSTR "cmpxchgl %1, %3\n\t" \
@@ -208,9 +256,30 @@ extern int __lll_mutex_unlock_wake (int *__futex)
result; })
+#define lll_robust_mutex_timedlock(futex, timeout, id) \
+ ({ int result, ignore1, ignore2; \
+ __asm __volatile (LOCK_INSTR "cmpxchgl %1, %3\n\t" \
+ "jnz _L_mutex_timedlock_%=\n\t" \
+ ".subsection 1\n\t" \
+ ".type _L_mutex_timedlock_%=,@function\n" \
+ "_L_mutex_timedlock_%=:\n\t" \
+ "leal %3, %%ecx\n\t" \
+ "movl %7, %%edx\n\t" \
+ "call __lll_robust_mutex_timedlock_wait\n\t" \
+ "jmp 1f\n\t" \
+ ".size _L_mutex_timedlock_%=,.-_L_mutex_timedlock_%=\n"\
+ ".previous\n" \
+ "1:" \
+ : "=a" (result), "=c" (ignore1), "=&d" (ignore2), \
+ "=m" (futex) \
+ : "0" (0), "1" (id), "m" (futex), "m" (timeout) \
+ : "memory"); \
+ result; })
+
+
#define lll_mutex_unlock(futex) \
(void) ({ int ignore; \
- __asm __volatile (LOCK_INSTR "subl $1,%0\n\t" \
+ __asm __volatile (LOCK_INSTR "subl $1, %0\n\t" \
"jne _L_mutex_unlock_%=\n\t" \
".subsection 1\n\t" \
".type _L_mutex_unlock_%=,@function\n" \
@@ -226,6 +295,53 @@ extern int __lll_mutex_unlock_wake (int *__futex)
: "memory"); })
+#define lll_robust_mutex_unlock(futex) \
+ (void) ({ int ignore; \
+ __asm __volatile (LOCK_INSTR "andl %2, %0\n\t" \
+ "jne _L_mutex_unlock_%=\n\t" \
+ ".subsection 1\n\t" \
+ ".type _L_mutex_unlock_%=,@function\n" \
+ "_L_mutex_unlock_%=:\n\t" \
+ "leal %0, %%eax\n\t" \
+ "call __lll_mutex_unlock_wake\n\t" \
+ "jmp 1f\n\t" \
+ ".size _L_mutex_unlock_%=,.-_L_mutex_unlock_%=\n" \
+ ".previous\n" \
+ "1:" \
+ : "=m" (futex), "=&a" (ignore) \
+ : "i" (FUTEX_TID_MASK), "m" (futex) \
+ : "memory"); })
+
+
+#define lll_robust_mutex_dead(futex) \
+ (void) ({ int __ignore; \
+ register int _nr asm ("edx") = 1; \
+ __asm __volatile (LOCK_INSTR "orl %5, (%2)\n\t" \
+ LLL_EBX_LOAD \
+ LLL_ENTER_KERNEL \
+ LLL_EBX_LOAD \
+ : "=a" (__ignore) \
+ : "0" (SYS_futex), LLL_EBX_REG (&(futex)), \
+ "c" (FUTEX_WAKE), "d" (_nr), \
+ "i" (FUTEX_OWNER_DIED), \
+ "i" (offsetof (tcbhead_t, sysinfo))); })
+
+
+#define lll_futex_wake(futex, nr) \
+ do { \
+ int __ignore; \
+ register __typeof (nr) _nr asm ("edx") = (nr); \
+ __asm __volatile (LLL_EBX_LOAD \
+ LLL_ENTER_KERNEL \
+ LLL_EBX_LOAD \
+ : "=a" (__ignore) \
+ : "0" (SYS_futex), LLL_EBX_REG (futex), \
+ "c" (FUTEX_WAKE), "d" (_nr), \
+ "i" (0) /* phony, to align next arg's number */, \
+ "i" (offsetof (tcbhead_t, sysinfo))); \
+ } while (0)
+
+
#define lll_mutex_islocked(futex) \
(futex != 0)
diff --git a/nptl/sysdeps/unix/sysv/linux/ia64/bits/pthreadtypes.h b/nptl/sysdeps/unix/sysv/linux/ia64/bits/pthreadtypes.h
index a13bb080a4..892769dca4 100644
--- a/nptl/sysdeps/unix/sysv/linux/ia64/bits/pthreadtypes.h
+++ b/nptl/sysdeps/unix/sysv/linux/ia64/bits/pthreadtypes.h
@@ -43,6 +43,13 @@ typedef union
} pthread_attr_t;
+typedef struct __pthread_internal_list
+{
+ struct __pthread_internal_list *__prev;
+ struct __pthread_internal_list *__next;
+} __pthread_list_t;
+
+
/* Data structures for mutex handling. The structure of the attribute
type is not exposed on purpose. */
typedef union
@@ -57,8 +64,7 @@ typedef union
binary compatibility. */
int __kind;
int __spins;
- struct __pthread_mutex_s *__next;
- struct __pthread_mutex_s *__prev;
+ __pthread_list_t __list;
#define __PTHREAD_MUTEX_HAVE_PREV 1
} __data;
char __size[__SIZEOF_PTHREAD_MUTEX_T];
diff --git a/nptl/sysdeps/unix/sysv/linux/lowlevelrobustlock.sym b/nptl/sysdeps/unix/sysv/linux/lowlevelrobustlock.sym
new file mode 100644
index 0000000000..2f1e9da52b
--- /dev/null
+++ b/nptl/sysdeps/unix/sysv/linux/lowlevelrobustlock.sym
@@ -0,0 +1,6 @@
+#include <stddef.h>
+#include <pthreadP.h>
+
+--
+
+TID offsetof (struct pthread, tid)
diff --git a/nptl/sysdeps/unix/sysv/linux/powerpc/bits/pthreadtypes.h b/nptl/sysdeps/unix/sysv/linux/powerpc/bits/pthreadtypes.h
index 493c2ab557..a7150f6aef 100644
--- a/nptl/sysdeps/unix/sysv/linux/powerpc/bits/pthreadtypes.h
+++ b/nptl/sysdeps/unix/sysv/linux/powerpc/bits/pthreadtypes.h
@@ -58,6 +58,20 @@ typedef union
} pthread_attr_t;
+#if __WORDSIZE == 64
+typedef struct __pthread_internal_list
+{
+ struct __pthread_internal_list *__prev;
+ struct __pthread_internal_list *__next;
+} __pthread_list_t;
+#else
+typedef struct __pthread_internal_slist
+{
+ struct __pthread_internal_slist *__next;
+} __pthread_slist_t;
+#endif
+
+
/* Data structures for mutex handling. The structure of the attribute
type is deliberately not exposed. */
typedef union
@@ -75,15 +89,14 @@ typedef union
int __kind;
#if __WORDSIZE == 64
int __spins;
- struct __pthread_mutex_s *__next;
- struct __pthread_mutex_s *__prev;
+ __pthread_list_t __list;
# define __PTHREAD_MUTEX_HAVE_PREV 1
#else
unsigned int __nusers;
__extension__ union
{
int __spins;
- struct __pthread_mutex_s *__next;
+ __pthread_slist_t __list;
};
#endif
} __data;
diff --git a/nptl/sysdeps/unix/sysv/linux/pthread_mutex_cond_lock.c b/nptl/sysdeps/unix/sysv/linux/pthread_mutex_cond_lock.c
index 990db87416..a97351f880 100644
--- a/nptl/sysdeps/unix/sysv/linux/pthread_mutex_cond_lock.c
+++ b/nptl/sysdeps/unix/sysv/linux/pthread_mutex_cond_lock.c
@@ -1,7 +1,8 @@
#include <pthreadP.h>
-#define LLL_MUTEX_LOCK(mutex) lll_mutex_cond_lock(mutex)
-#define LLL_MUTEX_TRYLOCK(mutex) lll_mutex_cond_trylock(mutex)
+#define LLL_MUTEX_LOCK(mutex) lll_mutex_cond_lock (mutex)
+#define LLL_MUTEX_TRYLOCK(mutex) lll_mutex_cond_trylock (mutex)
+#define LLL_ROBUST_MUTEX_LOCK(mutex, id) lll_robust_mutex_cond_lock (mutex, id)
#define __pthread_mutex_lock __pthread_mutex_cond_lock
#define NO_INCR
diff --git a/nptl/sysdeps/unix/sysv/linux/s390/bits/pthreadtypes.h b/nptl/sysdeps/unix/sysv/linux/s390/bits/pthreadtypes.h
index df8beeb685..c77031d7bb 100644
--- a/nptl/sysdeps/unix/sysv/linux/s390/bits/pthreadtypes.h
+++ b/nptl/sysdeps/unix/sysv/linux/s390/bits/pthreadtypes.h
@@ -57,6 +57,20 @@ typedef union
} pthread_attr_t;
+#if __WORDSIZE == 64
+typedef struct __pthread_internal_list
+{
+ struct __pthread_internal_list *__prev;
+ struct __pthread_internal_list *__next;
+} __pthread_list_t;
+#else
+typedef struct __pthread_internal_slist
+{
+ struct __pthread_internal_slist *__next;
+} __pthread_slist_t;
+#endif
+
+
/* Data structures for mutex handling. The structure of the attribute
type is not exposed on purpose. */
typedef union
@@ -74,15 +88,14 @@ typedef union
int __kind;
#if __WORDSIZE == 64
int __spins;
- struct __pthread_mutex_s *__next;
- struct __pthread_mutex_s *__prev;
+ __pthread_list_t __list;
# define __PTHREAD_MUTEX_HAVE_PREV 1
#else
unsigned int __nusers;
__extension__ union
{
int __spins;
- struct __pthread_mutex_s *__next;
+ __pthread_slist_t __list;
};
#endif
} __data;
diff --git a/nptl/sysdeps/unix/sysv/linux/sh/bits/pthreadtypes.h b/nptl/sysdeps/unix/sysv/linux/sh/bits/pthreadtypes.h
index 7642eccab0..969686dd5a 100644
--- a/nptl/sysdeps/unix/sysv/linux/sh/bits/pthreadtypes.h
+++ b/nptl/sysdeps/unix/sysv/linux/sh/bits/pthreadtypes.h
@@ -44,6 +44,12 @@ typedef union
} pthread_attr_t;
+typedef struct __pthread_internal_slist
+{
+ struct __pthread_internal_slist *__next;
+} __pthread_slist_t;
+
+
/* Data structures for mutex handling. The structure of the attribute
type is not exposed on purpose. */
typedef union
@@ -60,7 +66,7 @@ typedef union
__extension__ union
{
int __spins;
- struct __pthread_mutex_s *__next;
+ __pthread_slist_t __list;
};
} __data;
char __size[__SIZEOF_PTHREAD_MUTEX_T];
diff --git a/nptl/sysdeps/unix/sysv/linux/sparc/bits/pthreadtypes.h b/nptl/sysdeps/unix/sysv/linux/sparc/bits/pthreadtypes.h
index 7dc61c12e1..e734c1205e 100644
--- a/nptl/sysdeps/unix/sysv/linux/sparc/bits/pthreadtypes.h
+++ b/nptl/sysdeps/unix/sysv/linux/sparc/bits/pthreadtypes.h
@@ -58,6 +58,20 @@ typedef union
} pthread_attr_t;
+#if __WORDSIZE == 64
+typedef struct __pthread_internal_list
+{
+ struct __pthread_internal_list *__prev;
+ struct __pthread_internal_list *__next;
+} __pthread_list_t;
+#else
+typedef struct __pthread_internal_slist
+{
+ struct __pthread_internal_slist *__next;
+} __pthread_slist_t;
+#endif
+
+
/* Data structures for mutex handling. The structure of the attribute
type is deliberately not exposed. */
typedef union
@@ -75,15 +89,14 @@ typedef union
int __kind;
#if __WORDSIZE == 64
int __spins;
- struct __pthread_mutex_s *__next;
- struct __pthread_mutex_s *__prev;
+ __pthread_list_t __list;
# define __PTHREAD_MUTEX_HAVE_PREV 1
#else
unsigned int __nusers;
__extension__ union
{
int __spins;
- struct __pthread_mutex_s *__next;
+ __pthread_slist_t __list;
};
#endif
} __data;
diff --git a/nptl/sysdeps/unix/sysv/linux/x86_64/lowlevellock.h b/nptl/sysdeps/unix/sysv/linux/x86_64/lowlevellock.h
index ebcfe6eef1..f91ce9b8d9 100644
--- a/nptl/sysdeps/unix/sysv/linux/x86_64/lowlevellock.h
+++ b/nptl/sysdeps/unix/sysv/linux/x86_64/lowlevellock.h
@@ -111,6 +111,16 @@ extern int __lll_mutex_unlock_wait (int *__futex) attribute_hidden;
ret; })
+#define lll_robust_mutex_trylock(futex, id) \
+ ({ int ret; \
+ __asm __volatile (LOCK_INSTR "cmpxchgl %2, %1" \
+ : "=a" (ret), "=m" (futex) \
+ : "r" (id), "m" (futex), \
+ "0" (LLL_MUTEX_LOCK_INITIALIZER) \
+ : "memory"); \
+ ret; })
+
+
#define lll_mutex_cond_trylock(futex) \
({ int ret; \
__asm __volatile (LOCK_INSTR "cmpxchgl %2, %1" \
@@ -139,6 +149,25 @@ extern int __lll_mutex_unlock_wait (int *__futex) attribute_hidden;
: "cx", "r11", "cc", "memory"); })
+#define lll_robust_mutex_lock(futex, id) \
+ ({ int result, ignore1, ignore2; \
+ __asm __volatile (LOCK_INSTR "cmpxchgl %0, %2\n\t" \
+ "jnz 1f\n\t" \
+ ".subsection 1\n" \
+ "1:\tleaq %2, %%rdi\n\t" \
+ "subq $128, %%rsp\n\t" \
+ "callq __lll_robust_mutex_lock_wait\n\t" \
+ "addq $128, %%rsp\n\t" \
+ "jmp 2f\n\t" \
+ ".previous\n" \
+ "2:" \
+ : "=S" (ignore1), "=&D" (ignore2), "=m" (futex), \
+ "=a" (result) \
+ : "0" (id), "m" (futex), "3" (0) \
+ : "cx", "r11", "cc", "memory"); \
+ result; })
+
+
#define lll_mutex_cond_lock(futex) \
(void) ({ int ignore1, ignore2, ignore3; \
__asm __volatile (LOCK_INSTR "cmpxchgl %0, %2\n\t" \
@@ -157,6 +186,25 @@ extern int __lll_mutex_unlock_wait (int *__futex) attribute_hidden;
: "cx", "r11", "cc", "memory"); })
+#define lll_robust_mutex_cond_lock(futex, id) \
+ ({ int result, ignore1, ignore2; \
+ __asm __volatile (LOCK_INSTR "cmpxchgl %0, %2\n\t" \
+ "jnz 1f\n\t" \
+ ".subsection 1\n" \
+ "1:\tleaq %2, %%rdi\n\t" \
+ "subq $128, %%rsp\n\t" \
+ "callq __lll_robust_mutex_lock_wait\n\t" \
+ "addq $128, %%rsp\n\t" \
+ "jmp 2f\n\t" \
+ ".previous\n" \
+ "2:" \
+ : "=S" (ignore1), "=&D" (ignore2), "=m" (futex), \
+ "=a" (result) \
+ : "0" (id | FUTEX_WAITERS), "m" (futex), "3" (0) \
+ : "cx", "r11", "cc", "memory"); \
+ result; })
+
+
#define lll_mutex_timedlock(futex, timeout) \
({ int result, ignore1, ignore2, ignore3; \
__asm __volatile (LOCK_INSTR "cmpxchgl %2, %4\n\t" \
@@ -177,6 +225,26 @@ extern int __lll_mutex_unlock_wait (int *__futex) attribute_hidden;
result; })
+#define lll_robust_mutex_timedlock(futex, timeout, id) \
+ ({ int result, ignore1, ignore2, ignore3; \
+ __asm __volatile (LOCK_INSTR "cmpxchgl %2, %4\n\t" \
+ "jnz 1f\n\t" \
+ ".subsection 1\n" \
+ "1:\tleaq %4, %%rdi\n\t" \
+ "movq %8, %%rdx\n\t" \
+ "subq $128, %%rsp\n\t" \
+ "callq __lll_robust_mutex_timedlock_wait\n\t" \
+ "addq $128, %%rsp\n\t" \
+ "jmp 2f\n\t" \
+ ".previous\n" \
+ "2:" \
+ : "=a" (result), "=&D" (ignore1), "=S" (ignore2), \
+ "=&d" (ignore3), "=m" (futex) \
+ : "0" (0), "2" (id), "m" (futex), "m" (timeout) \
+ : "memory", "cx", "cc", "r10", "r11"); \
+ result; })
+
+
#define lll_mutex_unlock(futex) \
(void) ({ int ignore; \
__asm __volatile (LOCK_INSTR "decl %0\n\t" \
@@ -194,6 +262,34 @@ extern int __lll_mutex_unlock_wait (int *__futex) attribute_hidden;
: "ax", "cx", "r11", "cc", "memory"); })
+#define lll_robust_mutex_unlock(futex) \
+ (void) ({ int ignore; \
+ __asm __volatile (LOCK_INSTR "andl %2, %0\n\t" \
+ "jne 1f\n\t" \
+ ".subsection 1\n" \
+ "1:\tleaq %0, %%rdi\n\t" \
+ "subq $128, %%rsp\n\t" \
+ "callq __lll_mutex_unlock_wake\n\t" \
+ "addq $128, %%rsp\n\t" \
+ "jmp 2f\n\t" \
+ ".previous\n" \
+ "2:" \
+ : "=m" (futex), "=&D" (ignore) \
+ : "i" (FUTEX_TID_MASK), "m" (futex) \
+ : "ax", "cx", "r11", "cc", "memory"); })
+
+
+#define lll_robust_mutex_dead(futex) \
+ (void) ({ int ignore; \
+ __asm __volatile (LOCK_INSTR "orl %3, (%2)\n\t" \
+ "syscall" \
+ : "=m" (futex), "=a" (ignore) \
+ : "D" (&(futex)), "i" (FUTEX_OWNER_DIED), \
+ "S" (FUTEX_WAKE), "1" (__NR_futex), \
+ "d" (1) \
+ : "cx", "r11", "cc", "memory"); })
+
+
#define lll_mutex_islocked(futex) \
(futex != LLL_MUTEX_LOCK_INITIALIZER)
diff --git a/nptl/sysdeps/unix/sysv/linux/x86_64/lowlevelrobustlock.S b/nptl/sysdeps/unix/sysv/linux/x86_64/lowlevelrobustlock.S
new file mode 100644
index 0000000000..7bb9191691
--- /dev/null
+++ b/nptl/sysdeps/unix/sysv/linux/x86_64/lowlevelrobustlock.S
@@ -0,0 +1,192 @@
+/* Copyright (C) 2002, 2003, 2004, 2005, 2006 Free Software Foundation, Inc.
+ This file is part of the GNU C Library.
+ Contributed by Ulrich Drepper <drepper@redhat.com>, 2002.
+
+ The GNU C Library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 2.1 of the License, or (at your option) any later version.
+
+ The GNU C Library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with the GNU C Library; if not, write to the Free
+ Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA
+ 02111-1307 USA. */
+
+#include <sysdep.h>
+#include <pthread-errnos.h>
+#include <lowlevelrobustlock.h>
+
+ .text
+
+#ifndef LOCK
+# ifdef UP
+# define LOCK
+# else
+# define LOCK lock
+# endif
+#endif
+
+#define SYS_futex 202
+#define FUTEX_WAIT 0
+#define FUTEX_WAKE 1
+#define FUTEX_WAITERS 0x80000000
+#define FUTEX_OWNER_DIED 0x40000000
+
+/* For the calculation see asm/vsyscall.h. */
+#define VSYSCALL_ADDR_vgettimeofday 0xffffffffff600000
+
+
+ .globl __lll_robust_mutex_lock_wait
+ .type __lll_robust_mutex_lock_wait,@function
+ .hidden __lll_robust_mutex_lock_wait
+ .align 16
+__lll_robust_mutex_lock_wait:
+ pushq %r10
+ pushq %rdx
+
+ xorq %r10, %r10 /* No timeout. */
+#if FUTEX_WAIT == 0
+ xorl %esi, %esi
+#else
+ movl $FUTEX_WAIT, %esi
+#endif
+
+4: movl %eax, %edx
+ orl $FUTEX_WAITERS, %edx
+
+ testl $FUTEX_OWNER_DIED, %eax
+ jnz 3f
+
+ cmpl %edx, %eax
+ je 1f
+
+ LOCK
+ cmpxchgl %edx, (%rdi)
+ jnz 2f
+
+1: movl $SYS_futex, %eax
+ syscall
+
+ movl (%rdi), %eax
+
+2: testl %eax, %eax
+ jne 4b
+
+ movl %fs:TID, %edx
+ LOCK
+ cmpxchgl %edx, (%rdi)
+ jnz 4b
+ /* NB: %rax == 0 */
+
+3: popq %rdx
+ popq %r10
+ retq
+ .size __lll_robust_mutex_lock_wait,.-__lll_robust_mutex_lock_wait
+
+
+ .globl __lll_robust_mutex_timedlock_wait
+ .type __lll_robust_mutex_timedlock_wait,@function
+ .hidden __lll_robust_mutex_timedlock_wait
+ .align 16
+__lll_robust_mutex_timedlock_wait:
+ /* Check for a valid timeout value. */
+ cmpq $1000000000, 8(%rdx)
+ jae 3f
+
+ pushq %r8
+ pushq %r9
+ pushq %r12
+ pushq %r13
+
+ /* Stack frame for the timespec and timeval structs. */
+ subq $24, %rsp
+
+ movq %rdi, %r12
+ movq %rdx, %r13
+
+1: movq %rax, 16(%rsp)
+
+ /* Get current time. */
+ movq %rsp, %rdi
+ xorl %esi, %esi
+ movq $VSYSCALL_ADDR_vgettimeofday, %rax
+ /* This is a regular function call, all caller-save registers
+ might be clobbered. */
+ callq *%rax
+
+ /* Compute relative timeout. */
+ movq 8(%rsp), %rax
+ movl $1000, %edi
+ mul %rdi /* Milli seconds to nano seconds. */
+ movq (%r13), %rdi
+ movq 8(%r13), %rsi
+ subq (%rsp), %rdi
+ subq %rax, %rsi
+ jns 4f
+ addq $1000000000, %rsi
+ decq %rdi
+4: testq %rdi, %rdi
+ js 8f /* Time is already up. */
+
+ /* Futex call. */
+ movq %rdi, (%rsp) /* Store relative timeout. */
+ movq %rsi, 8(%rsp)
+
+ movq 16(%rsp), %rdx
+ movl %edx, %eax
+ orl $FUTEX_WAITERS, %edx
+
+ testl $FUTEX_OWNER_DIED, %eax
+ jnz 6f
+
+ cmpl %eax, %edx
+ je 2f
+
+ LOCK
+ cmpxchgl %edx, (%r12)
+ movq $0, %rcx /* Must use mov to avoid changing cc. */
+ jnz 5f
+
+2: movq %rsp, %r10
+#if FUTEX_WAIT == 0
+ xorl %esi, %esi
+#else
+ movl $FUTEX_WAIT, %esi
+#endif
+ movq %r12, %rdi
+ movl $SYS_futex, %eax
+ syscall
+ movq %rax, %rcx
+
+ movl (%r12), %eax
+
+5: testl %eax, %eax
+ jne 7f
+
+ movl %fs:TID, %edx
+ LOCK
+ cmpxchgl %edx, (%r12)
+ jnz 7f
+
+6: addq $24, %rsp
+ popq %r13
+ popq %r12
+ popq %r9
+ popq %r8
+ retq
+
+ /* Check whether the time expired. */
+7: cmpq $-ETIMEDOUT, %rcx
+ jne 1b
+
+8: movl $ETIMEDOUT, %eax
+ jmp 6b
+
+3: movl $EINVAL, %eax
+ retq
+ .size __lll_robust_mutex_timedlock_wait,.-__lll_robust_mutex_timedlock_wait