From cc25c8b4c1196a8c29e9a45b1e096b99a87b7f8c Mon Sep 17 00:00:00 2001 From: Torvald Riegel Date: Thu, 22 May 2014 16:00:12 +0200 Subject: New pthread rwlock that is more scalable. This replaces the pthread rwlock with a new implementation that uses a more scalable algorithm (primarily through not using a critical section anymore to make state changes). The fast path for rdlock acquisition and release is now basically a single atomic read-modify write or CAS and a few branches. See nptl/pthread_rwlock_common.c for details. * nptl/DESIGN-rwlock.txt: Remove. * nptl/lowlevelrwlock.sym: Remove. * nptl/Makefile: Add new tests. * nptl/pthread_rwlock_common.c: New file. Contains the new rwlock. * nptl/pthreadP.h (PTHREAD_RWLOCK_PREFER_READER_P): Remove. (PTHREAD_RWLOCK_WRPHASE, PTHREAD_RWLOCK_WRLOCKED, PTHREAD_RWLOCK_RWAITING, PTHREAD_RWLOCK_READER_SHIFT, PTHREAD_RWLOCK_READER_OVERFLOW, PTHREAD_RWLOCK_WRHANDOVER, PTHREAD_RWLOCK_FUTEX_USED): New. * nptl/pthread_rwlock_init.c (__pthread_rwlock_init): Adapt to new implementation. * nptl/pthread_rwlock_rdlock.c (__pthread_rwlock_rdlock_slow): Remove. (__pthread_rwlock_rdlock): Adapt. * nptl/pthread_rwlock_timedrdlock.c (pthread_rwlock_timedrdlock): Adapt. * nptl/pthread_rwlock_timedwrlock.c (pthread_rwlock_timedwrlock): Adapt. * nptl/pthread_rwlock_trywrlock.c (pthread_rwlock_trywrlock): Adapt. * nptl/pthread_rwlock_tryrdlock.c (pthread_rwlock_tryrdlock): Adapt. * nptl/pthread_rwlock_unlock.c (pthread_rwlock_unlock): Adapt. * nptl/pthread_rwlock_wrlock.c (__pthread_rwlock_wrlock_slow): Remove. (__pthread_rwlock_wrlock): Adapt. * nptl/tst-rwlock10.c: Adapt. * nptl/tst-rwlock11.c: Adapt. * nptl/tst-rwlock17.c: New file. * nptl/tst-rwlock18.c: New file. * nptl/tst-rwlock19.c: New file. * nptl/tst-rwlock2b.c: New file. * nptl/tst-rwlock8.c: Adapt. * nptl/tst-rwlock9.c: Adapt. * sysdeps/aarch64/nptl/bits/pthreadtypes.h (pthread_rwlock_t): Adapt. * sysdeps/arm/nptl/bits/pthreadtypes.h (pthread_rwlock_t): Adapt. * sysdeps/hppa/nptl/bits/pthreadtypes.h (pthread_rwlock_t): Adapt. * sysdeps/ia64/nptl/bits/pthreadtypes.h (pthread_rwlock_t): Adapt. * sysdeps/m68k/nptl/bits/pthreadtypes.h (pthread_rwlock_t): Adapt. * sysdeps/microblaze/nptl/bits/pthreadtypes.h (pthread_rwlock_t): Adapt. * sysdeps/mips/nptl/bits/pthreadtypes.h (pthread_rwlock_t): Adapt. * sysdeps/nios2/nptl/bits/pthreadtypes.h (pthread_rwlock_t): Adapt. * sysdeps/s390/nptl/bits/pthreadtypes.h (pthread_rwlock_t): Adapt. * sysdeps/sh/nptl/bits/pthreadtypes.h (pthread_rwlock_t): Adapt. * sysdeps/sparc/nptl/bits/pthreadtypes.h (pthread_rwlock_t): Adapt. * sysdeps/tile/nptl/bits/pthreadtypes.h (pthread_rwlock_t): Adapt. * sysdeps/unix/sysv/linux/alpha/bits/pthreadtypes.h (pthread_rwlock_t): Adapt. * sysdeps/unix/sysv/linux/powerpc/bits/pthreadtypes.h (pthread_rwlock_t): Adapt. * sysdeps/x86/bits/pthreadtypes.h (pthread_rwlock_t): Adapt. * nptl/nptl-printers.py (): Adapt. * nptl/nptl_lock_constants.pysym: Adapt. * nptl/test-rwlock-printers.py: Adapt. * nptl/test-rwlockattr-printers.c: Adapt. * nptl/test-rwlockattr-printers.py: Adapt. --- nptl/pthread_rwlock_wrlock.c | 106 ++----------------------------------------- 1 file changed, 5 insertions(+), 101 deletions(-) (limited to 'nptl/pthread_rwlock_wrlock.c') diff --git a/nptl/pthread_rwlock_wrlock.c b/nptl/pthread_rwlock_wrlock.c index 461ffdc06c..335fcd18cc 100644 --- a/nptl/pthread_rwlock_wrlock.c +++ b/nptl/pthread_rwlock_wrlock.c @@ -16,114 +16,18 @@ License along with the GNU C Library; if not, see . */ -#include -#include -#include -#include -#include -#include -#include -#include - - -/* Acquire write lock for RWLOCK. */ -static int __attribute__((noinline)) -__pthread_rwlock_wrlock_slow (pthread_rwlock_t *rwlock) -{ - int result = 0; - int futex_shared = - rwlock->__data.__shared == LLL_PRIVATE ? FUTEX_PRIVATE : FUTEX_SHARED; - - /* Caller has taken the lock. */ - - while (1) - { - /* Make sure we are not holding the rwlock as a writer. This is - a deadlock situation we recognize and report. */ - if (__builtin_expect (rwlock->__data.__writer - == THREAD_GETMEM (THREAD_SELF, tid), 0)) - { - result = EDEADLK; - break; - } - - /* Remember that we are a writer. */ - if (++rwlock->__data.__nr_writers_queued == 0) - { - /* Overflow on number of queued writers. */ - --rwlock->__data.__nr_writers_queued; - result = EAGAIN; - break; - } - - int waitval = rwlock->__data.__writer_wakeup; - - /* Free the lock. */ - lll_unlock (rwlock->__data.__lock, rwlock->__data.__shared); - - /* Wait for the writer or reader(s) to finish. We do not check the - return value because we decide how to continue based on the state of - the rwlock. */ - futex_wait_simple (&rwlock->__data.__writer_wakeup, waitval, - futex_shared); - - /* Get the lock. */ - lll_lock (rwlock->__data.__lock, rwlock->__data.__shared); - - /* To start over again, remove the thread from the writer list. */ - --rwlock->__data.__nr_writers_queued; - - /* Get the rwlock if there is no writer and no reader. */ - if (rwlock->__data.__writer == 0 && rwlock->__data.__nr_readers == 0) - { - /* Mark self as writer. */ - rwlock->__data.__writer = THREAD_GETMEM (THREAD_SELF, tid); - - LIBC_PROBE (wrlock_acquire_write, 1, rwlock); - break; - } - } - - /* We are done, free the lock. */ - lll_unlock (rwlock->__data.__lock, rwlock->__data.__shared); - - return result; -} - -/* Fast path of acquiring write lock for RWLOCK. */ +#include "pthread_rwlock_common.c" +/* See pthread_rwlock_common.c. */ int __pthread_rwlock_wrlock (pthread_rwlock_t *rwlock) { LIBC_PROBE (wrlock_entry, 1, rwlock); - if (ELIDE_LOCK (rwlock->__data.__rwelision, - rwlock->__data.__lock == 0 - && rwlock->__data.__writer == 0 - && rwlock->__data.__nr_readers == 0)) - return 0; - - /* Make sure we are alone. */ - lll_lock (rwlock->__data.__lock, rwlock->__data.__shared); - - /* Get the rwlock if there is no writer and no reader. */ - if (__glibc_likely((rwlock->__data.__writer | - rwlock->__data.__nr_readers) == 0)) - { - /* Mark self as writer. */ - rwlock->__data.__writer = THREAD_GETMEM (THREAD_SELF, tid); - - LIBC_PROBE (wrlock_acquire_write, 1, rwlock); - - /* We are done, free the lock. */ - lll_unlock (rwlock->__data.__lock, rwlock->__data.__shared); - - return 0; - } - - return __pthread_rwlock_wrlock_slow (rwlock); + int result = __pthread_rwlock_wrlock_full (rwlock, NULL); + LIBC_PROBE (wrlock_acquire_write, 1, rwlock); + return result; } - weak_alias (__pthread_rwlock_wrlock, pthread_rwlock_wrlock) hidden_def (__pthread_rwlock_wrlock) -- cgit v1.2.3