From e8c659d74e011346785355eeef03b7fb6f533c61 Mon Sep 17 00:00:00 2001 From: Andi Kleen Date: Sat, 22 Dec 2012 01:03:04 -0800 Subject: Add elision to pthread_mutex_{try,timed,un}lock Add elision paths to the basic mutex locks. The normal path has a check for RTM and upgrades the lock to RTM when available. Trylocks cannot automatically upgrade, so they check for elision every time. We use a 4 byte value in the mutex to store the lock elision adaptation state. This is separate from the adaptive spin state and uses a separate field. Condition variables currently do not support elision. Recursive mutexes and condition variables may be supported at some point, but are not in the current implementation. Also "trylock" will not automatically enable elision unless some other lock call has been already called on the lock. This version does not use IFUNC, so it means every lock has one additional check for elision. Benchmarking showed the overhead to be negligible. --- .../unix/sysv/linux/pthread_mutex_cond_lock.c | 7 +++++ .../unix/sysv/linux/x86/bits/pthreadtypes.h | 13 +++++++-- nptl/sysdeps/unix/sysv/linux/x86/elision-trylock.c | 5 ++-- nptl/sysdeps/unix/sysv/linux/x86/force-elision.h | 31 ++++++++++++++++++++++ .../unix/sysv/linux/x86/pthread_mutex_cond_lock.c | 21 +++++++++++++++ .../unix/sysv/linux/x86/pthread_mutex_lock.c | 21 +++++++++++++++ .../unix/sysv/linux/x86/pthread_mutex_timedlock.c | 20 ++++++++++++++ .../unix/sysv/linux/x86/pthread_mutex_trylock.c | 21 +++++++++++++++ 8 files changed, 134 insertions(+), 5 deletions(-) create mode 100644 nptl/sysdeps/unix/sysv/linux/x86/force-elision.h create mode 100644 nptl/sysdeps/unix/sysv/linux/x86/pthread_mutex_cond_lock.c create mode 100644 nptl/sysdeps/unix/sysv/linux/x86/pthread_mutex_lock.c create mode 100644 nptl/sysdeps/unix/sysv/linux/x86/pthread_mutex_timedlock.c create mode 100644 nptl/sysdeps/unix/sysv/linux/x86/pthread_mutex_trylock.c (limited to 'nptl/sysdeps/unix') diff --git a/nptl/sysdeps/unix/sysv/linux/pthread_mutex_cond_lock.c b/nptl/sysdeps/unix/sysv/linux/pthread_mutex_cond_lock.c index b417da5e22..7b6fbc18aa 100644 --- a/nptl/sysdeps/unix/sysv/linux/pthread_mutex_cond_lock.c +++ b/nptl/sysdeps/unix/sysv/linux/pthread_mutex_cond_lock.c @@ -2,8 +2,15 @@ #define LLL_MUTEX_LOCK(mutex) \ lll_cond_lock ((mutex)->__data.__lock, PTHREAD_MUTEX_PSHARED (mutex)) + +/* Not actually elided so far. Needed? */ +#define LLL_MUTEX_LOCK_ELISION(mutex) \ + ({ lll_cond_lock ((mutex)->__data.__lock, PTHREAD_MUTEX_PSHARED (mutex)); 0; }) + #define LLL_MUTEX_TRYLOCK(mutex) \ lll_cond_trylock ((mutex)->__data.__lock) +#define LLL_MUTEX_TRYLOCK_ELISION(mutex) LLL_MUTEX_TRYLOCK(mutex) + #define LLL_ROBUST_MUTEX_LOCK(mutex, id) \ lll_robust_cond_lock ((mutex)->__data.__lock, id, \ PTHREAD_ROBUST_MUTEX_PSHARED (mutex)) diff --git a/nptl/sysdeps/unix/sysv/linux/x86/bits/pthreadtypes.h b/nptl/sysdeps/unix/sysv/linux/x86/bits/pthreadtypes.h index ccd896ca03..1852e07842 100644 --- a/nptl/sysdeps/unix/sysv/linux/x86/bits/pthreadtypes.h +++ b/nptl/sysdeps/unix/sysv/linux/x86/bits/pthreadtypes.h @@ -101,14 +101,23 @@ typedef union binary compatibility. */ int __kind; #ifdef __x86_64__ - int __spins; + short __spins; + short __elision; __pthread_list_t __list; # define __PTHREAD_MUTEX_HAVE_PREV 1 +# define __PTHREAD_MUTEX_HAVE_ELISION 1 #else unsigned int __nusers; __extension__ union { - int __spins; + struct + { + short __espins; + short __elision; +# define __spins d.__espins +# define __elision d.__elision +# define __PTHREAD_MUTEX_HAVE_ELISION 2 + } d; __pthread_slist_t __list; }; #endif diff --git a/nptl/sysdeps/unix/sysv/linux/x86/elision-trylock.c b/nptl/sysdeps/unix/sysv/linux/x86/elision-trylock.c index 2939fc9640..689a6fbe2a 100644 --- a/nptl/sysdeps/unix/sysv/linux/x86/elision-trylock.c +++ b/nptl/sysdeps/unix/sysv/linux/x86/elision-trylock.c @@ -24,9 +24,8 @@ #define aconf __elision_aconf -/* Try to elide a futex trylock. FUTEX is the futex variable. TRY_LOCK is the - adaptation counter in the mutex. UPGRADED is != 0 when this is for an - automatically upgraded lock. */ +/* Try to elide a futex trylock. FUTEX is the futex variable. ADAPT_COUNT is the + adaptation counter in the mutex. */ int __lll_trylock_elision (int *futex, short *adapt_count) diff --git a/nptl/sysdeps/unix/sysv/linux/x86/force-elision.h b/nptl/sysdeps/unix/sysv/linux/x86/force-elision.h new file mode 100644 index 0000000000..703e930be1 --- /dev/null +++ b/nptl/sysdeps/unix/sysv/linux/x86/force-elision.h @@ -0,0 +1,31 @@ +/* force-elision.h: Automatic enabling of elision for mutexes + Copyright (C) 2013 Free Software Foundation, Inc. + This file is part of the GNU C Library. + + The GNU C Library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + The GNU C Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with the GNU C Library; if not, see + . */ + +/* Check for elision on this lock without upgrading. */ +#define DO_ELISION(m) \ + (__pthread_force_elision \ + && (m->__data.__kind & PTHREAD_MUTEX_NO_ELISION_NP) == 0) \ + +/* Automatically enable elision for existing user lock kinds. */ +#define FORCE_ELISION(m, s) \ + if (__pthread_force_elision \ + && (m->__data.__kind & PTHREAD_MUTEX_ELISION_FLAGS_NP) == 0) \ + { \ + mutex->__data.__kind |= PTHREAD_MUTEX_ELISION_NP; \ + s; \ + } diff --git a/nptl/sysdeps/unix/sysv/linux/x86/pthread_mutex_cond_lock.c b/nptl/sysdeps/unix/sysv/linux/x86/pthread_mutex_cond_lock.c new file mode 100644 index 0000000000..962f101073 --- /dev/null +++ b/nptl/sysdeps/unix/sysv/linux/x86/pthread_mutex_cond_lock.c @@ -0,0 +1,21 @@ +/* Copyright (C) 2013 Free Software Foundation, Inc. + This file is part of the GNU C Library. + + The GNU C Library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + The GNU C Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with the GNU C Library; if not, see + . */ + +/* The cond lock is not actually elided yet, but we still need to handle + already elided locks. */ +#include +#include "sysdeps/unix/sysv/linux/pthread_mutex_cond_lock.c" diff --git a/nptl/sysdeps/unix/sysv/linux/x86/pthread_mutex_lock.c b/nptl/sysdeps/unix/sysv/linux/x86/pthread_mutex_lock.c new file mode 100644 index 0000000000..e639460530 --- /dev/null +++ b/nptl/sysdeps/unix/sysv/linux/x86/pthread_mutex_lock.c @@ -0,0 +1,21 @@ +/* Elided version of pthread_mutex_lock. + Copyright (C) 2011, 2012, 2013 Free Software Foundation, Inc. + This file is part of the GNU C Library. + + The GNU C Library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + The GNU C Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with the GNU C Library; if not, see + . */ +#include +#include "force-elision.h" + +#include "nptl/pthread_mutex_lock.c" diff --git a/nptl/sysdeps/unix/sysv/linux/x86/pthread_mutex_timedlock.c b/nptl/sysdeps/unix/sysv/linux/x86/pthread_mutex_timedlock.c new file mode 100644 index 0000000000..5115d0304a --- /dev/null +++ b/nptl/sysdeps/unix/sysv/linux/x86/pthread_mutex_timedlock.c @@ -0,0 +1,20 @@ +/* Elided version of pthread_mutex_timedlock. + Copyright (C) 2011, 2012, 2013 Free Software Foundation, Inc. + This file is part of the GNU C Library. + + The GNU C Library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + The GNU C Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with the GNU C Library; if not, see + . */ +#include +#include "force-elision.h" +#include "nptl/pthread_mutex_timedlock.c" diff --git a/nptl/sysdeps/unix/sysv/linux/x86/pthread_mutex_trylock.c b/nptl/sysdeps/unix/sysv/linux/x86/pthread_mutex_trylock.c new file mode 100644 index 0000000000..9bb4a659d4 --- /dev/null +++ b/nptl/sysdeps/unix/sysv/linux/x86/pthread_mutex_trylock.c @@ -0,0 +1,21 @@ +/* Elided version of pthread_mutex_trylock. + Copyright (C) 2011, 2012, 2013 Free Software Foundation, Inc. + This file is part of the GNU C Library. + + The GNU C Library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + The GNU C Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with the GNU C Library; if not, see + . */ +#include +#include "force-elision.h" + +#include "nptl/pthread_mutex_trylock.c" -- cgit v1.2.3