From 65810f0ef05e8c9e333f17a44e77808b163ca298 Mon Sep 17 00:00:00 2001 From: Torvald Riegel Date: Thu, 22 Dec 2016 10:20:43 +0100 Subject: robust mutexes: Fix broken x86 assembly by removing it lll_robust_unlock on i386 and x86_64 first sets the futex word to FUTEX_WAITERS|0 before calling __lll_unlock_wake, which will set the futex word to 0. If the thread is killed between these steps, then the futex word will be FUTEX_WAITERS|0, and the kernel (at least current upstream) will not set it to FUTEX_OWNER_DIED|FUTEX_WAITERS because 0 is not equal to the TID of the crashed thread. The lll_robust_lock assembly code on i386 and x86_64 is not prepared to deal with this case because the fastpath tries to only CAS 0 to TID and not FUTEX_WAITERS|0 to TID; the slowpath simply waits until it can CAS 0 to TID or the futex_word has the FUTEX_OWNER_DIED bit set. This issue is fixed by removing the custom x86 assembly code and using the generic C code instead. However, instead of adding more duplicate code to the custom x86 lowlevellock.h, the code of the lll_robust* functions is inlined into the single call sites that exist for each of these functions in the pthread_mutex_* functions. The robust mutex paths in the latter have been slightly reorganized to make them simpler. This patch is meant to be easy to backport, so C11-style atomics are not used. [BZ #20985] * nptl/Makefile: Adapt. * nptl/pthread_mutex_cond_lock.c (LLL_ROBUST_MUTEX_LOCK): Remove. (LLL_ROBUST_MUTEX_LOCK_MODIFIER): New. * nptl/pthread_mutex_lock.c (LLL_ROBUST_MUTEX_LOCK): Remove. (LLL_ROBUST_MUTEX_LOCK_MODIFIER): New. (__pthread_mutex_lock_full): Inline lll_robust* functions and adapt. * nptl/pthread_mutex_timedlock.c (pthread_mutex_timedlock): Inline lll_robust* functions and adapt. * nptl/pthread_mutex_unlock.c (__pthread_mutex_unlock_full): Likewise. * sysdeps/nptl/lowlevellock.h (__lll_robust_lock_wait, __lll_robust_lock, lll_robust_cond_lock, __lll_robust_timedlock_wait, __lll_robust_timedlock, __lll_robust_unlock): Remove. * sysdeps/unix/sysv/linux/i386/lowlevellock.h (lll_robust_lock, lll_robust_cond_lock, lll_robust_timedlock, lll_robust_unlock): Remove. * sysdeps/unix/sysv/linux/x86_64/lowlevellock.h (lll_robust_lock, lll_robust_cond_lock, lll_robust_timedlock, lll_robust_unlock): Remove. * sysdeps/unix/sysv/linux/sparc/lowlevellock.h (__lll_robust_lock_wait, __lll_robust_lock, lll_robust_cond_lock, __lll_robust_timedlock_wait, __lll_robust_timedlock, __lll_robust_unlock): Remove. * nptl/lowlevelrobustlock.c: Remove file. * nptl/lowlevelrobustlock.sym: Likewise. * sysdeps/unix/sysv/linux/i386/lowlevelrobustlock.S: Likewise. * sysdeps/unix/sysv/linux/x86_64/lowlevelrobustlock.S: Likewise. --- sysdeps/unix/sysv/linux/i386/lowlevellock.h | 60 ------ sysdeps/unix/sysv/linux/i386/lowlevelrobustlock.S | 232 ---------------------- 2 files changed, 292 deletions(-) delete mode 100644 sysdeps/unix/sysv/linux/i386/lowlevelrobustlock.S (limited to 'sysdeps/unix/sysv/linux/i386') diff --git a/sysdeps/unix/sysv/linux/i386/lowlevellock.h b/sysdeps/unix/sysv/linux/i386/lowlevellock.h index 31946a5172..197bb1fcd9 100644 --- a/sysdeps/unix/sysv/linux/i386/lowlevellock.h +++ b/sysdeps/unix/sysv/linux/i386/lowlevellock.h @@ -132,20 +132,6 @@ } \ }) -#define lll_robust_lock(futex, id, private) \ - ({ int result, ignore1, ignore2; \ - __asm __volatile (LOCK_INSTR "cmpxchgl %1, %2\n\t" \ - "jz 18f\n\t" \ - "1:\tleal %2, %%edx\n" \ - "0:\tmovl %7, %%ecx\n" \ - "2:\tcall __lll_robust_lock_wait\n" \ - "18:" \ - : "=a" (result), "=c" (ignore1), "=m" (futex), \ - "=&d" (ignore2) \ - : "0" (0), "1" (id), "m" (futex), "g" ((int) (private))\ - : "memory"); \ - result; }) - /* Special version of lll_lock which causes the unlock function to always wakeup waiters. */ @@ -165,22 +151,6 @@ }) -#define lll_robust_cond_lock(futex, id, private) \ - ({ int result, ignore1, ignore2; \ - __asm __volatile (LOCK_INSTR "cmpxchgl %1, %2\n\t" \ - "jz 18f\n\t" \ - "1:\tleal %2, %%edx\n" \ - "0:\tmovl %7, %%ecx\n" \ - "2:\tcall __lll_robust_lock_wait\n" \ - "18:" \ - : "=a" (result), "=c" (ignore1), "=m" (futex), \ - "=&d" (ignore2) \ - : "0" (0), "1" (id | FUTEX_WAITERS), "m" (futex), \ - "g" ((int) (private)) \ - : "memory"); \ - result; }) - - #define lll_timedlock(futex, timeout, private) \ ({ int result, ignore1, ignore2, ignore3; \ __asm __volatile (LOCK_INSTR "cmpxchgl %1, %3\n\t" \ @@ -203,21 +173,6 @@ extern int __lll_timedlock_elision (int *futex, short *adapt_count, #define lll_timedlock_elision(futex, adapt_count, timeout, private) \ __lll_timedlock_elision(&(futex), &(adapt_count), timeout, private) -#define lll_robust_timedlock(futex, timeout, id, private) \ - ({ int result, ignore1, ignore2, ignore3; \ - __asm __volatile (LOCK_INSTR "cmpxchgl %1, %3\n\t" \ - "jz 18f\n\t" \ - "1:\tleal %3, %%ecx\n" \ - "0:\tmovl %8, %%edx\n" \ - "2:\tcall __lll_robust_timedlock_wait\n" \ - "18:" \ - : "=a" (result), "=c" (ignore1), "=&d" (ignore2), \ - "=m" (futex), "=S" (ignore3) \ - : "0" (0), "1" (id), "m" (futex), "m" (timeout), \ - "4" ((int) (private)) \ - : "memory"); \ - result; }) - #if !IS_IN (libc) || defined UP # define __lll_unlock_asm LOCK_INSTR "subl $1, %0\n\t" #else @@ -255,21 +210,6 @@ extern int __lll_timedlock_elision (int *futex, short *adapt_count, } \ }) -#define lll_robust_unlock(futex, private) \ - (void) \ - ({ int ignore, ignore2; \ - __asm __volatile (LOCK_INSTR "andl %3, %0\n\t" \ - "je 18f\n\t" \ - "1:\tleal %0, %%eax\n" \ - "0:\tmovl %5, %%ecx\n" \ - "2:\tcall __lll_unlock_wake\n" \ - "18:" \ - : "=m" (futex), "=&a" (ignore), "=&c" (ignore2) \ - : "i" (FUTEX_WAITERS), "m" (futex), \ - "g" ((int) (private)) \ - : "memory"); \ - }) - #define lll_islocked(futex) \ (futex != LLL_LOCK_INITIALIZER) diff --git a/sysdeps/unix/sysv/linux/i386/lowlevelrobustlock.S b/sysdeps/unix/sysv/linux/i386/lowlevelrobustlock.S deleted file mode 100644 index f3a68c0f44..0000000000 --- a/sysdeps/unix/sysv/linux/i386/lowlevelrobustlock.S +++ /dev/null @@ -1,232 +0,0 @@ -/* Copyright (C) 2002-2017 Free Software Foundation, Inc. - This file is part of the GNU C Library. - Contributed by Ulrich Drepper , 2002. - - The GNU C Library is free software; you can redistribute it and/or - modify it under the terms of the GNU Lesser General Public - License as published by the Free Software Foundation; either - version 2.1 of the License, or (at your option) any later version. - - The GNU C Library is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - Lesser General Public License for more details. - - You should have received a copy of the GNU Lesser General Public - License along with the GNU C Library; if not, see - . */ - -#include -#include -#include -#include -#include - - .text - -#define FUTEX_WAITERS 0x80000000 -#define FUTEX_OWNER_DIED 0x40000000 - -#ifdef __ASSUME_PRIVATE_FUTEX -# define LOAD_FUTEX_WAIT(reg) \ - xorl $(FUTEX_WAIT | FUTEX_PRIVATE_FLAG), reg -#else -# if FUTEX_WAIT == 0 -# define LOAD_FUTEX_WAIT(reg) \ - xorl $FUTEX_PRIVATE_FLAG, reg ; \ - andl %gs:PRIVATE_FUTEX, reg -# else -# define LOAD_FUTEX_WAIT(reg) \ - xorl $FUTEX_PRIVATE_FLAG, reg ; \ - andl %gs:PRIVATE_FUTEX, reg ; \ - orl $FUTEX_WAIT, reg -# endif -#endif - - .globl __lll_robust_lock_wait - .type __lll_robust_lock_wait,@function - .hidden __lll_robust_lock_wait - .align 16 -__lll_robust_lock_wait: - cfi_startproc - pushl %edx - cfi_adjust_cfa_offset(4) - pushl %ebx - cfi_adjust_cfa_offset(4) - pushl %esi - cfi_adjust_cfa_offset(4) - cfi_offset(%edx, -8) - cfi_offset(%ebx, -12) - cfi_offset(%esi, -16) - - movl %edx, %ebx - xorl %esi, %esi /* No timeout. */ - LOAD_FUTEX_WAIT (%ecx) - -4: movl %eax, %edx - orl $FUTEX_WAITERS, %edx - - testl $FUTEX_OWNER_DIED, %eax - jnz 3f - - cmpl %edx, %eax /* NB: %edx == 2 */ - je 1f - - LOCK - cmpxchgl %edx, (%ebx) - jnz 2f - -1: movl $SYS_futex, %eax - ENTER_KERNEL - - movl (%ebx), %eax - -2: test %eax, %eax - jne 4b - - movl %gs:TID, %edx - orl $FUTEX_WAITERS, %edx - LOCK - cmpxchgl %edx, (%ebx) - jnz 4b - /* NB: %eax == 0 */ - -3: popl %esi - cfi_adjust_cfa_offset(-4) - cfi_restore(%esi) - popl %ebx - cfi_adjust_cfa_offset(-4) - cfi_restore(%ebx) - popl %edx - cfi_adjust_cfa_offset(-4) - cfi_restore(%edx) - ret - cfi_endproc - .size __lll_robust_lock_wait,.-__lll_robust_lock_wait - - - .globl __lll_robust_timedlock_wait - .type __lll_robust_timedlock_wait,@function - .hidden __lll_robust_timedlock_wait - .align 16 -__lll_robust_timedlock_wait: - cfi_startproc - /* Check for a valid timeout value. */ - cmpl $1000000000, 4(%edx) - jae 3f - - pushl %edi - cfi_adjust_cfa_offset(4) - pushl %esi - cfi_adjust_cfa_offset(4) - pushl %ebx - cfi_adjust_cfa_offset(4) - pushl %ebp - cfi_adjust_cfa_offset(4) - cfi_offset(%edi, -8) - cfi_offset(%esi, -12) - cfi_offset(%ebx, -16) - cfi_offset(%ebp, -20) - - /* Stack frame for the timespec and timeval structs. */ - subl $12, %esp - cfi_adjust_cfa_offset(12) - - movl %ecx, %ebp - movl %edx, %edi - -1: movl %eax, 8(%esp) - - /* Get current time. */ - movl %esp, %ebx - xorl %ecx, %ecx - movl $__NR_gettimeofday, %eax - ENTER_KERNEL - - /* Compute relative timeout. */ - movl 4(%esp), %eax - movl $1000, %edx - mul %edx /* Milli seconds to nano seconds. */ - movl (%edi), %ecx - movl 4(%edi), %edx - subl (%esp), %ecx - subl %eax, %edx - jns 4f - addl $1000000000, %edx - subl $1, %ecx -4: testl %ecx, %ecx - js 8f /* Time is already up. */ - - /* Store relative timeout. */ - movl %ecx, (%esp) - movl %edx, 4(%esp) - - movl %ebp, %ebx - - movl 8(%esp), %edx - movl %edx, %eax - orl $FUTEX_WAITERS, %edx - - testl $FUTEX_OWNER_DIED, %eax - jnz 6f - - cmpl %eax, %edx - je 2f - - LOCK - cmpxchgl %edx, (%ebx) - movl $0, %ecx /* Must use mov to avoid changing cc. */ - jnz 5f - -2: - /* Futex call. */ - movl %esp, %esi - movl 20(%esp), %ecx - LOAD_FUTEX_WAIT (%ecx) - movl $SYS_futex, %eax - ENTER_KERNEL - movl %eax, %ecx - - movl (%ebx), %eax - -5: testl %eax, %eax - jne 7f - - movl %gs:TID, %edx - orl $FUTEX_WAITERS, %edx - LOCK - cmpxchgl %edx, (%ebx) - jnz 7f - -6: addl $12, %esp - cfi_adjust_cfa_offset(-12) - popl %ebp - cfi_adjust_cfa_offset(-4) - cfi_restore(%ebp) - popl %ebx - cfi_adjust_cfa_offset(-4) - cfi_restore(%ebx) - popl %esi - cfi_adjust_cfa_offset(-4) - cfi_restore(%esi) - popl %edi - cfi_adjust_cfa_offset(-4) - cfi_restore(%edi) - ret - -3: movl $EINVAL, %eax - ret - - cfi_adjust_cfa_offset(28) - cfi_offset(%edi, -8) - cfi_offset(%esi, -12) - cfi_offset(%ebx, -16) - cfi_offset(%ebp, -20) - /* Check whether the time expired. */ -7: cmpl $-ETIMEDOUT, %ecx - jne 1b - -8: movl $ETIMEDOUT, %eax - jmp 6b - cfi_endproc - .size __lll_robust_timedlock_wait,.-__lll_robust_timedlock_wait -- cgit v1.2.3