diff options
Diffstat (limited to 'nptl/sysdeps/unix/sysv/linux/x86_64')
11 files changed, 1202 insertions, 787 deletions
diff --git a/nptl/sysdeps/unix/sysv/linux/x86_64/lowlevellock.h b/nptl/sysdeps/unix/sysv/linux/x86_64/lowlevellock.h index 0b7e3bbaba..9b15bfbc57 100644 --- a/nptl/sysdeps/unix/sysv/linux/x86_64/lowlevellock.h +++ b/nptl/sysdeps/unix/sysv/linux/x86_64/lowlevellock.h @@ -54,6 +54,8 @@ #define FUTEX_TRYLOCK_PI 8 #define FUTEX_WAIT_BITSET 9 #define FUTEX_WAKE_BITSET 10 +#define FUTEX_WAIT_REQUEUE_PI 11 +#define FUTEX_CMP_REQUEUE_PI 12 #define FUTEX_PRIVATE_FLAG 128 #define FUTEX_CLOCK_REALTIME 256 diff --git a/nptl/sysdeps/unix/sysv/linux/x86_64/lowlevelrobustlock.S b/nptl/sysdeps/unix/sysv/linux/x86_64/lowlevelrobustlock.S index fa7516ef71..02db0a4f9d 100644 --- a/nptl/sysdeps/unix/sysv/linux/x86_64/lowlevelrobustlock.S +++ b/nptl/sysdeps/unix/sysv/linux/x86_64/lowlevelrobustlock.S @@ -1,4 +1,4 @@ -/* Copyright (C) 2002, 2003, 2004, 2005, 2006, 2007 +/* Copyright (C) 2002, 2003, 2004, 2005, 2006, 2007, 2009 Free Software Foundation, Inc. This file is part of the GNU C Library. Contributed by Ulrich Drepper <drepper@redhat.com>, 2002. @@ -32,6 +32,8 @@ #ifdef __ASSUME_PRIVATE_FUTEX # define LOAD_FUTEX_WAIT(reg) \ xorl $(FUTEX_WAIT | FUTEX_PRIVATE_FLAG), reg +# define LOAD_FUTEX_WAIT_ABS(reg) \ + xorl $(FUTEX_WAIT_BITSET | FUTEX_PRIVATE_FLAG | FUTEX_CLOCK_REALTIME), reg #else # if FUTEX_WAIT == 0 # define LOAD_FUTEX_WAIT(reg) \ @@ -43,6 +45,10 @@ andl %fs:PRIVATE_FUTEX, reg ; \ orl $FUTEX_WAIT, reg # endif +# define LOAD_FUTEX_WAIT_ABS(reg) \ + xorl $FUTEX_PRIVATE_FLAG, reg ; \ + andl %fs:PRIVATE_FUTEX, reg ; \ + orl $FUTEX_WAIT_BITSET | FUTEX_CLOCK_REALTIME, reg #endif /* For the calculation see asm/vsyscall.h. */ @@ -110,6 +116,73 @@ __lll_robust_lock_wait: .align 16 __lll_robust_timedlock_wait: cfi_startproc +# ifndef __ASSUME_FUTEX_CLOCK_REALTIME +# ifdef PIC + cmpl $0, __have_futex_clock_realtime(%rip) +# else + cmpl $0, __have_futex_clock_realtime +# endif + je .Lreltmo +# endif + + pushq %r9 + cfi_adjust_cfa_offset(8) + cfi_rel_offset(%r9, 0) + movq %rdx, %r10 + movl $0xffffffff, %r9d + LOAD_FUTEX_WAIT_ABS (%esi) + +1: testl $FUTEX_OWNER_DIED, %eax + jnz 3f + + movl %eax, %edx + orl $FUTEX_WAITERS, %edx + + cmpl %eax, %edx + je 5f + + LOCK + cmpxchgl %edx, (%rdi) + movq $0, %rcx /* Must use mov to avoid changing cc. */ + jnz 6f + +5: movl $SYS_futex, %eax + syscall + movl %eax, %ecx + + movl (%rdi), %eax + +6: testl %eax, %eax + jne 2f + + movl %fs:TID, %edx + orl $FUTEX_WAITERS, %edx + LOCK + cmpxchgl %edx, (%rdi) + jnz 2f + +3: popq %r9 + cfi_adjust_cfa_offset(-8) + cfi_restore(%r9) + retq + + cfi_adjust_cfa_offset(8) + cfi_rel_offset(%r9, 0) + /* Check whether the time expired. */ +2: cmpl $-ETIMEDOUT, %ecx + je 4f + cmpl $-EINVAL, %ecx + jne 1b + +4: movl %ecx, %eax + negl %eax + jmp 3b + cfi_adjust_cfa_offset(-8) + cfi_restore(%r9) + + +# ifndef __ASSUME_FUTEX_CLOCK_REALTIME +.Lreltmo: /* Check for a valid timeout value. */ cmpq $1000000000, 8(%rdx) jae 3f @@ -223,10 +296,11 @@ __lll_robust_timedlock_wait: cfi_offset(%r12, -32) cfi_offset(%r13, -40) /* Check whether the time expired. */ -7: cmpq $-ETIMEDOUT, %rcx +7: cmpl $-ETIMEDOUT, %ecx jne 1b 8: movl $ETIMEDOUT, %eax jmp 6b +#endif cfi_endproc .size __lll_robust_timedlock_wait,.-__lll_robust_timedlock_wait diff --git a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_broadcast.S b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_broadcast.S index 6155255eb0..0f10ec910c 100644 --- a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_broadcast.S +++ b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_broadcast.S @@ -1,4 +1,4 @@ -/* Copyright (C) 2002, 2003, 2004, 2005, 2006, 2007 +/* Copyright (C) 2002, 2003, 2004, 2005, 2006, 2007, 2009 Free Software Foundation, Inc. This file is part of the GNU C Library. Contributed by Ulrich Drepper <drepper@redhat.com>, 2002. @@ -70,12 +70,14 @@ __pthread_cond_broadcast: 8: cmpq $-1, %r8 je 9f - /* XXX: The kernel so far doesn't support requeue to PI futex. */ - /* XXX: The kernel only supports FUTEX_CMP_REQUEUE to the same - type of futex (private resp. shared). */ - testl $(PI_BIT | PS_BIT), MUTEX_KIND(%r8) + /* Do not use requeue for pshared condvars. */ + testl $PS_BIT, MUTEX_KIND(%r8) jne 9f + /* Requeue to a PI mutex if the PI bit is set. */ + testl $PI_BIT, MUTEX_KIND(%r8) + jne 81f + /* Wake up all threads. */ #ifdef __ASSUME_PRIVATE_FUTEX movl $(FUTEX_CMP_REQUEUE|FUTEX_PRIVATE_FLAG), %esi @@ -97,6 +99,20 @@ __pthread_cond_broadcast: 10: xorl %eax, %eax retq + /* Wake up all threads. */ +81: movl $(FUTEX_CMP_REQUEUE_PI|FUTEX_PRIVATE_FLAG), %esi + movl $SYS_futex, %eax + movl $1, %edx + movl $0x7fffffff, %r10d + syscall + + /* For any kind of error, which mainly is EAGAIN, we try again + with WAKE. The general test also covers running on old + kernels. */ + cmpq $-4095, %rax + jb 10b + jmp 9f + .align 16 /* Unlock. */ 4: LOCK diff --git a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_signal.S b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_signal.S index 8f65f2cd69..f1050fea7c 100644 --- a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_signal.S +++ b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_signal.S @@ -1,4 +1,4 @@ -/* Copyright (C) 2002, 2003, 2004, 2005, 2007 Free Software Foundation, Inc. +/* Copyright (C) 2002-2005, 2007, 2009 Free Software Foundation, Inc. This file is part of the GNU C Library. Contributed by Ulrich Drepper <drepper@redhat.com>, 2002. @@ -21,6 +21,7 @@ #include <shlib-compat.h> #include <lowlevellock.h> #include <lowlevelcond.h> +#include <pthread-pi-defines.h> #include <kernel-features.h> #include <pthread-errnos.h> @@ -56,19 +57,23 @@ __pthread_cond_signal: /* Wake up one thread. */ cmpq $-1, dep_mutex(%r8) + movl $FUTEX_WAKE_OP, %esi movl $1, %edx + movl $SYS_futex, %eax + je 8f + + /* Get the address of the mutex used. */ + movq dep_mutex(%r8), %rcx + testl $PI_BIT, MUTEX_KIND(%rcx) + jne 9f + #ifdef __ASSUME_PRIVATE_FUTEX - movl $FUTEX_WAKE_OP, %eax movl $(FUTEX_WAKE_OP|FUTEX_PRIVATE_FLAG), %esi - cmove %eax, %esi #else - movl $0, %eax - movl %fs:PRIVATE_FUTEX, %esi - cmove %eax, %esi - orl $FUTEX_WAKE_OP, %esi + orl %fs:PRIVATE_FUTEX, %esi #endif - movl $1, %r10d - movl $SYS_futex, %eax + +8: movl $1, %r10d #if cond_lock != 0 addq $cond_lock, %r8 #endif @@ -85,9 +90,27 @@ __pthread_cond_signal: xorl %eax, %eax retq -7: /* %esi should be either FUTEX_WAKE_OP or - FUTEX_WAKE_OP|FUTEX_PRIVATE_FLAG from the previous syscall. */ - xorl $(FUTEX_WAKE ^ FUTEX_WAKE_OP), %esi + /* Wake up one thread and requeue none in the PI Mutex case. */ +9: movl $(FUTEX_CMP_REQUEUE_PI|FUTEX_PRIVATE_FLAG), %esi + movq %rcx, %r8 + xorq %r10, %r10 + movl (%rdi), %r9d // XXX Can this be right? + syscall + + leaq -cond_futex(%rdi), %r8 + + /* For any kind of error, we try again with WAKE. + The general test also covers running on old kernels. */ + cmpq $-4095, %rax + jb 4f + +7: +#ifdef __ASSUME_PRIVATE_FUTEX + andl $FUTEX_PRIVATE_FLAG, %esi +#else + andl %fs:PRIVATE_FUTEX, %esi +#endif + orl $FUTEX_WAKE, %esi movl $SYS_futex, %eax /* %rdx should be 1 already from $FUTEX_WAKE_OP syscall. movl $1, %edx */ diff --git a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S index ddcf106a6d..f81466e1a5 100644 --- a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S +++ b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S @@ -21,6 +21,7 @@ #include <shlib-compat.h> #include <lowlevellock.h> #include <lowlevelcond.h> +#include <pthread-pi-defines.h> #include <pthread-errnos.h> #include <kernel-features.h> @@ -31,13 +32,24 @@ .text + /* int pthread_cond_timedwait (pthread_cond_t *cond, pthread_mutex_t *mutex, const struct timespec *abstime) */ .globl __pthread_cond_timedwait .type __pthread_cond_timedwait, @function .align 16 __pthread_cond_timedwait: +.LSTARTCODE: cfi_startproc +#ifdef SHARED + cfi_personality(DW_EH_PE_pcrel | DW_EH_PE_sdata4 | DW_EH_PE_indirect, + DW.ref.__gcc_personality_v0) + cfi_lsda(DW_EH_PE_pcrel | DW_EH_PE_sdata4, .LexceptSTART) +#else + cfi_personality(DW_EH_PE_udata4, __gcc_personality_v0) + cfi_lsda(DW_EH_PE_udata4, .LexceptSTART) +#endif + pushq %r12 cfi_adjust_cfa_offset(8) cfi_rel_offset(%r12, 0) @@ -47,23 +59,28 @@ __pthread_cond_timedwait: pushq %r14 cfi_adjust_cfa_offset(8) cfi_rel_offset(%r14, 0) -#define FRAME_SIZE 80 + pushq %r15 + cfi_adjust_cfa_offset(8) + cfi_rel_offset(%r15, 0) +#ifdef __ASSUME_FUTEX_CLOCK_REALTIME +# define FRAME_SIZE 32 +#else +# define FRAME_SIZE 48 +#endif subq $FRAME_SIZE, %rsp cfi_adjust_cfa_offset(FRAME_SIZE) cmpq $1000000000, 8(%rdx) movl $EINVAL, %eax - jae 18f + jae 48f /* Stack frame: - rsp + 80 - +--------------------------+ - rsp + 48 | cleanup buffer | + rsp + 48 +--------------------------+ - rsp + 40 | old wake_seq value | + rsp + 32 | timeout value | +--------------------------+ - rsp + 24 | timeout value | + rsp + 24 | old wake_seq value | +--------------------------+ rsp + 16 | mutex pointer | +--------------------------+ @@ -85,8 +102,18 @@ __pthread_cond_timedwait: je 22f movq %rsi, dep_mutex(%rdi) +22: +#ifndef __ASSUME_FUTEX_CLOCK_REALTIME +# ifdef PIC + cmpl $0, __have_futex_clock_realtime(%rip) +# else + cmpl $0, __have_futex_clock_realtime +# endif + je .Lreltmo +#endif + /* Get internal lock. */ -22: movl $1, %esi + movl $1, %esi xorl %eax, %eax LOCK #if cond_lock == 0 @@ -94,98 +121,29 @@ __pthread_cond_timedwait: #else cmpxchgl %esi, cond_lock(%rdi) #endif - jnz 1f + jnz 31f /* Unlock the mutex. */ -2: movq 16(%rsp), %rdi +32: movq 16(%rsp), %rdi xorl %esi, %esi callq __pthread_mutex_unlock_usercnt testl %eax, %eax - jne 16f + jne 46f movq 8(%rsp), %rdi incq total_seq(%rdi) incl cond_futex(%rdi) addl $(1 << nwaiters_shift), cond_nwaiters(%rdi) - /* Install cancellation handler. */ -#ifdef PIC - leaq __condvar_cleanup(%rip), %rsi -#else - leaq __condvar_cleanup, %rsi -#endif - leaq 48(%rsp), %rdi - movq %rsp, %rdx - callq __pthread_cleanup_push - /* Get and store current wakeup_seq value. */ movq 8(%rsp), %rdi movq wakeup_seq(%rdi), %r9 movl broadcast_seq(%rdi), %edx - movq %r9, 40(%rsp) + movq %r9, 24(%rsp) movl %edx, 4(%rsp) - /* Get the current time. */ -8: -#ifdef __NR_clock_gettime - /* Get the clock number. Note that the field in the condvar - structure stores the number minus 1. */ - movq 8(%rsp), %rdi - movl cond_nwaiters(%rdi), %edi - andl $((1 << nwaiters_shift) - 1), %edi - /* Only clocks 0 and 1 are allowed so far. Both are handled in the - kernel. */ - leaq 24(%rsp), %rsi -# ifdef SHARED - movq __vdso_clock_gettime@GOTPCREL(%rip), %rax - movq (%rax), %rax - PTR_DEMANGLE (%rax) - jz 26f - call *%rax - jmp 27f -# endif -26: movl $__NR_clock_gettime, %eax - syscall -27: -# ifndef __ASSUME_POSIX_TIMERS - cmpq $-ENOSYS, %rax - je 19f -# endif - - /* Compute relative timeout. */ - movq (%r13), %rcx - movq 8(%r13), %rdx - subq 24(%rsp), %rcx - subq 32(%rsp), %rdx -#else - leaq 24(%rsp), %rdi - xorl %esi, %esi - movq $VSYSCALL_ADDR_vgettimeofday, %rax - callq *%rax - - /* Compute relative timeout. */ - movq 32(%rsp), %rax - movl $1000, %edx - mul %rdx /* Milli seconds to nano seconds. */ - movq (%r13), %rcx - movq 8(%r13), %rdx - subq 24(%rsp), %rcx - subq %rax, %rdx -#endif - jns 12f - addq $1000000000, %rdx - decq %rcx -12: testq %rcx, %rcx - movq 8(%rsp), %rdi - movq $-ETIMEDOUT, %r14 - js 6f - - /* Store relative timeout. */ -21: movq %rcx, 24(%rsp) - movq %rdx, 32(%rsp) - - movl cond_futex(%rdi), %r12d +38: movl cond_futex(%rdi), %r12d /* Unlock. */ LOCK @@ -194,33 +152,67 @@ __pthread_cond_timedwait: #else decl cond_lock(%rdi) #endif - jne 3f + jne 33f -4: callq __pthread_enable_asynccancel +.LcleanupSTART1: +34: callq __pthread_enable_asynccancel movl %eax, (%rsp) + movq 8(%rsp), %rdi - leaq 24(%rsp), %r10 + movq %r13, %r10 cmpq $-1, dep_mutex(%rdi) - movq %r12, %rdx -#ifdef __ASSUME_PRIVATE_FUTEX - movl $FUTEX_WAIT, %eax - movl $(FUTEX_WAIT|FUTEX_PRIVATE_FLAG), %esi + movl $FUTEX_WAIT_BITSET, %eax + movl $(FUTEX_WAIT_BITSET|FUTEX_PRIVATE_FLAG), %esi cmove %eax, %esi + je 60f + + movq dep_mutex(%rdi), %r8 + /* Requeue to a PI mutex if the PI bit is set. */ + testl $PI_BIT, MUTEX_KIND(%r8) + je 60f + + movl $(FUTEX_WAIT_REQUEUE_PI|FUTEX_PRIVATE_FLAG), %esi + xorl %eax, %eax + /* The following only works like this because we only support + two clocks, represented using a single bit. */ + testl $1, cond_nwaiters(%rdi) + movl $FUTEX_CLOCK_REALTIME, %edx + cmove %edx, %eax + orl %eax, %esi + movq %r12, %rdx + addq $cond_futex, %rdi + movl $SYS_futex, %eax + syscall + + movl $1, %r15d +#ifdef __ASSUME_REQUEUE_PI + jmp 62f #else - movl $0, %eax - movl %fs:PRIVATE_FUTEX, %esi - cmove %eax, %esi -# if FUTEX_WAIT != 0 - orl $FUTEX_WAIT, %esi -# endif + cmpq $-4095, %rax + jnae 62f + + movl $(FUTEX_WAIT_BITSET|FUTEX_PRIVATE_FLAG), %esi + subq $cond_futex, %rdi #endif + +60: xorl %r15d, %r15d + xorl %eax, %eax + /* The following only works like this because we only support + two clocks, represented using a single bit. */ + testl $1, cond_nwaiters(%rdi) + movl $FUTEX_CLOCK_REALTIME, %edx + movl $0xffffffff, %r9d + cmove %edx, %eax + orl %eax, %esi + movq %r12, %rdx addq $cond_futex, %rdi movl $SYS_futex, %eax syscall - movq %rax, %r14 +62: movq %rax, %r14 movl (%rsp), %edi callq __pthread_disable_asynccancel +.LcleanupEND1: /* Lock. */ movq 8(%rsp), %rdi @@ -232,45 +224,45 @@ __pthread_cond_timedwait: #else cmpxchgl %esi, cond_lock(%rdi) #endif - jne 5f + jne 35f -6: movl broadcast_seq(%rdi), %edx +36: movl broadcast_seq(%rdi), %edx movq woken_seq(%rdi), %rax movq wakeup_seq(%rdi), %r9 cmpl 4(%rsp), %edx - jne 23f + jne 53f - cmpq 40(%rsp), %r9 - jbe 15f + cmpq 24(%rsp), %r9 + jbe 45f cmpq %rax, %r9 - ja 9f + ja 39f -15: cmpq $-ETIMEDOUT, %r14 - jne 8b +45: cmpq $-ETIMEDOUT, %r14 + jne 38b -13: incq wakeup_seq(%rdi) +99: incq wakeup_seq(%rdi) incl cond_futex(%rdi) movl $ETIMEDOUT, %r14d - jmp 14f + jmp 44f -23: xorq %r14, %r14 - jmp 24f +53: xorq %r14, %r14 + jmp 54f -9: xorq %r14, %r14 -14: incq woken_seq(%rdi) +39: xorq %r14, %r14 +44: incq woken_seq(%rdi) -24: subl $(1 << nwaiters_shift), cond_nwaiters(%rdi) +54: subl $(1 << nwaiters_shift), cond_nwaiters(%rdi) /* Wake up a thread which wants to destroy the condvar object. */ cmpq $0xffffffffffffffff, total_seq(%rdi) - jne 25f + jne 55f movl cond_nwaiters(%rdi), %eax andl $~((1 << nwaiters_shift) - 1), %eax - jne 25f + jne 55f addq $cond_nwaiters, %rdi cmpq $-1, dep_mutex-cond_nwaiters(%rdi) @@ -289,26 +281,31 @@ __pthread_cond_timedwait: syscall subq $cond_nwaiters, %rdi -25: LOCK +55: LOCK #if cond_lock == 0 decl (%rdi) #else decl cond_lock(%rdi) #endif - jne 10f + jne 40f - /* Remove cancellation handler. */ -11: movq 48+CLEANUP_PREV(%rsp), %rdx - movq %rdx, %fs:CLEANUP + /* If requeue_pi is used the kernel performs the locking of the + mutex. */ +41: xorl %eax, %eax + testl %r15d, %r15d + jnz 63f movq 16(%rsp), %rdi callq __pthread_mutex_cond_lock - testq %rax, %rax +63: testq %rax, %rax cmoveq %r14, %rax -18: addq $FRAME_SIZE, %rsp +48: addq $FRAME_SIZE, %rsp cfi_adjust_cfa_offset(-FRAME_SIZE) + popq %r15 + cfi_adjust_cfa_offset(-8) + cfi_restore(%r15) popq %r14 cfi_adjust_cfa_offset(-8) cfi_restore(%r14) @@ -322,11 +319,11 @@ __pthread_cond_timedwait: retq /* Initial locking failed. */ -1: - cfi_adjust_cfa_offset(3 * 8 + FRAME_SIZE) - cfi_rel_offset(%r12, FRAME_SIZE + 16) - cfi_rel_offset(%r13, FRAME_SIZE + 8) - cfi_rel_offset(%r14, FRAME_SIZE) +31: cfi_adjust_cfa_offset(4 * 8 + FRAME_SIZE) + cfi_rel_offset(%r12, FRAME_SIZE + 24) + cfi_rel_offset(%r13, FRAME_SIZE + 16) + cfi_rel_offset(%r14, FRAME_SIZE + 8) + cfi_rel_offset(%r15, FRAME_SIZE) #if cond_lock != 0 addq $cond_lock, %rdi #endif @@ -335,10 +332,10 @@ __pthread_cond_timedwait: movl $LLL_SHARED, %esi cmovne %eax, %esi callq __lll_lock_wait - jmp 2b + jmp 32b /* Unlock in loop requires wakeup. */ -3: +33: #if cond_lock != 0 addq $cond_lock, %rdi #endif @@ -347,10 +344,10 @@ __pthread_cond_timedwait: movl $LLL_SHARED, %esi cmovne %eax, %esi callq __lll_unlock_wake - jmp 4b + jmp 34b /* Locking in loop failed. */ -5: +35: #if cond_lock != 0 addq $cond_lock, %rdi #endif @@ -362,10 +359,10 @@ __pthread_cond_timedwait: #if cond_lock != 0 subq $cond_lock, %rdi #endif - jmp 6b + jmp 36b /* Unlock after loop requires wakeup. */ -10: +40: #if cond_lock != 0 addq $cond_lock, %rdi #endif @@ -374,10 +371,10 @@ __pthread_cond_timedwait: movl $LLL_SHARED, %esi cmovne %eax, %esi callq __lll_unlock_wake - jmp 11b + jmp 41b /* The initial unlocking of the mutex failed. */ -16: movq 8(%rsp), %rdi +46: movq 8(%rsp), %rdi movq %rax, (%rsp) LOCK #if cond_lock == 0 @@ -385,7 +382,7 @@ __pthread_cond_timedwait: #else decl cond_lock(%rdi) #endif - jne 17f + jne 47f #if cond_lock != 0 addq $cond_lock, %rdi @@ -396,23 +393,229 @@ __pthread_cond_timedwait: cmovne %eax, %esi callq __lll_unlock_wake -17: movq (%rsp), %rax - jmp 18b +47: movq (%rsp), %rax + jmp 48b + + +#ifndef __ASSUME_FUTEX_CLOCK_REALTIME +.Lreltmo: + xorl %r15d, %r15d + + /* Get internal lock. */ + movl $1, %esi + xorl %eax, %eax + LOCK +# if cond_lock == 0 + cmpxchgl %esi, (%rdi) +# else + cmpxchgl %esi, cond_lock(%rdi) +# endif + jnz 1f + + /* Unlock the mutex. */ +2: movq 16(%rsp), %rdi + xorl %esi, %esi + callq __pthread_mutex_unlock_usercnt + + testl %eax, %eax + jne 46b + + movq 8(%rsp), %rdi + incq total_seq(%rdi) + incl cond_futex(%rdi) + addl $(1 << nwaiters_shift), cond_nwaiters(%rdi) + + /* Get and store current wakeup_seq value. */ + movq 8(%rsp), %rdi + movq wakeup_seq(%rdi), %r9 + movl broadcast_seq(%rdi), %edx + movq %r9, 24(%rsp) + movl %edx, 4(%rsp) + + /* Get the current time. */ +8: +# ifdef __NR_clock_gettime + /* Get the clock number. Note that the field in the condvar + structure stores the number minus 1. */ + movq 8(%rsp), %rdi + movl cond_nwaiters(%rdi), %edi + andl $((1 << nwaiters_shift) - 1), %edi + /* Only clocks 0 and 1 are allowed so far. Both are handled in the + kernel. */ + leaq 32(%rsp), %rsi +# ifdef SHARED + movq __vdso_clock_gettime@GOTPCREL(%rip), %rax + movq (%rax), %rax + PTR_DEMANGLE (%rax) + jz 26f + call *%rax + jmp 27f +# endif +26: movl $__NR_clock_gettime, %eax + syscall +27: +# ifndef __ASSUME_POSIX_TIMERS + cmpq $-ENOSYS, %rax + je 19f +# endif + + /* Compute relative timeout. */ + movq (%r13), %rcx + movq 8(%r13), %rdx + subq 32(%rsp), %rcx + subq 40(%rsp), %rdx +# else + leaq 24(%rsp), %rdi + xorl %esi, %esi + movq $VSYSCALL_ADDR_vgettimeofday, %rax + callq *%rax + + /* Compute relative timeout. */ + movq 40(%rsp), %rax + movl $1000, %edx + mul %rdx /* Milli seconds to nano seconds. */ + movq (%r13), %rcx + movq 8(%r13), %rdx + subq 32(%rsp), %rcx + subq %rax, %rdx +# endif + jns 12f + addq $1000000000, %rdx + decq %rcx +12: testq %rcx, %rcx + movq 8(%rsp), %rdi + movq $-ETIMEDOUT, %r14 + js 6f + + /* Store relative timeout. */ +21: movq %rcx, 32(%rsp) + movq %rdx, 40(%rsp) + + movl cond_futex(%rdi), %r12d + + /* Unlock. */ + LOCK +# if cond_lock == 0 + decl (%rdi) +# else + decl cond_lock(%rdi) +# endif + jne 3f + +.LcleanupSTART2: +4: callq __pthread_enable_asynccancel + movl %eax, (%rsp) + movq 8(%rsp), %rdi + + leaq 32(%rsp), %r10 + cmpq $-1, dep_mutex(%rdi) + movq %r12, %rdx +# ifdef __ASSUME_PRIVATE_FUTEX + movl $FUTEX_WAIT, %eax + movl $(FUTEX_WAIT|FUTEX_PRIVATE_FLAG), %esi + cmove %eax, %esi +# else + movl $0, %eax + movl %fs:PRIVATE_FUTEX, %esi + cmove %eax, %esi +# if FUTEX_WAIT != 0 + orl $FUTEX_WAIT, %esi +# endif +# endif + addq $cond_futex, %rdi + movl $SYS_futex, %eax + syscall + movq %rax, %r14 + + movl (%rsp), %edi + callq __pthread_disable_asynccancel +.LcleanupEND2: + + /* Lock. */ + movq 8(%rsp), %rdi + movl $1, %esi + xorl %eax, %eax + LOCK +# if cond_lock == 0 + cmpxchgl %esi, (%rdi) +# else + cmpxchgl %esi, cond_lock(%rdi) +# endif + jne 5f + +6: movl broadcast_seq(%rdi), %edx + + movq woken_seq(%rdi), %rax + + movq wakeup_seq(%rdi), %r9 + + cmpl 4(%rsp), %edx + jne 53b + + cmpq 24(%rsp), %r9 + jbe 45b + + cmpq %rax, %r9 + ja 39b + + cmpq $-ETIMEDOUT, %r14 + jne 8b + + jmp 99b + + /* Initial locking failed. */ +1: +# if cond_lock != 0 + addq $cond_lock, %rdi +# endif + cmpq $-1, dep_mutex-cond_lock(%rdi) + movl $LLL_PRIVATE, %eax + movl $LLL_SHARED, %esi + cmovne %eax, %esi + callq __lll_lock_wait + jmp 2b + + /* Unlock in loop requires wakeup. */ +3: +# if cond_lock != 0 + addq $cond_lock, %rdi +# endif + cmpq $-1, dep_mutex-cond_lock(%rdi) + movl $LLL_PRIVATE, %eax + movl $LLL_SHARED, %esi + cmovne %eax, %esi + callq __lll_unlock_wake + jmp 4b + + /* Locking in loop failed. */ +5: +# if cond_lock != 0 + addq $cond_lock, %rdi +# endif + cmpq $-1, dep_mutex-cond_lock(%rdi) + movl $LLL_PRIVATE, %eax + movl $LLL_SHARED, %esi + cmovne %eax, %esi + callq __lll_lock_wait +# if cond_lock != 0 + subq $cond_lock, %rdi +# endif + jmp 6b -#if defined __NR_clock_gettime && !defined __ASSUME_POSIX_TIMERS +# if defined __NR_clock_gettime && !defined __ASSUME_POSIX_TIMERS /* clock_gettime not available. */ -19: leaq 24(%rsp), %rdi +19: leaq 32(%rsp), %rdi xorl %esi, %esi movq $VSYSCALL_ADDR_vgettimeofday, %rax callq *%rax /* Compute relative timeout. */ - movq 32(%rsp), %rax + movq 40(%rsp), %rax movl $1000, %edx mul %rdx /* Milli seconds to nano seconds. */ movq (%r13), %rcx movq 8(%r13), %rdx - subq 24(%rsp), %rcx + subq 32(%rsp), %rcx subq %rax, %rdx jns 20f addq $1000000000, %rdx @@ -422,8 +625,188 @@ __pthread_cond_timedwait: movq $-ETIMEDOUT, %r14 js 6b jmp 21b +# endif #endif - cfi_endproc .size __pthread_cond_timedwait, .-__pthread_cond_timedwait versioned_symbol (libpthread, __pthread_cond_timedwait, pthread_cond_timedwait, GLIBC_2_3_2) + + + .align 16 + .type __condvar_cleanup2, @function +__condvar_cleanup2: + /* Stack frame: + + rsp + 72 + +--------------------------+ + rsp + 64 | %r12 | + +--------------------------+ + rsp + 56 | %r13 | + +--------------------------+ + rsp + 48 | %r14 | + +--------------------------+ + rsp + 24 | unused | + +--------------------------+ + rsp + 16 | mutex pointer | + +--------------------------+ + rsp + 8 | condvar pointer | + +--------------------------+ + rsp + 4 | old broadcast_seq value | + +--------------------------+ + rsp + 0 | old cancellation mode | + +--------------------------+ + */ + + movq %rax, 24(%rsp) + + /* Get internal lock. */ + movq 8(%rsp), %rdi + movl $1, %esi + xorl %eax, %eax + LOCK +#if cond_lock == 0 + cmpxchgl %esi, (%rdi) +#else + cmpxchgl %esi, cond_lock(%rdi) +#endif + jz 1f + +#if cond_lock != 0 + addq $cond_lock, %rdi +#endif + cmpq $-1, dep_mutex-cond_lock(%rdi) + movl $LLL_PRIVATE, %eax + movl $LLL_SHARED, %esi + cmovne %eax, %esi + callq __lll_lock_wait +#if cond_lock != 0 + subq $cond_lock, %rdi +#endif + +1: movl broadcast_seq(%rdi), %edx + cmpl 4(%rsp), %edx + jne 3f + + /* We increment the wakeup_seq counter only if it is lower than + total_seq. If this is not the case the thread was woken and + then canceled. In this case we ignore the signal. */ + movq total_seq(%rdi), %rax + cmpq wakeup_seq(%rdi), %rax + jbe 6f + incq wakeup_seq(%rdi) + incl cond_futex(%rdi) +6: incq woken_seq(%rdi) + +3: subl $(1 << nwaiters_shift), cond_nwaiters(%rdi) + + /* Wake up a thread which wants to destroy the condvar object. */ + xorq %r12, %r12 + cmpq $0xffffffffffffffff, total_seq(%rdi) + jne 4f + movl cond_nwaiters(%rdi), %eax + andl $~((1 << nwaiters_shift) - 1), %eax + jne 4f + + cmpq $-1, dep_mutex(%rdi) + leaq cond_nwaiters(%rdi), %rdi + movl $1, %edx +#ifdef __ASSUME_PRIVATE_FUTEX + movl $FUTEX_WAKE, %eax + movl $(FUTEX_WAKE|FUTEX_PRIVATE_FLAG), %esi + cmove %eax, %esi +#else + movl $0, %eax + movl %fs:PRIVATE_FUTEX, %esi + cmove %eax, %esi + orl $FUTEX_WAKE, %esi +#endif + movl $SYS_futex, %eax + syscall + subq $cond_nwaiters, %rdi + movl $1, %r12d + +4: LOCK +#if cond_lock == 0 + decl (%rdi) +#else + decl cond_lock(%rdi) +#endif + je 2f +#if cond_lock != 0 + addq $cond_lock, %rdi +#endif + cmpq $-1, dep_mutex-cond_lock(%rdi) + movl $LLL_PRIVATE, %eax + movl $LLL_SHARED, %esi + cmovne %eax, %esi + callq __lll_unlock_wake + + /* Wake up all waiters to make sure no signal gets lost. */ +2: testq %r12, %r12 + jnz 5f + addq $cond_futex, %rdi + cmpq $-1, dep_mutex-cond_futex(%rdi) + movl $0x7fffffff, %edx +#ifdef __ASSUME_PRIVATE_FUTEX + movl $FUTEX_WAKE, %eax + movl $(FUTEX_WAKE|FUTEX_PRIVATE_FLAG), %esi + cmove %eax, %esi +#else + movl $0, %eax + movl %fs:PRIVATE_FUTEX, %esi + cmove %eax, %esi + orl $FUTEX_WAKE, %esi +#endif + movl $SYS_futex, %eax + syscall + +5: movq 16(%rsp), %rdi + callq __pthread_mutex_cond_lock + + movq 24(%rsp), %rdi + movq FRAME_SIZE(%rsp), %r15 + movq FRAME_SIZE+8(%rsp), %r14 + movq FRAME_SIZE+16(%rsp), %r13 + movq FRAME_SIZE+24(%rsp), %r12 +.LcallUR: + call _Unwind_Resume@PLT + hlt +.LENDCODE: + cfi_endproc + .size __condvar_cleanup2, .-__condvar_cleanup2 + + + .section .gcc_except_table,"a",@progbits +.LexceptSTART: + .byte DW_EH_PE_omit # @LPStart format + .byte DW_EH_PE_omit # @TType format + .byte DW_EH_PE_uleb128 # call-site format + .uleb128 .Lcstend-.Lcstbegin +.Lcstbegin: + .uleb128 .LcleanupSTART1-.LSTARTCODE + .uleb128 .LcleanupEND1-.LcleanupSTART1 + .uleb128 __condvar_cleanup2-.LSTARTCODE + .uleb128 0 +#ifndef __ASSUME_FUTEX_CLOCK_REALTIME + .uleb128 .LcleanupSTART2-.LSTARTCODE + .uleb128 .LcleanupEND2-.LcleanupSTART2 + .uleb128 __condvar_cleanup2-.LSTARTCODE + .uleb128 0 +#endif + .uleb128 .LcallUR-.LSTARTCODE + .uleb128 .LENDCODE-.LcallUR + .uleb128 0 + .uleb128 0 +.Lcstend: + + +#ifdef SHARED + .hidden DW.ref.__gcc_personality_v0 + .weak DW.ref.__gcc_personality_v0 + .section .gnu.linkonce.d.DW.ref.__gcc_personality_v0,"aw",@progbits + .align 8 + .type DW.ref.__gcc_personality_v0, @object + .size DW.ref.__gcc_personality_v0, 8 +DW.ref.__gcc_personality_v0: + .quad __gcc_personality_v0 +#endif diff --git a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S index e5e802d531..e6323ea3e2 100644 --- a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S +++ b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S @@ -1,4 +1,4 @@ -/* Copyright (C) 2002,2003,2004,2005,2006,2007 Free Software Foundation, Inc. +/* Copyright (C) 2002-2007, 2009 Free Software Foundation, Inc. This file is part of the GNU C Library. Contributed by Ulrich Drepper <drepper@redhat.com>, 2002. @@ -22,146 +22,42 @@ #include <lowlevellock.h> #include <lowlevelcond.h> #include <tcb-offsets.h> +#include <pthread-pi-defines.h> #include <kernel-features.h> .text - .align 16 - .type __condvar_cleanup, @function - .globl __condvar_cleanup - .hidden __condvar_cleanup -__condvar_cleanup: - pushq %r12 - - /* Get internal lock. */ - movq %rdi, %r8 - movq 8(%rdi), %rdi - movl $1, %esi - xorl %eax, %eax - LOCK -#if cond_lock == 0 - cmpxchgl %esi, (%rdi) -#else - cmpxchgl %esi, cond_lock(%rdi) -#endif - jz 1f - -#if cond_lock != 0 - addq $cond_lock, %rdi -#endif - cmpq $-1, dep_mutex-cond_lock(%rdi) - movl $LLL_PRIVATE, %eax - movl $LLL_SHARED, %esi - cmovne %eax, %esi - callq __lll_lock_wait -#if cond_lock != 0 - subq $cond_lock, %rdi -#endif - -1: movl broadcast_seq(%rdi), %edx - cmpl 4(%r8), %edx - jne 3f - - /* We increment the wakeup_seq counter only if it is lower than - total_seq. If this is not the case the thread was woken and - then canceled. In this case we ignore the signal. */ - movq total_seq(%rdi), %rax - cmpq wakeup_seq(%rdi), %rax - jbe 6f - incq wakeup_seq(%rdi) - incl cond_futex(%rdi) -6: incq woken_seq(%rdi) - -3: subl $(1 << nwaiters_shift), cond_nwaiters(%rdi) - - /* Wake up a thread which wants to destroy the condvar object. */ - xorq %r12, %r12 - cmpq $0xffffffffffffffff, total_seq(%rdi) - jne 4f - movl cond_nwaiters(%rdi), %eax - andl $~((1 << nwaiters_shift) - 1), %eax - jne 4f - - addq $cond_nwaiters, %rdi - cmpq $-1, dep_mutex-cond_nwaiters(%rdi) - movl $1, %edx -#ifdef __ASSUME_PRIVATE_FUTEX - movl $FUTEX_WAKE, %eax - movl $(FUTEX_WAKE|FUTEX_PRIVATE_FLAG), %esi - cmove %eax, %esi -#else - movl $0, %eax - movl %fs:PRIVATE_FUTEX, %esi - cmove %eax, %esi - orl $FUTEX_WAKE, %esi -#endif - movl $SYS_futex, %eax - syscall - subq $cond_nwaiters, %rdi - movl $1, %r12d - -4: LOCK -#if cond_lock == 0 - decl (%rdi) -#else - decl cond_lock(%rdi) -#endif - je 2f -#if cond_lock != 0 - addq $cond_lock, %rdi -#endif - cmpq $-1, dep_mutex-cond_lock(%rdi) - movl $LLL_PRIVATE, %eax - movl $LLL_SHARED, %esi - cmovne %eax, %esi - callq __lll_unlock_wake - - /* Wake up all waiters to make sure no signal gets lost. */ -2: testq %r12, %r12 - jnz 5f - addq $cond_futex, %rdi - cmpq $-1, dep_mutex-cond_futex(%rdi) - movl $0x7fffffff, %edx -#ifdef __ASSUME_PRIVATE_FUTEX - movl $FUTEX_WAKE, %eax - movl $(FUTEX_WAKE|FUTEX_PRIVATE_FLAG), %esi - cmove %eax, %esi -#else - movl $0, %eax - movl %fs:PRIVATE_FUTEX, %esi - cmove %eax, %esi - orl $FUTEX_WAKE, %esi -#endif - movl $SYS_futex, %eax - syscall - -5: movq 16(%r8), %rdi - callq __pthread_mutex_cond_lock - - popq %r12 - - retq - .size __condvar_cleanup, .-__condvar_cleanup - - /* int pthread_cond_wait (pthread_cond_t *cond, pthread_mutex_t *mutex) */ .globl __pthread_cond_wait .type __pthread_cond_wait, @function .align 16 __pthread_cond_wait: .LSTARTCODE: + cfi_startproc +#ifdef SHARED + cfi_personality(DW_EH_PE_pcrel | DW_EH_PE_sdata4 | DW_EH_PE_indirect, + DW.ref.__gcc_personality_v0) + cfi_lsda(DW_EH_PE_pcrel | DW_EH_PE_sdata4, .LexceptSTART) +#else + cfi_personality(DW_EH_PE_udata4, __gcc_personality_v0) + cfi_lsda(DW_EH_PE_udata4, .LexceptSTART) +#endif + pushq %r12 -.Lpush_r12: -#define FRAME_SIZE 64 + cfi_adjust_cfa_offset(8) + cfi_rel_offset(%r12, 0) + pushq %r13 + cfi_adjust_cfa_offset(8) + cfi_rel_offset(%r13, 0) +#define FRAME_SIZE 32 subq $FRAME_SIZE, %rsp -.Lsubq: + cfi_adjust_cfa_offset(FRAME_SIZE) + /* Stack frame: - rsp + 64 - +--------------------------+ - rsp + 32 | cleanup buffer | + rsp + 32 +--------------------------+ rsp + 24 | old wake_seq value | +--------------------------+ @@ -208,16 +104,6 @@ __pthread_cond_wait: incl cond_futex(%rdi) addl $(1 << nwaiters_shift), cond_nwaiters(%rdi) - /* Install cancellation handler. */ -#ifdef PIC - leaq __condvar_cleanup(%rip), %rsi -#else - leaq __condvar_cleanup, %rsi -#endif - leaq 32(%rsp), %rdi - movq %rsp, %rdx - callq __pthread_cleanup_push - /* Get and store current wakeup_seq value. */ movq 8(%rsp), %rdi movq wakeup_seq(%rdi), %r9 @@ -235,31 +121,57 @@ __pthread_cond_wait: #endif jne 3f +.LcleanupSTART: 4: callq __pthread_enable_asynccancel movl %eax, (%rsp) movq 8(%rsp), %rdi xorq %r10, %r10 movq %r12, %rdx - addq $cond_futex-cond_lock, %rdi + // XXX reverse + lea + addq $cond_futex, %rdi cmpq $-1, dep_mutex-cond_futex(%rdi) #ifdef __ASSUME_PRIVATE_FUTEX movl $FUTEX_WAIT, %eax movl $(FUTEX_WAIT|FUTEX_PRIVATE_FLAG), %esi cmove %eax, %esi #else - movl $FUTEX_WAIT, %eax + movl $0, %eax movl %fs:PRIVATE_FUTEX, %esi cmove %eax, %esi # if FUTEX_WAIT != 0 +# error "cc destroyed by following orl" orl $FUTEX_WAIT, %esi # endif #endif + je 60f + + movq dep_mutex-cond_futex(%rdi), %r8 + /* Requeue to a PI mutex if the PI bit is set. */ + testl $PI_BIT, MUTEX_KIND(%r8) + je 60f + + movl $(FUTEX_WAIT_REQUEUE_PI|FUTEX_PRIVATE_FLAG), %esi + movl $SYS_futex, %eax + syscall + + movl $1, %r13d +#ifdef __ASSUME_REQUEUE_PI + jmp 62f +#else + cmpq $-4095, %rax + jnae 62f + + movl $(FUTEX_WAIT|FUTEX_PRIVATE_FLAG), %esi +#endif + +60: xorl %r13d, %r13d movl $SYS_futex, %eax syscall - movl (%rsp), %edi +62: movl (%rsp), %edi callq __pthread_disable_asynccancel +.LcleanupEND: /* Lock. */ movq 8(%rsp), %rdi @@ -325,24 +237,33 @@ __pthread_cond_wait: #endif jne 10f - /* Remove cancellation handler. */ -11: movq 32+CLEANUP_PREV(%rsp), %rdx - movq %rdx, %fs:CLEANUP + /* If requeue_pi is used the kernel performs the locking of the + mutex. */ +11: xorl %eax, %eax + testl %r13d, %r13d + jnz 14f movq 16(%rsp), %rdi callq __pthread_mutex_cond_lock + 14: addq $FRAME_SIZE, %rsp -.Laddq: + cfi_adjust_cfa_offset(-FRAME_SIZE) + popq %r13 + cfi_adjust_cfa_offset(-8) + cfi_restore(%r13) popq %r12 -.Lpop_r12: + cfi_adjust_cfa_offset(-8) + cfi_restore(%r12) /* We return the result of the mutex_lock operation. */ retq /* Initial locking failed. */ 1: -.LSbl1: + cfi_adjust_cfa_offset(16 + FRAME_SIZE) + cfi_rel_offset(%r12, FRAME_SIZE + 8) + cfi_rel_offset(%r13, FRAME_SIZE) #if cond_lock != 0 addq $cond_lock, %rdi #endif @@ -414,75 +335,178 @@ __pthread_cond_wait: 13: movq %r10, %rax jmp 14b -.LENDCODE: .size __pthread_cond_wait, .-__pthread_cond_wait versioned_symbol (libpthread, __pthread_cond_wait, pthread_cond_wait, GLIBC_2_3_2) - .section .eh_frame,"a",@progbits -.LSTARTFRAME: - .long L(ENDCIE)-L(STARTCIE) # Length of the CIE. -.LSTARTCIE: - .long 0 # CIE ID. - .byte 1 # Version number. -#ifdef SHARED - .string "zR" # NUL-terminated augmentation - # string. + .align 16 + .type __condvar_cleanup1, @function + .globl __condvar_cleanup1 + .hidden __condvar_cleanup1 +__condvar_cleanup1: + /* Stack frame: + + rsp + 48 + +--------------------------+ + rsp + 40 | %r12 | + +--------------------------+ + rsp + 32 | %r13 | + +--------------------------+ + rsp + 24 | unused | + +--------------------------+ + rsp + 16 | mutex pointer | + +--------------------------+ + rsp + 8 | condvar pointer | + +--------------------------+ + rsp + 4 | old broadcast_seq value | + +--------------------------+ + rsp + 0 | old cancellation mode | + +--------------------------+ + */ + + movq %rax, 24(%rsp) + + /* Get internal lock. */ + movq 8(%rsp), %rdi + movl $1, %esi + xorl %eax, %eax + LOCK +#if cond_lock == 0 + cmpxchgl %esi, (%rdi) #else - .ascii "\0" # NUL-terminated augmentation - # string. + cmpxchgl %esi, cond_lock(%rdi) #endif - .uleb128 1 # Code alignment factor. - .sleb128 -8 # Data alignment factor. - .byte 16 # Return address register - # column. -#ifdef SHARED - .uleb128 1 # Augmentation value length. - .byte 0x1b # Encoding: DW_EH_PE_pcrel - # + DW_EH_PE_sdata4. + jz 1f + +#if cond_lock != 0 + addq $cond_lock, %rdi #endif - .byte 0x0c # DW_CFA_def_cfa - .uleb128 7 - .uleb128 8 - .byte 0x90 # DW_CFA_offset, column 0x8 - .uleb128 1 - .align 8 -.LENDCIE: - - .long .LENDFDE-.LSTARTFDE # Length of the FDE. -.LSTARTFDE: - .long .LSTARTFDE-.LSTARTFRAME # CIE pointer. -#ifdef SHARED - .long .LSTARTCODE-. # PC-relative start address - # of the code + cmpq $-1, dep_mutex-cond_lock(%rdi) + movl $LLL_PRIVATE, %eax + movl $LLL_SHARED, %esi + cmovne %eax, %esi + callq __lll_lock_wait +#if cond_lock != 0 + subq $cond_lock, %rdi +#endif + +1: movl broadcast_seq(%rdi), %edx + cmpl 4(%rsp), %edx + jne 3f + + /* We increment the wakeup_seq counter only if it is lower than + total_seq. If this is not the case the thread was woken and + then canceled. In this case we ignore the signal. */ + movq total_seq(%rdi), %rax + cmpq wakeup_seq(%rdi), %rax + jbe 6f + incq wakeup_seq(%rdi) + incl cond_futex(%rdi) +6: incq woken_seq(%rdi) + +3: subl $(1 << nwaiters_shift), cond_nwaiters(%rdi) + + /* Wake up a thread which wants to destroy the condvar object. */ + xorq %r12, %r12 + cmpq $0xffffffffffffffff, total_seq(%rdi) + jne 4f + movl cond_nwaiters(%rdi), %eax + andl $~((1 << nwaiters_shift) - 1), %eax + jne 4f + + cmpq $-1, dep_mutex(%rdi) + leaq cond_nwaiters(%rdi), %rdi + movl $1, %edx +#ifdef __ASSUME_PRIVATE_FUTEX + movl $FUTEX_WAKE, %eax + movl $(FUTEX_WAKE|FUTEX_PRIVATE_FLAG), %esi + cmove %eax, %esi +#else + movl $0, %eax + movl %fs:PRIVATE_FUTEX, %esi + cmove %eax, %esi + orl $FUTEX_WAKE, %esi +#endif + movl $SYS_futex, %eax + syscall + subq $cond_nwaiters, %rdi + movl $1, %r12d + +4: LOCK +#if cond_lock == 0 + decl (%rdi) #else - .long .LSTARTCODE # Start address of the code. + decl cond_lock(%rdi) #endif - .long .LENDCODE-.LSTARTCODE # Length of the code. -#ifdef SHARED - .uleb128 0 # No augmentation data. + je 2f +#if cond_lock != 0 + addq $cond_lock, %rdi #endif - .byte 0x40+.Lpush_r12-.LSTARTCODE # DW_CFA_advance_loc+N - .byte 14 # DW_CFA_def_cfa_offset - .uleb128 16 - .byte 0x8c # DW_CFA_offset %r12 - .uleb128 2 - .byte 0x40+.Lsubq-.Lpush_r12 # DW_CFA_advance_loc+N - .byte 14 # DW_CFA_def_cfa_offset - .uleb128 16+FRAME_SIZE - .byte 3 # DW_CFA_advance_loc2 - .2byte .Laddq-.Lsubq - .byte 14 # DW_CFA_def_cfa_offset - .uleb128 16 - .byte 0x40+.Lpop_r12-.Laddq # DW_CFA_advance_loc+N - .byte 14 # DW_CFA_def_cfa_offset - .uleb128 8 - .byte 0xcc # DW_CFA_restore %r12 - .byte 0x40+.LSbl1-.Lpop_r12 # DW_CFA_advance_loc+N - .byte 14 # DW_CFA_def_cfa_offset - .uleb128 80 - .byte 0x8c # DW_CFA_offset %r12 - .uleb128 2 + cmpq $-1, dep_mutex-cond_lock(%rdi) + movl $LLL_PRIVATE, %eax + movl $LLL_SHARED, %esi + cmovne %eax, %esi + callq __lll_unlock_wake + + /* Wake up all waiters to make sure no signal gets lost. */ +2: testq %r12, %r12 + jnz 5f + addq $cond_futex, %rdi + cmpq $-1, dep_mutex-cond_futex(%rdi) + movl $0x7fffffff, %edx +#ifdef __ASSUME_PRIVATE_FUTEX + movl $FUTEX_WAKE, %eax + movl $(FUTEX_WAKE|FUTEX_PRIVATE_FLAG), %esi + cmove %eax, %esi +#else + movl $0, %eax + movl %fs:PRIVATE_FUTEX, %esi + cmove %eax, %esi + orl $FUTEX_WAKE, %esi +#endif + movl $SYS_futex, %eax + syscall + +5: movq 16(%rsp), %rdi + callq __pthread_mutex_cond_lock + + movq 24(%rsp), %rdi + movq 40(%rsp), %r12 + movq 32(%rsp), %r13 +.LcallUR: + call _Unwind_Resume@PLT + hlt +.LENDCODE: + cfi_endproc + .size __condvar_cleanup1, .-__condvar_cleanup1 + + + .section .gcc_except_table,"a",@progbits +.LexceptSTART: + .byte DW_EH_PE_omit # @LPStart format + .byte DW_EH_PE_omit # @TType format + .byte DW_EH_PE_uleb128 # call-site format + .uleb128 .Lcstend-.Lcstbegin +.Lcstbegin: + .uleb128 .LcleanupSTART-.LSTARTCODE + .uleb128 .LcleanupEND-.LcleanupSTART + .uleb128 __condvar_cleanup1-.LSTARTCODE + .uleb128 0 + .uleb128 .LcallUR-.LSTARTCODE + .uleb128 .LENDCODE-.LcallUR + .uleb128 0 + .uleb128 0 +.Lcstend: + + +#ifdef SHARED + .hidden DW.ref.__gcc_personality_v0 + .weak DW.ref.__gcc_personality_v0 + .section .gnu.linkonce.d.DW.ref.__gcc_personality_v0,"aw",@progbits .align 8 -.LENDFDE: + .type DW.ref.__gcc_personality_v0, @object + .size DW.ref.__gcc_personality_v0, 8 +DW.ref.__gcc_personality_v0: + .quad __gcc_personality_v0 +#endif diff --git a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_once.S b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_once.S index c3b2b51bdb..ccc18493a2 100644 --- a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_once.S +++ b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_once.S @@ -1,4 +1,4 @@ -/* Copyright (C) 2002, 2003, 2005, 2007 Free Software Foundation, Inc. +/* Copyright (C) 2002, 2003, 2005, 2007, 2009 Free Software Foundation, Inc. This file is part of the GNU C Library. Contributed by Ulrich Drepper <drepper@redhat.com>, 2002. @@ -17,6 +17,7 @@ Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA. */ +#include <sysdep.h> #include <kernel-features.h> #include <tcb-offsets.h> #include <lowlevellock.h> @@ -32,6 +33,15 @@ .align 16 __pthread_once: .LSTARTCODE: + cfi_startproc +#ifdef SHARED + cfi_personality(DW_EH_PE_pcrel | DW_EH_PE_sdata4 | DW_EH_PE_indirect, + DW.ref.__gcc_personality_v0) + cfi_lsda(DW_EH_PE_pcrel | DW_EH_PE_sdata4, .LexceptSTART) +#else + cfi_personality(DW_EH_PE_udata4, __gcc_personality_v0) + cfi_lsda(DW_EH_PE_udata4, .LexceptSTART) +#endif testl $2, (%rdi) jz 1f xorl %eax, %eax @@ -39,7 +49,7 @@ __pthread_once: /* Preserve the function pointer. */ 1: pushq %rsi -.Lpush_rsi: + cfi_adjust_cfa_offset(8) xorq %r10, %r10 /* Not yet initialized or initialization in progress. @@ -86,9 +96,9 @@ __pthread_once: /* Preserve the pointer to the control variable. */ 3: pushq %rdi -.Lpush_rdi: + cfi_adjust_cfa_offset(8) pushq %rdi -.Lpush_rdi2: + cfi_adjust_cfa_offset(8) .LcleanupSTART: callq *16(%rsp) @@ -96,14 +106,14 @@ __pthread_once: /* Get the control variable address back. */ popq %rdi -.Lpop_rdi: + cfi_adjust_cfa_offset(-8) /* Sucessful run of the initializer. Signal that we are done. */ LOCK incl (%rdi) addq $8, %rsp -.Ladd1: + cfi_adjust_cfa_offset(-8) /* Wake up all other threads. */ movl $0x7fffffff, %edx @@ -117,10 +127,9 @@ __pthread_once: syscall 4: addq $8, %rsp -.Ladd2: + cfi_adjust_cfa_offset(-8) xorl %eax, %eax retq - .size __pthread_once,.-__pthread_once @@ -134,6 +143,7 @@ pthread_once = __pthread_once .type clear_once_control,@function .align 16 clear_once_control: + cfi_adjust_cfa_offset(3 * 8) movq (%rsp), %rdi movq %rax, %r8 movl $0, (%rdi) @@ -153,15 +163,15 @@ clear_once_control: call _Unwind_Resume@PLT hlt .LENDCODE: + cfi_endproc .size clear_once_control,.-clear_once_control .section .gcc_except_table,"a",@progbits .LexceptSTART: - .byte 0xff # @LPStart format (omit) - .byte 0xff # @TType format (omit) - .byte 0x01 # call-site format - # DW_EH_PE_uleb128 + .byte DW_EH_PE_omit # @LPStart format + .byte DW_EH_PE_omit # @TType format + .byte DW_EH_PE_uleb128 # call-site format .uleb128 .Lcstend-.Lcstbegin .Lcstbegin: .uleb128 .LcleanupSTART-.LSTARTCODE @@ -175,100 +185,6 @@ clear_once_control: .Lcstend: - .section .eh_frame,"a",@progbits -.LSTARTFRAME: - .long .LENDCIE-.LSTARTCIE # Length of the CIE. -.LSTARTCIE: - .long 0 # CIE ID. - .byte 1 # Version number. -#ifdef SHARED - .string "zPLR" # NUL-terminated augmentation - # string. -#else - .string "zPL" # NUL-terminated augmentation - # string. -#endif - .uleb128 1 # Code alignment factor. - .sleb128 -8 # Data alignment factor. - .byte 16 # Return address register - # column. -#ifdef SHARED - .uleb128 7 # Augmentation value length. - .byte 0x9b # Personality: DW_EH_PE_pcrel - # + DW_EH_PE_sdata4 - # + DW_EH_PE_indirect - .long DW.ref.__gcc_personality_v0-. - .byte 0x1b # LSDA Encoding: DW_EH_PE_pcrel - # + DW_EH_PE_sdata4. - .byte 0x1b # FDE Encoding: DW_EH_PE_pcrel - # + DW_EH_PE_sdata4. -#else - .uleb128 10 # Augmentation value length. - .byte 0x0 # Personality: absolute - .quad __gcc_personality_v0 - .byte 0x0 # LSDA Encoding: absolute -#endif - .byte 0x0c # DW_CFA_def_cfa - .uleb128 7 - .uleb128 8 - .byte 0x90 # DW_CFA_offset, column 0x10 - .uleb128 1 - .align 8 -.LENDCIE: - - .long .LENDFDE-.LSTARTFDE # Length of the FDE. -.LSTARTFDE: - .long .LSTARTFDE-.LSTARTFRAME # CIE pointer. -#ifdef SHARED - .long .LSTARTCODE-. # PC-relative start address - # of the code. - .long .LENDCODE-.LSTARTCODE # Length of the code. - .uleb128 4 # Augmentation size - .long .LexceptSTART-. -#else - .quad .LSTARTCODE # Start address of the code. - .quad .LENDCODE-.LSTARTCODE # Length of the code. - .uleb128 8 # Augmentation size - .quad .LexceptSTART -#endif - .byte 4 # DW_CFA_advance_loc4 - .long .Lpush_rsi-.LSTARTCODE - .byte 14 # DW_CFA_def_cfa_offset - .uleb128 16 - .byte 4 # DW_CFA_advance_loc4 - .long .Lpush_rdi-.Lpush_rsi - .byte 14 # DW_CFA_def_cfa_offset - .uleb128 24 - .byte 4 # DW_CFA_advance_loc4 - .long .Lpush_rdi2-.Lpush_rdi - .byte 14 # DW_CFA_def_cfa_offset - .uleb128 32 - .byte 4 # DW_CFA_advance_loc4 - .long .Lpop_rdi-.Lpush_rdi2 - .byte 14 # DW_CFA_def_cfa_offset - .uleb128 24 - .byte 4 # DW_CFA_advance_loc4 - .long .Ladd1-.Lpop_rdi - .byte 14 # DW_CFA_def_cfa_offset - .uleb128 16 - .byte 4 # DW_CFA_advance_loc4 - .long .Ladd2-.Ladd1 - .byte 14 # DW_CFA_def_cfa_offset - .uleb128 8 - .byte 4 # DW_CFA_advance_loc4 - .long clear_once_control-.Ladd2 - .byte 14 # DW_CFA_def_cfa_offset - .uleb128 32 -#if 0 - .byte 4 # DW_CFA_advance_loc4 - .long .Lpop_rdi3-clear_once_control - .byte 14 # DW_CFA_def_cfa_offset - .uleb128 16 -#endif - .align 8 -.LENDFDE: - - #ifdef SHARED .hidden DW.ref.__gcc_personality_v0 .weak DW.ref.__gcc_personality_v0 diff --git a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_timedrdlock.S b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_timedrdlock.S index 366c96fc36..23b218af34 100644 --- a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_timedrdlock.S +++ b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_timedrdlock.S @@ -1,4 +1,4 @@ -/* Copyright (C) 2002, 2003, 2004, 2005, 2007 Free Software Foundation, Inc. +/* Copyright (C) 2002-2005, 2007, 2009 Free Software Foundation, Inc. This file is part of the GNU C Library. Contributed by Ulrich Drepper <drepper@redhat.com>, 2002. @@ -36,16 +36,21 @@ pthread_rwlock_timedrdlock: cfi_startproc pushq %r12 cfi_adjust_cfa_offset(8) + cfi_rel_offset(%r12, 0) pushq %r13 cfi_adjust_cfa_offset(8) + cfi_rel_offset(%r13, 0) +#ifdef __ASSUME_FUTEX_CLOCK_REALTIME +# define VALREG %edx +#else pushq %r14 cfi_adjust_cfa_offset(8) - cfi_offset(%r12, -16) - cfi_offset(%r13, -24) - cfi_offset(%r14, -32) + cfi_rel_offset(%r14, 0) subq $16, %rsp cfi_adjust_cfa_offset(16) +# define VALREG %r14d +#endif movq %rdi, %r12 movq %rsi, %r13 @@ -76,7 +81,7 @@ pthread_rwlock_timedrdlock: incl READERS_QUEUED(%r12) je 4f - movl READERS_WAKEUP(%r12), %r14d + movl READERS_WAKEUP(%r12), VALREG /* Unlock. */ LOCK @@ -87,8 +92,33 @@ pthread_rwlock_timedrdlock: #endif jne 10f +11: +#ifndef __ASSUME_FUTEX_CLOCK_REALTIME +# ifdef PIC + cmpl $0, __have_futex_clock_realtime(%rip) +# else + cmpl $0, __have_futex_clock_realtime +# endif + je .Lreltmo +#endif + + movl $FUTEX_PRIVATE_FLAG|FUTEX_WAIT_BITSET|FUTEX_CLOCK_REALTIME, %esi + xorl PSHARED(%r12), %esi + movq %r13, %r10 + movl $0xffffffff, %r9d +#ifndef __ASSUME_FUTEX_CLOCK_REALTIME + movl %r14d, %edx +#endif +21: leaq READERS_WAKEUP(%r12), %rdi + movl $SYS_futex, %eax + syscall + movq %rax, %rdx + +#ifndef __ASSUME_FUTEX_CLOCK_REALTIME + .subsection 2 +.Lreltmo: /* Get current time. */ -11: movq %rsp, %rdi + movq %rsp, %rdi xorl %esi, %esi movq $VSYSCALL_ADDR_vgettimeofday, %rax callq *%rax @@ -111,27 +141,26 @@ pthread_rwlock_timedrdlock: movq %rcx, (%rsp) /* Store relative timeout. */ movq %rdi, 8(%rsp) -#ifdef __ASSUME_PRIVATE_FUTEX +# ifdef __ASSUME_PRIVATE_FUTEX movl $FUTEX_PRIVATE_FLAG|FUTEX_WAIT, %esi xorl PSHARED(%r12), %esi -#else -# if FUTEX_WAIT == 0 - movl PSHARED(%r12), %esi # else +# if FUTEX_WAIT == 0 + movl PSHARED(%r12), %esi +# else movl $FUTEX_WAIT, %esi orl PSHARED(%r12), %esi -# endif +# endif xorl %fs:PRIVATE_FUTEX, %esi -#endif +# endif movq %rsp, %r10 movl %r14d, %edx - leaq READERS_WAKEUP(%r12), %rdi - movl $SYS_futex, %eax - syscall - movq %rax, %rdx -17: - /* Reget the lock. */ + jmp 21b + .previous +#endif + +17: /* Reget the lock. */ movl $1, %esi xorl %eax, %eax LOCK @@ -163,11 +192,13 @@ pthread_rwlock_timedrdlock: 7: movq %rdx, %rax +#ifndef __ASSUME_FUTEX_CLOCK_REALTIME addq $16, %rsp cfi_adjust_cfa_offset(-16) popq %r14 cfi_adjust_cfa_offset(-8) cfi_restore(%r14) +#endif popq %r13 cfi_adjust_cfa_offset(-8) cfi_restore(%r13) @@ -176,10 +207,16 @@ pthread_rwlock_timedrdlock: cfi_restore(%r12) retq +#ifdef __ASSUME_PRIVATE_FUTEX + cfi_adjust_cfa_offset(16) + cfi_rel_offset(%r12, 8) + cfi_rel_offset(%r13, 0) +#else cfi_adjust_cfa_offset(40) cfi_offset(%r12, -16) cfi_offset(%r13, -24) cfi_offset(%r14, -32) +#endif 1: movl PSHARED(%rdi), %esi #if MUTEX != 0 addq $MUTEX, %rdi diff --git a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_timedwrlock.S b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_timedwrlock.S index dde6b58836..cd867b60dc 100644 --- a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_timedwrlock.S +++ b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_timedwrlock.S @@ -1,4 +1,4 @@ -/* Copyright (C) 2002, 2003, 2005, 2007 Free Software Foundation, Inc. +/* Copyright (C) 2002, 2003, 2005, 2007, 2009 Free Software Foundation, Inc. This file is part of the GNU C Library. Contributed by Ulrich Drepper <drepper@redhat.com>, 2002. @@ -36,16 +36,21 @@ pthread_rwlock_timedwrlock: cfi_startproc pushq %r12 cfi_adjust_cfa_offset(8) + cfi_rel_offset(%r12, 0) pushq %r13 cfi_adjust_cfa_offset(8) + cfi_rel_offset(%r13, 0) +#ifdef __ASSUME_FUTEX_CLOCK_REALTIME +# define VALREG %edx +#else pushq %r14 cfi_adjust_cfa_offset(8) - cfi_offset(%r12, -16) - cfi_offset(%r13, -24) - cfi_offset(%r14, -32) + cfi_rel_offset(%r14, 0) subq $16, %rsp cfi_adjust_cfa_offset(16) +# define VALREG %r14d +#endif movq %rdi, %r12 movq %rsi, %r13 @@ -74,7 +79,7 @@ pthread_rwlock_timedwrlock: incl WRITERS_QUEUED(%r12) je 4f - movl WRITERS_WAKEUP(%r12), %r14d + movl WRITERS_WAKEUP(%r12), VALREG LOCK #if MUTEX == 0 @@ -84,8 +89,33 @@ pthread_rwlock_timedwrlock: #endif jne 10f +11: +#ifndef __ASSUME_FUTEX_CLOCK_REALTIME +# ifdef PIC + cmpl $0, __have_futex_clock_realtime(%rip) +# else + cmpl $0, __have_futex_clock_realtime +# endif + je .Lreltmo +#endif + + movl $FUTEX_PRIVATE_FLAG|FUTEX_WAIT_BITSET|FUTEX_CLOCK_REALTIME, %esi + xorl PSHARED(%r12), %esi + movq %r13, %r10 + movl $0xffffffff, %r9d +#ifndef __ASSUME_FUTEX_CLOCK_REALTIME + movl %r14d, %edx +#endif +21: leaq WRITERS_WAKEUP(%r12), %rdi + movl $SYS_futex, %eax + syscall + movq %rax, %rdx + +#ifndef __ASSUME_FUTEX_CLOCK_REALTIME + .subsection 2 +.Lreltmo: /* Get current time. */ -11: movq %rsp, %rdi + movq %rsp, %rdi xorl %esi, %esi movq $VSYSCALL_ADDR_vgettimeofday, %rax callq *%rax @@ -108,27 +138,26 @@ pthread_rwlock_timedwrlock: movq %rcx, (%rsp) /* Store relative timeout. */ movq %rdi, 8(%rsp) -#ifdef __ASSUME_PRIVATE_FUTEX +# ifdef __ASSUME_PRIVATE_FUTEX movl $FUTEX_PRIVATE_FLAG|FUTEX_WAIT, %esi xorl PSHARED(%r12), %esi -#else -# if FUTEX_WAIT == 0 - movl PSHARED(%r12), %esi # else +# if FUTEX_WAIT == 0 + movl PSHARED(%r12), %esi +# else movl $FUTEX_WAIT, %esi orl PSHARED(%r12), %esi -# endif +# endif xorl %fs:PRIVATE_FUTEX, %esi -#endif +# endif movq %rsp, %r10 movl %r14d, %edx - leaq WRITERS_WAKEUP(%r12), %rdi - movl $SYS_futex, %eax - syscall - movq %rax, %rdx -17: - /* Reget the lock. */ + jmp 21b + .previous +#endif + +17: /* Reget the lock. */ movl $1, %esi xorl %eax, %eax LOCK @@ -160,11 +189,13 @@ pthread_rwlock_timedwrlock: 7: movq %rdx, %rax +#ifndef __ASSUME_PRIVATE_FUTEX addq $16, %rsp cfi_adjust_cfa_offset(-16) popq %r14 cfi_adjust_cfa_offset(-8) cfi_restore(%r14) +#endif popq %r13 cfi_adjust_cfa_offset(-8) cfi_restore(%r13) @@ -173,10 +204,16 @@ pthread_rwlock_timedwrlock: cfi_restore(%r12) retq +#ifdef __ASSUME_PRIVATE_FUTEX + cfi_adjust_cfa_offset(16) + cfi_rel_offset(%r12, 8) + cfi_rel_offset(%r13, 0) +#else cfi_adjust_cfa_offset(40) cfi_offset(%r12, -16) cfi_offset(%r13, -24) cfi_offset(%r14, -32) +#endif 1: movl PSHARED(%rdi), %esi #if MUTEX != 0 addq $MUTEX, %rdi diff --git a/nptl/sysdeps/unix/sysv/linux/x86_64/sem_timedwait.S b/nptl/sysdeps/unix/sysv/linux/x86_64/sem_timedwait.S index 88e99cf6a1..95762834d3 100644 --- a/nptl/sysdeps/unix/sysv/linux/x86_64/sem_timedwait.S +++ b/nptl/sysdeps/unix/sysv/linux/x86_64/sem_timedwait.S @@ -1,4 +1,4 @@ -/* Copyright (C) 2002, 2003, 2005, 2007 Free Software Foundation, Inc. +/* Copyright (C) 2002, 2003, 2005, 2007, 2009 Free Software Foundation, Inc. This file is part of the GNU C Library. Contributed by Ulrich Drepper <drepper@redhat.com>, 2002. @@ -18,6 +18,7 @@ 02111-1307 USA. */ #include <sysdep.h> +#include <kernel-features.h> #include <lowlevellock.h> #include <shlib-compat.h> #include <pthread-errnos.h> @@ -34,6 +35,15 @@ .align 16 sem_timedwait: .LSTARTCODE: + cfi_startproc +#ifdef SHARED + cfi_personality(DW_EH_PE_pcrel | DW_EH_PE_sdata4 | DW_EH_PE_indirect, + DW.ref.__gcc_personality_v0) + cfi_lsda(DW_EH_PE_pcrel | DW_EH_PE_sdata4, .LexceptSTART) +#else + cfi_personality(DW_EH_PE_udata4, __gcc_personality_v0) + cfi_lsda(DW_EH_PE_udata4, .LexceptSTART) +#endif #if VALUE == 0 movl (%rdi), %eax #else @@ -56,13 +66,21 @@ sem_timedwait: /* Check whether the timeout value is valid. */ 1: pushq %r12 -.Lpush_r12: + cfi_adjust_cfa_offset(8) + cfi_rel_offset(%r12, 0) pushq %r13 -.Lpush_r13: + cfi_adjust_cfa_offset(8) + cfi_rel_offset(%r13, 0) pushq %r14 -.Lpush_r14: - subq $24, %rsp -.Lsubq: + cfi_adjust_cfa_offset(8) + cfi_rel_offset(%r14, 0) +#ifdef __ASSUME_FUTEX_CLOCK_REALTIME +# define STACKFRAME 8 +#else +# define STACKFRAME 24 +#endif + subq $STACKFRAME, %rsp + cfi_adjust_cfa_offset(STACKFRAME) movq %rdi, %r12 movq %rsi, %r13 @@ -75,67 +93,50 @@ sem_timedwait: LOCK addq $1, NWAITERS(%r12) -7: xorl %esi, %esi - movq %rsp, %rdi - movq $VSYSCALL_ADDR_vgettimeofday, %rax - callq *%rax - - /* Compute relative timeout. */ - movq 8(%rsp), %rax - movl $1000, %edi - mul %rdi /* Milli seconds to nano seconds. */ - movq (%r13), %rdi - movq 8(%r13), %rsi - subq (%rsp), %rdi - subq %rax, %rsi - jns 5f - addq $1000000000, %rsi - decq %rdi -5: testq %rdi, %rdi - movl $ETIMEDOUT, %r14d - js 6f /* Time is already up. */ - - movq %rdi, (%rsp) /* Store relative timeout. */ - movq %rsi, 8(%rsp) +#ifndef __ASSUME_FUTEX_CLOCK_REALTIME +# ifdef PIC + cmpl $0, __have_futex_clock_realtime(%rip) +# else + cmpl $0, __have_futex_clock_realtime +# endif + je .Lreltmo +#endif .LcleanupSTART: - call __pthread_enable_asynccancel - movl %eax, 16(%rsp) +13: call __pthread_enable_asynccancel + movl %eax, (%rsp) - movq %rsp, %r10 + movq %r13, %r10 #if VALUE == 0 movq %r12, %rdi #else leaq VALUE(%r12), %rdi #endif -#if FUTEX_WAIT == 0 - movl PRIVATE(%rdi), %esi -#else - movl $FUTEX_WAIT, %esi + movl $0xffffffff, %r9d + movl $FUTEX_WAIT_BITSET|FUTEX_CLOCK_REALTIME, %esi orl PRIVATE(%rdi), %esi -#endif movl $SYS_futex, %eax xorl %edx, %edx syscall movq %rax, %r14 - movl 16(%rsp), %edi + movl (%rsp), %edi call __pthread_disable_asynccancel .LcleanupEND: testq %r14, %r14 - je 9f + je 11f cmpq $-EWOULDBLOCK, %r14 jne 3f -9: +11: #if VALUE == 0 movl (%r12), %eax #else movl VALUE(%r12), %eax #endif -8: testl %eax, %eax - je 7b +14: testl %eax, %eax + je 13b leaq -1(%rax), %rcx LOCK @@ -144,24 +145,30 @@ sem_timedwait: #else cmpxchgl %ecx, VALUE(%r12) #endif - jne 8b + jne 14b - xorl %eax, %eax +10: xorl %eax, %eax -10: LOCK +15: LOCK subq $1, NWAITERS(%r12) - addq $24, %rsp -.Laddq: + addq $STACKFRAME, %rsp + cfi_adjust_cfa_offset(-STACKFRAME) popq %r14 -.Lpop_r14: + cfi_adjust_cfa_offset(-8) + cfi_restore(%r14) popq %r13 -.Lpop_r13: + cfi_adjust_cfa_offset(-8) + cfi_restore(%r13) popq %r12 -.Lpop_r12: + cfi_adjust_cfa_offset(-8) + cfi_restore(%r12) retq -.Lafter_retq: + cfi_adjust_cfa_offset(STACKFRAME + 3 * 8) + cfi_rel_offset(%r12, STACKFRAME + 2 * 8) + cfi_rel_offset(%r13, STACKFRAME + 1 * 8) + cfi_rel_offset(%r14, STACKFRAME) 3: negq %r14 6: #if USE___THREAD @@ -173,7 +180,82 @@ sem_timedwait: #endif orl $-1, %eax + jmp 15b + +#ifndef __ASSUME_FUTEX_CLOCK_REALTIME +.Lreltmo: +7: xorl %esi, %esi + movq %rsp, %rdi + movq $VSYSCALL_ADDR_vgettimeofday, %rax + callq *%rax + + /* Compute relative timeout. */ + movq 8(%rsp), %rax + movl $1000, %edi + mul %rdi /* Milli seconds to nano seconds. */ + movq (%r13), %rdi + movq 8(%r13), %rsi + subq (%rsp), %rdi + subq %rax, %rsi + jns 5f + addq $1000000000, %rsi + decq %rdi +5: testq %rdi, %rdi + movl $ETIMEDOUT, %r14d + js 6b /* Time is already up. */ + + movq %rdi, (%rsp) /* Store relative timeout. */ + movq %rsi, 8(%rsp) + +.LcleanupSTART2: + call __pthread_enable_asynccancel + movl %eax, 16(%rsp) + + movq %rsp, %r10 +# if VALUE == 0 + movq %r12, %rdi +# else + leaq VALUE(%r12), %rdi +# endif +# if FUTEX_WAIT == 0 + movl PRIVATE(%rdi), %esi +# else + movl $FUTEX_WAIT, %esi + orl PRIVATE(%rdi), %esi +# endif + movl $SYS_futex, %eax + xorl %edx, %edx + syscall + movq %rax, %r14 + + movl 16(%rsp), %edi + call __pthread_disable_asynccancel +.LcleanupEND2: + + testq %r14, %r14 + je 9f + cmpq $-EWOULDBLOCK, %r14 + jne 3b + +9: +# if VALUE == 0 + movl (%r12), %eax +# else + movl VALUE(%r12), %eax +# endif +8: testl %eax, %eax + je 7b + + leaq -1(%rax), %rcx + LOCK +# if VALUE == 0 + cmpxchgl %ecx, (%r12) +# else + cmpxchgl %ecx, VALUE(%r12) +# endif + jne 8b jmp 10b +#endif .size sem_timedwait,.-sem_timedwait @@ -186,21 +268,27 @@ sem_timedwait_cleanup: call _Unwind_Resume@PLT hlt .LENDCODE: + cfi_endproc .size sem_timedwait_cleanup,.-sem_timedwait_cleanup .section .gcc_except_table,"a",@progbits .LexceptSTART: - .byte 0xff # @LPStart format (omit) - .byte 0xff # @TType format (omit) - .byte 0x01 # call-site format - # DW_EH_PE_uleb128 + .byte DW_EH_PE_omit # @LPStart format + .byte DW_EH_PE_omit # @TType format + .byte DW_EH_PE_uleb128 # call-site format .uleb128 .Lcstend-.Lcstbegin .Lcstbegin: .uleb128 .LcleanupSTART-.LSTARTCODE .uleb128 .LcleanupEND-.LcleanupSTART .uleb128 sem_timedwait_cleanup-.LSTARTCODE .uleb128 0 +#ifndef __ASSUME_FUTEX_CLOCK_REALTIME + .uleb128 .LcleanupSTART2-.LSTARTCODE + .uleb128 .LcleanupEND2-.LcleanupSTART2 + .uleb128 sem_timedwait_cleanup-.LSTARTCODE + .uleb128 0 +#endif .uleb128 .LcallUR-.LSTARTCODE .uleb128 .LENDCODE-.LcallUR .uleb128 0 @@ -208,118 +296,6 @@ sem_timedwait_cleanup: .Lcstend: - .section .eh_frame,"a",@progbits -.LSTARTFRAME: - .long .LENDCIE-.LSTARTCIE # Length of the CIE. -.LSTARTCIE: - .long 0 # CIE ID. - .byte 1 # Version number. -#ifdef SHARED - .string "zPLR" # NUL-terminated augmentation - # string. -#else - .string "zPL" # NUL-terminated augmentation - # string. -#endif - .uleb128 1 # Code alignment factor. - .sleb128 -8 # Data alignment factor. - .byte 16 # Return address register - # column. -#ifdef SHARED - .uleb128 7 # Augmentation value length. - .byte 0x9b # Personality: DW_EH_PE_pcrel - # + DW_EH_PE_sdata4 - # + DW_EH_PE_indirect - .long DW.ref.__gcc_personality_v0-. - .byte 0x1b # LSDA Encoding: DW_EH_PE_pcrel - # + DW_EH_PE_sdata4. - .byte 0x1b # FDE Encoding: DW_EH_PE_pcrel - # + DW_EH_PE_sdata4. -#else - .uleb128 10 # Augmentation value length. - .byte 0x0 # Personality: absolute - .quad __gcc_personality_v0 - .byte 0x0 # LSDA Encoding: absolute -#endif - .byte 0x0c # DW_CFA_def_cfa - .uleb128 7 - .uleb128 8 - .byte 0x90 # DW_CFA_offset, column 0x10 - .uleb128 1 - .align 8 -.LENDCIE: - - .long .LENDFDE-.LSTARTFDE # Length of the FDE. -.LSTARTFDE: - .long .LSTARTFDE-.LSTARTFRAME # CIE pointer. -#ifdef SHARED - .long .LSTARTCODE-. # PC-relative start address - # of the code. - .long .LENDCODE-.LSTARTCODE # Length of the code. - .uleb128 4 # Augmentation size - .long .LexceptSTART-. -#else - .quad .LSTARTCODE # Start address of the code. - .quad .LENDCODE-.LSTARTCODE # Length of the code. - .uleb128 8 # Augmentation size - .quad .LexceptSTART -#endif - - .byte 4 # DW_CFA_advance_loc4 - .long .Lpush_r12-.LSTARTCODE - .byte 14 # DW_CFA_def_cfa_offset - .uleb128 16 - .byte 0x8c # DW_CFA_offset %r12 - .uleb128 2 - .byte 4 # DW_CFA_advance_loc4 - .long .Lpush_r13-.Lpush_r12 - .byte 14 # DW_CFA_def_cfa_offset - .uleb128 24 - .byte 0x8d # DW_CFA_offset %r13 - .uleb128 3 - .byte 4 # DW_CFA_advance_loc4 - .long .Lpush_r14-.Lpush_r13 - .byte 14 # DW_CFA_def_cfa_offset - .uleb128 32 - .byte 0x8e # DW_CFA_offset %r14 - .uleb128 4 - .byte 4 # DW_CFA_advance_loc4 - .long .Lsubq-.Lpush_r14 - .byte 14 # DW_CFA_def_cfa_offset - .uleb128 56 - .byte 4 # DW_CFA_advance_loc4 - .long .Laddq-.Lsubq - .byte 14 # DW_CFA_def_cfa_offset - .uleb128 32 - .byte 4 # DW_CFA_advance_loc4 - .long .Lpop_r14-.Laddq - .byte 14 # DW_CFA_def_cfa_offset - .uleb128 24 - .byte 0xce # DW_CFA_restore %r14 - .byte 4 # DW_CFA_advance_loc4 - .long .Lpop_r13-.Lpop_r14 - .byte 14 # DW_CFA_def_cfa_offset - .uleb128 16 - .byte 0xcd # DW_CFA_restore %r13 - .byte 4 # DW_CFA_advance_loc4 - .long .Lpop_r12-.Lpop_r13 - .byte 14 # DW_CFA_def_cfa_offset - .uleb128 8 - .byte 0xcc # DW_CFA_restore %r12 - .byte 4 # DW_CFA_advance_loc4 - .long .Lafter_retq-.Lpop_r12 - .byte 14 # DW_CFA_def_cfa_offset - .uleb128 56 - .byte 0x8c # DW_CFA_offset %r12 - .uleb128 2 - .byte 0x8d # DW_CFA_offset %r13 - .uleb128 3 - .byte 0x8e # DW_CFA_offset %r14 - .uleb128 4 - .align 8 -.LENDFDE: - - #ifdef SHARED .hidden DW.ref.__gcc_personality_v0 .weak DW.ref.__gcc_personality_v0 diff --git a/nptl/sysdeps/unix/sysv/linux/x86_64/sem_wait.S b/nptl/sysdeps/unix/sysv/linux/x86_64/sem_wait.S index 5320a91e19..a01d745a17 100644 --- a/nptl/sysdeps/unix/sysv/linux/x86_64/sem_wait.S +++ b/nptl/sysdeps/unix/sysv/linux/x86_64/sem_wait.S @@ -1,4 +1,4 @@ -/* Copyright (C) 2002, 2003, 2005, 2007 Free Software Foundation, Inc. +/* Copyright (C) 2002, 2003, 2005, 2007, 2009 Free Software Foundation, Inc. This file is part of the GNU C Library. Contributed by Ulrich Drepper <drepper@redhat.com>, 2002. @@ -31,16 +31,20 @@ .align 16 sem_wait: .LSTARTCODE: - pushq %r12 -.Lpush_r12: - pushq %r13 -.Lpush_r13: - movq %rdi, %r13 + cfi_startproc +#ifdef SHARED + cfi_personality(DW_EH_PE_pcrel | DW_EH_PE_sdata4 | DW_EH_PE_indirect, + DW.ref.__gcc_personality_v0) + cfi_lsda(DW_EH_PE_pcrel | DW_EH_PE_sdata4, .LexceptSTART) +#else + cfi_personality(DW_EH_PE_udata4, __gcc_personality_v0) + cfi_lsda(DW_EH_PE_udata4, .LexceptSTART) +#endif #if VALUE == 0 - movl (%r13), %eax + movl (%rdi), %eax #else - movl VALUE(%r13), %eax + movl VALUE(%rdi), %eax #endif 2: testl %eax, %eax je 1f @@ -48,23 +52,24 @@ sem_wait: leal -1(%rax), %edx LOCK #if VALUE == 0 - cmpxchgl %edx, (%r13) + cmpxchgl %edx, (%rdi) #else - cmpxchgl %edx, VALUE(%r13) + cmpxchgl %edx, VALUE(%rdi) #endif jne 2b -7: xorl %eax, %eax - -9: popq %r13 -.Lpop_r13: - popq %r12 -.Lpop_r12: - + xorl %eax, %eax retq -.Lafter_retq: -1: LOCK +1: pushq %r12 + cfi_adjust_cfa_offset(8) + cfi_rel_offset(%r12, 0) + pushq %r13 + cfi_adjust_cfa_offset(8) + cfi_rel_offset(%r13, 0) + movq %rdi, %r13 + + LOCK addq $1, NWAITERS(%r13) .LcleanupSTART: @@ -113,8 +118,21 @@ sem_wait: LOCK subq $1, NWAITERS(%r13) - jmp 7b + xorl %eax, %eax + +9: popq %r13 + cfi_adjust_cfa_offset(-8) + cfi_restore(%r13) + popq %r12 + cfi_adjust_cfa_offset(-8) + cfi_restore(%r12) + + retq + + cfi_adjust_cfa_offset(2 * 8) + cfi_rel_offset(%r12, 8) + cfi_rel_offset(%r13, 0) 4: negq %r12 #if USE___THREAD movq errno@gottpoff(%rip), %rdx @@ -141,15 +159,15 @@ sem_wait_cleanup: call _Unwind_Resume@PLT hlt .LENDCODE: + cfi_endproc .size sem_wait_cleanup,.-sem_wait_cleanup .section .gcc_except_table,"a",@progbits .LexceptSTART: - .byte 0xff # @LPStart format (omit) - .byte 0xff # @TType format (omit) - .byte 0x01 # call-site format - # DW_EH_PE_uleb128 + .byte DW_EH_PE_omit # @LPStart format + .byte DW_EH_PE_omit # @TType format + .byte DW_EH_PE_uleb128 # call-site format .uleb128 .Lcstend-.Lcstbegin .Lcstbegin: .uleb128 .LcleanupSTART-.LSTARTCODE @@ -163,97 +181,6 @@ sem_wait_cleanup: .Lcstend: - .section .eh_frame,"a",@progbits -.LSTARTFRAME: - .long .LENDCIE-.LSTARTCIE # Length of the CIE. -.LSTARTCIE: - .long 0 # CIE ID. - .byte 1 # Version number. -#ifdef SHARED - .string "zPLR" # NUL-terminated augmentation - # string. -#else - .string "zPL" # NUL-terminated augmentation - # string. -#endif - .uleb128 1 # Code alignment factor. - .sleb128 -8 # Data alignment factor. - .byte 16 # Return address register - # column. -#ifdef SHARED - .uleb128 7 # Augmentation value length. - .byte 0x9b # Personality: DW_EH_PE_pcrel - # + DW_EH_PE_sdata4 - # + DW_EH_PE_indirect - .long DW.ref.__gcc_personality_v0-. - .byte 0x1b # LSDA Encoding: DW_EH_PE_pcrel - # + DW_EH_PE_sdata4. - .byte 0x1b # FDE Encoding: DW_EH_PE_pcrel - # + DW_EH_PE_sdata4. -#else - .uleb128 10 # Augmentation value length. - .byte 0x0 # Personality: absolute - .quad __gcc_personality_v0 - .byte 0x0 # LSDA Encoding: absolute -#endif - .byte 0x0c # DW_CFA_def_cfa - .uleb128 7 - .uleb128 8 - .byte 0x90 # DW_CFA_offset, column 0x10 - .uleb128 1 - .align 8 -.LENDCIE: - - .long .LENDFDE-.LSTARTFDE # Length of the FDE. -.LSTARTFDE: - .long .LSTARTFDE-.LSTARTFRAME # CIE pointer. -#ifdef SHARED - .long .LSTARTCODE-. # PC-relative start address - # of the code. - .long .LENDCODE-.LSTARTCODE # Length of the code. - .uleb128 4 # Augmentation size - .long .LexceptSTART-. -#else - .quad .LSTARTCODE # Start address of the code. - .quad .LENDCODE-.LSTARTCODE # Length of the code. - .uleb128 8 # Augmentation size - .quad .LexceptSTART -#endif - - .byte 4 # DW_CFA_advance_loc4 - .long .Lpush_r12-.LSTARTCODE - .byte 14 # DW_CFA_def_cfa_offset - .uleb128 16 - .byte 0x8c # DW_CFA_offset %r12 - .uleb128 2 - .byte 4 # DW_CFA_advance_loc4 - .long .Lpush_r13-.Lpush_r12 - .byte 14 # DW_CFA_def_cfa_offset - .uleb128 24 - .byte 0x8d # DW_CFA_offset %r13 - .uleb128 3 - .byte 4 # DW_CFA_advance_loc4 - .long .Lpop_r13-.Lpush_r13 - .byte 14 # DW_CFA_def_cfa_offset - .uleb128 16 - .byte 0xcd # DW_CFA_restore %r13 - .byte 4 # DW_CFA_advance_loc4 - .long .Lpop_r12-.Lpop_r13 - .byte 14 # DW_CFA_def_cfa_offset - .uleb128 8 - .byte 0xcc # DW_CFA_restore %r12 - .byte 4 # DW_CFA_advance_loc4 - .long .Lafter_retq-.Lpop_r12 - .byte 14 # DW_CFA_def_cfa_offset - .uleb128 24 - .byte 0x8c # DW_CFA_offset %r12 - .uleb128 2 - .byte 0x8d # DW_CFA_offset %r13 - .uleb128 3 - .align 8 -.LENDFDE: - - #ifdef SHARED .hidden DW.ref.__gcc_personality_v0 .weak DW.ref.__gcc_personality_v0 |