Fix bookkeeping in mutex when using requeue_pi.

5 files changed, 40 insertions, 20 deletions

diff --git a/nptl/sysdeps/unix/sysv/linux/pthread-pi-defines.sym b/nptl/sysdeps/unix/sysv/linux/pthread-pi-defines.sym
index d985c6a79b..46fbd0de74 100644
--- a/nptl/sysdeps/unix/sysv/linux/pthread-pi-defines.sym
+++ b/nptl/sysdeps/unix/sysv/linux/pthread-pi-defines.sym
@@ -3,5 +3,6 @@
 -- These PI macros are used by assembly code.
 
 MUTEX_KIND	offsetof (pthread_mutex_t, __data.__kind)
+ROBUST_BIT	PTHREAD_MUTEX_ROBUST_NORMAL_NP
 PI_BIT		PTHREAD_MUTEX_PRIO_INHERIT_NP
 PS_BIT		PTHREAD_MUTEX_PSHARED_BIT
diff --git a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_broadcast.S b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_broadcast.S
index 0f10ec910c..224a56088e 100644
--- a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_broadcast.S
+++ b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_broadcast.S
@@ -75,8 +75,10 @@ __pthread_cond_broadcast:
 	jne	9f
 
 	/* Requeue to a PI mutex if the PI bit is set.  */
-	testl	$PI_BIT, MUTEX_KIND(%r8)
-	jne	81f
+	movl	MUTEX_KIND(%r8), %eax
+	andl	$(ROBUST_BIT|PI_BIT), %eax
+	cmpl	$PI_BIT, %eax
+	je	81f
 
 	/* Wake up all threads.  */
 #ifdef __ASSUME_PRIVATE_FUTEX
diff --git a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_signal.S b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_signal.S
index f1050fea7c..4d001eec7f 100644
--- a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_signal.S
+++ b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_signal.S
@@ -64,8 +64,10 @@ __pthread_cond_signal:
 
 	/* Get the address of the mutex used.  */
 	movq    dep_mutex(%r8), %rcx
-	testl	$PI_BIT, MUTEX_KIND(%rcx)
-	jne	9f
+	movl	MUTEX_KIND(%rcx), %eax
+	andl	$(ROBUST_BIT|PI_BIT), %eax
+	cmpl	$PI_BIT, %eax
+	je	9f
 
 #ifdef __ASSUME_PRIVATE_FUTEX
 	movl	$(FUTEX_WAKE_OP|FUTEX_PRIVATE_FLAG), %esi
diff --git a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S
index 7486825d5f..4913beb8af 100644
--- a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S
+++ b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S
@@ -165,9 +165,12 @@ __pthread_cond_timedwait:
 	je	60f
 
 	movq	dep_mutex(%rdi), %r8
-	/* Requeue to a PI mutex if the PI bit is set.  */
-	testl	$PI_BIT, MUTEX_KIND(%r8)
-	je	61f
+	/* Requeue to a non-robust PI mutex if the PI bit is set and
+	the robust bit is not set.  */
+	movl	MUTEX_KIND(%r8), %eax
+	andl	$(ROBUST_BIT|PI_BIT), %eax
+	cmpl	$PI_BIT, %eax
+	jne	61f
 
 	movl	$(FUTEX_WAIT_REQUEUE_PI|FUTEX_PRIVATE_FLAG), %esi
 	xorl	%eax, %eax
@@ -289,11 +292,10 @@ __pthread_cond_timedwait:
 
 	/* If requeue_pi is used the kernel performs the locking of the
 	   mutex. */
-41:	xorl	%eax, %eax
+41:	movq	16(%rsp), %rdi
 	testl	%r15d, %r15d
-	jnz	63f
+	jnz	64f
 
-	movq	16(%rsp), %rdi
 	callq	__pthread_mutex_cond_lock
 
 63:	testq	%rax, %rax
@@ -316,12 +318,18 @@ __pthread_cond_timedwait:
 
 	retq
 
-	/* Initial locking failed.  */
-31:	cfi_adjust_cfa_offset(4 * 8 + FRAME_SIZE)
+	cfi_adjust_cfa_offset(4 * 8 + FRAME_SIZE)
 	cfi_rel_offset(%r12, FRAME_SIZE + 24)
 	cfi_rel_offset(%r13, FRAME_SIZE + 16)
 	cfi_rel_offset(%r14, FRAME_SIZE + 8)
 	cfi_rel_offset(%r15, FRAME_SIZE)
+
+64:	callq	__pthread_mutex_cond_lock_adjust
+	movq	%r14, %rax
+	jmp	48b
+
+	/* Initial locking failed.  */
+31:
 #if cond_lock != 0
 	addq	$cond_lock, %rdi
 #endif
diff --git a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S
index 2fab38e277..a66523eab6 100644
--- a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S
+++ b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S
@@ -134,9 +134,12 @@ __pthread_cond_wait:
 	je	60f
 
 	movq	dep_mutex-cond_futex(%rdi), %r8
-	/* Requeue to a PI mutex if the PI bit is set.  */
-	testl	$PI_BIT, MUTEX_KIND(%r8)
-	je	61f
+	/* Requeue to a non-robust PI mutex if the PI bit is set and
+	the robust bit is not set.  */
+	movl	MUTEX_KIND(%r8), %eax
+	andl	$(ROBUST_BIT|PI_BIT), %eax
+	cmpl	$PI_BIT, %eax
+	jne	61f
 
 	movl	$(FUTEX_WAIT_REQUEUE_PI|FUTEX_PRIVATE_FLAG), %esi
 	movl	$SYS_futex, %eax
@@ -234,11 +237,10 @@ __pthread_cond_wait:
 
 	/* If requeue_pi is used the kernel performs the locking of the
 	   mutex. */
-11:	xorl	%eax, %eax
+11:	movq	16(%rsp), %rdi
 	testl	%r13d, %r13d
-	jnz	14f
+	jnz	18f
 
-	movq	16(%rsp), %rdi
 	callq	__pthread_mutex_cond_lock
 
 14:	addq	$FRAME_SIZE, %rsp
@@ -254,11 +256,16 @@ __pthread_cond_wait:
 	/* We return the result of the mutex_lock operation.  */
 	retq
 
-	/* Initial locking failed.  */
-1:
 	cfi_adjust_cfa_offset(16 + FRAME_SIZE)
 	cfi_rel_offset(%r12, FRAME_SIZE + 8)
 	cfi_rel_offset(%r13, FRAME_SIZE)
+
+18:	callq	__pthread_mutex_cond_lock_adjust
+	xorl	%eax, %eax
+	jmp	14b
+
+	/* Initial locking failed.  */
+1:
 #if cond_lock != 0
 	addq	$cond_lock, %rdi
 #endif