Update.

	* sysdeps/unix/sysv/linux/x86_64/lowlevellock.h: Likewise.
	* sysdeps/unix/sysv/linux/x86_64/lowlevellock.S: Likewise.
	* sysdeps/unix/sysv/linux/x86_64/lowlevelmutex.S: Removed.
	* sysdeps/unix/sysv/linux/i386/i486/pthread_barrier_wait.S: Adjust
	for new mutex implementation.
	* sysdeps/unix/sysv/linux/i386/i486/pthread_cond_broadcast.S: Likewise.
	* sysdeps/unix/sysv/linux/i386/i486/pthread_cond_timedwait.S: Likewise.
	* sysdeps/unix/sysv/linux/i386/i486/pthread_cond_wait.S: Likewise.
	* sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_rdlock.S: Likewise.
	* sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_timedrdlock.S:
	Likewise.
	* sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_timedwrlock.S:
	Likewise.
	* sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_unlock.S: Likewise.
	* sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_wrlock.S: Likewise.
	* sysdeps/unix/sysv/linux/x86_64/pthread_barrier_wait.S: Likewise
	* sysdeps/unix/sysv/linux/x86_64/pthread_cond_broadcast.S: Likewise.
	* sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S: Likewise.
	* sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S: Likewise.
	* sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_rdlock.S: Likewise.
	* sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_timedrdlock.S:
	Likewise.
	* sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_timedwrlock.S:
	Likewise.
	* sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_unlock.S: Likewise.
	* sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_wrlock.S: Likewise.
	* sysdeps/unix/sysv/linux/i386/i486/pthread_cond_signal.S: Likewise.
	Don't use requeue.
	* sysdeps/unix/sysv/linux/x86_64/pthread_cond_signal.S: Likewise.

27 files changed, 292 insertions, 433 deletions

diff --git a/nptl/ChangeLog b/nptl/ChangeLog
index 0562ae947a..8bb48ddbc8 100644
--- a/nptl/ChangeLog
+++ b/nptl/ChangeLog
@@ -3,21 +3,40 @@
 	* sysdeps/unix/sysv/linux/i386/lowlevellock.h: Completely revamp the
 	locking macros.  No distinction between normal and mutex locking
 	anymore.
+	* sysdeps/unix/sysv/linux/x86_64/lowlevellock.h: Likewise.
 	* sysdeps/unix/sysv/linux/i386/i486/lowlevellock.S: Rewrite mutex
 	locking.  Merge bits from lowlevelmutex.S we still need.
+	* sysdeps/unix/sysv/linux/x86_64/lowlevellock.S: Likewise.
 	* sysdeps/unix/sysv/linux/i386/i486/lowlevelmutex.S: Removed.
+	* sysdeps/unix/sysv/linux/x86_64/lowlevelmutex.S: Removed.
 	* Makefile (routines): Remove libc-lowlevelmutex.
 	(libpthread-rountines): Remove lowlevelmutex.
-	* pthread_barrier_wait.S: Adjust for new mutex implementation.
-	* pthread_cond_broadcast.S: Likewise.
-	* pthread_cond_timedwait.S: Likewise.
-	* pthread_cond_wait.S: Likewise.
-	* pthread_rwlock_rdlock.S: Likewise.
-	* pthread_rwlock_timedrdlock.S: Likewise.
-	* pthread_rwlock_timedwrlock.S: Likewise.
-	* pthread_rwlock_unlock.S: Likewise.
-	* pthread_rwlock_wrlock.S: Likewise.
-	* pthread_cond_signal.S: Likewise.  Don't use requeue.
+	* sysdeps/unix/sysv/linux/i386/i486/pthread_barrier_wait.S: Adjust
+	for new mutex implementation.
+	* sysdeps/unix/sysv/linux/i386/i486/pthread_cond_broadcast.S: Likewise.
+	* sysdeps/unix/sysv/linux/i386/i486/pthread_cond_timedwait.S: Likewise.
+	* sysdeps/unix/sysv/linux/i386/i486/pthread_cond_wait.S: Likewise.
+	* sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_rdlock.S: Likewise.
+	* sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_timedrdlock.S:
+	Likewise.
+	* sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_timedwrlock.S:
+	Likewise.
+	* sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_unlock.S: Likewise.
+	* sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_wrlock.S: Likewise.
+	* sysdeps/unix/sysv/linux/x86_64/pthread_barrier_wait.S: Likewise
+	* sysdeps/unix/sysv/linux/x86_64/pthread_cond_broadcast.S: Likewise.
+	* sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S: Likewise.
+	* sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S: Likewise.
+	* sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_rdlock.S: Likewise.
+	* sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_timedrdlock.S:
+	Likewise.
+	* sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_timedwrlock.S:
+	Likewise.
+	* sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_unlock.S: Likewise.
+	* sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_wrlock.S: Likewise.
+	* sysdeps/unix/sysv/linux/i386/i486/pthread_cond_signal.S: Likewise.
+	Don't use requeue.
+	* sysdeps/unix/sysv/linux/x86_64/pthread_cond_signal.S: Likewise.
 
 2003-09-20  Ulrich Drepper  <drepper@redhat.com>
 
diff --git a/nptl/Makefile b/nptl/Makefile
index ba568dffba..d592137caf 100644
--- a/nptl/Makefile
+++ b/nptl/Makefile
@@ -29,8 +29,7 @@ extra-libs := libpthread
 extra-libs-others := $(extra-libs)
 install-lib-ldscripts := libpthread.so
 
-routines = alloca_cutoff forward libc-lowlevellock libc-lowlevelmutex \
-	   libc-cancellation
+routines = alloca_cutoff forward libc-lowlevellock libc-cancellation
 shared-only-routines = forward
 
 libpthread-routines = init events version \
@@ -102,7 +101,7 @@ libpthread-routines = init events version \
 		      cleanup_defer_compat unwind \
 		      pt-longjmp \
 		      cancellation \
-		      lowlevellock lowlevelmutex \
+		      lowlevellock \
 		      pt-vfork \
 		      ptw-write ptw-read ptw-close ptw-fcntl ptw-accept \
 		      ptw-connect ptw-recv ptw-recvfrom ptw-recvmsg ptw-send \
diff --git a/nptl/sysdeps/unix/sysv/linux/i386/i486/lowlevellock.S b/nptl/sysdeps/unix/sysv/linux/i386/i486/lowlevellock.S
index 6e4b077295..c73970d45f 100644
--- a/nptl/sysdeps/unix/sysv/linux/i386/i486/lowlevellock.S
+++ b/nptl/sysdeps/unix/sysv/linux/i386/i486/lowlevellock.S
@@ -65,8 +65,7 @@ __lll_mutex_lock_wait:
 2:	LOCK
 	cmpxchgl %edx, (%ebx)
 
-	testl	%eax, %eax
-	jne,pn	1b
+	jnz,pn	1b
 
 	popl	%edx
 	popl	%ebx
@@ -144,8 +143,7 @@ __lll_mutex_timedlock_wait:
 	LOCK
 	cmpxchgl %edx, (%ebx)
 
-	testl	%eax, %eax
-	jne	7f
+	jnz	7f
 
 6:	addl	$8, %esp
 	popl	%ebp
diff --git a/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_barrier_wait.S b/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_barrier_wait.S
index 08c6e915b9..d4b61a10c1 100644
--- a/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_barrier_wait.S
+++ b/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_barrier_wait.S
@@ -46,8 +46,7 @@ pthread_barrier_wait:
 	xorl	%eax, %eax
 	LOCK
 	cmpxchgl %edx, MUTEX(%ebx)
-	testl	%eax, %eax
-	jne	1f
+	jnz	1f
 
 	/* One less waiter.  If this was the last one needed wake
 	   everybody.  */
diff --git a/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_broadcast.S b/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_broadcast.S
index 1049e0f4dd..cdcf6d2914 100644
--- a/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_broadcast.S
+++ b/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_broadcast.S
@@ -59,8 +59,7 @@ __pthread_cond_broadcast:
 #else
 	cmpxchgl %edx, cond_lock(%ebx)
 #endif
-	testl	%eax, %eax
-	jne	1f
+	jnz	1f
 
 2:	addl	$wakeup_seq, %ebx
 	movl	total_seq+4-wakeup_seq(%ebx), %eax
diff --git a/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_signal.S b/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_signal.S
index d86ee054a9..3a24adf911 100644
--- a/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_signal.S
+++ b/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_signal.S
@@ -58,8 +58,7 @@ __pthread_cond_signal:
 #else
 	cmpxchgl %edx, cond_lock(%edi)
 #endif
-	testl	%eax, %eax
-	jne	1f
+	jnz	1f
 
 2:	leal	wakeup_seq(%edi), %ebx
 	movl	total_seq+4(%edi), %eax
diff --git a/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_timedwait.S b/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_timedwait.S
index 5799d8f862..ca5c1423c0 100644
--- a/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_timedwait.S
+++ b/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_timedwait.S
@@ -64,8 +64,7 @@ __pthread_cond_timedwait:
 #else
 	cmpxchgl %edx, cond_lock(%ebx)
 #endif
-	testl	%eax, %eax
-	jne	1f
+	jnz	1f
 
 	/* Store the reference to the mutex.  If there is already a
 	   different value in there this is a bad user bug.  */
@@ -179,8 +178,7 @@ __pthread_cond_timedwait:
 #else
 	cmpxchgl %edx, cond_lock(%ebx)
 #endif
-	testl	%eax, %eax
-	jne	5f
+	jnz	5f
 
 6:	movl	woken_seq(%ebx), %eax
 	movl	woken_seq+4(%ebx), %ecx
@@ -363,8 +361,7 @@ __condvar_tw_cleanup:
 #else
 	cmpxchgl %edx, cond_lock(%ebx)
 #endif
-	testl	%eax, %eax
-	je	1f
+	jz	1f
 
 #if cond_lock == 0
 	movl	%ebx, %ecx
diff --git a/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_wait.S b/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_wait.S
index db0428f527..ff3a1ccef3 100644
--- a/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_wait.S
+++ b/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_wait.S
@@ -61,8 +61,7 @@ __pthread_cond_wait:
 #else
 	cmpxchgl %edx, cond_lock(%ebx)
 #endif
-	testl	%eax, %eax
-	jne	1f
+	jnz	1f
 
 	/* Store the reference to the mutex.  If there is already a
 	   different value in there this is a bad user bug.  */
@@ -123,8 +122,7 @@ __pthread_cond_wait:
 #else
 	cmpxchgl %edx, cond_lock(%ebx)
 #endif
-	testl	%eax, %eax
-	jne	5f
+	jnz	5f
 
 6:	movl	woken_seq(%ebx), %eax
 	movl	woken_seq+4(%ebx), %ecx
@@ -256,8 +254,7 @@ __condvar_w_cleanup:
 #else
 	cmpxchgl %edx, cond_lock(%ebx)
 #endif
-	testl	%eax, %eax
-	je	1f
+	jz	1f
 
 #if cond_lock == 0
 	movl	%ebx, %ecx
diff --git a/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_rdlock.S b/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_rdlock.S
index 1c149433f0..db0639d21c 100644
--- a/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_rdlock.S
+++ b/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_rdlock.S
@@ -54,8 +54,7 @@ __pthread_rwlock_rdlock:
 #else
 	cmpxchgl %edx, MUTEX(%ebx)
 #endif
-	testl	%eax, %eax
-	jne	1f
+	jnz	1f
 
 2:	movl	WRITER(%ebx), %eax
 	testl	%eax, %eax
@@ -94,8 +93,7 @@ __pthread_rwlock_rdlock:
 #else
 	cmpxchgl %edx, MUTEX(%ebx)
 #endif
-	testl	%eax, %eax
-	jne	12f
+	jnz	12f
 
 13:	subl	$1, READERS_QUEUED(%ebx)
 	jmp	2b
diff --git a/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_timedrdlock.S b/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_timedrdlock.S
index 409c046b7b..eb5665b432 100644
--- a/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_timedrdlock.S
+++ b/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_timedrdlock.S
@@ -58,8 +58,7 @@ pthread_rwlock_timedrdlock:
 #else
 	cmpxchgl %edx, MUTEX(%ebp)
 #endif
-	testl	%eax, %eax
-	jne	1f
+	jnz	1f
 
 2:	movl	WRITER(%ebp), %eax
 	testl	%eax, %eax
@@ -127,8 +126,7 @@ pthread_rwlock_timedrdlock:
 #else
 	cmpxchgl %edx, MUTEX(%ebp)
 #endif
-	testl	%eax, %eax
-	jne	12f
+	jnz	12f
 
 13:	subl	$1, READERS_QUEUED(%ebp)
 	cmpl	$-ETIMEDOUT, %ecx
diff --git a/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_timedwrlock.S b/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_timedwrlock.S
index 00d9906e59..d9db77ba05 100644
--- a/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_timedwrlock.S
+++ b/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_timedwrlock.S
@@ -58,8 +58,7 @@ pthread_rwlock_timedwrlock:
 #else
 	cmpxchgl %edx, MUTEX(%ebp)
 #endif
-	testl	%eax, %eax
-	jne	1f
+	jnz	1f
 
 2:	movl	WRITER(%ebp), %eax
 	testl	%eax, %eax
@@ -125,8 +124,7 @@ pthread_rwlock_timedwrlock:
 #else
 	cmpxchgl %edx, MUTEX(%ebp)
 #endif
-	testl	%eax, %eax
-	jne	12f
+	jnz	12f
 
 13:	subl	$1, WRITERS_QUEUED(%ebp)
 	cmpl	$-ETIMEDOUT, %ecx
diff --git a/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_unlock.S b/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_unlock.S
index 839a5eb336..64aac3255a 100644
--- a/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_unlock.S
+++ b/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_unlock.S
@@ -52,8 +52,7 @@ __pthread_rwlock_unlock:
 #else
 	cmpxchgl %edx, MUTEX(%edi)
 #endif
-	testl	%eax, %eax
-	jne	1f
+	jnz	1f
 
 2:	cmpl	$0, WRITER(%edi)
 	jne	5f
diff --git a/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_wrlock.S b/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_wrlock.S
index 89ad6a3774..ea9cc170db 100644
--- a/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_wrlock.S
+++ b/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_wrlock.S
@@ -54,8 +54,7 @@ __pthread_rwlock_wrlock:
 #else
 	cmpxchgl %edx, MUTEX(%ebx)
 #endif
-	testl	%eax, %eax
-	jne	1f
+	jnz	1f
 
 2:	movl	WRITER(%ebx), %eax
 	testl	%eax, %eax
@@ -92,8 +91,7 @@ __pthread_rwlock_wrlock:
 #else
 	cmpxchgl %edx, MUTEX(%ebx)
 #endif
-	testl	%eax, %eax
-	jne	12f
+	jnz	12f
 
 13:	subl	$1, WRITERS_QUEUED(%ebx)
 	jmp	2b
diff --git a/nptl/sysdeps/unix/sysv/linux/i386/lowlevellock.h b/nptl/sysdeps/unix/sysv/linux/i386/lowlevellock.h
index ba32591623..be00e89638 100644
--- a/nptl/sysdeps/unix/sysv/linux/i386/lowlevellock.h
+++ b/nptl/sysdeps/unix/sysv/linux/i386/lowlevellock.h
@@ -115,8 +115,7 @@ extern int __lll_mutex_unlock_wake (int *__futex)
 #define lll_mutex_lock(futex) \
   (void) ({ int ignore1, ignore2;					      \
 	    __asm __volatile (LOCK_INSTR "cmpxchgl %1, %2\n\t"		      \
-			      "testl %0, %0\n\t"			      \
-			      "jne _L_mutex_lock_%=\n\t"		      \
+			      "jnz _L_mutex_lock_%=\n\t"		      \
 			      ".subsection 1\n\t"			      \
 			      ".type _L_mutex_lock_%=,@function\n"	      \
 			      "_L_mutex_lock_%=:\n\t"			      \
@@ -136,8 +135,7 @@ extern int __lll_mutex_unlock_wake (int *__futex)
 #define lll_mutex_cond_lock(futex) \
   (void) ({ int ignore1, ignore2;					      \
 	    __asm __volatile (LOCK_INSTR "cmpxchgl %1, %2\n\t"		      \
-			      "testl %0, %0\n\t"			      \
-			      "jne _L_mutex_cond_lock_%=\n\t"		      \
+			      "jnz _L_mutex_cond_lock_%=\n\t"		      \
 			      ".subsection 1\n\t"			      \
 			      ".type _L_mutex_cond_lock_%=,@function\n"	      \
 			      "_L_mutex_cond_lock_%=:\n\t"		      \
@@ -155,8 +153,7 @@ extern int __lll_mutex_unlock_wake (int *__futex)
 #define lll_mutex_timedlock(futex, timeout) \
   ({ int result, ignore1, ignore2;					      \
      __asm __volatile (LOCK_INSTR "cmpxchgl %1, %3\n\t"			      \
-		       "testl %0, %0\n\t"				      \
-		       "jne _L_mutex_timedlock_%=\n\t"			      \
+		       "jnz _L_mutex_timedlock_%=\n\t"			      \
 		       ".subsection 1\n\t"				      \
 		       ".type _L_mutex_timedlock_%=,@function\n"	      \
 		       "_L_mutex_timedlock_%=:\n\t"			      \
@@ -251,8 +248,7 @@ extern int lll_unlock_wake_cb (int *__futex) attribute_hidden;
 			      "je,pt 0f\n\t"				      \
 			      "lock\n"					      \
 			      "0:\tcmpxchgl %1, %2\n\t"			      \
-			      "testl %0, %0\n\t"			      \
-			      "jne _L_mutex_lock_%=\n\t"		      \
+			      "jnz _L_mutex_lock_%=\n\t"		      \
 			      ".subsection 1\n\t"			      \
 			      ".type _L_mutex_lock_%=,@function\n"	      \
 			      "_L_mutex_lock_%=:\n\t"			      \
diff --git a/nptl/sysdeps/unix/sysv/linux/x86_64/lowlevellock.S b/nptl/sysdeps/unix/sysv/linux/x86_64/lowlevellock.S
index aff449b13a..a56c714aeb 100644
--- a/nptl/sysdeps/unix/sysv/linux/x86_64/lowlevellock.S
+++ b/nptl/sysdeps/unix/sysv/linux/x86_64/lowlevellock.S
@@ -38,34 +38,129 @@
 #define VSYSCALL_ADDR_vgettimeofday	0xffffffffff600000
 
 
-	/* Modified: %rax, %rsi.  */
-	.globl	__lll_lock_wait
-	.type	__lll_lock_wait,@function
-	.hidden	__lll_lock_wait
+	.globl	__lll_mutex_lock_wait
+	.type	__lll_mutex_lock_wait,@function
+	.hidden	__lll_mutex_lock_wait
 	.align	16
-__lll_lock_wait:
+__lll_mutex_lock_wait:
 	pushq	%r10
 	pushq	%rdx
 
 	xorq	%r10, %r10	/* No timeout.  */
+	movl	$2, %edx
+	movq	%r10, %rsi	/* movq $FUTEX_WAIT, %rsi */
 
 1:
-	leaq	-1(%rsi), %rdx	/* account for the preceeded xadd.  */
-	movq	%r10, %rsi	/* m