Update.

2004-06-03  Ulrich Drepper  <drepper@redhat.com>

	* sysdeps/i386/i486/bits/atomic.h: Optimize a bit.

30 files changed, 327 insertions, 209 deletions

diff --git a/ChangeLog b/ChangeLog
index 4f38604c52..1fd3646e3f 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,7 @@
+2004-06-03  Ulrich Drepper  <drepper@redhat.com>
+
+	* sysdeps/i386/i486/bits/atomic.h: Optimize a bit.
+
 2004-05-07  Dmitry V. Levin  <ldv@altlinux.org>
 
 	* argp/argp-help.c (__argp_error, __argp_failure): Check result
diff --git a/nptl/ChangeLog b/nptl/ChangeLog
index 071e7048cc..1ff058e21b 100644
--- a/nptl/ChangeLog
+++ b/nptl/ChangeLog
@@ -1,3 +1,95 @@
+2004-05-21  Jakub Jelinek  <jakub@redhat.com>
+
+	* Makefile (tests): Add tst-cond16.
+	* sysdeps/unix/sysv/linux/lowlevelcond.sym (cond_futex): Add.
+	* pthread_cond_init.c (__pthread_cond_init): Clear __data.__futex.
+	* sysdeps/unix/sysv/linux/i386/bits/pthreadtypes.h (pthread_cond_t):
+	Add __data.__futex field, reshuffle __data.__clock.
+	* sysdeps/unix/sysv/linux/i386/pthread_cond_signal.S
+	(__pthread_cond_signal): Increment __futex at the same time as
+	__wakeup_seq or __total_seq.  Pass address of __futex instead of
+	address of low 32-bits of __wakeup_seq to futex syscall.
+	* sysdeps/unix/sysv/linux/i386/pthread_cond_wait.S
+	(__pthread_cond_wait): Likewise.  Pass __futex value from before
+	releasing internal lock to FUTEX_WAIT.
+	* sysdeps/unix/sysv/linux/i386/pthread_cond_timedwait.S
+	(__pthread_cond_timedwait): Likewise.
+	* sysdeps/unix/sysv/linux/i386/pthread_cond_broadcast.S
+	(FUTEX_CMP_REQUEUE): Define.
+	(__pthread_cond_broadcast): Set __futex to 2 * __total_seq.
+	Use FUTEX_CMP_REQUEUE operation instead of FUTEX_REQUEUE.
+	Pass __futex value from before the unlock and __futex address instead
+	of address of low 32-bits of __wakeup_seq to futex syscall.
+	Fallback to FUTEX_WAKE all on any errors.
+	* sysdeps/unix/sysv/linux/alpha/lowlevellock.h (FUTEX_CMP_REQUEUE):
+	Define.
+	(lll_futex_requeue): Add val argument, use FUTEX_CMP_REQUEUE
+	internally.  Return non-zero if error, zero if success.
+	* sysdeps/unix/sysv/linux/alpha/bits/pthreadtypes.h (pthread_cond_t):
+	Add __data.__futex field, reshuffle __data.__clock.
+	* sysdeps/unix/sysv/linux/s390/lowlevellock.h (FUTEX_CMP_REQUEUE):
+	Define.
+	(lll_futex_requeue): Add val argument, return 1 unconditionally
+	for the time being.
+	* sysdeps/unix/sysv/linux/s390/bits/pthreadtypes.h (pthread_cond_t):
+	Add __data.__futex field, reshuffle __data.__clock.
+	* sysdeps/unix/sysv/linux/powerpc/lowlevellock.h (FUTEX_CMP_REQUEUE):
+	Define.
+	(lll_futex_requeue): Add val argument, use FUTEX_CMP_REQUEUE
+	internally.  Return non-zero if error, zero if success.
+	* sysdeps/unix/sysv/linux/powerpc/bits/pthreadtypes.h
+	(pthread_cond_t): Add __data.__futex field, reshuffle __data.__clock.
+	* sysdeps/unix/sysv/linux/sparc/lowlevellock.h (FUTEX_CMP_REQUEUE):
+	Define.
+	(lll_futex_requeue): Add val argument, use FUTEX_CMP_REQUEUE
+	internally.  Return non-zero if error, zero if success.
+	* sysdeps/unix/sysv/linux/sparc/bits/pthreadtypes.h (pthread_cond_t):
+	Add __data.__futex field, reshuffle __data.__clock.
+	* sysdeps/unix/sysv/linux/ia64/lowlevellock.h (FUTEX_CMP_REQUEUE):
+	Define.
+	(lll_futex_requeue): Add val argument, use FUTEX_CMP_REQUEUE
+	internally.  Return non-zero if error, zero if success.
+	* sysdeps/unix/sysv/linux/ia64/bits/pthreadtypes.h (pthread_cond_t):
+	Add __data.__futex field, reshuffle __data.__clock.
+	* sysdeps/unix/sysv/linux/x86_64/bits/pthreadtypes.h (pthread_cond_t):
+	Add __data.__futex field, reshuffle __data.__clock.
+	* sysdeps/pthread/pthread_cond_signal.c (__pthread_cond_signal):
+	Increment __futex at the same time as __wakeup_seq or __total_seq.
+	Pass address of __futex instead of address of low 32-bits of
+	__wakeup_seq to futex syscall.
+	* sysdeps/pthread/pthread_cond_wait.c (__pthread_cond_wait): Likewise.
+	Pass __futex value from before releasing internal lock
+	to FUTEX_WAIT.
+	* sysdeps/pthread/pthread_cond_timedwait.c (__pthread_cond_timedwait):
+	Likewise.  Avoid unnecessary shadowing of variables.
+	* sysdeps/pthread/pthread_cond_broadcast.c (__pthread_cond_broadcast):
+	Set __futex to 2 * __total_seq.  Pass __futex value from before the
+	unlock and __futex address instead of address of low 32-bits of
+	__wakeup_seq to futex_requeue macro, adjust for new return value
+	meaning.
+	* sysdeps/unix/sysv/linux/x86_64/pthread_cond_signal.S
+	(__pthread_cond_signal): Increment __futex at the same time as
+	__wakeup_seq or __total_seq.  Pass address of __futex instead of
+	address of low 32-bits of __wakeup_seq to futex syscall.
+	* sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S
+	(__pthread_cond_wait): Likewise.  Pass __futex value from before
+	releasing internal lock to FUTEX_WAIT.
+	* sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S
+	(__pthread_cond_timedwait): Likewise.
+	* sysdeps/unix/sysv/linux/x86_64/pthread_cond_broadcast.S
+	(FUTEX_CMP_REQUEUE): Define.
+	(__pthread_cond_broadcast): Set __futex to 2 * __total_seq.
+	Use FUTEX_CMP_REQUEUE operation instead of FUTEX_REQUEUE.
+	Pass __futex value from before the unlock and __futex address instead
+	of address of low 32-bits of __wakeup_seq to futex syscall.
+	Fallback to FUTEX_WAKE all on any errors.
+
+2004-06-03  Kaz Kojima  <kkojima@rr.iij4u.or.jp>
+
+	* sysdeps/unix/sysv/linux/sh/lowlevellock.h (lll_mutex_lock):
+	Add nop to align the end of critical section.
+	(lll_mutex_cond_lock, lll_mutex_timedlock): Likewise.
+
 2004-06-01  Kaz Kojima  <kkojima@rr.iij4u.or.jp>
 
 	* sysdeps/unix/sysv/linux/sh/bits/pthreadtypes.h (pthread_cond_t):
diff --git a/nptl/Makefile b/nptl/Makefile
index 687a6a0cfc..fd3ed9f699 100644
--- a/nptl/Makefile
+++ b/nptl/Makefile
@@ -194,7 +194,7 @@ tests = tst-attr1 tst-attr2 tst-attr3 \
 	tst-spin1 tst-spin2 tst-spin3 \
 	tst-cond1 tst-cond2 tst-cond3 tst-cond4 tst-cond5 tst-cond6 tst-cond7 \
 	tst-cond8 tst-cond9 tst-cond10 tst-cond11 tst-cond12 tst-cond13 \
-	tst-cond14 tst-cond15 tst-cond17 tst-cond18 \
+	tst-cond14 tst-cond15 tst-cond16 tst-cond17 tst-cond18 \
 	tst-rwlock1 tst-rwlock2 tst-rwlock3 tst-rwlock4 tst-rwlock5 \
 	tst-rwlock6 tst-rwlock7 tst-rwlock8 tst-rwlock9 tst-rwlock10 \
 	tst-rwlock11 tst-rwlock12 tst-rwlock13 \
diff --git a/nptl/pthread_cond_init.c b/nptl/pthread_cond_init.c
index d059da2315..f5fbd64e96 100644
--- a/nptl/pthread_cond_init.c
+++ b/nptl/pthread_cond_init.c
@@ -31,6 +31,7 @@ __pthread_cond_init (cond, cond_attr)
   struct pthread_condattr *icond_attr = (struct pthread_condattr *) cond_attr;
 
   cond->__data.__lock = LLL_MUTEX_LOCK_INITIALIZER;
+  cond->__data.__futex = 0;
   cond->__data.__clock = (icond_attr == NULL
 			  ? CLOCK_REALTIME : (icond_attr->value & 0xfe) >> 1);
   cond->__data.__total_seq = 0;
diff --git a/nptl/sysdeps/pthread/pthread_cond_broadcast.c b/nptl/sysdeps/pthread/pthread_cond_broadcast.c
index a42c579658..1eac8ecf83 100644
--- a/nptl/sysdeps/pthread/pthread_cond_broadcast.c
+++ b/nptl/sysdeps/pthread/pthread_cond_broadcast.c
@@ -41,35 +41,29 @@ __pthread_cond_broadcast (cond)
       /* Yes.  Mark them all as woken.  */
       cond->__data.__wakeup_seq = cond->__data.__total_seq;
       cond->__data.__woken_seq = cond->__data.__total_seq;
+      cond->__data.__futex = (unsigned int) cond->__data.__total_seq * 2;
+      int futex_val = cond->__data.__futex;
       /* Signal that a broadcast happened.  */
       ++cond->__data.__broadcast_seq;
 
       /* We are done.  */
       lll_mutex_unlock (cond->__data.__lock);
 
-      /* The futex syscall operates on a 32-bit word.  That is fine,
-	 we just use the low 32 bits of the sequence counter.  */
-#if BYTE_ORDER == LITTLE_ENDIAN
-      int *futex = ((int *) (&cond->__data.__wakeup_seq));
-#elif BYTE_ORDER == BIG_ENDIAN
-      int *futex = ((int *) (&cond->__data.__wakeup_seq)) + 1;
-#else
-# error "No valid byte order"
-#endif
-
       /* Do not use requeue for pshared condvars.  */
       if (cond->__data.__mutex == (void *) ~0l)
 	goto wake_all;
 
       /* Wake everybody.  */
       pthread_mutex_t *mut = (pthread_mutex_t *) cond->__data.__mutex;
-      if (__builtin_expect (lll_futex_requeue (futex, 1, INT_MAX,
-					       &mut->__data.__lock) == -EINVAL,
-			    0))
+      /* lll_futex_requeue returns 0 for success and non-zero
+	 for errors.  */
+      if (__builtin_expect (lll_futex_requeue (&cond->__data.__futex, 1,
+					       INT_MAX, &mut->__data.__lock,
+					       futex_val), 0))
 	{
 	  /* The requeue functionality is not available.  */
 	wake_all:
-	  lll_futex_wake (futex, INT_MAX);
+	  lll_futex_wake (&cond->__data.__futex, INT_MAX);
 	}
 
       /* That's all.  */
diff --git a/nptl/sysdeps/pthread/pthread_cond_signal.c b/nptl/sysdeps/pthread/pthread_cond_signal.c
index 76203ac2cc..f5623480f8 100644
--- a/nptl/sysdeps/pthread/pthread_cond_signal.c
+++ b/nptl/sysdeps/pthread/pthread_cond_signal.c
@@ -1,4 +1,4 @@
-/* Copyright (C) 2003 Free Software Foundation, Inc.
+/* Copyright (C) 2003, 2004 Free Software Foundation, Inc.
    This file is part of the GNU C Library.
    Contributed by Martin Schwidefsky <schwidefsky@de.ibm.com>, 2003.
 
@@ -40,19 +40,10 @@ __pthread_cond_signal (cond)
     {
       /* Yes.  Mark one of them as woken.  */
       ++cond->__data.__wakeup_seq;
-
-      /* The futex syscall operates on a 32-bit word.  That is fine,
-	 we just use the low 32 bits of the sequence counter.  */
-#if BYTE_ORDER == LITTLE_ENDIAN
-      int *futex = ((int *) (&cond->__data.__wakeup_seq));
-#elif BYTE_ORDER == BIG_ENDIAN
-      int *futex = ((int *) (&cond->__data.__wakeup_seq)) + 1;
-#else
-# error "No valid byte order"
-#endif
+      ++cond->__data.__futex;
 
       /* Wake one.  */
-      lll_futex_wake (futex, 1);
+      lll_futex_wake (&cond->__data.__futex, 1);
     }
 
   /* We are done.  */
diff --git a/nptl/sysdeps/pthread/pthread_cond_timedwait.c b/nptl/sysdeps/pthread/pthread_cond_timedwait.c
index 96b1029cf5..940b51b4be 100644
--- a/nptl/sysdeps/pthread/pthread_cond_timedwait.c
+++ b/nptl/sysdeps/pthread/pthread_cond_timedwait.c
@@ -66,6 +66,7 @@ __pthread_cond_timedwait (cond, mutex, abstime)
 
   /* We have one new user of the condvar.  */
   ++cond->__data.__total_seq;
+  ++cond->__data.__futex;
 
   /* Remember the mutex we are using here.  If there is already a
      different address store this is a bad user bug.  Do not store
@@ -89,27 +90,17 @@ __pthread_cond_timedwait (cond, mutex, abstime)
   /* Remember the broadcast counter.  */
   cbuffer.bc_seq = cond->__data.__broadcast_seq;
 
-  /* The futex syscall operates on a 32-bit word.  That is fine, we
-     just use the low 32 bits of the sequence counter.  */
-#if BYTE_ORDER == LITTLE_ENDIAN
-  int *futex = ((int *) (&cond->__data.__wakeup_seq));
-#elif BYTE_ORDER == BIG_ENDIAN
-  int *futex = ((int *) (&cond->__data.__wakeup_seq)) + 1;
-#else
-# error "No valid byte order"
-#endif
-
   while (1)
     {
       struct timespec rt;
       {
 #ifdef __NR_clock_gettime
 	INTERNAL_SYSCALL_DECL (err);
-	int val;
-	val = INTERNAL_SYSCALL (clock_gettime, err, 2,
+	int ret;
+	ret = INTERNAL_SYSCALL (clock_gettime, err, 2,
 				cond->__data.__clock, &rt);
 # ifndef __ASSUME_POSIX_TIMERS
-	if (__builtin_expect (INTERNAL_SYSCALL_ERROR_P (val, err), 0))
+	if (__builtin_expect (INTERNAL_SYSCALL_ERROR_P (ret, err), 0))
 	  {
 	    struct timeval tv;
 	    (void) gettimeofday (&tv, NULL);
@@ -149,15 +140,17 @@ __pthread_cond_timedwait (cond, mutex, abstime)
 	  goto timeout;
 	}
 
+      unsigned int futex_val = cond->__data.__futex;
+
       /* Prepare to wait.  Release the condvar futex.  */
       lll_mutex_unlock (cond->__data.__lock);
 
       /* Enable asynchronous cancellation.  Required by the standard.  */
       cbuffer.oldtype = __pthread_enable_asynccancel ();
 
-      /* Wait until woken by signal or broadcast.  Note that we
-	 truncate the 'val' value to 32 bits.  */
-      err = lll_futex_timed_wait (futex, (unsigned int) val, &rt);
+      /* Wait until woken by signal or broadcast.  */
+      err = lll_futex_timed_wait (&cond->__data.__futex,
+				  futex_val, &rt);
 
       /* Disable asynchronous cancellation.  */
       __pthread_disable_asynccancel (cbuffer.oldtype);
@@ -180,6 +173,7 @@ __pthread_cond_timedwait (cond, mutex, abstime)
 	timeout:
 	  /* Yep.  Adjust the counters.  */
 	  ++cond->__data.__wakeup_seq;
+	  ++cond->__data.__futex;
 
 	  /* The error value.  */
 	  result = ETIMEDOUT;
diff --git a/nptl/sysdeps/pthread/pthread_cond_wait.c b/nptl/sysdeps/pthread/pthread_cond_wait.c
index 128f5a2d99..45187b5240 100644
--- a/nptl/sysdeps/pthread/pthread_cond_wait.c
+++ b/nptl/sysdeps/pthread/pthread_cond_wait.c
@@ -52,20 +52,14 @@ __condvar_cleanup (void *arg)
 	 appropriately.  */
       ++cbuffer->cond->__data.__wakeup_seq;
       ++cbuffer->cond->__data.__woken_seq;
+      ++cbuffer->cond->__data.__futex;
     }
 
   /* We are done.  */
   lll_mutex_unlock (cbuffer->cond->__data.__lock);
 
   /* Wake everybody to make sure no condvar signal gets lost.  */
-#if BYTE_ORDER == LITTLE_ENDIAN
-  int *futex = ((int *) (&cbuffer->cond->__data.__wakeup_seq));
-#elif BYTE_ORDER == BIG_ENDIAN
-  int *futex = ((int *) (&cbuffer->cond->__data.__wakeup_seq)) + 1;
-#else
-# error "No valid byte order"
-#endif
-  lll_futex_wake (futex, INT_MAX);
+  lll_futex_wake (&cbuffer->cond->__data.__futex, INT_MAX);
 
   /* Get the mutex before returning unless asynchronous cancellation
      is in effect.  */
@@ -95,6 +89,7 @@ __pthread_cond_wait (cond, mutex)
 
   /* We have one new user of the condvar.  */
   ++cond->__data.__total_seq;
+  ++cond->__data.__futex;
 
   /* Remember the mutex we are using here.  If there is already a
      different address store this is a bad user bug.  Do not store
@@ -118,27 +113,18 @@ __pthread_cond_wait (cond, mutex)
   /* Remember the broadcast counter.  */
   cbuffer.bc_seq = cond->__data.__broadcast_seq;
 
-  /* The futex syscall operates on a 32-bit word.  That is fine, we
-     just use the low 32 bits of the sequence counter.  */
-#if BYTE_ORDER == LITTLE_ENDIAN
-  int *futex = ((int *) (&cond->__data.__wakeup_seq));
-#elif BYTE_ORDER == BIG_ENDIAN
-  int *futex = ((int *) (&cond->__data.__wakeup_seq)) + 1;
-#else
-# error "No valid byte order"
-#endif
-
   do
     {
+      unsigned int futex_val = cond->__data.__futex;
+
       /* Prepare to wait.  Release the condvar futex.  */
       lll_mutex_unlock (cond->__data.__lock);
 
       /* Enable asynchronous cancellation.  Required by the standard.  */
       cbuffer.oldtype = __pthread_enable_asynccancel ();
 
-      /* Wait until woken by signal or broadcast.  Note that we
-	 truncate the 'val' value to 32 bits.  */
-      lll_futex_wait (futex, (unsigned int) val);
+      /* Wait until woken by signal or broadcast.  */
+      lll_futex_wait (&cond->__data.__futex, futex_val);
 
       /* Disable asynchronous cancellation.  */
       __pthread_disable_asynccancel (cbuffer.oldtype);
diff --git a/nptl/sysdeps/unix/sysv/linux/alpha/bits/pthreadtypes.h b/nptl/sysdeps/unix/sysv/linux/alpha/bits/pthreadtypes.h
index 1bb2968120..62c853cee9 100644
--- a/nptl/sysdeps/unix/sysv/linux/alpha/bits/pthreadtypes.h
+++ b/nptl/sysdeps/unix/sysv/linux/alpha/bits/pthreadtypes.h
@@ -76,11 +76,12 @@ typedef union
   struct
   {
     int __lock;
-    int __clock;
+    unsigned int __futex;
     unsigned long long int __total_seq;
     unsigned long long int __wakeup_seq;
     unsigned long long int __woken_seq;
     void *__mutex;
+    int __clock;
     unsigned int __broadcast_seq;
   } __data;
   char __size[__SIZEOF_PTHREAD_COND_T];
diff --git a/nptl/sysdeps/unix/sysv/linux/alpha/lowlevellock.h b/nptl/sysdeps/unix/sysv/linux/alpha/lowlevellock.h
index a7079a8490..fd4a7ca4bb 100644
--- a/nptl/sysdeps/unix/sysv/linux/alpha/lowlevellock.h
+++ b/nptl/sysdeps/unix/sysv/linux/alpha/lowlevellock.h
@@ -30,6 +30,7 @@
 #define FUTEX_WAIT		0
 #define FUTEX_WAKE		1
 #define FUTEX_REQUEUE		3
+#define FUTEX_CMP_REQUEUE	4