Update.

	* sysdeps/unix/sysv/linux/smp.h: New file.
	* sysdeps/unix/sysv/linux/sh/smp.h: New file.
	* init.c: Define __is_smp.
	(__pthread_initialize_minimal_internal): Call is_smp_system to
	initialize __is_smp.
	* pthreadP.h: Declare __is_smp.
	Define MAX_ADAPTIVE_COUNT is necessary.
	* pthread_mutex_init.c: Add comment regarding __spins field.
	* pthread_mutex_lock.c: Implement adaptive mutex type.
	* pthread_mutex_timedlock.c: Likewise.
	* sysdeps/unix/sysv/linux/pthread_mutex_cond_lock.c: Likewise.
	* sysdeps/unix/sysv/linux/alpha/bits/pthreadtypes.h (pthread_mutex_t):
	Add __spins field.
	* sysdeps/unix/sysv/linux/i386/bits/pthreadtypes.h: Likewise.
	* sysdeps/unix/sysv/linux/ia64/bits/pthreadtypes.h: Likewise.
	* sysdeps/unix/sysv/linux/powerpc/bits/pthreadtypes.h: Likewise.
	* sysdeps/unix/sysv/linux/sh/bits/pthreadtypes.h: Likewise.
	* sysdeps/unix/sysv/linux/s390/bits/pthreadtypes.h: Likewise.
	* sysdeps/unix/sysv/linux/sparc/bits/pthreadtypes.h: Likewise.
	* sysdeps/unix/sysv/linux/x86_64/bits/pthreadtypes.h: Likewise.
	* sysdeps/unix/sysv/linux/alpha/bits/pthreadtypes.h: Define
	lll_mutex_cond_trylock.
	* sysdeps/unix/sysv/linux/powerpc/bits/pthreadtypes.h: Likewise.
	* sysdeps/unix/sysv/linux/sh/bits/pthreadtypes.h: Likewise.
	* sysdeps/unix/sysv/linux/s390/bits/pthreadtypes.h: Likewise.
	* sysdeps/unix/sysv/linux/sparc/bits/pthreadtypes.h: Likewise.
	* sysdeps/unix/sysv/linux/i386/bits/pthreadtypes.h: Likewise.
	Define BUSY_WAIT_NOP.
	* sysdeps/unix/sysv/linux/ia64/bits/pthreadtypes.h: Likewise.
	* sysdeps/unix/sysv/linux/x86_64/bits/pthreadtypes.h: Likewise.

	* tst-mutex5.c: Add support for testing adaptive mutexes.
	* tst-mutex7.c: Likewise.
	* tst-mutex5a.c: New file.
	* tst-mutex7a.c: New file.
	* Makefile (tests): Add tst-mutex5a and tst-mutex7a.

31 files changed, 364 insertions, 28 deletions

diff --git a/nptl/Banner b/nptl/Banner
index 5f10f81731..53444807bb 100644
--- a/nptl/Banner
+++ b/nptl/Banner
@@ -1 +1 @@
-NPTL 0.60 by Ulrich Drepper
+NPTL 0.61 by Ulrich Drepper
diff --git a/nptl/ChangeLog b/nptl/ChangeLog
index a10c6c12ec..05906737e7 100644
--- a/nptl/ChangeLog
+++ b/nptl/ChangeLog
@@ -1,5 +1,42 @@
 2004-03-23  Ulrich Drepper  <drepper@redhat.com>
 
+	* sysdeps/unix/sysv/linux/smp.h: New file.
+	* sysdeps/unix/sysv/linux/sh/smp.h: New file.
+	* init.c: Define __is_smp.
+	(__pthread_initialize_minimal_internal): Call is_smp_system to
+	initialize __is_smp.
+	* pthreadP.h: Declare __is_smp.
+	Define MAX_ADAPTIVE_COUNT is necessary.
+	* pthread_mutex_init.c: Add comment regarding __spins field.
+	* pthread_mutex_lock.c: Implement adaptive mutex type.
+	* pthread_mutex_timedlock.c: Likewise.
+	* sysdeps/unix/sysv/linux/pthread_mutex_cond_lock.c: Likewise.
+	* sysdeps/unix/sysv/linux/alpha/bits/pthreadtypes.h (pthread_mutex_t):
+	Add __spins field.
+	* sysdeps/unix/sysv/linux/i386/bits/pthreadtypes.h: Likewise.
+	* sysdeps/unix/sysv/linux/ia64/bits/pthreadtypes.h: Likewise.
+	* sysdeps/unix/sysv/linux/powerpc/bits/pthreadtypes.h: Likewise.
+	* sysdeps/unix/sysv/linux/sh/bits/pthreadtypes.h: Likewise.
+	* sysdeps/unix/sysv/linux/s390/bits/pthreadtypes.h: Likewise.
+	* sysdeps/unix/sysv/linux/sparc/bits/pthreadtypes.h: Likewise.
+	* sysdeps/unix/sysv/linux/x86_64/bits/pthreadtypes.h: Likewise.
+	* sysdeps/unix/sysv/linux/alpha/bits/pthreadtypes.h: Define
+	lll_mutex_cond_trylock.
+	* sysdeps/unix/sysv/linux/powerpc/bits/pthreadtypes.h: Likewise.
+	* sysdeps/unix/sysv/linux/sh/bits/pthreadtypes.h: Likewise.
+	* sysdeps/unix/sysv/linux/s390/bits/pthreadtypes.h: Likewise.
+	* sysdeps/unix/sysv/linux/sparc/bits/pthreadtypes.h: Likewise.
+	* sysdeps/unix/sysv/linux/i386/bits/pthreadtypes.h: Likewise.
+	Define BUSY_WAIT_NOP.
+	* sysdeps/unix/sysv/linux/ia64/bits/pthreadtypes.h: Likewise.
+	* sysdeps/unix/sysv/linux/x86_64/bits/pthreadtypes.h: Likewise.
+
+	* tst-mutex5.c: Add support for testing adaptive mutexes.
+	* tst-mutex7.c: Likewise.
+	* tst-mutex5a.c: New file.
+	* tst-mutex7a.c: New file.
+	* Makefile (tests): Add tst-mutex5a and tst-mutex7a.
+
 	* sysdeps/unix/sysv/linux/x86_64/lowlevellock.S
 	(__lll_mutex_timedlock_wait): Preserve r8 and r9 since the
 	vgettimeofday call miht destroy the content.
diff --git a/nptl/Makefile b/nptl/Makefile
index 114d4a3ad8..c3bc89cb43 100644
--- a/nptl/Makefile
+++ b/nptl/Makefile
@@ -190,7 +190,7 @@ omit-deps = $(unix-syscalls:%=ptw-%)
 
 tests = tst-attr1 tst-attr2 tst-attr3 \
 	tst-mutex1 tst-mutex2 tst-mutex3 tst-mutex4 tst-mutex5 tst-mutex6 \
-	tst-mutex7 tst-mutex8 tst-mutex9 \
+	tst-mutex7 tst-mutex8 tst-mutex9 tst-mutex5a tst-mutex7a \
 	tst-spin1 tst-spin2 tst-spin3 \
 	tst-cond1 tst-cond2 tst-cond3 tst-cond4 tst-cond5 tst-cond6 tst-cond7 \
 	tst-cond8 tst-cond9 tst-cond10 tst-cond11 tst-cond12 tst-cond13 \
diff --git a/nptl/init.c b/nptl/init.c
index f46e0bb531..47bb0a6f72 100644
--- a/nptl/init.c
+++ b/nptl/init.c
@@ -31,6 +31,7 @@
 #include <fork.h>
 #include <version.h>
 #include <shlib-compat.h>
+#include <smp.h>
 
 
 #ifndef __NR_set_tid_address
@@ -61,6 +62,9 @@ size_t __default_stacksize attribute_hidden;
 size_t __static_tls_size;
 size_t __static_tls_align_m1;
 
+/* Flag whether the machine is SMP or not.  */
+int __is_smp attribute_hidden;
+
 /* Version of the library, used in libthread_db to detect mismatches.  */
 static const char nptl_version[] __attribute_used__ = VERSION;
 
@@ -301,6 +305,9 @@ __pthread_initialize_minimal_internal (void)
 #endif
     __libc_pthread_init (&__fork_generation, __reclaim_stacks,
 			 ptr_pthread_functions);
+
+  /* Determine whether the machine is SMP or not.  */
+  __is_smp = is_smp_system ();
 }
 strong_alias (__pthread_initialize_minimal_internal,
 	      __pthread_initialize_minimal)
diff --git a/nptl/pthreadP.h b/nptl/pthreadP.h
index 560457908a..ba9529f01b 100644
--- a/nptl/pthreadP.h
+++ b/nptl/pthreadP.h
@@ -45,6 +45,12 @@
 #endif
 
 
+/* Adaptive mutex definitions.  */
+#ifndef MAX_ADAPTIVE_COUNT
+# define MAX_ADAPTIVE_COUNT 100
+#endif
+
+
 /* Internal variables.  */
 
 
@@ -55,6 +61,9 @@ extern size_t __default_stacksize attribute_hidden;
 extern size_t __static_tls_size attribute_hidden;
 extern size_t __static_tls_align_m1 attribute_hidden;
 
+/* Flag whether the machine is SMP or not.  */
+extern int __is_smp attribute_hidden;
+
 /* Thread descriptor handling.  */
 extern list_t __stack_user;
 hidden_proto (__stack_user)
diff --git a/nptl/pthread_mutex_init.c b/nptl/pthread_mutex_init.c
index a95f97ad4f..074941daf3 100644
--- a/nptl/pthread_mutex_init.c
+++ b/nptl/pthread_mutex_init.c
@@ -1,4 +1,4 @@
-/* Copyright (C) 2002, 2003 Free Software Foundation, Inc.
+/* Copyright (C) 2002, 2003, 2004 Free Software Foundation, Inc.
    This file is part of the GNU C Library.
    Contributed by Ulrich Drepper <drepper@redhat.com>, 2002.
 
@@ -50,6 +50,7 @@ __pthread_mutex_init (mutex, mutexattr)
   // mutex->__count = 0;	already done by memset
   // mutex->__owner = 0;	already done by memset
   // mutex->__nusers = 0;	already done by memset
+  // mutex->__spins = 0;	already done by memset
 
   return 0;
 }
diff --git a/nptl/pthread_mutex_lock.c b/nptl/pthread_mutex_lock.c
index ac770c5c00..ee39f20820 100644
--- a/nptl/pthread_mutex_lock.c
+++ b/nptl/pthread_mutex_lock.c
@@ -1,4 +1,4 @@
-/* Copyright (C) 2002, 2003 Free Software Foundation, Inc.
+/* Copyright (C) 2002, 2003, 2004 Free Software Foundation, Inc.
    This file is part of the GNU C Library.
    Contributed by Ulrich Drepper <drepper@redhat.com>, 2002.
 
@@ -25,6 +25,7 @@
 
 #ifndef LLL_MUTEX_LOCK
 # define LLL_MUTEX_LOCK(mutex) lll_mutex_lock (mutex)
+# define LLL_MUTEX_TRYLOCK(mutex) lll_mutex_trylock (mutex)
 #endif
 
 
@@ -32,6 +33,8 @@ int
 __pthread_mutex_lock (mutex)
      pthread_mutex_t *mutex;
 {
+  assert (sizeof (mutex->__size) >= sizeof (mutex->__data));
+
   pid_t id = THREAD_GETMEM (THREAD_SELF, tid);
 
   switch (__builtin_expect (mutex->__data.__kind, PTHREAD_MUTEX_TIMED_NP))
@@ -68,10 +71,37 @@ __pthread_mutex_lock (mutex)
     default:
       /* Correct code cannot set any other type.  */
     case PTHREAD_MUTEX_TIMED_NP:
-    case PTHREAD_MUTEX_ADAPTIVE_NP:
+    simple:
       /* Normal mutex.  */
       LLL_MUTEX_LOCK (mutex->__data.__lock);
       break;
+
+    case PTHREAD_MUTEX_ADAPTIVE_NP:
+      if (! __is_smp)
+	goto simple;
+
+      if (LLL_MUTEX_TRYLOCK (mutex->__data.__lock) != 0)
+	{
+	  int cnt = 0;
+	  int max_cnt = MIN (MAX_ADAPTIVE_COUNT,
+			     mutex->__data.__spins * 2 + 10);
+	  do
+	    {
+	      if (cnt++ >= max_cnt)
+		{
+		  LLL_MUTEX_LOCK (mutex->__data.__lock);
+		  break;
+		}
+
+#ifdef BUSY_WAIT_NOP
+	      BUSY_WAIT_NOP;
+#endif
+	    }
+	  while (LLL_MUTEX_TRYLOCK (mutex->__data.__lock) != 0);
+
+	  mutex->__data.__spins += (cnt - mutex->__data.__spins) / 8;
+	}
+      break;
     }
 
   /* Record the ownership.  */
diff --git a/nptl/pthread_mutex_timedlock.c b/nptl/pthread_mutex_timedlock.c
index 8efc422042..1cd2c7e606 100644
--- a/nptl/pthread_mutex_timedlock.c
+++ b/nptl/pthread_mutex_timedlock.c
@@ -1,4 +1,4 @@
-/* Copyright (C) 2002, 2003 Free Software Foundation, Inc.
+/* Copyright (C) 2002, 2003, 2004 Free Software Foundation, Inc.
    This file is part of the GNU C Library.
    Contributed by Ulrich Drepper <drepper@redhat.com>, 2002.
 
@@ -73,10 +73,37 @@ pthread_mutex_timedlock (mutex, abstime)
     default:
       /* Correct code cannot set any other type.  */
     case PTHREAD_MUTEX_TIMED_NP:
-    case PTHREAD_MUTEX_ADAPTIVE_NP:
+    simple:
       /* Normal mutex.  */
       result = lll_mutex_timedlock (mutex->__data.__lock, abstime);
       break;
+
+    case PTHREAD_MUTEX_ADAPTIVE_NP:
+      if (! __is_smp)
+	goto simple;
+
+      if (lll_mutex_trylock (mutex->__data.__lock) != 0)
+	{
+	  int cnt = 0;
+	  int max_cnt = MIN (MAX_ADAPTIVE_COUNT,
+			     mutex->__data.__spins * 2 + 10);
+	  do
+	    {
+	      if (cnt++ >= max_cnt)
+		{
+		  result = lll_mutex_timedlock (mutex->__data.__lock, abstime);
+		  break;
+		}
+
+#ifdef BUSY_WAIT_NOP
+	      BUSY_WAIT_NOP;
+#endif
+	    }
+	  while (lll_mutex_trylock (mutex->__data.__lock) != 0);
+
+	  mutex->__data.__spins += (cnt - mutex->__data.__spins) / 8;
+	}
+      break;
     }
 
   if (result == 0)
diff --git a/nptl/sysdeps/unix/sysv/linux/alpha/bits/pthreadtypes.h b/nptl/sysdeps/unix/sysv/linux/alpha/bits/pthreadtypes.h
index 2d8a00b93f..0fed5cc918 100644
--- a/nptl/sysdeps/unix/sysv/linux/alpha/bits/pthreadtypes.h
+++ b/nptl/sysdeps/unix/sysv/linux/alpha/bits/pthreadtypes.h
@@ -1,5 +1,5 @@
 /* Machine-specific pthread type layouts.  Alpha version.
-   Copyright (C) 2003 Free Software Foundation, Inc.
+   Copyright (C) 2003, 2004 Free Software Foundation, Inc.
    This file is part of the GNU C Library.
 
    The GNU C Library is free software; you can redistribute it and/or
@@ -56,6 +56,7 @@ typedef union
     /* KIND must stay at this position in the structure to maintain
        binary compatibility.  */
     int __kind;
+    int __spins;
   } __data;
   char __size[__SIZEOF_PTHREAD_MUTEX_T];
   long int __align;
diff --git a/nptl/sysdeps/unix/sysv/linux/alpha/lowlevellock.h b/nptl/sysdeps/unix/sysv/linux/alpha/lowlevellock.h
index 9d125e0d6d..a7079a8490 100644
--- a/nptl/sysdeps/unix/sysv/linux/alpha/lowlevellock.h
+++ b/nptl/sysdeps/unix/sysv/linux/alpha/lowlevellock.h
@@ -1,4 +1,4 @@
-/* Copyright (C) 2003 Free Software Foundation, Inc.
+/* Copyright (C) 2003, 2004 Free Software Foundation, Inc.
    This file is part of the GNU C Library.
 
    The GNU C Library is free software; you can redistribute it and/or
@@ -80,6 +80,14 @@ __lll_mutex_trylock(int *futex)
 #define lll_mutex_trylock(lock)	__lll_mutex_trylock (&(lock))
 
 
+static inline int __attribute__((always_inline))
+__lll_mutex_cond_trylock(int *futex)
+{
+  return atomic_compare_and_exchange_val_acq (futex, 2, 0) != 0;
+}
+#define lll_mutex_cond_trylock(lock)	__lll_mutex_cond_trylock (&(lock))
+
+
 extern void __lll_lock_wait (int *futex) attribute_hidden;
 
 static inline void __attribute__((always_inline))
diff --git a/nptl/sysdeps/unix/sysv/linux/i386/bits/pthreadtypes.h b/nptl/sysdeps/unix/sysv/linux/i386/bits/pthreadtypes.h
index fc0a54c036..f998138d7e 100644
--- a/nptl/sysdeps/unix/sysv/linux/i386/bits/pthreadtypes.h
+++ b/nptl/sysdeps/unix/sysv/linux/i386/bits/pthreadtypes.h
@@ -1,4 +1,4 @@
-/* Copyright (C) 2002, 2003 Free Software Foundation, Inc.
+/* Copyright (C) 2002, 2003, 2004 Free Software Foundation, Inc.
    This file is part of the GNU C Library.
 
    The GNU C Library is free software; you can redistribute it and/or
@@ -56,6 +56,7 @@ typedef union
        binary compatibility.  */
     int __kind;
     unsigned int __nusers;
+    int __spins;
   } __data;
   char __size[__SIZEOF_PTHREAD_MUTEX_T];
   long int __align;
diff --git a/nptl/sysdeps/unix/sysv/linux/i386/lowlevellock.h b/nptl/sysdeps/unix/sysv/linux/i386/lowlevellock.h
index a9823d8d4b..b86f11c9b4 100644
--- a/nptl/sysdeps/unix/sysv/linux/i386/lowlevellock.h
+++ b/nptl/sysdeps/unix/sysv/linux/i386/lowlevellock.h
@@ -40,6 +40,7 @@
 /* Initializer for compatibility lock.  */
 #define LLL_MUTEX_LOCK_INITIALIZER		(0)
 #define LLL_MUTEX_LOCK_INITIALIZER_LOCKED	(1)
+#define LLL_MUTEX_LOCK_INITIALIZER_WAITERS	(2)
 
 
 #ifdef PIC
@@ -60,6 +61,9 @@
 # define LLL_ENTER_KERNEL	"int $0x80\n\t"
 #endif
 
+/* Delay in spinlock loop.  */
+#define BUSY_WAIT_NOP          asm ("rep; nop")
+
 
 #define lll_futex_wait(futex, val) \
   do {									      \
@@ -117,6 +121,16 @@ extern int __lll_mutex_unlock_wake (int *__futex)
      ret; })
 
 
+#define lll_mutex_cond_trylock(futex) \
+  ({ int ret;								      \
+     __asm __volatile (LOCK_INSTR "cmpxchgl %2, %1"			      \
+		       : "=a" (ret), "=m" (futex)			      \
+		       : "r" (LLL_MUTEX_LOCK_INITIALIZER_WAITERS),	      \
+			  "m" (futex), "0" (LLL_MUTEX_LOCK_INITIALIZER)	      \
+		       : "memory");					      \
+     ret; })
+
+
 #define lll_mutex_lock(futex) \
   (void) ({ int ignore1, ignore2;					      \
 	    __asm __volatile (LOCK_INSTR "cmpxchgl %1, %2\n\t"		      \
diff --git a/nptl/sysdeps/unix/sysv/linux/ia64/bits/pthreadtypes.h b/nptl/sysdeps/unix/sysv/linux/ia64/bits/pthreadtypes.h
index 6cd6c05705..e4c15d1e20 100644
--- a/nptl/sysdeps/unix/sysv/linux/ia64/bits/pthreadtypes.h
+++ b/nptl/sysdeps/unix/sysv/linux/ia64/bits/pthreadtypes.h
@@ -1,4 +1,4 @@
-/* Copyright (C) 2003 Free Software Foundation, Inc.
+/* Copyright (C) 2003, 2004 Free Software Foundation, Inc.
    This file is part of the GNU C Library.
    Contributed by Jakub Jelinek <jakub@redhat.com>, 2003.
 
@@ -56,6 +56,7 @@ typedef union
     /* KIND must stay at this position in the structure to maintain
        binary compatibility.  */
     int __kind;
+    int __spins;
   } __data;
   char __size[__SIZEOF_PTHREAD_MUTEX_T];
   long int __align;
diff --git a/nptl/sysdeps/unix/sysv/linux/ia64/lowlevellock.h b/nptl/sysdeps/unix/sysv/linux/ia64/lowlevellock.h
index e462776c02..6ed21fdc14 100644
--- a/nptl/sysdeps/unix/sysv/linux/ia64/lowlevellock.h
+++ b/nptl/sysdeps/unix/sysv/linux/ia64/lowlevellock.h
@@ -1,4 +1,4 @@
-/* Copyright (C) 2003 Free Software Foundation, Inc.
+/* Copyright (C) 2003, 2004 Free Software Foundation, Inc.
    This file is part of the GNU C Library.
    Contributed by Jakub Jelinek <jakub@redhat.com>, 2003.
 
@@ -31,6 +31,9 @@
 #define FUTEX_WAKE		1
 #define FUTEX_REQUEUE		3
 
+/* Delay in spinlock loop.  */
+#define BUSY_WAIT_NOP          asm ("hint @pause")
+