* sysdeps/unix/sysv/linux/not-cancel.h (__openat_not_cancel,

	__openat64_not_cancel): Remove prototypes.
	(__openat_nocancel, __openat64_nocancel): New prototypes or defines.
	(openat_not_cancel, openat_not_cancel_3, openat64_not_cancel,
	openat64_not_cancel_3): Use them.

28 files changed, 1020 insertions, 208 deletions

diff --git a/ChangeLog b/ChangeLog
index a0c4c17274..ba21c348e3 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,11 @@
+2006-02-13  Jakub Jelinek  <jakub@redhat.com>
+
+	* sysdeps/unix/sysv/linux/not-cancel.h (__openat_not_cancel,
+	__openat64_not_cancel): Remove prototypes.
+	(__openat_nocancel, __openat64_nocancel): New prototypes or defines.
+	(openat_not_cancel, openat_not_cancel_3, openat64_not_cancel,
+	openat64_not_cancel_3): Use them.
+
 2006-02-12  Ulrich Drepper  <drepper@redhat.com>
 
 	* io/ftw.c: Start using *at functions.
diff --git a/nptl/ChangeLog b/nptl/ChangeLog
index 073b3649b7..c280d8dbc8 100644
--- a/nptl/ChangeLog
+++ b/nptl/ChangeLog
@@ -1,3 +1,30 @@
+2006-02-15  Ulrich Drepper  <drepper@redhat.com>
+
+	* pthreadp.h: Define PTHREAD_MUTEX_INCONSISTENT instead of
+	PTHREAD_MUTEX_OWNERDEAD.
+	(PTHREAD_MUTEX_ROBUST_PRIVATE_NP): Define as 16, not 256.
+	Define FUTEX_WAITERS, FUTEX_OWNER_DIED, FUTEX_TID_MASK.
+	* Makefile (libpthread-routines): Add lowlevelrobustlock.
+	* pthread_create.c (start_thread): Very much simplify robust_list loop.
+	* pthread_mutex_consistent.c: Inconsistent mutex have __owner now set
+	to PTHREAD_MUTEX_INCONSISTENT.
+	* pthread_mutex_destroy.c: Allow destroying of inconsistent mutexes.
+	* pthread_mutex_lock.c: Reimplement robust mutex handling.
+	* pthread_mutex_trylock.c: Likewise.
+	* pthread_mutex_timedlock.c: Likewise.
+	* pthread_mutex_unlock.c: Likewise.
+	* sysdeps/unix/sysv/linux/pthread_mutex_cond_lock.c: Likewise.
+	* sysdeps/unix/sysv/linux/Makefile (gen-as-const-headers): Add
+	lowlevelrobustlock.sym.
+	* sysdeps/unix/sysv/linux/lowlevelrobustlock.sym: New file.
+	* sysdeps/unix/sysv/linux/i386/lowlevellock.h: Add lll_robust_mutex_*
+	definitions.
+	* sysdeps/unix/sysv/linux/x86_64/lowlevellock.h: Likewise.
+	* sysdeps/unix/sysv/linux/i386/i486/lowlevelrobustlock.S: New file.
+	* sysdeps/unix/sysv/linux/i386/i586/lowlevelrobustlock.S: New file.
+	* sysdeps/unix/sysv/linux/i386/i686/lowlevelrobustlock.S: New file.
+	* sysdeps/unix/sysv/linux/x86_64/lowlevelrobustlock.S: New file.
+
 2006-02-12  Ulrich Drepper  <drepper@redhat.com>
 
 	* allocatestack.c (allocate_stack): Initialize robust_list.
diff --git a/nptl/Makefile b/nptl/Makefile
index d7b3aaf1ce..e484c6ab61 100644
--- a/nptl/Makefile
+++ b/nptl/Makefile
@@ -100,7 +100,7 @@ libpthread-routines = init vars events version \
 		      cleanup_defer_compat unwind \
 		      pt-longjmp pt-cleanup\
 		      cancellation \
-		      lowlevellock \
+		      lowlevellock lowlevelrobustlock \
 		      pt-vfork \
 		      ptw-write ptw-read ptw-close ptw-fcntl ptw-accept \
 		      ptw-connect ptw-recv ptw-recvfrom ptw-recvmsg ptw-send \
diff --git a/nptl/descr.h b/nptl/descr.h
index 6984138537..d5491c1355 100644
--- a/nptl/descr.h
+++ b/nptl/descr.h
@@ -166,7 +166,7 @@ struct pthread
   do {									      \
     __pthread_slist_t *runp = THREAD_GETMEM (THREAD_SELF, robust_list.__next);\
     if (runp == &mutex->__data.__list)					      \
-      THREAD_SETMEM (THREAD_SELF, robust_list, runp->__next);		      \
+      THREAD_SETMEM (THREAD_SELF, robust_list.__next, runp->__next);	      \
     else								      \
       {									      \
 	while (runp->__next != &mutex->__data.__list)			      \
diff --git a/nptl/pthreadP.h b/nptl/pthreadP.h
index 61b7176159..77d8f5ad24 100644
--- a/nptl/pthreadP.h
+++ b/nptl/pthreadP.h
@@ -1,4 +1,4 @@
-/* Copyright (C) 2002, 2003, 2004, 2005 Free Software Foundation, Inc.
+/* Copyright (C) 2002, 2003, 2004, 2005, 2006 Free Software Foundation, Inc.
    This file is part of the GNU C Library.
    Contributed by Ulrich Drepper <drepper@redhat.com>, 2002.
 
@@ -52,7 +52,7 @@
 
 
 /* Magic cookie representing robust mutex with dead owner.  */
-#define PTHREAD_MUTEX_OWNERDEAD		INT_MAX
+#define PTHREAD_MUTEX_INCONSISTENT	INT_MAX
 /* Magic cookie representing not recoverable robust mutex.  */
 #define PTHREAD_MUTEX_NOTRECOVERABLE	(INT_MAX - 1)
 
@@ -60,7 +60,7 @@
 /* Internal mutex type value.  */
 enum
 {
-  PTHREAD_MUTEX_ROBUST_PRIVATE_NP = 256,
+  PTHREAD_MUTEX_ROBUST_PRIVATE_NP = 16,
   PTHREAD_MUTEX_ROBUST_PRIVATE_RECURSIVE_NP
   = PTHREAD_MUTEX_ROBUST_PRIVATE_NP | PTHREAD_MUTEX_RECURSIVE_NP,
   PTHREAD_MUTEX_ROBUST_PRIVATE_ERRORCHECK_NP
@@ -77,6 +77,12 @@ enum
   (PTHREAD_MUTEXATTR_FLAG_ROBUST | PTHREAD_MUTEXATTR_FLAG_PSHARED)
 
 
+/* Bits used in robust mutex implementation.  */
+#define FUTEX_WAITERS		0x80000000
+#define FUTEX_OWNER_DIED	0x40000000
+#define FUTEX_TID_MASK		0x1fffffff
+
+
 /* Internal variables.  */
 
 
diff --git a/nptl/pthread_create.c b/nptl/pthread_create.c
index b1253b2243..f3d90ecebf 100644
--- a/nptl/pthread_create.c
+++ b/nptl/pthread_create.c
@@ -324,17 +324,12 @@ start_thread (void *arg)
 	    ((char *) robust - offsetof (struct __pthread_mutex_s, __list));
 	  robust = robust->__next;
 
-	  assert (lll_mutex_islocked (this->__lock));
-	  this->__count = 0;
-	  --this->__nusers;
-	  assert (this->__owner != PTHREAD_MUTEX_NOTRECOVERABLE);
-	  this->__owner = PTHREAD_MUTEX_OWNERDEAD;
 	  this->__list.__next = NULL;
 #ifdef __PTHREAD_MUTEX_HAVE_PREV
 	  this->__list.__prev = NULL;
 #endif
 
-	  lll_mutex_unlock (this->__lock);
+	  lll_robust_mutex_dead (this->__lock);
 	}
       while (robust != &pd->robust_list);
 
diff --git a/nptl/pthread_mutex_consistent.c b/nptl/pthread_mutex_consistent.c
index 2edfe8a5a6..0cfe972da0 100644
--- a/nptl/pthread_mutex_consistent.c
+++ b/nptl/pthread_mutex_consistent.c
@@ -1,4 +1,4 @@
-/* Copyright (C) 2005 Free Software Foundation, Inc.
+/* Copyright (C) 2005, 2006 Free Software Foundation, Inc.
    This file is part of the GNU C Library.
    Contributed by Ulrich Drepper <drepper@redhat.com>, 2005.
 
@@ -27,10 +27,10 @@ pthread_mutex_consistent_np (mutex)
 {
   /* Test whether this is a robust mutex with a dead owner.  */
   if ((mutex->__data.__kind & PTHREAD_MUTEX_ROBUST_PRIVATE_NP) == 0
-      || mutex->__data.__owner != -THREAD_GETMEM (THREAD_SELF, tid))
+      || mutex->__data.__owner != PTHREAD_MUTEX_INCONSISTENT)
     return EINVAL;
 
-  mutex->__data.__owner = -mutex->__data.__owner;
+  mutex->__data.__owner = THREAD_GETMEM (THREAD_SELF, tid);
 
   return 0;
 }
diff --git a/nptl/pthread_mutex_destroy.c b/nptl/pthread_mutex_destroy.c
index 2bf76a9da3..19a647a846 100644
--- a/nptl/pthread_mutex_destroy.c
+++ b/nptl/pthread_mutex_destroy.c
@@ -1,4 +1,4 @@
-/* Copyright (C) 2002, 2003, 2005 Free Software Foundation, Inc.
+/* Copyright (C) 2002, 2003, 2005, 2006 Free Software Foundation, Inc.
    This file is part of the GNU C Library.
    Contributed by Ulrich Drepper <drepper@redhat.com>, 2002.
 
@@ -26,9 +26,17 @@ __pthread_mutex_destroy (mutex)
      pthread_mutex_t *mutex;
 {
   if (mutex->__data.__nusers != 0)
-    return EBUSY;
+    {
+      if ((mutex->__data.__kind & PTHREAD_MUTEX_ROBUST_PRIVATE_NP) != 0
+	  && (mutex->__data.__lock & FUTEX_OWNER_DIED) != 0
+	  && mutex->__data.__nusers == 1)
+	goto dead_robust_mutex;
+
+      return EBUSY;
+    }
 
   /* Set to an invalid value.  */
+ dead_robust_mutex:
   mutex->__data.__kind = -1;
 
   return 0;
diff --git a/nptl/pthread_mutex_lock.c b/nptl/pthread_mutex_lock.c
index 420711a4d4..dd22567c71 100644
--- a/nptl/pthread_mutex_lock.c
+++ b/nptl/pthread_mutex_lock.c
@@ -1,4 +1,4 @@
-/* Copyright (C) 2002, 2003, 2004, 2005 Free Software Foundation, Inc.
+/* Copyright (C) 2002, 2003, 2004, 2005, 2006 Free Software Foundation, Inc.
    This file is part of the GNU C Library.
    Contributed by Ulrich Drepper <drepper@redhat.com>, 2002.
 
@@ -27,6 +27,7 @@
 #ifndef LLL_MUTEX_LOCK
 # define LLL_MUTEX_LOCK(mutex) lll_mutex_lock (mutex)
 # define LLL_MUTEX_TRYLOCK(mutex) lll_mutex_trylock (mutex)
+# define LLL_ROBUST_MUTEX_LOCK(mutex, id) lll_robust_mutex_lock (mutex, id)
 #endif
 
 
@@ -36,6 +37,7 @@ __pthread_mutex_lock (mutex)
 {
   assert (sizeof (mutex->__size) >= sizeof (mutex->__data));
 
+  int oldval;
   pid_t id = THREAD_GETMEM (THREAD_SELF, tid);
 
   int retval = 0;
@@ -107,60 +109,83 @@ __pthread_mutex_lock (mutex)
       break;
 
     case PTHREAD_MUTEX_ROBUST_PRIVATE_RECURSIVE_NP:
-      /* Check whether we already hold the mutex.  */
-      if (abs (mutex->__data.__owner) == id)
+    case PTHREAD_MUTEX_ROBUST_PRIVATE_ERRORCHECK_NP:
+    case PTHREAD_MUTEX_ROBUST_PRIVATE_NP:
+    case PTHREAD_MUTEX_ROBUST_PRIVATE_ADAPTIVE_NP:
+      oldval = mutex->__data.__lock;
+      do
 	{
-	  /* Just bump the counter.  */
-	  if (__builtin_expect (mutex->__data.__count + 1 == 0, 0))
-	    /* Overflow of the counter.  */
-	    return EAGAIN;
-
-	  ++mutex->__data.__count;
-
-	  return 0;
-	}
-
-      /* We have to get the mutex.  */
-      LLL_MUTEX_LOCK (mutex->__data.__lock);
+	  if ((oldval & FUTEX_OWNER_DIED) != 0)
+	    {
+	      /* The previous owner died.  Try locking the mutex.  */
+	      int newval;
+	      while ((newval
+		      = atomic_compare_and_exchange_val_acq (&mutex->__data.__lock,
+							     id, oldval))
+		     != oldval)
+		{
+		  if ((newval & FUTEX_OWNER_DIED) == 0)
+		    goto normal;
+		  oldval = newval;
+		}
 
-      mutex->__data.__count = 1;
+	      /* We got the mutex.  */
+	      mutex->__data.__count = 1;
+	      /* But it is inconsistent unless marked otherwise.  */
+	      mutex->__data.__owner = PTHREAD_MUTEX_INCONSISTENT;
+
+	      ENQUEUE_MUTEX (mutex);
+
+	      /* Note that we deliberately exit here.  If we fall
+		 through to the end of the function __nusers would be
+		 incremented which is not correct because the old
+		 owner has to be discounted.  If we are not supposed
+		 to increment __nusers we actually have to decrement
+		 it here.  */
+#ifdef NO_INCR
+	      --mutex->__data.__nusers;
+#endif
 
-      goto robust;
+	      return EOWNERDEAD;
+	    }
 
-    case PTHREAD_MUTEX_ROBUST_PRIVATE_ERRORCHECK_NP:
-      /* Check whether we already hold the mutex.  */
-      if (__builtin_expect (abs (mutex->__data.__owner) == id, 0))
-	return EDEADLK;
+	normal:
+	  /* Check whether we already hold the mutex.  */
+	  if (__builtin_expect ((mutex->__data.__lock & FUTEX_TID_MASK)
+				== id, 0))
+	    {
+	      if (mutex->__data.__kind
+		  == PTHREAD_MUTEX_ROBUST_PRIVATE_ERRORCHECK_NP)
+		return EDEADLK;
 
-      /* FALLTHROUGH */
+	      if (mutex->__data.__kind
+		  == PTHREAD_MUTEX_ROBUST_PRIVATE_RECURSIVE_NP)
+		{
+		  /* Just bump the counter.  */
+		  if (__builtin_expect (mutex->__data.__count + 1 == 0, 0))
+		    /* Overflow of the counter.  */
+		    return EAGAIN;
 
-    case PTHREAD_MUTEX_ROBUST_PRIVATE_NP:
-    case PTHREAD_MUTEX_ROBUST_PRIVATE_ADAPTIVE_NP:
-      LLL_MUTEX_LOCK (mutex->__data.__lock);
+		  ++mutex->__data.__count;
 
-    robust:
-      if (__builtin_expect (mutex->__data.__owner
-			    == PTHREAD_MUTEX_NOTRECOVERABLE, 0))
-	{
-	  /* This mutex is now not recoverable.  */
-	  mutex->__data.__count = 0;
-	  lll_mutex_unlock (mutex->__data.__lock);
-	  return ENOTRECOVERABLE;
-	}
+		  return 0;
+		}
+	    }
 
-      /* This mutex is either healthy or we can try to recover it.  */
-      assert (mutex->__data.__owner == 0
-	      || mutex->__data.__owner == PTHREAD_MUTEX_OWNERDEAD);
+	  oldval = LLL_ROBUST_MUTEX_LOCK (mutex->__data.__lock, id);
 
-      if (__builtin_expect (mutex->__data.__owner
-			    == PTHREAD_MUTEX_OWNERDEAD, 0))
-	{
-	  retval = EOWNERDEAD;
-	  /* We signal ownership of a not yet recovered robust mutex
-	     by storing the negative thread ID.  */
-	  id = -id;
+	  if (__builtin_expect (mutex->__data.__owner
+				== PTHREAD_MUTEX_NOTRECOVERABLE, 0))
+	    {
+	      /* This mutex is now not recoverable.  */
+	      mutex->__data.__count = 0;
+	      lll_mutex_unlock (mutex->__data.__lock);
+	      return ENOTRECOVERABLE;
+	    }
 	}
+      while ((oldval & FUTEX_OWNER_DIED) != 0);
 
+      mutex->__data.__count = 1;
       ENQUEUE_MUTEX (mutex);
       break;
 
diff --git a/nptl/pthread_mutex_timedlock.c b/nptl/pthread_mutex_timedlock.c
index bc4ead765d..b69caedc7c 100644
--- a/nptl/pthread_mutex_timedlock.c
+++ b/nptl/pthread_mutex_timedlock.c
@@ -1,4 +1,4 @@
-/* Copyright (C) 2002, 2003, 2004, 2005 Free Software Foundation, Inc.
+/* Copyright (C) 2002, 2003, 2004, 2005, 2006 Free Software Foundation, Inc.
    This file is part of the GNU C Library.
    Contributed by Ulrich Drepper <drepper@redhat.com>, 2002.
 
@@ -28,6 +28,7 @@ pthread_mutex_timedlock (mutex, abstime)
      pthread_mutex_t *mutex;
      const struct timespec *abstime;
 {
+  int oldval;
   pid_t id = THREAD_GETMEM (THREAD_SELF, tid);
   int result = 0;
 
@@ -103,67 +104,83 @@ pthread_mutex_timedlock (mutex, abstime)
       break;
 
     case PTHREAD_MUTEX_ROBUST_PRIVATE_RECURSIVE_NP:
-      /* Check whether we already hold the mutex.  */
-      if (abs (mutex->__data.__owner) == id)
+    case PTHREAD_MUTEX_ROBUST_PRIVATE_ERRORCHECK_NP:
+    case PTHREAD_MUTEX_ROBUST_PRIVATE_NP:
+    case PTHREAD_MUTEX_ROBUST_PRIVATE_ADAPTIVE_NP:
+      oldval = mutex->__data.__lock;
+      do
 	{
-	  /* Just bump the counter.  */
-	  if (__builtin_expect (mutex->__data.__count + 1 == 0, 0))
-	    /* Overflow of the counter.  */
-	    return EAGAIN;
-
-	  ++mutex->__data.__count;
+	  if ((oldval & FUTEX_OWNER_DIED) != 0)
+	    {
+	      /* The previous owner died.  Try locking the mutex.  */
+	      int newval;
+	      while ((newval
+		      = atomic_compare_and_exchange_val_acq (&mutex->__data.__lock,
+							     id, oldval))
+		     != oldval)
+		{
+		  if ((newval & FUTEX_OWNER_DIED) == 0)
+		    goto normal;
+		  oldval = newval;
+		}
 
-	  goto out;
-	}
+	      /* We got the mutex.  */
+	      mutex->__data.__count = 1;
+	      /* But it is inconsistent unless marked otherwise.  */
+	      mutex->__data.__owner = PTHREAD_MUTEX_INCONSISTENT;
 
-      /* We have to get the mutex.  */
-      result = lll_mutex_timedlock (mutex->__data.__lock, abstime);
+	      ENQUEUE_MUTEX (mutex);
 
-      if (result != 0)
-	goto out;
+	      /* Note that we deliberately exist here.  If we fall
+		 through to the end of the function __nusers would be
+		 incremented which is not correct because the old
+		 owner has to be discounted.  */
+	      return EOWNERDEAD;
+	    }
 
-      /* Only locked once so far.  */
-      mutex->__data.__count = 1;
-      goto robust;
+	normal:
+	  /* Check whether we already hold the mutex.  */
+	  if (__builtin_expect ((mutex->__data.__lock & FUTEX_TID_MASK)
+				== id, 0))
+	    {
+	      if (mutex->__data.__kind
+		  == PTHREAD_MUTEX_ROBUST_PRIVATE_ERRORCHECK_NP)
+		return EDEADLK;
 
-    case PTHREAD_MUTEX_ROBUST_PRIVATE_ERRORCHECK_NP:
-      /* Check whether we already hold the mutex.  */
-      if (__builtin_expect (abs (mutex->__data.__owner) == id, 0))
-	return EDEADLK;
+	      if (mutex->__data.__kind
+		  == PTHREAD_MUTEX_ROBUST_PRIVATE_RECURSIVE_NP)
+		{
+		  /* Just bump the counter.  */
+		  if (__builtin_expect (mutex->__data.__count + 1 == 0, 0))
+		    /* Overflow of the counter.  */
+		    return EAGAIN;
 
-      /* FALLTHROUGH */
+		  ++mutex->__data.__count;
 
-    case PTHREAD_MUTEX_ROBUST_PRIVATE_NP:
-    case PTHREAD_MUTEX_ROBUST_PRIVATE_ADAPTIVE_NP:
-      result = lll_mutex_timedlock (mutex->__data.__lock, abstime);
+		  return 0;
+		}
+	    }
 
-      if (result != 0)
-	goto out;
+	  result = lll_robust_mutex_timedlock (mutex->__data.__lock, abstime,
+					       id);
 
-    robust:
-      if (__builtin_expect (mutex->__data.__owner
-			    == PTHREAD_MUTEX_NOTRECOVERABLE, 0))
-	{
-	  /* This mutex is now not recoverable.  */
-	  mutex->__data.__count = 0;
-	  lll_mutex_unlock (mutex->__data._