mcheck: Wean away from malloc hooks [BZ #23489]

Split the mcheck implementation into the debugging hooks and API so
that the API can be replicated in libc and libc_malloc_debug.so.  The
libc APIs always result in failure.

The mcheck implementation has also been moved entirely into
libc_malloc_debug.so and with it, all of the hook initialization code
can now be moved into the debug library.  Now the initialization can
be done independently of libc internals.

With this patch, libc_malloc_debug.so can no longer be used with older
libcs, which is not its goal anyway.  tst-vfork3 breaks due to this
since it spawns shell scripts, which in turn execute using the system
glibc.  Move the test to tests-container so that only the built glibc
is used.

This move also fixes bugs in the mcheck version of memalign and
realloc, thus allowing removal of the tests from tests-mcheck
exclusion list.

Reviewed-by: Carlos O'Donell <carlos@redhat.com>
Tested-by: Carlos O'Donell <carlos@redhat.com>

41 files changed, 861 insertions, 465 deletions

diff --git a/include/mcheck.h b/include/mcheck.h
index 8883c3d53e..5ad7cd1313 100644
--- a/include/mcheck.h
+++ b/include/mcheck.h
@@ -3,9 +3,5 @@
 #include <malloc/mcheck.h>
 
 # ifndef _ISOMAC
-
-libc_hidden_proto (mcheck)
-libc_hidden_proto (mcheck_check_all)
-
 # endif /* !_ISOMAC */
 #endif
diff --git a/malloc/Makefile b/malloc/Makefile
index cafe427097..eb5f5560bb 100644
--- a/malloc/Makefile
+++ b/malloc/Makefile
@@ -92,9 +92,6 @@ tests-exclude-mcheck = tst-mallocstate \
 	tst-malloc-thread-fail \
 	tst-malloc-usable-tunables \
 	tst-malloc_info \
-	tst-memalign \
-	tst-posix_memalign \
-	tst-realloc \
 	tst-pvalloc-fortify \
 	tst-reallocarray \
 	tst-compathooks-off tst-compathooks-on
diff --git a/malloc/Versions b/malloc/Versions
index c87f6df8ca..6548970419 100644
--- a/malloc/Versions
+++ b/malloc/Versions
@@ -110,8 +110,17 @@ libc_malloc_debug {
     realloc;
     valloc;
 
+    __free_hook;
+    __malloc_hook;
+    __memalign_hook;
+    __realloc_hook;
+
+    mcheck;
+    mprobe;
   }
   GLIBC_2.2 {
+    mcheck_check_all;
+    mcheck_pedantic;
     posix_memalign;
   }
   GLIBC_2.16 {
diff --git a/malloc/hooks.c b/malloc/hooks.c
index 3cd44eeb84..8e9fefe6c3 100644
--- a/malloc/hooks.c
+++ b/malloc/hooks.c
@@ -34,65 +34,10 @@ void *(*__morecore)(ptrdiff_t);
 compat_symbol (libc, __morecore, __morecore, GLIBC_2_0);
 #endif
 
-static void *malloc_hook_ini (size_t, const void *) __THROW;
-static void *realloc_hook_ini (void *, size_t, const void *) __THROW;
-static void *memalign_hook_ini (size_t, size_t, const void *) __THROW;
-
 void weak_variable (*__free_hook) (void *, const void *) = NULL;
-void *weak_variable (*__malloc_hook)
-  (size_t, const void *) = malloc_hook_ini;
-void *weak_variable (*__realloc_hook)
-  (void *, size_t, const void *) = realloc_hook_ini;
-void *weak_variable (*__memalign_hook)
-  (size_t, size_t, const void *) = memalign_hook_ini;
-
-/* Hooks for debugging versions.  The initial hooks just call the
-   initialization routine, then do the normal work. */
-
-/* These hooks will get executed only through the interposed allocator
-   functions in libc_malloc_debug.so.  This means that the calls to malloc,
-   realloc, etc. will lead back into the interposed functions, which is what we
-   want.
-
-   These initial hooks are assumed to be called in a single-threaded context,
-   so it is safe to reset all hooks at once upon initialization.  */
-
-static void
-generic_hook_ini (void)
-{
-  __malloc_hook = NULL;
-  __realloc_hook = NULL;
-  __memalign_hook = NULL;
-  ptmalloc_init ();
-
-#if SHLIB_COMPAT (libc, GLIBC_2_0, GLIBC_2_24)
-  void (*hook) (void) = atomic_forced_read (__malloc_initialize_hook);
-  if (hook != NULL)
-    (*hook)();
-#endif
-  __malloc_initialized = 1;
-}
-
-static void *
-malloc_hook_ini (size_t sz, const void *caller)
-{
-  generic_hook_ini ();
-  return malloc (sz);
-}
-
-static void *
-realloc_hook_ini (void *ptr, size_t sz, const void *caller)
-{
-  generic_hook_ini ();
-  return realloc (ptr, sz);
-}
-
-static void *
-memalign_hook_ini (size_t alignment, size_t sz, const void *caller)
-{
-  generic_hook_ini ();
-  return memalign (alignment, sz);
-}
+void *weak_variable (*__malloc_hook) (size_t, const void *) = NULL;
+void *weak_variable (*__realloc_hook) (void *, size_t, const void *) = NULL;
+void *weak_variable (*__memalign_hook) (size_t, size_t, const void *) = NULL;
 
 #include "malloc-check.c"
 
diff --git a/malloc/malloc-debug.c b/malloc/malloc-debug.c
index 41dfcd3369..7c3a1e26b5 100644
--- a/malloc/malloc-debug.c
+++ b/malloc/malloc-debug.c
@@ -43,14 +43,96 @@ DEBUG_FN(valloc);
 DEBUG_FN(pvalloc);
 DEBUG_FN(calloc);
 
-extern void (*__free_hook) (void *, const void *);
-compat_symbol_reference (libc, __free_hook, __free_hook, GLIBC_2_0);
-extern void * (*__malloc_hook) (size_t, const void *);
-compat_symbol_reference (libc, __malloc_hook, __malloc_hook, GLIBC_2_0);
-extern void * (*__realloc_hook) (void *, size_t, const void *);
-compat_symbol_reference (libc, __realloc_hook, __realloc_hook, GLIBC_2_0);
-extern void * (*__memalign_hook) (size_t, size_t, const void *);
-compat_symbol_reference (libc, __memalign_hook, __memalign_hook, GLIBC_2_0);
+static int debug_initialized = -1;
+
+enum malloc_debug_hooks
+{
+  MALLOC_NONE_HOOK = 0,
+  MALLOC_MCHECK_HOOK = 1 << 0, /* mcheck()  */
+};
+static unsigned __malloc_debugging_hooks;
+
+static __always_inline bool
+__is_malloc_debug_enabled (enum malloc_debug_hooks flag)
+{
+  return __malloc_debugging_hooks & flag;
+}
+
+static __always_inline void
+__malloc_debug_enable (enum malloc_debug_hooks flag)
+{
+  __malloc_debugging_hooks |= flag;
+}
+
+static __always_inline void
+__malloc_debug_disable (enum malloc_debug_hooks flag)
+{
+  __malloc_debugging_hooks &= ~flag;
+}
+
+#include "mcheck.c"
+
+extern void (*__malloc_initialize_hook) (void);
+compat_symbol_reference (libc, __malloc_initialize_hook,
+			 __malloc_initialize_hook, GLIBC_2_0);
+
+static void *malloc_hook_ini (size_t, const void *) __THROW;
+static void *realloc_hook_ini (void *, size_t, const void *) __THROW;
+static void *memalign_hook_ini (size_t, size_t, const void *) __THROW;
+
+void (*__free_hook) (void *, const void *) = NULL;
+void *(*__malloc_hook) (size_t, const void *) = malloc_hook_ini;
+void *(*__realloc_hook) (void *, size_t, const void *) = realloc_hook_ini;
+void *(*__memalign_hook) (size_t, size_t, const void *) = memalign_hook_ini;
+
+/* Hooks for debugging versions.  The initial hooks just call the
+   initialization routine, then do the normal work. */
+
+/* These hooks will get executed only through the interposed allocator
+   functions in libc_malloc_debug.so.  This means that the calls to malloc,
+   realloc, etc. will lead back into the interposed functions, which is what we
+   want.
+
+   These initial hooks are assumed to be called in a single-threaded context,
+   so it is safe to reset all hooks at once upon initialization.  */
+
+static void
+generic_hook_ini (void)
+{
+  debug_initialized = 0;
+  __malloc_hook = NULL;
+  __realloc_hook = NULL;
+  __memalign_hook = NULL;
+  /* The compiler does not know that these functions are allocators, so it will
+     not try to optimize it away.  */
+  __libc_free (__libc_malloc (0));
+
+  void (*hook) (void) = __malloc_initialize_hook;
+  if (hook != NULL)
+    (*hook)();
+  debug_initialized = 1;
+}
+
+static void *
+malloc_hook_ini (size_t sz, const void *caller)
+{
+  generic_hook_ini ();
+  return __debug_malloc (sz);
+}
+
+static void *
+realloc_hook_ini (void *ptr, size_t sz, const void *caller)
+{
+  generic_hook_ini ();
+  return __debug_realloc (ptr, sz);
+}
+
+static void *
+memalign_hook_ini (size_t alignment, size_t sz, const void *caller)
+{
+  generic_hook_ini ();
+  return __debug_memalign (alignment, sz);
+}
 
 static size_t pagesize;
 
@@ -63,7 +145,17 @@ __debug_malloc (size_t bytes)
   if (__builtin_expect (hook != NULL, 0))
     return (*hook)(bytes, RETURN_ADDRESS (0));
 
-  return __libc_malloc (bytes);
+  void *victim = NULL;
+  size_t orig_bytes = bytes;
+  if (!__is_malloc_debug_enabled (MALLOC_MCHECK_HOOK)
+      || !malloc_mcheck_before (&bytes, &victim))
+    {
+      victim = __libc_malloc (bytes);
+    }
+  if (__is_malloc_debug_enabled (MALLOC_MCHECK_HOOK) && victim != NULL)
+    victim = malloc_mcheck_after (victim, orig_bytes);
+
+  return victim;
 }
 strong_alias (__debug_malloc, malloc)
 
@@ -76,6 +168,10 @@ __debug_free (void *mem)
       (*hook)(mem, RETURN_ADDRESS (0));
       return;
     }
+
+  if (__is_malloc_debug_enabled (MALLOC_MCHECK_HOOK))
+    mem = free_mcheck (mem);
+
   __libc_free (mem);
 }
 strong_alias (__debug_free, free)
@@ -88,7 +184,19 @@ __debug_realloc (void *oldmem, size_t bytes)
   if (__builtin_expect (hook != NULL, 0))
     return (*hook)(oldmem, bytes, RETURN_ADDRESS (0));
 
-  return __libc_realloc (oldmem, bytes);
+  size_t orig_bytes = bytes, oldsize = 0;
+  void *victim = NULL;
+
+  if (!__is_malloc_debug_enabled (MALLOC_MCHECK_HOOK)
+      || !realloc_mcheck_before (&oldmem, &bytes, &oldsize, &victim))
+    {
+      victim = __libc_realloc (oldmem, bytes);
+    }
+  if (__is_malloc_debug_enabled (MALLOC_MCHECK_HOOK) && victim != NULL)
+    victim = realloc_mcheck_after (victim, oldmem, orig_bytes,
+				   oldsize);
+
+  return victim;
 }
 strong_alias (__debug_realloc, realloc)
 
@@ -100,7 +208,18 @@ _debug_mid_memalign (size_t alignment, size_t bytes, const void *address)
   if (__builtin_expect (hook != NULL, 0))
     return (*hook)(alignment, bytes, address);
 
-  return __libc_memalign (alignment, bytes);
+  void *victim = NULL;
+  size_t orig_bytes = bytes;
+
+  if (!__is_malloc_debug_enabled (MALLOC_MCHECK_HOOK)
+      || !memalign_mcheck_before (alignment, &bytes, &victim))
+    {
+      victim = __libc_memalign (alignment, bytes);
+    }
+  if (__is_malloc_debug_enabled (MALLOC_MCHECK_HOOK) && victim != NULL)
+    victim = memalign_mcheck_after (victim, alignment, orig_bytes);
+
+  return victim;
 }
 
 static void *
@@ -165,17 +284,17 @@ strong_alias (__debug_posix_memalign, posix_memalign)
 static void *
 __debug_calloc (size_t nmemb, size_t size)
 {
+  size_t bytes;
+
+  if (__glibc_unlikely (__builtin_mul_overflow (nmemb, size, &bytes)))
+    {
+      errno = ENOMEM;
+      return NULL;
+    }
+
   void *(*hook) (size_t, const void *) = atomic_forced_read (__malloc_hook);
   if (__builtin_expect (hook != NULL, 0))
     {
-      size_t bytes;
-
-      if (__glibc_unlikely (__builtin_mul_overflow (nmemb, size, &bytes)))
-	{
-	  errno = ENOMEM;
-	  return NULL;
-	}
-
       void *mem = (*hook)(bytes, RETURN_ADDRESS (0));
 
       if (mem != NULL)
@@ -184,6 +303,20 @@ __debug_calloc (size_t nmemb, size_t size)
       return mem;
     }
 
-  return __libc_calloc (nmemb, size);
+  size_t orig_bytes = bytes;
+  void *victim = NULL;
+
+  if (!__is_malloc_debug_enabled (MALLOC_MCHECK_HOOK)
+      || !malloc_mcheck_before (&bytes, &victim))
+    {
+      victim = __libc_malloc (bytes);
+    }
+  if (victim != NULL)
+    {
+      if (__is_malloc_debug_enabled (MALLOC_MCHECK_HOOK))
+	victim = malloc_mcheck_after (victim, orig_bytes);
+      memset (victim, 0, orig_bytes);
+    }
+  return victim;
 }
 strong_alias (__debug_calloc, calloc)
diff --git a/malloc/mcheck-impl.c b/malloc/mcheck-impl.c
new file mode 100644
index 0000000000..8857e6b179
--- /dev/null
+++ b/malloc/mcheck-impl.c
@@ -0,0 +1,406 @@
+/* mcheck debugging hooks for malloc.
+   Copyright (C) 1990-2021 Free Software Foundation, Inc.
+   This file is part of the GNU C Library.
+   Written May 1989 by Mike Haertel.
+
+   The GNU C Library is free software; you can redistribute it and/or
+   modify it under the terms of the GNU Lesser General Public
+   License as published by the Free Software Foundation; either
+   version 2.1 of the License, or (at your option) any later version.
+
+   The GNU C Library is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MER