Move malloc hooks into a compat DSO

Remove all malloc hook uses from core malloc functions and move it
into a new library libc_malloc_debug.so.  With this, the hooks now no
longer have any effect on the core library.

libc_malloc_debug.so is a malloc interposer that needs to be preloaded
to get hooks functionality back so that the debugging features that
depend on the hooks, i.e. malloc-check, mcheck and mtrace work again.
Without the preloaded DSO these debugging features will be nops.
These features will be ported away from hooks in subsequent patches.

Similarly, legacy applications that need hooks functionality need to
preload libc_malloc_debug.so.

The symbols exported by libc_malloc_debug.so are maintained at exactly
the same version as libc.so.

Finally, static binaries will no longer be able to use malloc
debugging features since they cannot preload the debugging DSO.

Reviewed-by: Carlos O'Donell <carlos@redhat.com>
Tested-by: Carlos O'Donell <carlos@redhat.com>

71 files changed, 865 insertions, 158 deletions

diff --git a/NEWS b/NEWS
index 75ad8a9b95..fa80c9685b 100644
--- a/NEWS
+++ b/NEWS
@@ -132,6 +132,12 @@ Deprecated and removed features, and other changes affecting compatibility:
   applications will continue to link against these symbols but the interfaces
   no longer have any effect on malloc.
 
+* Debugging features in malloc such as the MALLOC_CHECK_ environment variable
+  (or the glibc.malloc.check tunable), mtrace() and mcheck() have now been
+  disabled by default in the main C library.  Users looking to use these
+  features now need to preload a new debugging DSO libc_malloc_debug.so to get
+  this functionality back.
+
 Changes to build and runtime requirements:
 
 * On Linux, the shm_open, sem_open, and related functions now expect the
diff --git a/Rules b/Rules
index ba13598df6..b1137afe71 100644
--- a/Rules
+++ b/Rules
@@ -279,10 +279,17 @@ endif
 
 # All malloc-check tests will be run with MALLOC_CHECK_=3
 define malloc-check-ENVS
-$(1)-malloc-check-ENV = MALLOC_CHECK_=3
+$(1)-malloc-check-ENV = MALLOC_CHECK_=3 \
+			LD_PRELOAD=$(common-objpfx)/malloc/libc_malloc_debug.so
 endef
 $(foreach t,$(tests-malloc-check),$(eval $(call malloc-check-ENVS,$(t))))
 
+# mcheck tests need the debug DSO to support -lmcheck.
+define mcheck-ENVS
+$(1)-mcheck-ENV = LD_PRELOAD=$(common-objpfx)/malloc/libc_malloc_debug.so
+endef
+$(foreach t,$(tests-mcheck),$(eval $(call mcheck-ENVS,$(t))))
+
 ifneq "$(strip $(tests) $(tests-internal) $(xtests) $(test-srcs))" ""
 # These are the implicit rules for making test outputs
 # from the test programs and whatever input files are present.
diff --git a/catgets/Makefile b/catgets/Makefile
index 54da9a985f..215965b352 100644
--- a/catgets/Makefile
+++ b/catgets/Makefile
@@ -56,7 +56,9 @@ generated += tst-catgets.mtrace tst-catgets-mem.out
 
 generated-dirs += de
 
-tst-catgets-ENV = NLSPATH="$(objpfx)%l/%N.cat" LANG=de MALLOC_TRACE=$(objpfx)tst-catgets.mtrace
+tst-catgets-ENV = NLSPATH="$(objpfx)%l/%N.cat" LANG=de \
+		  MALLOC_TRACE=$(objpfx)tst-catgets.mtrace \
+		  LD_PRELOAD=$(common-objpfx)/malloc/libc_malloc_debug.so
 
 ifeq ($(run-built-tests),yes)
 # This test just checks whether the program produces any error or not.
diff --git a/elf/Makefile b/elf/Makefile
index 4fe60947ad..87a70d6c7a 100644
--- a/elf/Makefile
+++ b/elf/Makefile
@@ -156,7 +156,7 @@ $(inst_auditdir)/sotruss-lib.so: $(objpfx)sotruss-lib.so $(+force)
 	$(do-install-program)
 endif
 
-tests-static-normal := tst-leaks1-static tst-array1-static tst-array5-static \
+tests-static-normal := tst-array1-static tst-array5-static \
 	       tst-dl-iter-static \
 	       tst-tlsalign-static tst-tlsalign-extern-static \
 	       tst-linkall-static tst-env-setuid tst-env-setuid-tunables \
@@ -242,7 +242,7 @@ endif
 tests += $(tests-execstack-$(have-z-execstack))
 ifeq ($(run-built-tests),yes)
 tests-special += $(objpfx)tst-leaks1-mem.out \
-		 $(objpfx)tst-leaks1-static-mem.out $(objpfx)noload-mem.out \
+		 $(objpfx)noload-mem.out \
 		 $(objpfx)tst-ldconfig-X.out $(objpfx)tst-rtld-help.out
 endif
 tlsmod17a-suffixes = 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19
@@ -904,7 +904,8 @@ $(objpfx)noload.out: $(objpfx)testobj5.so
 $(objpfx)noload-mem.out: $(objpfx)noload.out
 	$(common-objpfx)malloc/mtrace $(objpfx)noload.mtrace > $@; \
 	$(evaluate-test)
-noload-ENV = MALLOC_TRACE=$(objpfx)noload.mtrace
+noload-ENV = MALLOC_TRACE=$(objpfx)noload.mtrace \
+	     LD_PRELOAD=$(common-objpfx)/malloc/libc_malloc_debug.so
 
 LDFLAGS-nodelete = -rdynamic
 LDFLAGS-nodelmod1.so = -Wl,--enable-new-dtags,-z,nodelete
@@ -1282,12 +1283,8 @@ $(objpfx)tst-leaks1-mem.out: $(objpfx)tst-leaks1.out
 	$(common-objpfx)malloc/mtrace $(objpfx)tst-leaks1.mtrace > $@; \
 	$(evaluate-test)
 
-$(objpfx)tst-leaks1-static-mem.out: $(objpfx)tst-leaks1-static.out
-	$(common-objpfx)malloc/mtrace $(objpfx)tst-leaks1-static.mtrace > $@; \
-	$(evaluate-test)
-
-tst-leaks1-ENV = MALLOC_TRACE=$(objpfx)tst-leaks1.mtrace
-tst-leaks1-static-ENV = MALLOC_TRACE=$(objpfx)tst-leaks1-static.mtrace
+tst-leaks1-ENV = MALLOC_TRACE=$(objpfx)tst-leaks1.mtrace \
+		 LD_PRELOAD=$(common-objpfx)/malloc/libc_malloc_debug.so
 
 $(objpfx)tst-thrlock: $(shared-thread-library)
 $(objpfx)tst-thrlock.out: $(libm)
diff --git a/elf/tst-leaks1-static.c b/elf/tst-leaks1-static.c
deleted file mode 100644
index b956d66905..0000000000
--- a/elf/tst-leaks1-static.c
+++ /dev/null
@@ -1 +0,0 @@
-#include "tst-leaks1.c"
diff --git a/iconvdata/Makefile b/iconvdata/Makefile
index bb3f621b49..c216f959df 100644
--- a/iconvdata/Makefile
+++ b/