Move malloc hooks into a compat DSO

Remove all malloc hook uses from core malloc functions and move it
into a new library libc_malloc_debug.so.  With this, the hooks now no
longer have any effect on the core library.

libc_malloc_debug.so is a malloc interposer that needs to be preloaded
to get hooks functionality back so that the debugging features that
depend on the hooks, i.e. malloc-check, mcheck and mtrace work again.
Without the preloaded DSO these debugging features will be nops.
These features will be ported away from hooks in subsequent patches.

Similarly, legacy applications that need hooks functionality need to
preload libc_malloc_debug.so.

The symbols exported by libc_malloc_debug.so are maintained at exactly
the same version as libc.so.

Finally, static binaries will no longer be able to use malloc
debugging features since they cannot preload the debugging DSO.

Reviewed-by: Carlos O'Donell <carlos@redhat.com>
Tested-by: Carlos O'Donell <carlos@redhat.com>

1 files changed, 29 insertions, 18 deletions

diff --git a/malloc/Makefile b/malloc/Makefile
index bbb70f6f87..cafe427097 100644
--- a/malloc/Makefile
+++ b/malloc/Makefile
@@ -46,12 +46,11 @@ tests := mallocbug tst-malloc tst-valloc tst-calloc tst-obstack \
 
 tests-static := \
 	 tst-interpose-static-nothread \
-	 tst-interpose-static-thread \
-	 tst-malloc-usable-static \
+	 tst-interpose-static-thread
 
 # Test for the malloc_set_state symbol removed in glibc 2.25.
-ifeq ($(have-GLIBC_2.24)$(build-shared),yesyes)
-tests += tst-mallocstate
+ifeq ($(have-GLIBC_2.23)$(build-shared),yesyes)
+tests += tst-mallocstate tst-compathooks-off tst-compathooks-on
 endif
 
 tests-internal := tst-scratch_buffer
@@ -64,7 +63,6 @@ tests-internal += \
 
 ifneq (no,$(have-tunables))
 tests += tst-malloc-usable-tunables tst-mxfast
-tests-static += tst-malloc-usable-static-tunables
 endif
 
 tests += $(tests-static)
@@ -73,7 +71,8 @@ test-srcs = tst-mtrace
 # These tests either are run with MALLOC_CHECK_=3 by default or do not work
 # with MALLOC_CHECK_=3 because they expect a specific failure.
 tests-exclude-malloc-check = tst-malloc-check tst-malloc-usable \
-	tst-mxfast tst-safe-linking
+	tst-mxfast tst-safe-linking \
+	tst-compathooks-off tst-compathooks-on
 
 # Run all tests with MALLOC_CHECK_=3
 tests-malloc-check = $(filter-out $(tests-exclude-malloc-check),$(tests))
@@ -91,15 +90,14 @@ tests-exclude-mcheck = tst-mallocstate \
 	tst-malloc-tcache-leak \
 	tst-malloc-thread-exit \
 	tst-malloc-thread-fail \
-	tst-malloc-usable-static \
-	tst-malloc-usable-static-tunables \
 	tst-malloc-usable-tunables \
 	tst-malloc_info \
 	tst-memalign \
 	tst-posix_memalign \
 	tst-realloc \
 	tst-pvalloc-fortify \
-	tst-reallocarray
+	tst-reallocarray \
+	tst-compathooks-off tst-compathooks-on
 
 tests-mcheck = $(filter-out $(tests-exclude-mcheck), $(tests))
 endif
@@ -122,8 +120,8 @@ routines = malloc mcheck mtrace obstack reallocarray \
 install-lib := libmcheck.a
 non-lib.a := libmcheck.a
 
-# Additional library.
-extra-libs = libmemusage
+# Additional libraries.
+extra-libs = libmemusage libc_malloc_debug
 extra-libs-others = $(extra-libs)
 
 # Helper objects for some tests.
@@ -138,6 +136,9 @@ test-extras = \
 libmemusage-routines = memusage
 libmemusage-inhibit-o = $(filter-out .os,$(object-suffixes))
 
+libc_malloc_debug-routines = malloc-debug
+libc_malloc_debug-inhibit-o = $(filter-out .os,$(object-suffixes))
+
 $(objpfx)tst-malloc-backtrace: $(shared-thread-library)
 $(objpfx)tst-malloc-thread-exit: $(shared-thread-library)
 $(objpfx)tst-malloc-thread-fail: $(shared-thread-library)
@@ -244,11 +245,12 @@ endif
 endif
 endif
 
-tst-malloc-check-ENV = MALLOC_CHECK_=3
-tst-malloc-usable-ENV = MALLOC_CHECK_=3
-tst-malloc-usable-static-ENV = $(tst-malloc-usable-ENV)
-tst-malloc-usable-tunables-ENV = GLIBC_TUNABLES=glibc.malloc.check=3
-tst-malloc-usable-static-tunables-ENV = $(tst-malloc-usable-tunables-ENV)
+tst-malloc-check-ENV = MALLOC_CHECK_=3 \
+		       LD_PRELOAD=$(objpfx)/libc_malloc_debug.so
+tst-malloc-usable-ENV = MALLOC_CHECK_=3 \
+		       LD_PRELOAD=$(objpfx)/libc_malloc_debug.so
+tst-malloc-usable-tunables-ENV = GLIBC_TUNABLES=glibc.malloc.check=3 \
+				 LD_PRELOAD=$(objpfx)/libc_malloc_debug.so
 
 tst-mxfast-ENV = GLIBC_TUNABLES=glibc.malloc.tcache_count=0:glibc.malloc.mxfast=0
 
@@ -303,12 +305,14 @@ $(objpfx)tst-interpose-static-thread-mcheck: \
 $(objpfx)tst-interpose-static-thread-malloc-check: \
   $(objpfx)tst-interpose-aux-thread.o $(static-thread-library)
 
-tst-dynarray-ENV = MALLOC_TRACE=$(objpfx)tst-dynarray.mtrace
+tst-dynarray-ENV = MALLOC_TRACE=$(objpfx)tst-dynarray.mtrace \
+		   LD_PRELOAD=$(objpfx)libc_malloc_debug.so
 $(objpfx)tst-dynarray-mem.out: $(objpfx)tst-dynarray.out
 	$(common-objpfx)malloc/mtrace $(objpfx)tst-dynarray.mtrace > $@; \
 	$(evaluate-test)
 
-tst-dynarray-fail-ENV = MALLOC_TRACE=$(objpfx)tst-dynarray-fail.mtrace
+tst-dynarray-fail-ENV = MALLOC_TRACE=$(objpfx)tst-dynarray-fail.mtrace \
+			LD_PRELOAD=$(objpfx)libc_malloc_debug.so
 $(objpfx)tst-dynarray-fail-mem.out: $(objpfx)tst-dynarray-fail.out
 	$(common-objpfx)malloc/mtrace $(objpfx)tst-dynarray-fail.mtrace > $@; \
 	$(evaluate-test)
@@ -322,3 +326,10 @@ $(objpfx)tst-mallocfork2-mcheck: $(shared-thread-library)
 $(objpfx)tst-malloc-tcache-leak-malloc-check: $(shared-thread-library)
 $(objpfx)tst-malloc_info-malloc-check: $(shared-thread-library)
 $(objpfx)tst-mallocfork2-malloc-check: $(shared-thread-library)
+
+tst-compathooks-on-ENV = LD_PRELOAD=$(objpfx)libc_malloc_debug.so
+tst-compathooks-on-mcheck-ENV = LD_PRELOAD=$(objpfx)libc_malloc_debug.so
+tst-compathooks-on-malloc-check-ENV = \
+	LD_PRELOAD=$(objpfx)libc_malloc_debug.so
+tst-mallocstate-ENV = LD_PRELOAD=$(objpfx)libc_malloc_debug.so
+tst-mallocstate-malloc-check-ENV = LD_PRELOAD=$(objpfx)libc_malloc_debug.so