Remove malloc hooks [BZ #23328]

Make malloc hooks symbols compat-only so that new applications cannot
link against them and remove the declarations from the API.  Also
remove the unused malloc-hooks.h.

Finally, mark all symbols in libc_malloc_debug.so as compat so that
the library cannot be linked against.

Add a note about the deprecation in NEWS.

Reviewed-by: Carlos O'Donell <carlos@redhat.com>
Tested-by: Carlos O'Donell <carlos@redhat.com>

6 files changed, 57 insertions, 226 deletions

diff --git a/NEWS b/NEWS
index e26a9e2c17..ea54518142 100644
--- a/NEWS
+++ b/NEWS
@@ -143,6 +143,16 @@ Deprecated and removed features, and other changes affecting compatibility:
   that still use these functions will now need to preload libc_malloc_debug.so
   in their environment using the LD_PRELOAD environment variable.
 
+* The deprecated memory allocation hooks __malloc_hook, __realloc_hook,
+  __memalign_hook and __free_hook are now removed from the API.  Compatibility
+  symbols are present to support legacy programs but new applications can no
+  longer link to these symbols.  These hooks no longer have any effect on glibc
+  functionality.  The malloc debugging DSO libc_malloc_debug.so currently
+  supports hooks and can be preloaded to get this functionality back for older
+  programs.  However this is a transitional measure and may be removed in a
+  future release of the GNU C Library.  Users may port away from these hooks by
+  writing and preloading their own malloc interposition library.
+
 Changes to build and runtime requirements:
 
 * On Linux, the shm_open, sem_open, and related functions now expect the
diff --git a/malloc/hooks.c b/malloc/hooks.c
index 8e1afe55e5..2761e4739b 100644
--- a/malloc/hooks.c
+++ b/malloc/hooks.c
@@ -32,12 +32,16 @@ void weak_variable (*__after_morecore_hook) (void) = NULL;
 compat_symbol (libc, __after_morecore_hook, __after_morecore_hook, GLIBC_2_0);
 void *(*__morecore)(ptrdiff_t);
 compat_symbol (libc, __morecore, __morecore, GLIBC_2_0);
-#endif
 
 void weak_variable (*__free_hook) (void *, const void *) = NULL;
 void *weak_variable (*__malloc_hook) (size_t, const void *) = NULL;
 void *weak_variable (*__realloc_hook) (void *, size_t, const void *) = NULL;
 void *weak_variable (*__memalign_hook) (size_t, size_t, const void *) = NULL;
+compat_symbol (libc, __free_hook, __free_hook, GLIBC_2_0);
+compat_symbol (libc, __malloc_hook, __malloc_hook, GLIBC_2_0);
+compat_symbol (libc, __realloc_hook, __realloc_hook, GLIBC_2_0);
+compat_symbol (libc, __memalign_hook, __memalign_hook, GLIBC_2_0);
+#endif
 
 /*
  * Local variables:
diff --git a/malloc/malloc-debug.c b/malloc/malloc-debug.c
index b7744460e9..34523b0cc3 100644
--- a/malloc/malloc-debug.c
+++ b/malloc/malloc-debug.c
@@ -23,6 +23,7 @@
 #include <unistd.h>
 #include <sys/param.h>
 
+#if SHLIB_COMPAT (libc_malloc_debug, GLIBC_2_0, GLIBC_2_34)
 /* Support only the glibc allocators.  */
 extern void *__libc_malloc (size_t);
 extern void __libc_free (void *);
@@ -76,9 +77,11 @@ __malloc_debug_disable (enum malloc_debug_hooks flag)
 #include "mtrace.c"
 #include "malloc-check.c"
 
+#if SHLIB_COMPAT (libc_malloc_debug, GLIBC_2_0, GLIBC_2_24)
 extern void (*__malloc_initialize_hook) (void);
 compat_symbol_reference (libc, __malloc_initialize_hook,
 			 __malloc_initialize_hook, GLIBC_2_0);
+#endif
 
 static void *malloc_hook_ini (size_t, const void *) __THROW;
 static void *realloc_hook_ini (void *, size_t, const void *) __THROW;
@@ -115,9 +118,11 @@ generic_hook_ini (void)
        will not try to optimize it away.  */
     __libc_free (__libc_malloc (0));
 
+#if SHLIB_COMPAT (libc_malloc_debug, GLIBC_2_0, GLIBC_2_24)
   void (*hook) (void) = __malloc_initialize_hook;
   if (hook != NULL)
     (*hook)();
+#endif
 
   debug_initialized = 1;
 }
@@ -631,3 +636,34 @@ malloc_set_state (void *msptr)
 compat_symbol (libc_malloc_debug, malloc_set_state, malloc_set_state,
 	       GLIBC_2_0);
 #endif
+
+/* Do not allow linking against the library.  */
+compat_symbol (libc_malloc_debug, aligned_alloc, aligned_alloc, GLIBC_2_16);
+compat_symbol (libc_malloc_debug, calloc, calloc, GLIBC_2_0);
+compat_symbol (libc_malloc_debug, free, free, GLIBC_2_0);
+compat_symbol (libc_malloc_debug, mallinfo2, mallinfo2, GLIBC_2_33);
+compat_symbol (libc_malloc_debug, mallinfo, mallinfo, GLIBC_2_0);
+compat_symbol (libc_malloc_debug, malloc_info, malloc_info, GLIBC_2_10);
+compat_symbol (libc_malloc_debug, malloc, malloc, GLIBC_2_0);
+compat_symbol (libc_malloc_debug, malloc_stats, malloc_stats, GLIBC_2_0);
+compat_symbol (libc_malloc_debug, malloc_trim, malloc_trim, GLIBC_2_0);
+compat_symbol (libc_malloc_debug, malloc_usable_size, malloc_usable_size,
+	       GLIBC_2_0);
+compat_symbol (libc_malloc_debug, mallopt, mallopt, GLIBC_2_0);
+compat_symbol (libc_malloc_debug, mcheck_check_all, mcheck_check_all,
+	       GLIBC_2_2);
+compat_symbol (libc_malloc_debug, mcheck, mcheck, GLIBC_2_0);
+compat_symbol (libc_malloc_debug, mcheck_pedantic, mcheck_pedantic, GLIBC_2_2);
+compat_symbol (libc_malloc_debug, memalign, memalign, GLIBC_2_0);
+compat_symbol (libc_malloc_debug, mprobe, mprobe, GLIBC_2_0);
+compat_symbol (libc_malloc_debug, mtrace, mtrace, GLIBC_2_0);
+compat_symbol (libc_malloc_debug, muntrace, muntrace, GLIBC_2_0);
+compat_symbol (libc_malloc_debug, posix_memalign, posix_memalign, GLIBC_2_2);
+compat_symbol (libc_malloc_debug, pvalloc, pvalloc, GLIBC_2_0);
+compat_symbol (libc_malloc_debug, realloc, realloc, GLIBC_2_0);
+compat_symbol (libc_malloc_debug, valloc, valloc, GLIBC_2_0);
+compat_symbol (libc_malloc_debug, __free_hook, __free_hook, GLIBC_2_0);
+compat_symbol (libc_malloc_debug, __malloc_hook, __malloc_hook, GLIBC_2_0);
+compat_symbol (libc_malloc_debug, __realloc_hook, __realloc_hook, GLIBC_2_0);
+compat_symbol (libc_malloc_debug, __memalign_hook, __memalign_hook, GLIBC_2_0);
+#endif
diff --git a/malloc/malloc-hooks.h b/malloc/malloc-hooks.h
deleted file mode 100644
index 0133331b83..0000000000
--- a/malloc/malloc-hooks.h
+++ /dev/null
@@ -1,24 +0,0 @@
-/* Internal declarations of malloc hooks no longer in the public API.
-   Copyright (C) 2016-2021 Free Software Foundation, Inc.
-   This file is part of the GNU C Library.
-
-   The GNU C Library is free software; you can redistribute it and/or
-   modify it under the terms of the GNU Lesser General Public License as
-   published by the Free Software Foundation; either version 2.1 of the
-   License, or (at your option) any later version.
-
-   The GNU C Library is distributed in the hope that it will be useful,
-   but WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   Lesser General Public License for more details.
-
-   You should have received a copy of the GNU Lesser General Public
-   License along with the GNU C Library; see the file COPYING.LIB.  If
-   not, see <https://www.gnu.org/licenses/>.  */
-
-#ifndef _MALLOC_HOOKS_H
-#define _MALLOC_HOOKS_H
-
-void (*__malloc_initialize_hook) (void);
-
-#endif  /* _MALLOC_HOOKS_H */
diff --git a/malloc/malloc.h b/malloc/malloc.h
index 17ab9ee345..2df0b38050 100644
--- a/malloc/malloc.h
+++ b/malloc/malloc.h
@@ -156,21 +156,5 @@ extern void malloc_stats (void) __THROW;
 /* Output information about state of allocator to stream FP.  */
 extern int malloc_info (int __options, FILE *__fp) __THROW;
 
-/* Hooks for debugging and user-defined versions. */
-extern void (*__MALLOC_HOOK_VOLATILE __free_hook) (void *__ptr,
-                                                   const void *)
-__MALLOC_DEPRECATED;
-extern void *(*__MALLOC_HOOK_VOLATILE __malloc_hook)(size_t __size,
-                                                     const void *)
-__MALLOC_DEPRECATED;
-extern void *(*__MALLOC_HOOK_VOLATILE __realloc_hook)(void *__ptr,
-                                                      size_t __size,
-                                                      const void *)
-__MALLOC_DEPRECATED;
-extern void *(*__MALLOC_HOOK_VOLATILE __memalign_hook)(size_t __alignment,
-                                                       size_t __size,
-                                                       const void *)
-__MALLOC_DEPRECATED;
-
 __END_DECLS
 #endif /* malloc.h */
diff --git a/manual/memory.texi b/manual/memory.texi
index 93305f289b..69c8894602 100644
--- a/manual/memory.texi
+++ b/manual/memory.texi
@@ -328,8 +328,6 @@ any time (or never).
 * Malloc Tunable Parameters::   Use @code{mallopt} to adjust allocation
                                  parameters.
 * Heap Consistency Checking::   Automatic checking for errors.
-* Hooks for Malloc::            You can use these hooks for debugging
-				 programs that use @code{malloc}.
 * Statistics of Malloc::        Getting information about how much
 				 memory your program is using.
 * Summary of Malloc::           Summary of @code{malloc} and related functions.
@@ -1392,170 +1390,6 @@ compatibility.  Both @code{MALLOC_CHECK_} and @samp{-lmcheck} should
 uncover the same bugs - but using @code{MALLOC_CHECK_} you don't need to
 recompile your application.
 
-@node Hooks for Malloc
-@subsubsection Memory Allocation Hooks
-@cindex allocation hooks, for @code{malloc}
-
-@Theglibc{} lets you modify the behavior of @code{malloc},
-@code{realloc}, and @code{free} by specifying appropriate hook
-functions.  You can use these hooks to help you debug programs that use
-dynamic memory allocation, for example.
-
-The hook variables are declared in @file{malloc.h}.
-@pindex malloc.h
-
-@defvar __malloc_hook
-@standards{GNU, malloc.h}
-The value of this variable is a pointer to the function that
-@code{malloc} uses whenever it is called.  You should define this
-function to look like @code{malloc}; that is, like:
-
-@smallexample
-void *@var{function} (size_t @var{size}, const void *@var{caller})
-@end smallexample
-
-The value of @var{caller} is the return address found on the stack when
-the @code{malloc} function was called.  This value allows you to trace
-the memory consumption of the program.
-@end defvar
-
-@defvar __realloc_hook
-@standards{GNU, malloc.h}
-The value of this variable is a pointer to function that @code{realloc}
-uses whenever it is called.  You should define this function to look
-like @code{realloc}; that is, like:
-
-@smallexample
-void *@var{function} (void *@var{ptr}, size_t @var{size}, const void *@var{caller})
-@end smallexample
-
-The value of @var{caller} is the return address found on the stack when
-the @code{realloc} function was called.  This value allows you to trace the
-memory consumption of the program.
-@end defvar
-
-@defvar __free_hook
-@standards{GNU, malloc.h}
-The value of this variable is a pointer to function that @code{free}
-uses whenever it is called.  You should define this function to look
-like @code{free}; that is, like:
-
-@smallexample
-void @var{function} (void *@var{ptr}, const void *@var{caller})
-@end smallexample
-
-The value of @var{caller} is the return address found on the stack when
-the @code{free} function was called.  This value allows you to trace the
-memory consumption of the program.
-@end defvar
-
-@defvar __memalign_hook
-@standards{GNU, malloc.h}
-The value of this variable is a pointer to function that @code{aligned_alloc},
-@code{memalign}, @code{posix_memalign} and @code{valloc} use whenever they
-are called.  You should define this function to look like @code{aligned_alloc};
-that is, like:
-
-@smallexample
-void *@var{function} (size_t @var{alignment}, size_t @var{size}, const void *@var{caller})
-@end smallexample
-
-The value of @var{caller} is the return address found on the stack when
-the @code{aligned_alloc}, @code{memalign}, @code{posix_memalign} or
-@code{valloc} functions are called.  This value allows you to trace the
-memory consumption of the program.
-@end defvar
-
-You must make sure that the function you install as a hook for one of
-these functions does not call that function recursively without restoring
-the old value of the hook first!  Otherwise, your program will get stuck
-in an infinite recursion.  Before calling the function recursively, one
-should make sure to restore all the hooks to their previous value.  When
-coming back from the recursive call, all the hooks should be resaved
-since a hook might modify itself.
-
-An issue to look out for is the time at which the hook functions
-can be safely installed.  If the hook functions call the @code{malloc}-related
-functions recursively, it is necessary that @code{malloc} has already properly
-initialized itself at the time when @code{__malloc_hook} etc. is
-assigned to.  On the other hand, if the hook functions provide a
-complete @code{malloc} implementation of their own, it is vital that the hooks
-are assigned to @emph{before} the very first @code{malloc} call has
-completed, because otherwise a chunk obtained from the ordinary,
-un-hooked @code{malloc} may later be handed to @code{__free_hook}, for example.
-
-Here is an example showing how to use @code{__malloc_hook} and
-@code{__free_hook} properly.  It installs a function that prints out
-information every time @code{malloc} or @code{free} is called.  We just
-assume here that @code{realloc} and @code{memalign} are not used in our
-program.
-
-@smallexample
-/* Prototypes for __malloc_hook, __free_hook */
-#include <malloc.h>
-
-/* Prototypes for our hooks.  */
-static void my_init_hook (void);
-static void *my_malloc_hook (size_t, const void *);
-static void my_free_hook (void*, const void *);
-
-static void
-my_init (void)
-@{
-  old_malloc_hook = __malloc_hook;
-  old_free_hook = __free_hook;
-  __malloc_hook = my_malloc_hook;
-  __free_hook = my_free_hook;
-@}
-
-static void *
-my_malloc_hook (size_t size, const void *caller)
-@{
-  void *result;
-  /* Restore all old hooks */
-  __malloc_hook = old_malloc_hook;
-  __free_hook = old_free_hook;
-  /* Call recursively */
-  result = malloc (size);
-  /* Save underlying hooks */
-  old_malloc_hook = __malloc_hook;
-  old_free_hook = __free_hook;
-  /* @r{@code{printf} might call @code{malloc}, so protect it too.} */
-  printf ("malloc (%u) returns %p\n", (unsigned int) size, result);
-  /* Restore our own hooks */
-  __malloc_hook = my_malloc_hook;
-  __free_hook = my_free_hook;
-  return result;
-@}
-
-static void
-my_free_hook (void *ptr, const void *caller)
-@{
-  /* Restore all old hooks */
-  __malloc_hook = old_malloc_hook;
-  __free_hook = old_free_hook;
-  /* Call recursively */
-  free (ptr);
-  /* Save underlying hooks */
-  old_malloc_hook = __malloc_hook;
-  old_free_hook = __free_hook;
-  /* @r{@code{printf} might call @code{free}, so protect it too.} */
-  printf ("freed pointer %p\n", ptr);
-  /* Restore our own hooks */
-  __malloc_hook = my_malloc_hook;
-  __free_hook = my_free_hook;
-@}
-
-main ()
-@{
-  my_init ();
-  @dots{}
-@}
-@end smallexample
-
-The @code{mcheck} function (@pxref{Heap Consistency Checking}) works by
-installing such hooks.
-
 @c __morecore, __after_morecore_hook are undocumented
 @c It's not clear whether to document them.
 
@@ -1690,19 +1524,6 @@ Tell @code{malloc} to perform occasional consistency checks on
 dynamically allocated memory, and to call @var{abortfn} when an
 inconsistency is found.  @xref{Heap Consistency Checking}.
 
-@item void *(*__malloc_hook) (size_t @var{size}, const void *@var{caller})
-A pointer to a function that @code{malloc} uses whenever it is called.
-
-@item void *(*__realloc_hook) (void *@var{ptr}, size_t @var{size}, const void *@var{caller})
-A pointer to a function that @code{realloc} uses whenever it is called.
-
-@item void (*__free_hook) (void *@var{ptr}, const void *@var{caller})
-A pointer to a function that @code{free} uses whenever it is called.
-
-@item void (*__memalign_hook) (size_t @var{size}, size_t @var{alignment}, const void *@var{caller})
-A pointer to a function that @code{aligned_alloc}, @code{memalign},
-@code{posix_memalign} and @code{valloc} use whenever they are called.
-
 @item struct mallinfo2 mallinfo2 (void)
 Return information about the current dynamic memory usage.
 @xref{Statistics of Malloc}.
@@ -1737,7 +1558,7 @@ penalties for the program if the debugging mode is not enabled.
 
 @deftypefun void mtrace (void)
 @standards{GNU, mcheck.h}
-@safety{@prelim{}@mtunsafe{@mtsenv{} @mtasurace{:mtrace} @mtasuconst{:malloc_hooks} @mtuinit{}}@asunsafe{@asuinit{} @ascuheap{} @asucorrupt{} @asulock{}}@acunsafe{@acuinit{} @acucorrupt{} @aculock{} @acsfd{} @acsmem{}}}
+@safety{@prelim{}@mtunsafe{@mtsenv{} @mtasurace{:mtrace} @mtuinit{}}@asunsafe{@asuinit{} @ascuheap{} @asucorrupt{} @asulock{}}@acunsafe{@acuinit{} @acucorrupt{} @aculock{} @acsfd{} @acsmem{}}}
 @c Like the mcheck hooks, these are not designed with thread safety in
 @c mind, because the hook pointers are temporarily modified without
 @c regard to other threads, signals or cancellation.
@@ -1768,10 +1589,10 @@ with the SUID or SGID bit set.
 
 If the named file is successfully opened, @code{mtrace} installs special
 handlers for the functions @code{malloc}, @code{realloc}, and
-@code{free} (@pxref{Hooks for Malloc}).  From then on, all uses of these
-functions are traced and protocolled into the file.  There is now of
-course a speed penalty for all calls to the traced functions so tracing
-should not be enabled during normal use.
+@code{free}.  From then on, all uses of these functions are traced and
+protocolled into the file.  There is now of course a speed penalty for all
+calls to the traced functions so tracing should not be enabled during normal
+use.
 
 This function is a GNU extension and generally not available on other
 systems.  The prototype can be found in @file{mcheck.h}.
@@ -1779,7 +1600,7 @@ systems.  The prototype can be found in @file{mcheck.h}.
 
 @deftypefun void muntrace (void)
 @standards{GNU, mcheck.h}
-@safety{@prelim{}@mtunsafe{@mtasurace{:mtrace} @mtasuconst{:malloc_hooks} @mtslocale{}}@asunsafe{@asucorrupt{} @ascuheap{}}@acunsafe{@acucorrupt{} @acsmem{} @aculock{} @acsfd{}}}
+@safety{@prelim{}@mtunsafe{@mtasurace{:mtrace} @mtslocale{}}@asunsafe{@asucorrupt{} @ascuheap{}}@acunsafe{@acucorrupt{} @acsmem{} @aculock{} @acsfd{}}}
 
 @c muntrace @mtasurace:mtrace @mtslocale @asucorrupt @ascuheap @acucorrupt @acsmem @aculock @acsfd
 @c  fprintf (fputs) dup @mtslocale @asucorrupt @ascuheap @acsmem @aculock @acucorrupt