nptl: Move legacy unwinding implementation into libc

It is still used internally.  Since unwinding is now available
unconditionally, avoid indirect calls through function pointers loaded
from the stack by inlining the non-cancellation cleanup code.  This
avoids a regression in security hardening.

The out-of-line  __libc_cleanup_routine implementation is no longer
needed because the inline definition is now static __always_inline.

Reviewed-by: Adhemerval Zanella  <adhemerval.zanella@linaro.org>

4 files changed, 66 insertions, 58 deletions

diff --git a/nptl/Versions b/nptl/Versions
index 60202b4969..2e5a964b11 100644
--- a/nptl/Versions
+++ b/nptl/Versions
@@ -87,6 +87,8 @@ libc {
     __futex_abstimed_wait64;
     __futex_abstimed_wait_cancelable64;
     __libc_alloca_cutoff;
+    __libc_cleanup_pop_restore;
+    __libc_cleanup_push_defer;
     __libc_dl_error_tsd;
     __libc_pthread_init;
     __lll_clocklock_elision;
diff --git a/nptl/cleanup_defer_compat.c b/nptl/cleanup_defer_compat.c
index 49ef53ea60..1957318208 100644
--- a/nptl/cleanup_defer_compat.c
+++ b/nptl/cleanup_defer_compat.c
@@ -17,41 +17,15 @@
    <https://www.gnu.org/licenses/>.  */
 
 #include "pthreadP.h"
-
+#include <libc-lock.h>
 
 void
 _pthread_cleanup_push_defer (struct _pthread_cleanup_buffer *buffer,
 			     void (*routine) (void *), void *arg)
 {
-  struct pthread *self = THREAD_SELF;
-
   buffer->__routine = routine;
   buffer->__arg = arg;
-  buffer->__prev = THREAD_GETMEM (self, cleanup);
-
-  int cancelhandling = THREAD_GETMEM (self, cancelhandling);
-
-  /* Disable asynchronous cancellation for now.  */
-  if (__glibc_unlikely (cancelhandling & CANCELTYPE_BITMASK))
-    while (1)
-      {
-	int curval = THREAD_ATOMIC_CMPXCHG_VAL (self, cancelhandling,
-						cancelhandling
-						& ~CANCELTYPE_BITMASK,
-						cancelhandling);
-	if (__glibc_likely (curval == cancelhandling))
-	  /* Successfully replaced the value.  */
-	  break;
-
-	/* Prepare for the next round.  */
-	cancelhandling = curval;
-      }
-
-  buffer->__canceltype = (cancelhandling & CANCELTYPE_BITMASK
-			  ? PTHREAD_CANCEL_ASYNCHRONOUS
-			  : PTHREAD_CANCEL_DEFERRED);
-
-  THREAD_SETMEM (self, cleanup, buffer);
+  __libc_cleanup_push_defer (buffer);
 }
 strong_alias (_pthread_cleanup_push_defer, __pthread_cleanup_push_defer)
 
@@ -60,31 +34,7 @@ void
 _pthread_cleanup_pop_restore (struct _pthread_cleanup_buffer *buffer,
 			      int execute)
 {
-  struct pthread *self = THREAD_SELF;
-
-  THREAD_SETMEM (self, cleanup, buffer->__prev);
-
-  int cancelhandling;
-  if (__builtin_expect (buffer->__canceltype != PTHREAD_CANCEL_DEFERRED, 0)
-      && ((cancelhandling = THREAD_GETMEM (self, cancelhandling))
-	  & CANCELTYPE_BITMASK) == 0)
-    {
-      while (1)
-	{
-	  int curval = THREAD_ATOMIC_CMPXCHG_VAL (self, cancelhandling,
-						  cancelhandling
-						  | CANCELTYPE_BITMASK,
-						  cancelhandling);
-	  if (__glibc_likely (curval == cancelhandling))
-	    /* Successfully replaced the value.  */
-	    break;
-
-	  /* Prepare for the next round.  */
-	  cancelhandling = curval;
-	}
-
-      CANCELLATION_P (self);
-    }
+  __libc_cleanup_pop_restore (buffer);
 
   /* If necessary call the cleanup routine after we removed the
      current cleanup block from the list.  */
diff --git a/nptl/libc-cleanup.c b/nptl/libc-cleanup.c
index 61f97eceda..14ccfe9285 100644
--- a/nptl/libc-cleanup.c
+++ b/nptl/libc-cleanup.c
@@ -17,11 +17,69 @@
    <https://www.gnu.org/licenses/>.  */
 
 #include "pthreadP.h"
+#include <tls.h>
+#include <libc-lock.h>
 
+void
+__libc_cleanup_push_defer (struct _pthread_cleanup_buffer *buffer)
+{
+  struct pthread *self = THREAD_SELF;
+
+  buffer->__prev = THREAD_GETMEM (self, cleanup);
+
+  int cancelhandling = THREAD_GETMEM (self, cancelhandling);
+
+  /* Disable asynchronous cancellation for now.  */
+  if (__glibc_unlikely (cancelhandling & CANCELTYPE_BITMASK))
+    while (1)
+      {
+	int curval = THREAD_ATOMIC_CMPXCHG_VAL (self, cancelhandling,
+						cancelhandling
+						& ~CANCELTYPE_BITMASK,
+						cancelhandling);
+	if (__glibc_likely (curval == cancelhandling))
+	  /* Successfully replaced the value.  */
+	  break;
+
+	/* Prepare for the next round.  */
+	cancelhandling = curval;
+      }
+
+  buffer->__canceltype = (cancelhandling & CANCELTYPE_BITMASK
+			  ? PTHREAD_CANCEL_ASYNCHRONOUS
+			  : PTHREAD_CANCEL_DEFERRED);
+
+  THREAD_SETMEM (self, cleanup, buffer);
+}
+libc_hidden_def (__libc_cleanup_push_defer)
 
 void
-__libc_cleanup_routine (struct __pthread_cleanup_frame *f)
+__libc_cleanup_pop_restore (struct _pthread_cleanup_buffer *buffer)
 {
-  if (f->__do_it)
-    f->__cancel_routine (f->__cancel_arg);
+  struct pthread *self = THREAD_SELF;
+
+  THREAD_SETMEM (self, cleanup, buffer->__prev);
+
+  int cancelhandling;
+  if (__builtin_expect (buffer->__canceltype != PTHREAD_CANCEL_DEFERRED, 0)
+      && ((cancelhandling = THREAD_GETMEM (self, cancelhandling))
+	  & CANCELTYPE_BITMASK) == 0)
+    {
+      while (1)
+	{
+	  int curval = THREAD_ATOMIC_CMPXCHG_VAL (self, cancelhandling,
+						  cancelhandling
+						  | CANCELTYPE_BITMASK,
+						  cancelhandling);
+	  if (__glibc_likely (curval == cancelhandling))
+	    /* Successfully replaced the value.  */
+	    break;
+
+	  /* Prepare for the next round.  */
+	  cancelhandling = curval;
+	}
+
+      CANCELLATION_P (self);
+    }
 }
+libc_hidden_def (__libc_cleanup_pop_restore)
diff --git a/nptl/nptl-init.c b/nptl/nptl-init.c
index 2c7e2222d4..43e2564e59 100644
--- a/nptl/nptl-init.c
+++ b/nptl/nptl-init.c
@@ -95,8 +95,6 @@ static const struct pthread_functions pthread_functions =
     .ptr___pthread_key_create = __pthread_key_create,
     .ptr___pthread_getspecific = __pthread_getspecific,
     .ptr___pthread_setspecific = __pthread_setspecific,
-    .ptr__pthread_cleanup_push_defer = __pthread_cleanup_push_defer,
-    .ptr__pthread_cleanup_pop_restore = __pthread_cleanup_pop_restore,
     .ptr_nthreads = &__nptl_nthreads,
     .ptr___pthread_unwind = &__pthread_unwind,
     .ptr__nptl_deallocate_tsd = __nptl_deallocate_tsd,