x86/cet: Enable shadow stack during startup

Previously, CET was enabled by kernel before passing control to user
space and the startup code must disable CET if applications or shared
libraries aren't CET enabled.  Since the current kernel only supports
shadow stack and won't enable shadow stack before passing control to
user space, we need to enable shadow stack during startup if the
application and all shared library are shadow stack enabled.  There
is no need to disable shadow stack at startup.  Shadow stack can only
be enabled in a function which will never return.  Otherwise, shadow
stack will underflow at the function return.

1. GL(dl_x86_feature_1) is set to the CET features which are supported
by the processor and are not disabled by the tunable.  Only non-zero
features in GL(dl_x86_feature_1) should be enabled.  After enabling
shadow stack with ARCH_SHSTK_ENABLE, ARCH_SHSTK_STATUS is used to check
if shadow stack is really enabled.
2. Use ARCH_SHSTK_ENABLE in RTLD_START in dynamic executable.  It is
safe since RTLD_START never returns.
3. Call arch_prctl (ARCH_SHSTK_ENABLE) from ARCH_SETUP_TLS in static
executable.  Since the start function using ARCH_SETUP_TLS never returns,
it is safe to enable shadow stack in ARCH_SETUP_TLS.

3 files changed, 72 insertions, 51 deletions

diff --git a/sysdeps/unix/sysv/linux/x86/cpu-features.c b/sysdeps/unix/sysv/linux/x86/cpu-features.c
deleted file mode 100644
index 0e6e2bf855..0000000000
--- a/sysdeps/unix/sysv/linux/x86/cpu-features.c
+++ /dev/null
@@ -1,49 +0,0 @@
-/* Initialize CPU feature data for Linux/x86.
-   This file is part of the GNU C Library.
-   Copyright (C) 2018-2023 Free Software Foundation, Inc.
-
-   The GNU C Library is free software; you can redistribute it and/or
-   modify it under the terms of the GNU Lesser General Public
-   License as published by the Free Software Foundation; either
-   version 2.1 of the License, or (at your option) any later version.
-
-   The GNU C Library is distributed in the hope that it will be useful,
-   but WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   Lesser General Public License for more details.
-
-   You should have received a copy of the GNU Lesser General Public
-   License along with the GNU C Library; if not, see
-   <https://www.gnu.org/licenses/>.  */
-
-#if CET_ENABLED
-# include <sys/prctl.h>
-# include <asm/prctl.h>
-
-static inline int __attribute__ ((always_inline))
-get_cet_status (void)
-{
-  unsigned long long kernel_feature;
-  unsigned int status = 0;
-  if (INTERNAL_SYSCALL_CALL (arch_prctl, ARCH_SHSTK_STATUS,
-			     &kernel_feature) == 0)
-    {
-      if ((kernel_feature & ARCH_SHSTK_SHSTK) != 0)
-	status = GNU_PROPERTY_X86_FEATURE_1_SHSTK;
-    }
-  return status;
-}
-
-# ifndef SHARED
-static inline void
-x86_setup_tls (void)
-{
-  __libc_setup_tls ();
-  THREAD_SETMEM (THREAD_SELF, header.feature_1, GL(dl_x86_feature_1));
-}
-
-#  define ARCH_SETUP_TLS() x86_setup_tls ()
-# endif
-#endif
-
-#include <sysdeps/x86/cpu-features.c>
diff --git a/sysdeps/unix/sysv/linux/x86/dl-cet.h b/sysdeps/unix/sysv/linux/x86/dl-cet.h
index da220ac627..6ad5e03f69 100644
--- a/sysdeps/unix/sysv/linux/x86/dl-cet.h
+++ b/sysdeps/unix/sysv/linux/x86/dl-cet.h
@@ -18,7 +18,7 @@
 #include <sys/prctl.h>
 #include <asm/prctl.h>
 
-static inline int __attribute__ ((always_inline))
+static __always_inline int
 dl_cet_disable_cet (unsigned int cet_feature)
 {
   if (cet_feature != GNU_PROPERTY_X86_FEATURE_1_SHSTK)
@@ -28,7 +28,7 @@ dl_cet_disable_cet (unsigned int cet_feature)
 				      kernel_feature);
 }
 
-static inline int __attribute__ ((always_inline))
+static __always_inline int
 dl_cet_lock_cet (unsigned int cet_feature)
 {
   if (cet_feature != GNU_PROPERTY_X86_FEATURE_1_SHSTK)
@@ -38,3 +38,26 @@ dl_cet_lock_cet (unsigned int cet_feature)
   return (int) INTERNAL_SYSCALL_CALL (arch_prctl, ARCH_SHSTK_LOCK,
 				      kernel_feature);
 }
+
+static __always_inline unsigned int
+dl_cet_get_cet_status (void)
+{
+  unsigned long long kernel_feature;
+  unsigned int status = 0;
+  if (INTERNAL_SYSCALL_CALL (arch_prctl, ARCH_SHSTK_STATUS,
+			     &kernel_feature) == 0)
+    {
+      if ((kernel_feature & ARCH_SHSTK_SHSTK) != 0)
+	status = GNU_PROPERTY_X86_FEATURE_1_SHSTK;
+    }
+  return status;
+}
+
+/* Enable shadow stack with a macro to avoid shadow stack underflow.  */
+#define ENABLE_X86_CET(cet_feature)				\
+  if ((cet_feature & GNU_PROPERTY_X86_FEATURE_1_SHSTK))		\
+    {								\
+      long long int kernel_feature = ARCH_SHSTK_SHSTK;		\
+      INTERNAL_SYSCALL_CALL (arch_prctl, ARCH_SHSTK_ENABLE,	\
+			     kernel_feature);			\
+    }
diff --git a/sysdeps/unix/sysv/linux/x86_64/dl-cet.h b/sysdeps/unix/sysv/linux/x86_64/dl-cet.h
new file mode 100644
index 0000000000..e23e05c6b8
--- /dev/null
+++ b/sysdeps/unix/sysv/linux/x86_64/dl-cet.h
@@ -0,0 +1,47 @@
+/* Linux/x86-64 CET initializers function.
+   Copyright (C) 2023 Free Software Foundation, Inc.
+
+   The GNU C Library is free software; you can redistribute it and/or
+   modify it under the terms of the GNU Lesser General Public
+   License as published by the Free Software Foundation; either
+   version 2.1 of the License, or (at your option) any later version.
+
+   The GNU C Library is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+   Lesser General Public License for more details.
+
+   You should have received a copy of the GNU Lesser General Public
+   License along with the GNU C Library; if not, see
+   <https://www.gnu.org/licenses/>.  */
+
+#include <cpu-features-offsets.h>
+#include_next <dl-cet.h>
+
+#define X86_STRINGIFY_1(x)	#x
+#define X86_STRINGIFY(x)	X86_STRINGIFY_1 (x)
+
+/* Enable shadow stack before calling _dl_init if it is enabled in
+   GL(dl_x86_feature_1).  Call _dl_setup_x86_features to setup shadow
+   stack.  */
+#define RTLD_START_ENABLE_X86_FEATURES \
+"\
+	# Check if shadow stack is enabled in GL(dl_x86_feature_1).\n\
+	movl _rtld_local+" X86_STRINGIFY (RTLD_GLOBAL_DL_X86_FEATURE_1_OFFSET) "(%rip), %edx\n\
+	testl $" X86_STRINGIFY (X86_FEATURE_1_SHSTK) ", %edx\n\
+	jz 1f\n\
+	# Enable shadow stack if enabled in GL(dl_x86_feature_1).\n\
+	movl $" X86_STRINGIFY (ARCH_SHSTK_SHSTK) ", %esi\n\
+	movl $" X86_STRINGIFY (ARCH_SHSTK_ENABLE) ", %edi\n\
+	movl $" X86_STRINGIFY (__NR_arch_prctl) ", %eax\n\
+	syscall\n\
+1:\n\
+	# Pass GL(dl_x86_feature_1) to _dl_cet_setup_features.\n\
+	movl %edx, %edi\n\
+	# Align stack for the _dl_cet_setup_features call.\n\
+	andq $-16, %rsp\n\
+	call _dl_cet_setup_features\n\
+	# Restore %rax and %rsp from %r12 and %r13.\n\
+	movq %r12, %rax\n\
+	movq %r13, %rsp\n\
+"