Linux: Support __IPC_64 in sysvctl *ctl command arguments (bug 29771)

Old applications pass __IPC_64 as part of the command argument because
old glibc did not check for unknown commands, and passed through the
arguments directly to the kernel, without adding __IPC_64.
Applications need to continue doing that for old glibc compatibility,
so this commit enables this approach in current glibc.

For msgctl and shmctl, if no translation is required, make
direct system calls, as we did before the time64 changes.  If
translation is required, mask __IPC_64 from the command argument.

For semctl, the union-in-vararg argument handling means that
translation is needed on all architectures.

Reviewed-by: Adhemerval Zanella  <adhemerval.zanella@linaro.org>
(cherry picked from commit 22a46dee24351fd5f4f188ad80554cad79c82524)

5 files changed, 64 insertions, 26 deletions

diff --git a/NEWS b/NEWS
index ed59136e55..1864714d29 100644
--- a/NEWS
+++ b/NEWS
@@ -709,6 +709,7 @@ The following bugs are resolved with this release:
   [27237] malloc: deadlock in malloc/tst-malloc-stats-cancellation
   [27256] locale: Assertion failure in ISO-2022-JP-3 gconv module
     related to combining characters (CVE-2021-3326)
+  [29771] Restore IPC_64 support in sysvipc *ctl functions
 
 
 Version 2.32
diff --git a/sysdeps/unix/sysv/linux/ipc_priv.h b/sysdeps/unix/sysv/linux/ipc_priv.h
index f9852367a4..d4efb9f348 100644
--- a/sysdeps/unix/sysv/linux/ipc_priv.h
+++ b/sysdeps/unix/sysv/linux/ipc_priv.h
@@ -63,4 +63,10 @@ struct __old_ipc_perm
 # define __IPC_TIME64 0
 #endif
 
+#if __IPC_TIME64 || defined __ASSUME_SYSVIPC_BROKEN_MODE_T
+# define IPC_CTL_NEED_TRANSLATION 1
+#else
+# define IPC_CTL_NEED_TRANSLATION 0
+#endif
+
 #include <ipc_ops.h>
diff --git a/sysdeps/unix/sysv/linux/msgctl.c b/sysdeps/unix/sysv/linux/msgctl.c
index 9f38c06d53..ba7b94c22d 100644
--- a/sysdeps/unix/sysv/linux/msgctl.c
+++ b/sysdeps/unix/sysv/linux/msgctl.c
@@ -86,11 +86,19 @@ msgctl_syscall (int msqid, int cmd, msgctl_arg_t *buf)
 int
 __msgctl64 (int msqid, int cmd, struct __msqid64_ds *buf)
 {
-#if __IPC_TIME64
+#if IPC_CTL_NEED_TRANSLATION
+# if __IPC_TIME64
   struct kernel_msqid64_ds ksemid, *arg = NULL;
-#else
+# else
   msgctl_arg_t *arg;
-#endif
+# endif
+
+  /* Some applications pass the __IPC_64 flag in cmd, to invoke
+     previously unsupported commands back when there was no EINVAL
+     error checking in glibc.  Mask the flag for the switch statements
+     below.  msgctl_syscall adds back the __IPC_64 flag for the actual
+     system call.  */
+  cmd &= ~__IPC_64;
 
   switch (cmd)
     {
@@ -102,19 +110,19 @@ __msgctl64 (int msqid, int cmd, struct __msqid64_ds *buf)
     case IPC_STAT:
     case MSG_STAT:
     case MSG_STAT_ANY:
-#if __IPC_TIME64
+# if __IPC_TIME64
       if (buf != NULL)
 	{
 	  msqid64_to_kmsqid64 (buf, &ksemid);
 	  arg = &ksemid;
 	}
-# ifdef __ASSUME_SYSVIPC_BROKEN_MODE_T
+#  ifdef __ASSUME_SYSVIPC_BROKEN_MODE_T
       if (cmd == IPC_SET)
 	arg->msg_perm.mode *= 0x10000U;
-# endif
-#else
+#  endif
+# else
       arg = buf;
-#endif
+# endif
       break;
 
     case IPC_INFO:
@@ -138,21 +146,25 @@ __msgctl64 (int msqid, int cmd, struct __msqid64_ds *buf)
     case IPC_STAT:
     case MSG_STAT:
     case MSG_STAT_ANY:
-#ifdef __ASSUME_SYSVIPC_BROKEN_MODE_T
+# ifdef __ASSUME_SYSVIPC_BROKEN_MODE_T
       arg->msg_perm.mode >>= 16;
-#else
+# else
       /* Old Linux kernel versions might not clear the mode padding.  */
       if (sizeof ((struct msqid_ds){0}.msg_perm.mode)
           != sizeof (__kernel_mode_t))
 	arg->msg_perm.mode &= 0xFFFF;
-#endif
+# endif
 
-#if __IPC_TIME64
+# if __IPC_TIME64
       kmsqid64_to_msqid64 (arg, buf);
-#endif
+# endif
     }
 
   return ret;
+
+#else /* !IPC_CTL_NEED_TRANSLATION */
+  return msgctl_syscall (msqid, cmd, buf);
+#endif
 }
 #if __TIMESIZE != 64
 libc_hidden_def (__msgctl64)
diff --git a/sysdeps/unix/sysv/linux/semctl.c b/sysdeps/unix/sysv/linux/semctl.c
index bb2690d30f..97fa411547 100644
--- a/sysdeps/unix/sysv/linux/semctl.c
+++ b/sysdeps/unix/sysv/linux/semctl.c
@@ -141,6 +141,13 @@ __semctl64 (int semid, int semnum, int cmd, ...)
   union semun64 arg64 = { 0 };
   va_list ap;
 
+  /* Some applications pass the __IPC_64 flag in cmd, to invoke
+     previously unsupported commands back when there was no EINVAL
+     error checking in glibc.  Mask the flag for the switch statements
+     below.  semctl_syscall adds back the __IPC_64 flag for the actual
+     system call.  */
+  cmd &= ~__IPC_64;
+
   /* Get the argument only if required.  */
   switch (cmd)
     {
diff --git a/sysdeps/unix/sysv/linux/shmctl.c b/sysdeps/unix/sysv/linux/shmctl.c
index f52018bfae..c44cbd6e4a 100644
--- a/sysdeps/unix/sysv/linux/shmctl.c
+++ b/sysdeps/unix/sysv/linux/shmctl.c
@@ -86,11 +86,19 @@ shmctl_syscall (int shmid, int cmd, shmctl_arg_t *buf)
 int
 __shmctl64 (int shmid, int cmd, struct __shmid64_ds *buf)
 {
-#if __IPC_TIME64
+#if IPC_CTL_NEED_TRANSLATION
+# if __IPC_TIME64
   struct kernel_shmid64_ds kshmid, *arg = NULL;
-#else
+# else
   shmctl_arg_t *arg;
-#endif
+# endif
+
+  /* Some applications pass the __IPC_64 flag in cmd, to invoke
+     previously unsupported commands back when there was no EINVAL
+     error checking in glibc.  Mask the flag for the switch statements
+     below.  shmctl_syscall adds back the __IPC_64 flag for the actual
+     system call.  */
+  cmd &= ~__IPC_64;
 
   switch (cmd)
     {
@@ -104,19 +112,19 @@ __shmctl64 (int shmid, int cmd, struct __shmid64_ds *buf)
     case IPC_STAT:
     case SHM_STAT:
     case SHM_STAT_ANY:
-#if __IPC_TIME64
+# if __IPC_TIME64
       if (buf != NULL)
 	{
 	  shmid64_to_kshmid64 (buf, &kshmid);
 	  arg = &kshmid;
 	}
-# ifdef __ASSUME_SYSVIPC_BROKEN_MODE_T
+#  ifdef __ASSUME_SYSVIPC_BROKEN_MODE_T
       if (cmd == IPC_SET)
         arg->shm_perm.mode *= 0x10000U;
-# endif
-#else
+#  endif
+# else
       arg = buf;
-#endif
+# endif
       break;
 
     case IPC_INFO:
@@ -141,21 +149,25 @@ __shmctl64 (int shmid, int cmd, struct __shmid64_ds *buf)
       case IPC_STAT:
       case SHM_STAT:
       case SHM_STAT_ANY:
-#ifdef __ASSUME_SYSVIPC_BROKEN_MODE_T
+# ifdef __ASSUME_SYSVIPC_BROKEN_MODE_T
         arg->shm_perm.mode >>= 16;
-#else
+# else
       /* Old Linux kernel versions might not clear the mode padding.  */
       if (sizeof ((struct shmid_ds){0}.shm_perm.mode)
 	  != sizeof (__kernel_mode_t))
 	arg->shm_perm.mode &= 0xFFFF;
-#endif
+# endif
 
-#if __IPC_TIME64
+# if __IPC_TIME64
       kshmid64_to_shmid64 (arg, buf);
-#endif
+# endif
     }
 
   return ret;
+
+#else /* !IPC_CTL_NEED_TRANSLATION */
+  return shmctl_syscall (shmid, cmd, buf);
+#endif
 }
 #if __TIMESIZE != 64
 libc_hidden_def (__shmctl64)