rt: fix shm_open not set ENAMETOOLONG when name exceeds {_POSIX_PATH_MAX}

according to man-pages-posix-2017, shm_open() function may fail if the length
of the name argument exceeds {_POSIX_PATH_MAX} and set ENAMETOOLONG

Signed-off-by: abushwang <abushwangs@gmail.com>
Reviewed-by: Adhemerval Zanella  <adhemerval.zanella@linaro.org>

3 files changed, 15 insertions, 7 deletions

diff --git a/posix/shm-directory.c b/posix/shm-directory.c
index 86d9fd8e4f..f130bab3c2 100644
--- a/posix/shm-directory.c
+++ b/posix/shm-directory.c
@@ -25,6 +25,7 @@
 #include <string.h>
 #include <sys/mman.h>
 #include <fcntl.h>
+#include <errno.h>
 
 int
 __shm_get_name (struct shmdir_name *result, const char *name, bool sem_prefix)
@@ -54,9 +55,14 @@ __shm_get_name (struct shmdir_name *result, const char *name, bool sem_prefix)
   if (sem_prefix)
     alloc_buffer_copy_bytes (&buffer, "sem.", strlen ("sem."));
   alloc_buffer_copy_bytes (&buffer, name, namelen + 1);
-  if (namelen == 0 || memchr (name, '/', namelen) != NULL
-      || alloc_buffer_has_failed (&buffer))
-    return -1;
+  if (namelen == 0 || memchr (name, '/', namelen) != NULL)
+    return EINVAL;
+  if (alloc_buffer_has_failed (&buffer))
+    {
+      if (namelen > NAME_MAX)
+        return ENAMETOOLONG;
+      return EINVAL;
+    }
   return 0;
 }
 libc_hidden_def (__shm_get_name)
diff --git a/rt/shm_open.c b/rt/shm_open.c
index 6c1f4d604f..fc1dc96bb4 100644
--- a/rt/shm_open.c
+++ b/rt/shm_open.c
@@ -30,9 +30,10 @@ int
 __shm_open (const char *name, int oflag, mode_t mode)
 {
   struct shmdir_name dirname;
-  if (__shm_get_name (&dirname, name, false) != 0)
+  int ret =__shm_get_name (&dirname, name, false);
+  if (ret != 0)
     {
-      __set_errno (EINVAL);
+      __set_errno (ret);
       return -1;
     }
 
diff --git a/sysdeps/pthread/sem_open.c b/sysdeps/pthread/sem_open.c
index 2551b035e1..2d32a13557 100644
--- a/sysdeps/pthread/sem_open.c
+++ b/sysdeps/pthread/sem_open.c
@@ -47,9 +47,10 @@ __sem_open (const char *name, int oflag, ...)
     }
 
   struct shmdir_name dirname;
-  if (__shm_get_name (&dirname, name, true) != 0)
+  int ret = __shm_get_name (&dirname, name, true);
+  if (ret != 0)
     {
-      __set_errno (EINVAL);
+      __set_errno (ret);
       return SEM_FAILED;
     }