linux ttyname{_r}: Make tty checks consistent

In the ttyname and ttyname_r routines on Linux, at several points it needs to
check if a given TTY is the TTY we are looking for. It used to be that this
check was (to see if `maybe` is `mytty`):

       __xstat64(_STAT_VER, maybe_filename, &maybe) == 0
    #ifdef _STATBUF_ST_RDEV
       && S_ISCHR(maybe.st_mode) && maybe.st_rdev == mytty.st_rdev
    #else
       && maybe.st_ino == mytty.st_ino && maybe.st_dev == mytty.st_dev
    #endif

This check appears in several places.

Then, one of the changes made in commit 15e9a4f378c8607c2ae1aa465436af4321db0e23
was to change that check to:

       __xstat64(_STAT_VER, maybe_filename, &maybe) == 0
    #ifdef _STATBUF_ST_RDEV
       && S_ISCHR(maybe.st_mode) && maybe.st_rdev == mytty.st_rdev
    #endif
       && maybe.st_ino == mytty.st_ino && maybe.st_dev == mytty.st_dev

That is, it made the st_ino and st_dev parts of the check happen even if we have
the st_rdev member. This is an important change, because the kernel allows
multiple devpts filesystem instances to be created; a device file in one devpts
instance may share the same st_rdev with a file in another devpts instance, but
they aren't the same file.

This check appears twice in each file (ttyname.c and ttyname_r.c), once (in
ttyname and __ttyname_r) to check if a candidate file found by inspecting /proc
is the desired TTY, and once (in getttyname and getttyname_r) to check if a
candidate file found by searching /dev is the desired TTY. However, 15e9a4f
only updated the checks for files found via /proc; but the concern about
collisions between devpts instances is just as valid for files found via /dev.

So, update all 4 occurrences the check to be consistent with the version of the
check introduced in 15e9a4f. Make it easy to keep all 4 occurrences of the
check consistent by pulling it in to a static inline function, is_mytty.

Reviewed-by: Christian Brauner <christian.brauner@ubuntu.com>

1 files changed, 8 insertions, 33 deletions

diff --git a/sysdeps/unix/sysv/linux/ttyname_r.c b/sysdeps/unix/sysv/linux/ttyname_r.c
index 18f35ef2b7..58eb919c3f 100644
--- a/sysdeps/unix/sysv/linux/ttyname_r.c
+++ b/sysdeps/unix/sysv/linux/ttyname_r.c
@@ -31,12 +31,12 @@
 #include "ttyname.h"
 
 static int getttyname_r (char *buf, size_t buflen,
-			 dev_t mydev, ino64_t myino, int save,
+			 const struct stat64 *mytty, int save,
 			 int *dostat);
 
 static int
 attribute_compat_text_section
-getttyname_r (char *buf, size_t buflen, dev_t mydev, ino64_t myino,
+getttyname_r (char *buf, size_t buflen, const struct stat64 *mytty,
 	      int save, int *dostat)
 {
   struct stat64 st;
@@ -52,7 +52,7 @@ getttyname_r (char *buf, size_t buflen, dev_t mydev, ino64_t myino,
     }
 
   while ((d = __readdir64 (dirstream)) != NULL)
-    if ((d->d_fileno == myino || *dostat)
+    if ((d->d_fileno == mytty->st_ino || *dostat)
 	&& strcmp (d->d_name, "stdin")
 	&& strcmp (d->d_name, "stdout")
 	&& strcmp (d->d_name, "stderr"))
@@ -72,12 +72,7 @@ getttyname_r (char *buf, size_t buflen, dev_t mydev, ino64_t myino,
 	cp[0] = '\0';
 
 	if (__xstat64 (_STAT_VER, buf, &st) == 0
-#ifdef _STATBUF_ST_RDEV
-	    && S_ISCHR (st.st_mode) && st.st_rdev == mydev
-#else
-	    && d->d_fileno == myino && st.st_dev == mydev
-#endif
-	   )
+	    && is_mytty (mytty, &st))
 	  {
 	    (void) __closedir (dirstream);
 	    __set_errno (save);
@@ -151,12 +146,7 @@ __ttyname_r (int fd, char *buf, size_t buflen)
       /* Verify readlink result, fall back on iterating through devices.  */
       if (buf[0] == '/'
 	  && __xstat64 (_STAT_VER, buf, &st1) == 0
-#ifdef _STATBUF_ST_RDEV
-	  && S_ISCHR (st1.st_mode)
-	  && st1.st_rdev == st.st_rdev
-#endif
-	  && st1.st_ino == st.st_ino
-	  && st1.st_dev == st.st_dev)
+	  && is_mytty (&st, &st1))
 	return 0;
 
       /* If the link doesn't exist, then it points to a device in another
@@ -175,13 +165,8 @@ __ttyname_r (int fd, char *buf, size_t buflen)
 
   if (__xstat64 (_STAT_VER, buf, &st1) == 0 && S_ISDIR (st1.st_mode))
     {
-#ifdef _STATBUF_ST_RDEV
-      ret = getttyname_r (buf, buflen, st.st_rdev, st.st_ino, save,
+      ret = getttyname_r (buf, buflen, &st, save,
 			  &dostat);
-#else
-      ret = getttyname_r (buf, buflen, st.st_dev, st.st_ino, save,
-			  &dostat);
-#endif
     }
   else
     {
@@ -193,26 +178,16 @@ __ttyname_r (int fd, char *buf, size_t buflen)
     {
       buf[sizeof ("/dev/") - 1] = '\0';
       buflen += sizeof ("pts/") - 1;
-#ifdef _STATBUF_ST_RDEV
-      ret = getttyname_r (buf, buflen, st.st_rdev, st.st_ino, save,
-			  &dostat);
-#else
-      ret = getttyname_r (buf, buflen, st.st_dev, st.st_ino, save,
+      ret = getttyname_r (buf, buflen, &st, save,
 			  &dostat);
-#endif
     }
 
   if (ret && dostat != -1)
     {
       buf[sizeof ("/dev/") - 1] = '\0';
       dostat = 1;
-#ifdef _STATBUF_ST_RDEV
-      ret = getttyname_r (buf, buflen, st.st_rdev, st.st_ino,
-			  save, &dostat);
-#else
-      ret = getttyname_r (buf, buflen, st.st_dev, st.st_ino,
+      ret = getttyname_r (buf, buflen, &st,
 			  save, &dostat);
-#endif
     }
 
   return ret;