aboutsummaryrefslogtreecommitdiff
path: root/conform/conformtest.pl
diff options
context:
space:
mode:
authorJoseph Myers <joseph@codesourcery.com>2015-06-19 20:05:41 +0000
committerJoseph Myers <joseph@codesourcery.com>2015-06-19 20:05:41 +0000
commit5675961d15cf3bb9bf7b0096704d5fe09569e875 (patch)
tree11218bec533c538c61221ce8e5cc69662fce8bd9 /conform/conformtest.pl
parentc34248820017fc9f132965aeb36f795e4298f2c0 (diff)
downloadglibc-5675961d15cf3bb9bf7b0096704d5fe09569e875.tar
glibc-5675961d15cf3bb9bf7b0096704d5fe09569e875.tar.gz
glibc-5675961d15cf3bb9bf7b0096704d5fe09569e875.tar.bz2
glibc-5675961d15cf3bb9bf7b0096704d5fe09569e875.zip
conformtest: Support xfail markers on individual assertions.
Various conformtest tests fail because of known issues, filed in Bugzilla, that are hard to fix (requiring new features, kernel cooperation or involving changing types in ways that may involve care around padding when interfacing to the kernel). Such an issue has the effect of making the whole test for the (standard, header) pair fail, so hiding any other issues with that header for that standard (possibly regressions or architecture-specific issues). This patch adds a mechanism for individual conformtest test expectations to start with xfail-, meaning that a failure of that particular assertion does not cause the whole test to fail at the makefile level and so failure at the makefile level can be used to detect other issues that are likely to be easier to fix. This is similar to the whitelisted symbols in the linknamespace tests, or the marking of particular libm tests as allowing spurious or missing exceptions, for example. The bugs filed in Bugzilla should still be fixed, and the xfail- markers removed at that point, but xfail- renders the tests more useful until that happens. Note that there is no way to add such a marker for the assertion that a header uses only symbols in the namespace of symbols it's meant to use. I don't think there's any need for a way to xfail those namespace tests other than xfailing the whole (standard, header) pair at the makefile level, since they are generally straightforward to fix (add appropriate conditionals on the problem definitions). The xfails in this patch do not necessarily cover all cases of hard-to-fix header bugs filed in Bugzilla that currently show up in conformtest failures; there may be more yet to add for existing open bugs. Tested for x86_64 and x86. * conform/conformtest.pl ($xerrors): New variable. (note_error): New function. (compiletest): New argument $xfail. Use not_error. (runtest): Likewise. (top level): Handle xfail- lines. Update calls to compiletest and runtest. Handle xfail- and optional- in headers listed with allow-header. * conform/data/fcntl.h-data (O_TTY_INIT): Use xfail-. (O_EXEC): Likewise. (O_SEARCH): Likewise. * conform/data/stropts.h-data (ioctl): Likewise. * conform/data/sys/ipc.h-data (ipc_perm.mode): Likewise. * conform/data/sys/sem.h-data (semid_ds.sem_nsems): Likewise. * conform/data/sys/socket.h-data (msghdr.msg_iovlen): Likewise. (msghdr.msg_controllen): Likewise. (cmsghdr.cmsg_len): Likewise. * conform/data/utmpx.h-data (utmpx.ut_tv): Likewise. * conform/Makefile (test-xfail-XPG3/sys/ipc.h/conform): Remove variable. (test-xfail-XPG3/sys/sem.h/conform): Likewise. (test-xfail-XPG4/stropts.h/conform): Likewise. (test-xfail-XPG4/sys/ipc.h/conform): Likewise. (test-xfail-XPG4/sys/sem.h/conform): Likewise. (test-xfail-XPG4/sys/socket.h/conform): Likewise. (test-xfail-UNIX98/stropts.h/conform): Likewise. (test-xfail-UNIX98/sys/ipc.h/conform): Likewise. (test-xfail-UNIX98/sys/sem.h/conform): Likewise. (test-xfail-UNIX98/sys/socket.h/conform): Likewise. (test-xfail-XOPEN2K/stropts.h/conform): Likewise. (test-xfail-XOPEN2K/sys/ipc.h/conform): Likewise. (test-xfail-XOPEN2K/sys/sem.h/conform): Likewise. (test-xfail-XOPEN2K/sys/socket.h/conform): Likewise. (test-xfail-XOPEN2K/utmpx.h/conform): Likewise. (test-xfail-POSIX2008/fcntl.h/conform): Likewise. (test-xfail-POSIX2008/stropts.h/conform): Likewise. (test-xfail-XOPEN2K8/fcntl.h/conform): Likewise. (test-xfail-XOPEN2K8/stropts.h/conform): Likewise. (test-xfail-XOPEN2K8/sys/ipc.h/conform): Likewise. (test-xfail-XOPEN2K8/sys/sem.h/conform): Likewise. (test-xfail-XOPEN2K8/sys/socket.h/conform): Likewise.
Diffstat (limited to 'conform/conformtest.pl')
-rw-r--r--conform/conformtest.pl88
1 files changed, 61 insertions, 27 deletions
diff --git a/conform/conformtest.pl b/conform/conformtest.pl
index 8d61016bbd..a8a27f5d2a 100644
--- a/conform/conformtest.pl
+++ b/conform/conformtest.pl
@@ -75,6 +75,17 @@ $verbose = 1;
$total = 0;
$skipped = 0;
$errors = 0;
+$xerrors = 0;
+
+sub note_error {
+ my($xfail) = @_;
+ if ($xfail) {
+ $xerrors++;
+ printf ("Ignoring this failure.\n");
+ } else {
+ $errors++;
+ }
+}
sub poorfnmatch {
@@ -107,7 +118,7 @@ sub poorfnmatch {
sub compiletest
{
- my($fnamebase, $msg, $errmsg, $skip, $optional) = @_;
+ my($fnamebase, $msg, $errmsg, $skip, $optional, $xfail) = @_;
my($result) = $skip;
my($printlog) = 0;
@@ -129,7 +140,7 @@ sub compiletest
printf (" $errmsg Compiler message:\n");
$printlog = 1;
}
- ++$errors;
+ note_error($xfail);
$result = 1;
}
} else {
@@ -159,7 +170,7 @@ sub compiletest
sub runtest
{
- my($fnamebase, $msg, $errmsg, $skip) = @_;
+ my($fnamebase, $msg, $errmsg, $skip, $xfail) = @_;
my($result) = $skip;
my($printlog) = 0;
@@ -177,7 +188,7 @@ sub runtest
printf (" $errmsg Compiler message:\n");
$printlog = 1;
}
- ++$errors;
+ note_error($xfail);
$result = 1;
} else {
# Now run the program. If the exit code is not zero something is wrong.
@@ -191,7 +202,7 @@ sub runtest
}
} else {
printf (" FAIL\n");
- ++$errors;
+ note_error($xfail);
$printlog = 1;
unlink "$fnamebase.out";
rename "$fnamebase.out2", "$fnamebase.out";
@@ -328,11 +339,16 @@ while ($#headers >= 0) {
close (TESTFILE);
$missing = compiletest ($fnamebase, "Checking whether <$h> is available",
- "Header <$h> not available", 0, 0);
+ "Header <$h> not available", 0, 0, 0);
printf ("\n");
last control if ($missing);
}
+ my($xfail) = 0;
+ if (/^xfail-/) {
+ s/^xfail-//;
+ $xfail = 1;
+ }
my($optional) = 0;
if (/^optional-/) {
s/^optional-//;
@@ -364,7 +380,7 @@ while ($#headers >= 0) {
($optional
? "NOT AVAILABLE."
: "Member \"$member\" not available."), $res,
- $optional);
+ $optional, $xfail);
if ($res == 0 || $missing != 0 || !$optional) {
# Test the types of the members.
@@ -378,7 +394,7 @@ while ($#headers >= 0) {
compiletest ($fnamebase, "Testing for type of member $member",
"Member \"$member\" does not have the correct type.",
- $res, 0);
+ $res, 0, $xfail);
}
} elsif (/^(macro|constant|macro-constant|macro-int-constant) +([a-zA-Z0-9_]*) *(?:{([^}]*)} *)?(?:([>=<!]+) ([A-Za-z0-9_\\'-]*))?/) {
my($symbol_type) = $1;
@@ -407,7 +423,7 @@ while ($#headers >= 0) {
($optional
? "NOT PRESENT"
: "Macro \"$symbol\" is not available."), $res,
- $optional);
+ $optional, $xfail);
}
if ($symbol_type =~ /constant/) {
@@ -422,7 +438,7 @@ while ($#headers >= 0) {
($optional
? "NOT PRESENT"
: "Constant \"$symbol\" not available."), $res,
- $optional);
+ $optional, $xfail);
}
$res = $res || $mres || $cres;
@@ -461,7 +477,7 @@ while ($#headers >= 0) {
close (TESTFILE);
runtest ($fnamebase, "Testing for #if usability of symbol $symbol",
- "Symbol \"$symbol\" not usable in #if.", $res);
+ "Symbol \"$symbol\" not usable in #if.", $res, $xfail);
}
if (defined ($type) && ($res == 0 || !$optional)) {
@@ -480,7 +496,7 @@ while ($#headers >= 0) {
compiletest ($fnamebase, "Testing for type of symbol $symbol",
"Symbol \"$symbol\" does not have the correct type.",
- $res, 0);
+ $res, 0, $xfail);
}
if (defined ($op) && ($res == 0 || !$optional)) {
@@ -493,7 +509,8 @@ while ($#headers >= 0) {
close (TESTFILE);
$res = runtest ($fnamebase, "Testing for value of symbol $symbol",
- "Symbol \"$symbol\" has not the right value.", $res);
+ "Symbol \"$symbol\" has not the right value.", $res,
+ $xfail);
}
} elsif (/^symbol *([a-zA-Z0-9_]*) *([A-Za-z0-9_-]*)?/) {
my($symbol) = $1;
@@ -513,7 +530,7 @@ while ($#headers >= 0) {
close (TESTFILE);
$res = compiletest ($fnamebase, "Testing for symbol $symbol",
- "Symbol \"$symbol\" not available.", $res, 0);
+ "Symbol \"$symbol\" not available.", $res, 0, $xfail);
if ($value ne "") {
# Generate a program to test for the value of this constant.
@@ -524,7 +541,8 @@ while ($#headers >= 0) {
close (TESTFILE);
$res = runtest ($fnamebase, "Testing for value of symbol $symbol",
- "Symbol \"$symbol\" has not the right value.", $res);
+ "Symbol \"$symbol\" has not the right value.", $res,
+ $xfail);
}
} elsif (/^type *({([^}]*)|([a-zA-Z0-9_]*))/) {
my($type) = "$2$3";
@@ -554,7 +572,8 @@ while ($#headers >= 0) {
compiletest ($fnamebase, "Testing for type $type",
($optional
? "NOT AVAILABLE"
- : "Type \"$type\" not available."), $missing, $optional);
+ : "Type \"$type\" not available."), $missing, $optional,
+ $xfail);
} elsif (/^tag *({([^}]*)|([a-zA-Z0-9_]*))/) {
my($type) = "$2$3";
@@ -575,7 +594,7 @@ while ($#headers >= 0) {
close (TESTFILE);
compiletest ($fnamebase, "Testing for type $type",
- "Type \"$type\" not available.", $missing, 0);
+ "Type \"$type\" not available.", $missing, 0, $xfail);
} elsif (/^function *({([^}]*)}|([a-zA-Z0-9_]*)) [(][*]([a-zA-Z0-9_]*) ([(].*[)])/) {
my($rettype) = "$2$3";
my($fname) = "$4";
@@ -597,7 +616,7 @@ while ($#headers >= 0) {
($optional
? "NOT AVAILABLE"
: "Function \"$fname\" is not available."), $res,
- $optional);
+ $optional, $xfail);
if ($res == 0 || $missing == 1 || !$optional) {
# Generate a program to test for the type of this function.
@@ -610,7 +629,8 @@ while ($#headers >= 0) {
close (TESTFILE);
compiletest ($fnamebase, "Test for type of function $fname",
- "Function \"$fname\" has incorrect type.", $res, 0);
+ "Function \"$fname\" has incorrect type.", $res, 0,
+ $xfail);
}
} elsif (/^function *({([^}]*)}|([a-zA-Z0-9_]*)) ([a-zA-Z0-9_]*) ([(].*[)])/) {
my($rettype) = "$2$3";
@@ -633,7 +653,7 @@ while ($#headers >= 0) {
($optional
? "NOT AVAILABLE"
: "Function \"$fname\" is not available."), $res,
- $optional);
+ $optional, $xfail);
if ($res == 0 || $missing != 0 || !$optional) {
# Generate a program to test for the type of this function.
@@ -646,7 +666,8 @@ while ($#headers >= 0) {
close (TESTFILE);
compiletest ($fnamebase, "Test for type of function $fname",
- "Function \"$fname\" has incorrect type.", $res, 0);
+ "Function \"$fname\" has incorrect type.", $res, 0,
+ $xfail);
}
} elsif (/^variable *({([^}]*)}|([a-zA-Z0-9_]*)) ([a-zA-Z0-9_]*) *(.*)/) {
my($type) = "$2$3";
@@ -667,7 +688,8 @@ while ($#headers >= 0) {
close (TESTFILE);
$res = compiletest ($fnamebase, "Test availability of variable $vname",
- "Variable \"$vname\" is not available.", $res, 0);
+ "Variable \"$vname\" is not available.", $res, 0,
+ $xfail);
# Generate a program to test for the type of this function.
open (TESTFILE, ">$fnamebase.c");
@@ -678,7 +700,7 @@ while ($#headers >= 0) {
close (TESTFILE);
compiletest ($fnamebase, "Test for type of variable $fname",
- "Variable \"$vname\" has incorrect type.", $res, 0);
+ "Variable \"$vname\" has incorrect type.", $res, 0, $xfail);
} elsif (/^macro-function *({([^}]*)}|([a-zA-Z0-9_]*)) ([a-zA-Z0-9_]*) ([(].*[)])/) {
my($rettype) = "$2$3";
my($fname) = "$4";
@@ -698,7 +720,8 @@ while ($#headers >= 0) {
close (TESTFILE);
$res = compiletest ($fnamebase, "Test availability of macro $fname",
- "Function \"$fname\" is not available.", $res, 0);
+ "Function \"$fname\" is not available.", $res, 0,
+ $xfail);
# Generate a program to test for the type of this function.
open (TESTFILE, ">$fnamebase.c");
@@ -711,7 +734,7 @@ while ($#headers >= 0) {
close (TESTFILE);
compiletest ($fnamebase, "Test for type of macro $fname",
- "Function \"$fname\" has incorrect type.", $res, 0);
+ "Function \"$fname\" has incorrect type.", $res, 0, $xfail);
} elsif (/^macro-str *([^ ]*) *(\".*\")/) {
# The above regex doesn't handle a \" in a string.
my($macro) = "$1";
@@ -731,7 +754,7 @@ while ($#headers >= 0) {
close (TESTFILE);
compiletest ($fnamebase, "Test availability of macro $macro",
- "Macro \"$macro\" is not available.", $missing, 0);
+ "Macro \"$macro\" is not available.", $missing, 0, $xfail);
# Generate a program to test for the value of this macro.
open (TESTFILE, ">$fnamebase.c");
@@ -743,7 +766,8 @@ while ($#headers >= 0) {
close (TESTFILE);
$res = runtest ($fnamebase, "Testing for value of macro $macro",
- "Macro \"$macro\" has not the right value.", $res);
+ "Macro \"$macro\" has not the right value.", $res,
+ $xfail);
} elsif (/^allow-header *(.*)/) {
my($pattern) = $1;
if ($seenheader{$pattern} != 1) {
@@ -774,6 +798,8 @@ while ($#headers >= 0) {
next acontrol if (/^#/);
next acontrol if (/^[ ]*$/);
+ s/^xfail-//;
+ s/^optional-//;
if (/^element *({([^}]*)}|([^ ]*)) *({([^}]*)}|([^ ]*)) *([A-Za-z0-9_]*) *(.*)/) {
push @allow, $7;
} elsif (/^(macro|constant|macro-constant|macro-int-constant) +([a-zA-Z0-9_]*) *(?:{([^}]*)} *)?(?:([>=<!]+) ([A-Za-z0-9_-]*))?/) {
@@ -838,6 +864,14 @@ if ($errors > 0 && $percent < 1.0) {
printf ("%3d%%)\n", $percent);
}
+printf (" Number of xfailed tests : %4d (", $xerrors);
+$percent = ($xerrors * 100) / $total;
+if ($xerrors > 0 && $percent < 1.0) {
+ printf (" <1%%)\n");
+} else {
+ printf ("%3d%%)\n", $percent);
+}
+
printf (" Number of skipped tests : %4d (", $skipped);
$percent = ($skipped * 100) / $total;
if ($skipped > 0 && $percent < 1.0) {