aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorJoseph Myers <joseph@codesourcery.com>2014-02-27 03:25:27 +0000
committerJoseph Myers <joseph@codesourcery.com>2014-02-27 03:25:27 +0000
commitf8c17e79fab13a3d1de976c1c3564df7f8c2a175 (patch)
tree3192a7221761cb0a42279698044f338c961ccc1e
parenta5f891ac8df09e56ecc64bf18a839a06470deacd (diff)
downloadglibc-f8c17e79fab13a3d1de976c1c3564df7f8c2a175.tar
glibc-f8c17e79fab13a3d1de976c1c3564df7f8c2a175.tar.gz
glibc-f8c17e79fab13a3d1de976c1c3564df7f8c2a175.tar.bz2
glibc-f8c17e79fab13a3d1de976c1c3564df7f8c2a175.zip
Support expected failures in .test-result files.
This patch, an updated version of <https://sourceware.org/ml/libc-alpha/2014-01/msg00195.html>, makes it possible for .test-result files for individual tests to contain XPASS and XFAIL rather than PASS and FAIL in cases where failure is expected. This replaces the marking of two individual tests with "-" to cause them to be expected at makefile level to fail; evaluate-test.sh will ensure it exits with status 0 for an expected failure. Tested x86_64. * scripts/evaluate-test.sh: Take new argument indicating whether failure is expected. * Makeconfig (evaluate-test): Pass argument to evaluate-test.sh indicating whether failure is expected. * conform/Makefile (test-xfail-run-conformtest): New variable. ($(objpfx)run-conformtest.out): Don't expect to fail at makefile level. * posix/Makefile (test-xfail-annexc): New variable. ($(objpfx)annexc.out): Don't expect to fail at makefile level.
-rw-r--r--ChangeLog12
-rw-r--r--Makeconfig5
-rw-r--r--conform/Makefile3
-rw-r--r--posix/Makefile3
-rwxr-xr-xscripts/evaluate-test.sh11
5 files changed, 29 insertions, 5 deletions
diff --git a/ChangeLog b/ChangeLog
index 4ef5ccc317..7a64c5e6af 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,15 @@
+2014-02-27 Joseph Myers <joseph@codesourcery.com>
+
+ * scripts/evaluate-test.sh: Take new argument indicating whether
+ failure is expected.
+ * Makeconfig (evaluate-test): Pass argument to evaluate-test.sh
+ indicating whether failure is expected.
+ * conform/Makefile (test-xfail-run-conformtest): New variable.
+ ($(objpfx)run-conformtest.out): Don't expect to fail at makefile
+ level.
+ * posix/Makefile (test-xfail-annexc): New variable.
+ ($(objpfx)annexc.out): Don't expect to fail at makefile level.
+
2014-02-26 Joseph Myers <joseph@codesourcery.com>
* argp/Makefile: Include Makeconfig immediately after defining
diff --git a/Makeconfig b/Makeconfig
index 21a7d6ea65..3709c88a45 100644
--- a/Makeconfig
+++ b/Makeconfig
@@ -1083,8 +1083,11 @@ endif
# The name to give to a test in test results summaries.
test-name = $(strip $(patsubst %.out, %, $(patsubst $(common-objpfx)%, %, $@)))
-# Command to output a test status line (such as PASS: test-name).
+# Command to output a test status line (such as PASS: test-name). If
+# test-xfail-$(@F:.out=) has a nonempty value, the status will be
+# XPASS or XFAIL rather than PASS or FAIL.
evaluate-test = $(..)scripts/evaluate-test.sh $(test-name) $$? \
+ $(if $(test-xfail-$(@F:.out=)),true,false) \
> $(common-objpfx)$(test-name).test-result
endif # Makeconfig not yet included
diff --git a/conform/Makefile b/conform/Makefile
index 41f0bb3f57..40081f304b 100644
--- a/conform/Makefile
+++ b/conform/Makefile
@@ -30,10 +30,11 @@ tests: $(objpfx)run-conformtest.out
endif
endif
+test-xfail-run-conformtest = yes
$(objpfx)run-conformtest.out: run-conformtest.sh conformtest.pl \
$(wildcard data/*.h-data) \
$(wildcard data/*/*.h-data)
- -$(BASH) -e $< $(objpfx) $(PERL) '$(CC)' \
+ $(BASH) -e $< $(objpfx) $(PERL) '$(CC)' \
'-I../include $(+sysdep-includes) $(sysincludes) -I..'; \
$(evaluate-test)
diff --git a/posix/Makefile b/posix/Makefile
index 304238ffe8..b41055a509 100644
--- a/posix/Makefile
+++ b/posix/Makefile
@@ -233,8 +233,9 @@ tests: $(objpfx)bug-regex2-mem $(objpfx)bug-regex14-mem \
xtests: $(objpfx)bug-ga2-mem
endif
+test-xfail-annexc = yes
$(objpfx)annexc.out: $(objpfx)annexc
- -$(dir $<)$(notdir $<) '$(CC)' \
+ $(dir $<)$(notdir $<) '$(CC)' \
'$(patsubst %,-I../%,$(sorted-subdirs)) -I../include $(+sysdep-includes) $(sysincludes) -I..' > $@; \
$(evaluate-test)
diff --git a/scripts/evaluate-test.sh b/scripts/evaluate-test.sh
index 9cb6435de2..c8f5012c5a 100755
--- a/scripts/evaluate-test.sh
+++ b/scripts/evaluate-test.sh
@@ -17,10 +17,12 @@
# License along with the GNU C Library; if not, see
# <http://www.gnu.org/licenses/>.
-# usage: evaluate-test.sh test_name rc
+# usage: evaluate-test.sh test_name rc xfail
test_name=$1
rc=$2
+orig_rc=$rc
+xfail=$3
if [ $rc -eq 0 ]; then
result="PASS"
@@ -28,6 +30,11 @@ else
result="FAIL"
fi
+if $xfail; then
+ result="X$result"
+ rc=0
+fi
+
echo "$result: $test_name"
-echo "original exit status $rc"
+echo "original exit status $orig_rc"
exit $rc