[LTP] [PATCH 1/4] testcases/lib/run_tests.sh: Check expected results

[Petr Vorel]

This verification helps 1) see if anything broke 2) be able to run in CI.

Also:
1) Allow to run tests outside of the test directory (call just by
relative PATH).
2) Allow to pass build directory (useful for out of tree build).
3) Allow to skip tests (useful for github CI).

shell_loader_all_filesystems.sh shell_loader_supported_archs.sh
shell_loader_filesystems.sh fails on Github Actions due broken loop
device therefore skip them:

tst_tmpdir.c:317: TINFO: Using /tmp/LTP_sheHtNv5R as tmpdir (overlayfs filesystem)
tst_device.c:147: TINFO: No free devices found
tst_device.c:360: TBROK: Failed to acquire device

Signed-off-by: Petr Vorel <pvorel@suse.cz>
---
NOTE: it's not perfect (we could finally add check which compares whole
output), but checking exit code is better than nothing.

lib/newlib_tests/runtest.sh uses a different approach, but I did not
bother trying to unify them. But it would be worth to add support for
tests which TBROK on GitHub also to lib/newlib_tests/runtest.sh. That
allows us to easily run 'make test' locally to get higher coverage (e.g.
before the release).

 testcases/lib/run_tests.sh | 115 ++++++++++++++++++++++++++++++-------
 1 file changed, 95 insertions(+), 20 deletions(-)

diff --git a/testcases/lib/run_tests.sh b/testcases/lib/run_tests.sh
index 40d415e6c4..380870ae55 100755
--- a/testcases/lib/run_tests.sh
+++ b/testcases/lib/run_tests.sh
@@ -1,32 +1,107 @@
 #!/bin/sh
 
-testdir=$(realpath $(dirname $0))
-export PATH="$PATH:$testdir:$testdir/tests/"
-
-for i in `seq -w 01 06`; do
-	echo
-	echo "*** Running shell_test$i ***"
-	echo
-	./tests/shell_test$i
-done
+TESTS_PASS="shell_test01 shell_test02 shell_test03 shell_test04 shell_test05
+shell_loader.sh shell_loader_tcnt.sh shell_loader_c_child.sh"
+
+TESTS_PASS_GITHUB_TBROK="shell_loader_all_filesystems.sh shell_loader_supported_archs.sh shell_loader_filesystems.sh shell_loader_kconfigs.sh"
+
+TESTS_FAIL="shell_loader_tags.sh"
+
+TESTS_TBROK="shell_loader_wrong_metadata.sh shell_loader_no_metadata.sh
+shell_loader_invalid_metadata.sh shell_loader_invalid_block.sh"
+
+TESTS_TCONF="shell_test06"
 
-for i in shell_loader.sh shell_loader_all_filesystems.sh shell_loader_no_metadata.sh \
-	 shell_loader_wrong_metadata.sh shell_loader_invalid_metadata.sh\
-	 shell_loader_supported_archs.sh shell_loader_filesystems.sh\
-	 shell_loader_tcnt.sh shell_loader_kconfigs.sh shell_loader_tags.sh \
-	 shell_loader_invalid_block.sh shell_loader_c_child.sh; do
-	echo
-	echo "*** Running $i ***"
-	echo
-	$i
+SKIP_GITHUB=
+FAIL=
+
+srcdir="$(realpath $(dirname $0))"
+builddir="$srcdir"
+
+usage()
+{
+	cat << EOF
+Usage: $0 [-b DIR ] [-s TESTS]
+-b DIR   build directory (required for out-of-tree build)
+-h       print this help
+EOF
+}
+
+while getopts b:h opt; do
+	case $opt in
+		'h') usage; exit 0;;
+		'b')
+			builddir="$OPTARG/testcases/lib/"
+			if [ ! -d "$builddir" ]; then
+				echo "directory '$builddir' does not exist!" >&2
+				exit 1
+			fi
+			;;
+		*) usage; runtest_brk TBROK "Error: invalid option";;
+	esac
 done
 
+# srcdir is for *.sh, builddir for *.c
+export PATH="$PATH:$srcdir:$builddir:$srcdir/tests/:$builddir/tests/"
+
+
+tst_mask2flag()
+{
+	case "$1" in
+	0) echo TPASS;;
+	1) echo TFAIL;;
+	2) echo TBROK;;
+	4) echo TWARN;;
+	16) echo TINFO;;
+	32) echo TCONF;;
+	esac
+}
+
+run_tests()
+{
+	local exp="$1"
+	local test rc
+	shift
+
+	for test in "$@"; do
+		echo "*** Running '$test' (exp: $(tst_mask2flag $exp)) ***"
+		$test
+		rc=$?
+		if [ $rc = 127 ]; then
+			echo "Test '$test' not found, maybe out-of-tree build and unset builddir?" >&2
+			exit 1
+		elif [ $rc = 2 -a $WHITELIST_GITHUB = 1 -a "$GITHUB_ACTIONS" ]; then
+			SKIP_GITHUB="$SKIP_GITHUB\n*$test"
+		elif [ $rc != $exp ]; then
+			FAIL="$FAIL\n* $test ($(tst_mask2flag $rc), exp: $(tst_mask2flag $exp))"
+		fi
+	done
+}
+
+run_tests 0 $TESTS_PASS
+WHITELIST_GITHUB=1 run_tests 0 $TESTS_PASS_GITHUB_TBROK
+run_tests 32 $TESTS_TCONF
+
 echo
 echo "*** Testing LTP test -h option ***"
 echo
-shell_loader.sh -h
+run_tests 0 "shell_loader.sh -h"
 
 echo
 echo "*** Testing LTP test -i option ***"
 echo
-shell_loader.sh -i 2
+run_tests 0 "shell_loader.sh -i 2"
+
+echo
+echo "***** RESULTS *****"
+
+if [ "$SKIP_GITHUB" ]; then
+	printf "Test skipped on GitHub Actions:$SKIP_GITHUB\n"
+fi
+
+if [ "$FAIL" ]; then
+	printf "Failed tests:$FAIL\n"
+	exit 1
+fi
+
+echo "All tests passed"
-- 
2.45.2