util/romcc/test.sh

#!/bin/sh
#
# This file is part of the coreboot project.
#
# Copyright 2016 Jonathan Neuschäfer <j.neuschaefer@gmx.net>
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; version 2 of the License.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.


# These tests are currently known to be broken and should not result in a
# build failure:
XFAIL_TESTS="
	simple_test4.c
	simple_test6.c
	simple_test25.c
	simple_test26.c
	simple_test46.c
	simple_test47.c
	simple_test54.c
	simple_test72.c
	simple_test73.c
	linux_test2.c
	linux_test5.c
	linux_test10.c
	linux_test11.c
	linux_test12.c
"

# ------------------------------------------------------------------- #

BASEDIR="$(dirname "$0")"
BUILDDIR="$BASEDIR/build"
LOGDIR="$BUILDDIR/logs"
mkdir -p "$BUILDDIR"
mkdir -p "$LOGDIR"

usage () {
	echo "Usage: test.sh [--nocolor] CLASS"
	echo ""
	echo "CLASS selects a group of tests to run. It must be one of the following:"
	echo "  all     - all tests"
	echo "  simple  - simple tests"
	echo "  linux   - linux programs whose output is checked against a reference"
	echo ""
	echo "--nocolor disables colors."
	exit 1
}

COLORS=1
if [ "$1" = "--nocolor" ]; then
	shift
	COLORS=0
fi


if [ -t 1 -a "$COLORS" -eq 1 ]; then
	red()   { printf "\033[1;31m%s\033[0m" "$*"; }
	green() { printf "\033[1;32m%s\033[0m" "$*"; }
	blue()  { printf "\033[1;34m%s\033[0m" "$*"; }
else
	red()   { printf "%s" "$*"; }
	green() { printf "%s" "$*"; }
	blue()  { printf "%s" "$*"; }
fi

init_stats() {
	NUM_TOTAL=0	# Number of tests that were run
	NUM_FAIL=0	# Number of tests that failed unexpectedly
	NUM_BROKEN=0	# Number of tests that failed expectedly
	NUM_PASS=0	# Number of tests that passed expectedly
	NUM_FIXED=0	# Number of tests that passed unexpectedly
}

get_romcc() {
	ROMCC="$BUILDDIR/romcc"
	if [ ! -f "$ROMCC" ]; then
		echo "romcc not found! Please run \"make\"."
		exit 1
	fi
}

init_testing() {
	init_stats
	get_romcc
}

show_stats() {
	printf "passed: %s\t(%s newly fixed)\n" $NUM_PASS $NUM_FIXED
	printf "failed: %s\t(%s known broken)\n" $NUM_FAIL $NUM_BROKEN
	printf "total:  %s\n" $NUM_TOTAL
}

is_xfail() {
	local t
	for t in $XFAIL_TESTS; do
		if [ "$t" = "$1" ]; then
			return 0
		fi
	done
	return 1
}

pass() {
	NUM_TOTAL=$((NUM_TOTAL + 1))
	NUM_PASS=$((NUM_PASS + 1))

	green "passed"
	if is_xfail "$(basename "$1")"; then
		blue " (fixed)"
		NUM_FIXED=$((NUM_FIXED + 1))
	fi
	echo
}

fail() {
	NUM_TOTAL=$((NUM_TOTAL + 1))
	NUM_FAIL=$((NUM_FAIL + 1))

	red "failed"
	if is_xfail "$(basename "$1")"; then
		blue " (known broken)"
		NUM_BROKEN=$((NUM_BROKEN + 1))
	fi
	echo
}

run_simple_test() {
	# TODO: "timeout" is not POSIX compliant. Use something that is.
	timeout 60 "$ROMCC" $1 "$2" -o "$BUILDDIR/dummy.S"
}

run_simple_tests() {
	echo "Running simple tests..."

	local t
	for t in $(find "$BASEDIR/tests" -name 'simple_test*.c'); do
		printf "%s" "$(basename "$t")"

		local result=pass
		local logfile="$LOGDIR/$(basename "$t").log"
		rm "$logfile" >/dev/null 2>&1
		for opt in "" "-O" "-O2" "-mmmx" "-msse" "-mmmx -msse" \
		           "-O -mmmx" "-O -msse" "-O  -mmmx -msse" \
		           "-O2 -mmmx" "-O2 -msse" "-O2 -mmmx -msse"; do
			if run_simple_test "$opt" "$t" \
					>> "$logfile" 2>&1; then
				printf .
			else
				result=fail
				break
			fi
		done
		printf " "
		$result "$t"
	done

	echo
}

run_linux_test() {
	local base="$(basename "$1")"

	# TODO: "timeout" is not POSIX compliant. Use something that is.

	timeout 60 "$ROMCC" "$1" -o "$BUILDDIR/$base.S" || return 1
	as --32 "$BUILDDIR/$base.S" -o "$BUILDDIR/$base.o" || return 1
	ld -m elf_i386 -T "$BASEDIR/tests/ldscript.ld" \
		"$BUILDDIR/$base.o" -o "$BUILDDIR/$base.elf" || return 1
	timeout 60 "$BUILDDIR/$base.elf" > "$BUILDDIR/$base.out" || return 1

	diff -u "$BASEDIR/results/${base%.c}.out" "$BUILDDIR/$base.out"
}

run_linux_tests() {
	echo "Running linux tests..."

	local t
	for t in $(find "$BASEDIR/tests" -name 'linux_test*.c'); do
		printf "%s... " "$(basename "$t")"

		local logfile="$LOGDIR/$(basename "$t").log"
		if run_linux_test "$t" > "$logfile" 2>&1; then
			pass "$t"
		else
			fail "$t"
		fi
	done

	echo
}


if [ $# -ne 1 ]; then
	usage
fi

CLASS="$1"

case "$CLASS" in
	all)
		init_testing
		run_simple_tests
		run_linux_tests
		show_stats
		;;
	simple)
		init_testing
		run_simple_tests
		show_stats
		;;
	linux)
		init_testing
		run_linux_tests
		show_stats
		;;
	*)
		echo "Invalid test class $CLASS"
		echo
		usage
		;;
esac

if [ $NUM_FAIL -ne $NUM_BROKEN ]; then
	exit 1
fi