From 1188bea4447ad0a13831527af54ca603993d2206 Mon Sep 17 00:00:00 2001 From: nirenjan Date: Sat, 6 Jun 2020 08:33:30 -0700 Subject: [PATCH] Use TAP for tests Prior to this change, the tests were using the automake simple test harness. The limitation was that for each set of test cases (e.g. timezone tests), even if one test failed, it would report every test in that set as failed. Migrating to TAP allows fine-grained reporting on every single test case, and allows better investigation into checking which individual tests failed. This change also updates the timezone tests to explicitly mark the ones that check when the timezone is Pacific Daylight Time as expected to fail. It also updates the clock tests to improve the formatting of the test case identifier. --- .gitignore | 4 ++ configure.ac | 2 + tests/Makefile.am | 21 +++--- tests/common_infra.sh | 66 ++++++++----------- .../{test_brightness.sh => test_brightness} | 0 tests/libx52/{test_clock.sh => test_clock} | 4 +- .../{test_indicator.sh => test_indicator} | 0 tests/libx52/{test_leds.sh => test_leds} | 0 tests/libx52/{test_mfd.sh => test_mfd} | 0 .../{test_timezone.sh => test_timezone} | 3 + 10 files changed, 46 insertions(+), 54 deletions(-) rename tests/libx52/{test_brightness.sh => test_brightness} (100%) mode change 100644 => 100755 rename tests/libx52/{test_clock.sh => test_clock} (96%) mode change 100644 => 100755 rename tests/libx52/{test_indicator.sh => test_indicator} (100%) mode change 100644 => 100755 rename tests/libx52/{test_leds.sh => test_leds} (100%) mode change 100644 => 100755 rename tests/libx52/{test_mfd.sh => test_mfd} (100%) mode change 100644 => 100755 rename tests/libx52/{test_timezone.sh => test_timezone} (94%) mode change 100644 => 100755 diff --git a/.gitignore b/.gitignore index 4391acf..a9ba97a 100644 --- a/.gitignore +++ b/.gitignore @@ -41,8 +41,12 @@ Makefile.in *.la *.lo *.m4 +stamp-h1 +tap-driver.sh test-driver tests/test-suite.log +tests/**/*.log +tests/**/*.trs *.pc # Autotools Gettext objects diff --git a/configure.ac b/configure.ac index 13f0f7a..5f201c2 100644 --- a/configure.ac +++ b/configure.ac @@ -7,8 +7,10 @@ AC_INIT([x52pro-linux], [0.2.1], [nirenjan@gmail.com]) AC_CONFIG_MACRO_DIR([m4]) AM_INIT_AUTOMAKE([-Wall foreign subdir-objects]) +AC_REQUIRE_AUX_FILE([tap-driver.sh]) AC_PROG_CC AC_PROG_CC_STDC +AC_PROG_AWK AM_PROG_AR AM_PATH_PYTHON([2.6]) LT_INIT diff --git a/tests/Makefile.am b/tests/Makefile.am index 25719a7..f42a932 100644 --- a/tests/Makefile.am +++ b/tests/Makefile.am @@ -4,21 +4,16 @@ # # SPDX-License-Identifier: GPL-2.0-only WITH Classpath-exception-2.0 -TEST_EXTENSIONS = .sh -SH_LOG_COMPILER = $(SHELL) -AM_SH_LOG_FLAGS = +# Source: https://stackoverflow.com/a/20831065 +LOG_DRIVER = env AM_TAP_AWK='$(AWK)' $(SHELL) $(top_srcdir)/tap-driver.sh TESTS = \ - libx52/test_leds.sh \ - libx52/test_brightness.sh \ - libx52/test_indicator.sh \ - libx52/test_mfd.sh \ - libx52/test_clock.sh \ - libx52/test_timezone.sh - -# NOTE: Timezone tests currently fail when local time is daylight time. Once -# this is fixed, remove the line below. -XFAIL_TESTS = libx52/test_timezone.sh + libx52/test_leds \ + libx52/test_brightness \ + libx52/test_indicator \ + libx52/test_mfd \ + libx52/test_clock \ + libx52/test_timezone EXTRA_DIST = common_infra.sh $(TESTS) diff --git a/tests/common_infra.sh b/tests/common_infra.sh index 6e3f24f..51ba327 100644 --- a/tests/common_infra.sh +++ b/tests/common_infra.sh @@ -5,12 +5,6 @@ # # SPDX-License-Identifier: GPL-2.0-only WITH Classpath-exception-2.0 -# Set up exit status codes -EXIT_SUCCESS=0 -EXIT_SKIP=77 -EXIT_HARD_ERROR=99 -EXIT_FAILURE=1 - # Set up some command sequences X52_LED_COMMAND_INDEX='00b8' X52_LED_FIRE_ON='0101' @@ -80,7 +74,8 @@ find_programs() # Tests and distcheck do not work on OSX, skip the tests if [[ `uname -s` == [Dd]arwin* ]] then - exit $EXIT_SKIP + echo "1..0 # skip Tests not supported on OSX" + exit 0 fi # Find the X52cli script @@ -88,21 +83,21 @@ find_programs() if [[ -z "$X52CLI" ]] then - exit $EXIT_HARD_ERROR + exit 1 fi # Find the x52test_log_actions program X52LOGACT=$(find .. -path '*/libusbx52/x52test_log_actions' -executable) if [[ -z "$X52LOGACT" ]] then - exit $EXIT_HARD_ERROR + exit 1 fi # Find the x52test_create_device_list program X52DEVLIST=$(find .. -path '*/libusbx52/x52test_create_device_list' -executable) if [[ -z "$X52DEVLIST" ]] then - exit $EXIT_HARD_ERROR + exit 1 fi } @@ -114,14 +109,14 @@ require_programs() do if ! command -v "$prog" then - echo "Required program '$prog' not found, skipping test suite" + echo "1..0 # skip Required program '$prog' not found" skip=true fi done if $skip then - exit $EXIT_SKIP + exit 0 fi } @@ -132,7 +127,7 @@ setup_preload() if [[ -z "$LIBUSB" ]] then - exit $EXIT_HARD_ERROR + exit 1 fi export LD_PRELOAD=$(realpath $LIBUSB) @@ -154,6 +149,7 @@ setup_test() expect_pattern() { + TEST_NUM=$((TEST_NUM + 1)) # Save pattern to expected output file export LIBUSBX52_OUTPUT_DATA=$EXPECTED_OUTPUT $X52LOGACT $@ @@ -164,23 +160,22 @@ expect_pattern() verify_output() { + TEST_COUNT=$(($TEST_COUNT + 1)) if diff -q $EXPECTED_OUTPUT $OBSERVED_OUTPUT then - echo "PASS: $TEST_ID" + echo "ok $TEST_COUNT $TEST_ID" TEST_PASS=$(($TEST_PASS + 1)) else - echo "FAIL: $TEST_ID" - echo 'Expected:' - echo '=========' - sed 's/^/\t/' $EXPECTED_OUTPUT - echo - echo 'Observed:' - echo '=========' - sed 's/^/\t/' $OBSERVED_OUTPUT + echo "not ok $TEST_COUNT $TEST_ID" + echo '# Expected:' + echo '# =========' + sed 's/^/#\t/' $EXPECTED_OUTPUT + echo '#' + echo '# Observed:' + echo '# =========' + sed 's/^/#\t/' $OBSERVED_OUTPUT TEST_FAIL=$(($TEST_FAIL + 1)) fi - - TEST_COUNT=$(($TEST_COUNT + 1)) } verify_test_suite() @@ -190,22 +185,15 @@ verify_test_suite() sep="$sep$sep" sep="$sep$sep" - echo - echo $sep - echo $TEST_SUITE_ID - echo $sep - echo -e "Total Tests:\t$TEST_COUNT" - echo -e "Tests Passed:\t$TEST_PASS" - echo -e "Tests Failed:\t$TEST_FAIL" - echo $sep - echo + echo '#' $sep + echo '#' $TEST_SUITE_ID + echo '#' $sep + echo -e "# Total Tests:\t$TEST_COUNT" + echo -e "# Tests Passed:\t$TEST_PASS" + echo -e "# Tests Failed:\t$TEST_FAIL" + echo '#' $sep - if [[ "$TEST_FAIL" != 0 ]] - then - exit $EXIT_FAILURE - else - exit $EXIT_SUCCESS - fi + echo "1..$TEST_COUNT" } set -e diff --git a/tests/libx52/test_brightness.sh b/tests/libx52/test_brightness old mode 100644 new mode 100755 similarity index 100% rename from tests/libx52/test_brightness.sh rename to tests/libx52/test_brightness diff --git a/tests/libx52/test_clock.sh b/tests/libx52/test_clock old mode 100644 new mode 100755 similarity index 96% rename from tests/libx52/test_clock.sh rename to tests/libx52/test_clock index 608156f..7e5b0d1 --- a/tests/libx52/test_clock.sh +++ b/tests/libx52/test_clock @@ -93,7 +93,7 @@ raw_time_test() local hh=$1 local mm=$2 local time_format=$3 - TEST_ID="Test setting time to $hh:$mm, $time_format" + TEST_ID=$(printf "Test setting time to %02d:%02d, %s" $hh $mm $time_format) local value=$(($hh * 256 + $mm)) if [[ $time_format == 24hr ]] @@ -133,7 +133,7 @@ raw_date_test() ;; esac - TEST_ID="Test setting date in $date_format format to ${dd1}-${mm1}-${yy1}" + TEST_ID=$(printf "Test setting date in %s format to %02d-%02d-%02d" $date_format $dd1 $mm1 $yy1) local date_value=$(printf '%04x' $(($mm1 * 256 + $dd1))) local year_value=$(printf '%04x' $yy1) diff --git a/tests/libx52/test_indicator.sh b/tests/libx52/test_indicator old mode 100644 new mode 100755 similarity index 100% rename from tests/libx52/test_indicator.sh rename to tests/libx52/test_indicator diff --git a/tests/libx52/test_leds.sh b/tests/libx52/test_leds old mode 100644 new mode 100755 similarity index 100% rename from tests/libx52/test_leds.sh rename to tests/libx52/test_leds diff --git a/tests/libx52/test_mfd.sh b/tests/libx52/test_mfd old mode 100644 new mode 100755 similarity index 100% rename from tests/libx52/test_mfd.sh rename to tests/libx52/test_mfd diff --git a/tests/libx52/test_timezone.sh b/tests/libx52/test_timezone old mode 100644 new mode 100755 similarity index 94% rename from tests/libx52/test_timezone.sh rename to tests/libx52/test_timezone index 083e350..0a071c9 --- a/tests/libx52/test_timezone.sh +++ b/tests/libx52/test_timezone @@ -33,6 +33,9 @@ timezone_test() # Pacific Daylight Time # Default offset for clocks 2 & 3 should be +420 minutes off=420 + + # TODO: Remove the following line once #20 is fixed. + TEST_ID="$TEST_ID # TODO Fix issue #20" else # Pacific Standard time # Default offset for clocks 2 & 3 should be +480 minutes