Use TAP for tests

Prior to this change, the tests were using the automake simple test
harness. The limitation was that for each set of test cases (e.g.
timezone tests), even if one test failed, it would report every test in
that set as failed. Migrating to TAP allows fine-grained reporting on
every single test case, and allows better investigation into checking
which individual tests failed.

This change also updates the timezone tests to explicitly mark the ones
that check when the timezone is Pacific Daylight Time as expected to
fail. It also updates the clock tests to improve the formatting of the
test case identifier.

.gitignore

@@ -41,8 +41,12 @@ Makefile.in
 *.la
 *.lo
 *.m4
+stamp-h1
+tap-driver.sh
 test-driver
 tests/test-suite.log
+tests/**/*.log
+tests/**/*.trs
 *.pc
 
 # Autotools Gettext objects

configure.ac

@@ -7,8 +7,10 @@
 AC_INIT([x52pro-linux], [0.2.1], [nirenjan@gmail.com])
 AC_CONFIG_MACRO_DIR([m4])
 AM_INIT_AUTOMAKE([-Wall foreign subdir-objects])
+AC_REQUIRE_AUX_FILE([tap-driver.sh])
 AC_PROG_CC
 AC_PROG_CC_STDC
+AC_PROG_AWK
 AM_PROG_AR
 AM_PATH_PYTHON([2.6])
 LT_INIT

tests/Makefile.am

@@ -4,21 +4,16 @@
 #
 # SPDX-License-Identifier: GPL-2.0-only WITH Classpath-exception-2.0
 
-TEST_EXTENSIONS = .sh
+# Source: https://stackoverflow.com/a/20831065
-SH_LOG_COMPILER = $(SHELL)
+LOG_DRIVER = env AM_TAP_AWK='$(AWK)' $(SHELL) $(top_srcdir)/tap-driver.sh
-AM_SH_LOG_FLAGS =
 
 TESTS = \
-	libx52/test_leds.sh \
+	libx52/test_leds \
-	libx52/test_brightness.sh \
+	libx52/test_brightness \
-	libx52/test_indicator.sh \
+	libx52/test_indicator \
-	libx52/test_mfd.sh \
+	libx52/test_mfd \
-	libx52/test_clock.sh \
+	libx52/test_clock \
-	libx52/test_timezone.sh
+	libx52/test_timezone
-
-# NOTE: Timezone tests currently fail when local time is daylight time. Once
-# this is fixed, remove the line below.
-XFAIL_TESTS = libx52/test_timezone.sh
 
 EXTRA_DIST = common_infra.sh $(TESTS)
 

tests/common_infra.sh

@@ -5,12 +5,6 @@
 #
 # SPDX-License-Identifier: GPL-2.0-only WITH Classpath-exception-2.0
 
-# Set up exit status codes
-EXIT_SUCCESS=0
-EXIT_SKIP=77
-EXIT_HARD_ERROR=99
-EXIT_FAILURE=1
-
 # Set up some command sequences
 X52_LED_COMMAND_INDEX='00b8'
 X52_LED_FIRE_ON='0101'
@@ -80,7 +74,8 @@ find_programs()
     # Tests and distcheck do not work on OSX, skip the tests
     if [[ `uname -s` == [Dd]arwin* ]]
     then
-        exit $EXIT_SKIP
+        echo "1..0 # skip Tests not supported on OSX"
+        exit 0
     fi
 
     # Find the X52cli script
@@ -88,21 +83,21 @@ find_programs()
 
     if [[ -z "$X52CLI" ]]
     then
-        exit $EXIT_HARD_ERROR
+        exit 1
     fi
 
     # Find the x52test_log_actions program
     X52LOGACT=$(find .. -path '*/libusbx52/x52test_log_actions' -executable)
     if [[ -z "$X52LOGACT" ]]
     then
-        exit $EXIT_HARD_ERROR
+        exit 1
     fi
 
     # Find the x52test_create_device_list program
     X52DEVLIST=$(find .. -path '*/libusbx52/x52test_create_device_list' -executable)
     if [[ -z "$X52DEVLIST" ]]
     then
-        exit $EXIT_HARD_ERROR
+        exit 1
     fi
 }
 
@@ -114,14 +109,14 @@ require_programs()
     do
         if ! command -v "$prog"
         then
-            echo "Required program '$prog' not found, skipping test suite"
+            echo "1..0 # skip Required program '$prog' not found"
             skip=true
         fi
     done
 
     if $skip
     then
-        exit $EXIT_SKIP
+        exit 0
     fi
 }
 
@@ -132,7 +127,7 @@ setup_preload()
 
     if [[ -z "$LIBUSB" ]]
     then
-        exit $EXIT_HARD_ERROR
+        exit 1
     fi
 
     export LD_PRELOAD=$(realpath $LIBUSB)
@@ -154,6 +149,7 @@ setup_test()
 
 expect_pattern()
 {
+    TEST_NUM=$((TEST_NUM + 1))
     # Save pattern to expected output file
     export LIBUSBX52_OUTPUT_DATA=$EXPECTED_OUTPUT
     $X52LOGACT $@
@@ -164,23 +160,22 @@ expect_pattern()
 
 verify_output()
 {
+    TEST_COUNT=$(($TEST_COUNT + 1))
     if diff -q $EXPECTED_OUTPUT $OBSERVED_OUTPUT
     then
-        echo "PASS: $TEST_ID"
+        echo "ok $TEST_COUNT $TEST_ID"
         TEST_PASS=$(($TEST_PASS + 1))
     else
-        echo "FAIL: $TEST_ID"
+        echo "not ok $TEST_COUNT $TEST_ID"
-        echo 'Expected:'
+        echo '# Expected:'
-        echo '========='
+        echo '# ========='
-        sed 's/^/\t/' $EXPECTED_OUTPUT
+        sed 's/^/#\t/' $EXPECTED_OUTPUT
-        echo
+        echo '#'
-        echo 'Observed:'
+        echo '# Observed:'
-        echo '========='
+        echo '# ========='
-        sed 's/^/\t/' $OBSERVED_OUTPUT
+        sed 's/^/#\t/' $OBSERVED_OUTPUT
         TEST_FAIL=$(($TEST_FAIL + 1))
     fi
-
-    TEST_COUNT=$(($TEST_COUNT + 1))
 }
 
 verify_test_suite()
@@ -190,22 +185,15 @@ verify_test_suite()
     sep="$sep$sep"
     sep="$sep$sep"
 
-    echo
+    echo '#' $sep
-    echo $sep
+    echo '#' $TEST_SUITE_ID
-    echo $TEST_SUITE_ID
+    echo '#' $sep
-    echo $sep
+    echo -e "# Total Tests:\t$TEST_COUNT"
-    echo -e "Total Tests:\t$TEST_COUNT"
+    echo -e "# Tests Passed:\t$TEST_PASS"
-    echo -e "Tests Passed:\t$TEST_PASS"
+    echo -e "# Tests Failed:\t$TEST_FAIL"
-    echo -e "Tests Failed:\t$TEST_FAIL"
+    echo '#' $sep
-    echo $sep
-    echo
 
-    if [[ "$TEST_FAIL" != 0 ]]
+    echo "1..$TEST_COUNT"
-    then
-        exit $EXIT_FAILURE
-    else
-        exit $EXIT_SUCCESS
-    fi
 }
 
 set -e

tests/libx52/test_clock

@@ -93,7 +93,7 @@ raw_time_test()
     local hh=$1
     local mm=$2
     local time_format=$3
-    TEST_ID="Test setting time to $hh:$mm, $time_format"
+    TEST_ID=$(printf "Test setting time to %02d:%02d, %s" $hh $mm $time_format)
 
     local value=$(($hh * 256 + $mm))
     if [[ $time_format == 24hr ]]
@@ -133,7 +133,7 @@ raw_date_test()
         ;;
     esac
 
-    TEST_ID="Test setting date in $date_format format to ${dd1}-${mm1}-${yy1}"
+    TEST_ID=$(printf "Test setting date in %s format to %02d-%02d-%02d" $date_format $dd1 $mm1 $yy1)
 
     local date_value=$(printf '%04x' $(($mm1 * 256 + $dd1)))
     local year_value=$(printf '%04x' $yy1)

tests/libx52/test_timezone

@@ -33,6 +33,9 @@ timezone_test()
         # Pacific Daylight Time
         # Default offset for clocks 2 & 3 should be +420 minutes
         off=420
+
+        # TODO: Remove the following line once #20 is fixed.
+        TEST_ID="$TEST_ID # TODO Fix issue #20"
     else
         # Pacific Standard time
         # Default offset for clocks 2 & 3 should be +480 minutes