Merge pull request #15771 from aabadie/review_test_manual

tests: add test-with-config and test-as-root targets

.murdock

@@ -27,6 +27,8 @@ tests/driver_v*"}
 : ${TEST_KCONFIG_native:="examples/hello-world tests/periph_*
 tests/xtimer_* tests/ztimer_* tests/driver_ws281x"}
 
+: ${TEST_WITH_CONFIG_SUPPORTED:="examples/suit_update tests/driver_at86rf2xx_aes"}
+
 export RIOT_CI_BUILD=1
 export CC_NOCOLOR=1
 export STATIC_TESTS=0
@@ -382,7 +384,11 @@ run_test() {
     BOARD=$board TOOLCHAIN=${toolchain} make -C$appdir flash-only termdeps -j2
 
     # now run the actual test
-    BOARD=$board TOOLCHAIN=${toolchain} make -C$appdir test
+    if is_in_list "${appdir}" "${TEST_WITH_CONFIG_SUPPORTED}"; then
+        BOARD=${board} TOOLCHAIN=${toolchain} make -C${appdir} test-with-config
+    else
+        BOARD=$board TOOLCHAIN=${toolchain} make -C$appdir test
+    fi
     RES=$?
 
     if [ $RES -eq 0 -a -n "$TEST_HASH" ]; then

Makefile.include

@@ -761,42 +761,8 @@ reset:
 	$(call check_cmd,$(RESET),Reset program)
 	$(RESET) $(RESET_FLAGS)
 
-.PHONY: test test/available
+# tests related targets and variables
-TESTS ?= $(foreach file,$(wildcard $(APPDIR)/tests/*[^~]),\
+include $(RIOTMAKE)/tests/tests.inc.mk
-                        $(shell test -f $(file) -a -x $(file) && echo $(file)))
-
-# "make test" calls "make term" which would implicitly build it's dependencies,
-# but that increases the time "make test" needs to get ready. That can cause
-# problems ("make term" missing some lines) as terminal startup is not properly
-# sychronized, but depends on a static timeout (TESTRUNNER_START_DELAY).
-# Murdock builds the term dependencies before running "make test" to circumvent
-# this. In order to make local builds behave similar, add the term deps here.
-# See #11762.
-TEST_DEPS += $(TERMDEPS)
-
-# Export TESTRUNNER_RESET_AFTER_TERM only for the test target. This allows for
-# it to be accessed through the environment from python test script.
-# This is currently needed only by `examples/%/tests` and should be removed in
-# the future since `make reset` after `term` is not a valid synch method across
-# all platforms.
-$(call target-export-variables,test,TESTRUNNER_RESET_AFTER_TERM)
-test: $(TEST_DEPS)
-	$(Q) for t in $(TESTS); do \
-		$$t || exit 1; \
-	done
-
-test/available:
-	$(Q)test -n "$(strip $(TESTS))"
-
-# this target only makes sense if an ELFFILE is actually created, thus guard by
-# RIOTNOLINK="".
-ifeq (,$(RIOTNOLINK))
-test-input-hash: $(TESTS) $(ELFFILE) $(TEST_EXTRA_FILES)
-	sha1sum $^ > $(BINDIR)/test-input-hash.sha1
-else
-test-input-hash:
-	true
-endif
 
 .PHONY: fuzz
 fuzz:

examples/lorawan/Makefile

@@ -46,7 +46,4 @@ ifneq (,$(filter test,$(MAKECMDGOALS)))
   DEFAULT_MODULE += test_utils_interactive_sync
 endif
 
-# Can't be run on ci since it requires gateway + lora node
-TEST_ON_CI_BLACKLIST = all
-
 include $(RIOTBASE)/Makefile.include

makefiles/tests/tests.inc.mk

@@ -0,0 +1,63 @@
+# Export TESTRUNNER_RESET_AFTER_TERM only for the test target. This allows for
+# it to be accessed through the environment from python test script.
+# This is currently needed only by `examples/%/tests` and should be removed in
+# the future since `make reset` after `term` is not a valid synch method across
+# all platforms.
+$(call target-export-variables,test test-as-root test-with-config,TESTRUNNER_RESET_AFTER_TERM)
+
+.PHONY: test test/available
+TESTS ?= $(foreach file,$(wildcard $(APPDIR)/tests/*[^~]),\
+                        $(shell test -f $(file) -a -x $(file) && echo $(file)))
+
+# "make test" calls "make term" which would implicitly build it's dependencies,
+# but that increases the time "make test" needs to get ready. That can cause
+# problems ("make term" missing some lines) as terminal startup is not properly
+# sychronized, but depends on a static timeout (TESTRUNNER_START_DELAY).
+# Murdock builds the term dependencies before running "make test" to circumvent
+# this. In order to make local builds behave similar, add the term deps here.
+# See #11762.
+TEST_DEPS += $(TERMDEPS)
+
+test: $(TEST_DEPS)
+	$(Q) for t in $(TESTS); do \
+		$$t || exit 1; \
+	done
+
+test/available:
+	$(Q)test -n "$(strip $(TESTS))"
+
+# Tests that require root privileges
+.PHONY: test-as-root test-as-root/available
+TESTS_AS_ROOT ?= $(foreach file,$(wildcard $(APPDIR)/tests-as-root/*[^~]),\
+                    $(shell test -f $(file) -a -x $(file) && echo $(file)))
+
+test-as-root: $(TEST_DEPS)
+	$(Q) for t in $(TESTS_AS_ROOT); do \
+		sudo -E PYTHONPATH=$(PYTHONPATH) $$t || exit 1; \
+	done
+
+test-as-root/available:
+	$(Q)test -n "$(strip $(TESTS_AS_ROOT))"
+
+# Tests that require specific configuration
+.PHONY: test-with-config test-with-config/available
+TESTS_WITH_CONFIG ?= $(foreach file,$(wildcard $(APPDIR)/tests-with-config/*[^~]),\
+                        $(shell test -f $(file) -a -x $(file) && echo $(file)))
+
+test-with-config: $(TEST_DEPS)
+	$(Q) for t in $(TESTS_WITH_CONFIG); do \
+		$$t || exit 1; \
+	done
+
+test-with-config/available:
+	$(Q)test -n "$(strip $(TESTS_WITH_CONFIG))"
+
+# this target only makes sense if an ELFFILE is actually created, thus guard by
+# RIOTNOLINK="".
+ifeq (,$(RIOTNOLINK))
+test-input-hash: $(TESTS) $(TESTS_WITH_CONFIG) $(TESTS_AS_ROOT) $(ELFFILE) $(TEST_EXTRA_FILES)
+	sha1sum $^ > $(BINDIR)/test-input-hash.sha1
+else
+test-input-hash:
+	true
+endif

tests/Makefile.tests_common

@@ -1,6 +1,6 @@
 APPLICATION ?= tests_$(notdir $(patsubst %/,%,$(CURDIR)))
 
-ifneq (,$(wildcard $(CURDIR)/tests/.))
+ifneq (,$(wildcard $(CURDIR)/tests*/.))
   DEFAULT_MODULE += test_utils_interactive_sync
   # add interactive test configuration when testing Kconfig
   ifeq (1,$(TEST_KCONFIG))

tests/README.md

@@ -117,3 +117,22 @@ The expected behavior is verified with the test in `tests/test_tools`.
 Tests cannot rely on having on all boards and terminal programs:
 * unbuffered input
 * allowing sending special characters like `ctrl+c/ctrl+d`
+
+
+Running tests that require a preliminary manual configuration
+-------------------------------------------------------------
+
+Some tests need active monitoring or manual setup steps but still have some
+automated scripts. The test automation scripts are defined in the
+`<test_application>/tests-with-config/` folder.
+For running them, follow the setup or analysis documentation and use the
+`test-with-config` target.
+
+Running tests that require root privileges
+------------------------------------------
+
+Some tests require root privileges to launch their automated script. In this
+case, the test automation scripts are defined in the
+`<test_application>/tests-as-root/` folder.
+For running them, follow the setup or analysis documentation and use the
+`test-as-root` target.

tests/driver_apds99xx/tests-with-config/01-run.py

@@ -11,11 +11,8 @@ from testrunner import run
 
 
 def testfunc(child):
-    child.expect('Initializing APDS99XX sensor')
+    child.expect_exact('Initializing APDS99XX sensor')
-    i = child.expect(['[OK]', '[Failed]'])
+    child.expect_exact('[OK]')
-    if i == 1:
-        print('FAILED')
-        return
     child.expect(r'proximity = \d+ \[cnts\]')
     child.expect(r'ambient = \d+ \[cnts\]')
     child.expect([r'red = \d+ \[cnts\], green = \d+ \[cnts\], blue = \d+ \[cnts\]',

tests/driver_apds99xx_full/tests-with-config/01-run.py

@@ -11,11 +11,8 @@ from testrunner import run
 
 
 def testfunc(child):
-    child.expect('Initializing APDS99XX sensor')
+    child.expect_exact('Initializing APDS99XX sensor')
-    i = child.expect(['[OK]', '[Failed]'])
+    child.expect_exact('[OK]')
-    if i == 1:
-        print('FAILED')
-        return
     child.expect(r'ambient = \d+ \[cnts\]')
     child.expect([r'red = \d+ \[cnts\], green = \d+ \[cnts\], blue = \d+ \[cnts\]',
                   r'illuminance = %d [lux]'])

tests/driver_bme680/tests-with-config/01-run.py

@@ -11,11 +11,7 @@ from testrunner import run
 
 
 def testfunc(child):
-    child.expect('Initialize BME680 sensor 0 ... ')
+    child.expect_exact('Initialize BME680 sensor 0 ... OK')
-    i = child.expect(['[OK]', '[failed]'])
-    if i == 1:
-        print('FAILED')
-        return
     child.expect(r'\[bme680\]: dev=0, '
                  r'T = \d+.\d+ degC, '
                  r'P = \d+ Pa, '

tests/driver_ds1307/Makefile

@@ -1,9 +1,5 @@
 include ../Makefile.tests_common
 
-# Blacklist iotlab boards since a different device has the same i2c address
-BOARD_BLACKLIST := iotlab-a8-m3 \
-                   iotlab-m3
-
 USEMODULE += ds1307
 USEMODULE += embunit
 USEMODULE += xtimer

tests/driver_ds3231/Makefile

@@ -1,12 +1,5 @@
 include ../Makefile.tests_common
 
-# the test needs real hardware to run
-TEST_ON_CI_BLACKLIST += all
-
-# Blacklist iotlab boards since a different device has the same i2c address
-BOARD_BLACKLIST := iotlab-a8-m3 \
-                   iotlab-m3
-
 USEMODULE += ds3231
 USEMODULE += xtimer
 USEMODULE += shell

tests/driver_hd44780/Makefile

@@ -11,8 +11,4 @@ ifeq (native,$(BOARD))
   endif
 endif
 
-# Fails on esp32 because the driver defines default GPIOs that are used for the
-# SPI flash interface.
-TEST_ON_CI_BLACKLIST += esp32-wroom-32
-
 include $(RIOTBASE)/Makefile.include

tests/driver_si1133/tests-with-config/01-run.py

@@ -12,10 +12,7 @@ from testrunner import run
 
 def testfunc(child):
     child.expect_exact('Testing Si1133 in blocking mode:')
-    i = child.expect([r'.*Result: OK\s', r'.*Result: FAILED (\d+)\s'])
+    child.expect_exact('Result: OK')
-    if i == 1:
-        print('FAILED')
-        return
     print('SUCCESS')
 
 

tests/pkg_fatfs_vfs/Makefile

@@ -45,10 +45,6 @@ BOARD_WHITELIST := airfy-beacon arduino-due arduino-duemilanove \
 
 TEST_DEPS += image
 
-# The test requires some manual setup to work
-# So it cannot currently be run
-TEST_ON_CI_BLACKLIST += all
-
 include $(RIOTBASE)/Makefile.include
 
 image:

tests/pkg_fatfs_vfs/README.md

@@ -21,6 +21,8 @@ FAT) That implies it doesn't show any modifications in RIOT that you perform on
 your OS and the other way round. So always remember to mount/unmount correctly
 or your FS will probably get damaged.
 
+    make flash test-with-config
+
 # Real Hardware
 
 Currently the test defaults to sdcard_spi on real hardware. But generally any
@@ -30,3 +32,5 @@ storage device (e.g. your SD-card). To copy the image onto the card you can use
 something like `make image && dd if=bin/riot_fatfs_disk.img
 of=/dev/<your_sdcard>`. After that you can connect the card to your RIOT device
 and check the test output via terminal.
+
+    make flash test-with-config

tests/pkg_semtech-loramac/Makefile

@@ -34,7 +34,4 @@ ifneq (,$(filter iotlab%,$(MAKECMDGOALS)))
   include $(RIOTBASE)/dist/testbed-support/Makefile.iotlab
 endif
 
-# Can't be run on ci since it requires gateway + lora node
-TEST_ON_CI_BLACKLIST = all
-
 include $(RIOTBASE)/Makefile.include