diff --git a/.murdock b/.murdock index 1e2fb2d7e6..b3e7b7e16b 100755 --- a/.murdock +++ b/.murdock @@ -27,6 +27,8 @@ tests/driver_v*"} : ${TEST_KCONFIG_native:="examples/hello-world tests/periph_* tests/xtimer_* tests/ztimer_* tests/driver_ws281x"} +: ${TEST_WITH_CONFIG_SUPPORTED:="examples/suit_update tests/driver_at86rf2xx_aes"} + export RIOT_CI_BUILD=1 export CC_NOCOLOR=1 export STATIC_TESTS=0 @@ -382,7 +384,11 @@ run_test() { BOARD=$board TOOLCHAIN=${toolchain} make -C$appdir flash-only termdeps -j2 # now run the actual test - BOARD=$board TOOLCHAIN=${toolchain} make -C$appdir test + if is_in_list "${appdir}" "${TEST_WITH_CONFIG_SUPPORTED}"; then + BOARD=${board} TOOLCHAIN=${toolchain} make -C${appdir} test-with-config + else + BOARD=$board TOOLCHAIN=${toolchain} make -C$appdir test + fi RES=$? if [ $RES -eq 0 -a -n "$TEST_HASH" ]; then diff --git a/Makefile.include b/Makefile.include index 07eee030d5..3a63080c23 100644 --- a/Makefile.include +++ b/Makefile.include @@ -761,42 +761,8 @@ reset: $(call check_cmd,$(RESET),Reset program) $(RESET) $(RESET_FLAGS) -.PHONY: test test/available -TESTS ?= $(foreach file,$(wildcard $(APPDIR)/tests/*[^~]),\ - $(shell test -f $(file) -a -x $(file) && echo $(file))) - -# "make test" calls "make term" which would implicitly build it's dependencies, -# but that increases the time "make test" needs to get ready. That can cause -# problems ("make term" missing some lines) as terminal startup is not properly -# sychronized, but depends on a static timeout (TESTRUNNER_START_DELAY). -# Murdock builds the term dependencies before running "make test" to circumvent -# this. In order to make local builds behave similar, add the term deps here. -# See #11762. -TEST_DEPS += $(TERMDEPS) - -# Export TESTRUNNER_RESET_AFTER_TERM only for the test target. This allows for -# it to be accessed through the environment from python test script. -# This is currently needed only by `examples/%/tests` and should be removed in -# the future since `make reset` after `term` is not a valid synch method across -# all platforms. -$(call target-export-variables,test,TESTRUNNER_RESET_AFTER_TERM) -test: $(TEST_DEPS) - $(Q) for t in $(TESTS); do \ - $$t || exit 1; \ - done - -test/available: - $(Q)test -n "$(strip $(TESTS))" - -# this target only makes sense if an ELFFILE is actually created, thus guard by -# RIOTNOLINK="". -ifeq (,$(RIOTNOLINK)) -test-input-hash: $(TESTS) $(ELFFILE) $(TEST_EXTRA_FILES) - sha1sum $^ > $(BINDIR)/test-input-hash.sha1 -else -test-input-hash: - true -endif +# tests related targets and variables +include $(RIOTMAKE)/tests/tests.inc.mk .PHONY: fuzz fuzz: diff --git a/examples/lorawan/Makefile b/examples/lorawan/Makefile index b18d388841..76a803707a 100644 --- a/examples/lorawan/Makefile +++ b/examples/lorawan/Makefile @@ -46,7 +46,4 @@ ifneq (,$(filter test,$(MAKECMDGOALS))) DEFAULT_MODULE += test_utils_interactive_sync endif -# Can't be run on ci since it requires gateway + lora node -TEST_ON_CI_BLACKLIST = all - include $(RIOTBASE)/Makefile.include diff --git a/examples/lorawan/tests/01-run.py b/examples/lorawan/tests-with-config/01-run.py similarity index 100% rename from examples/lorawan/tests/01-run.py rename to examples/lorawan/tests-with-config/01-run.py diff --git a/examples/suit_update/tests/01-run.py b/examples/suit_update/tests-with-config/01-run.py similarity index 100% rename from examples/suit_update/tests/01-run.py rename to examples/suit_update/tests-with-config/01-run.py diff --git a/makefiles/tests/tests.inc.mk b/makefiles/tests/tests.inc.mk new file mode 100644 index 0000000000..c2152ccb21 --- /dev/null +++ b/makefiles/tests/tests.inc.mk @@ -0,0 +1,63 @@ +# Export TESTRUNNER_RESET_AFTER_TERM only for the test target. This allows for +# it to be accessed through the environment from python test script. +# This is currently needed only by `examples/%/tests` and should be removed in +# the future since `make reset` after `term` is not a valid synch method across +# all platforms. +$(call target-export-variables,test test-as-root test-with-config,TESTRUNNER_RESET_AFTER_TERM) + +.PHONY: test test/available +TESTS ?= $(foreach file,$(wildcard $(APPDIR)/tests/*[^~]),\ + $(shell test -f $(file) -a -x $(file) && echo $(file))) + +# "make test" calls "make term" which would implicitly build it's dependencies, +# but that increases the time "make test" needs to get ready. That can cause +# problems ("make term" missing some lines) as terminal startup is not properly +# sychronized, but depends on a static timeout (TESTRUNNER_START_DELAY). +# Murdock builds the term dependencies before running "make test" to circumvent +# this. In order to make local builds behave similar, add the term deps here. +# See #11762. +TEST_DEPS += $(TERMDEPS) + +test: $(TEST_DEPS) + $(Q) for t in $(TESTS); do \ + $$t || exit 1; \ + done + +test/available: + $(Q)test -n "$(strip $(TESTS))" + +# Tests that require root privileges +.PHONY: test-as-root test-as-root/available +TESTS_AS_ROOT ?= $(foreach file,$(wildcard $(APPDIR)/tests-as-root/*[^~]),\ + $(shell test -f $(file) -a -x $(file) && echo $(file))) + +test-as-root: $(TEST_DEPS) + $(Q) for t in $(TESTS_AS_ROOT); do \ + sudo -E PYTHONPATH=$(PYTHONPATH) $$t || exit 1; \ + done + +test-as-root/available: + $(Q)test -n "$(strip $(TESTS_AS_ROOT))" + +# Tests that require specific configuration +.PHONY: test-with-config test-with-config/available +TESTS_WITH_CONFIG ?= $(foreach file,$(wildcard $(APPDIR)/tests-with-config/*[^~]),\ + $(shell test -f $(file) -a -x $(file) && echo $(file))) + +test-with-config: $(TEST_DEPS) + $(Q) for t in $(TESTS_WITH_CONFIG); do \ + $$t || exit 1; \ + done + +test-with-config/available: + $(Q)test -n "$(strip $(TESTS_WITH_CONFIG))" + +# this target only makes sense if an ELFFILE is actually created, thus guard by +# RIOTNOLINK="". +ifeq (,$(RIOTNOLINK)) +test-input-hash: $(TESTS) $(TESTS_WITH_CONFIG) $(TESTS_AS_ROOT) $(ELFFILE) $(TEST_EXTRA_FILES) + sha1sum $^ > $(BINDIR)/test-input-hash.sha1 +else +test-input-hash: + true +endif diff --git a/tests/Makefile.tests_common b/tests/Makefile.tests_common index a9a798c3b9..506cc6ef65 100644 --- a/tests/Makefile.tests_common +++ b/tests/Makefile.tests_common @@ -1,6 +1,6 @@ APPLICATION ?= tests_$(notdir $(patsubst %/,%,$(CURDIR))) -ifneq (,$(wildcard $(CURDIR)/tests/.)) +ifneq (,$(wildcard $(CURDIR)/tests*/.)) DEFAULT_MODULE += test_utils_interactive_sync # add interactive test configuration when testing Kconfig ifeq (1,$(TEST_KCONFIG)) diff --git a/tests/README.md b/tests/README.md index f57b7b9eeb..2db0ac5faa 100644 --- a/tests/README.md +++ b/tests/README.md @@ -117,3 +117,22 @@ The expected behavior is verified with the test in `tests/test_tools`. Tests cannot rely on having on all boards and terminal programs: * unbuffered input * allowing sending special characters like `ctrl+c/ctrl+d` + + +Running tests that require a preliminary manual configuration +------------------------------------------------------------- + +Some tests need active monitoring or manual setup steps but still have some +automated scripts. The test automation scripts are defined in the +`/tests-with-config/` folder. +For running them, follow the setup or analysis documentation and use the +`test-with-config` target. + +Running tests that require root privileges +------------------------------------------ + +Some tests require root privileges to launch their automated script. In this +case, the test automation scripts are defined in the +`/tests-as-root/` folder. +For running them, follow the setup or analysis documentation and use the +`test-as-root` target. diff --git a/tests/driver_apds99xx/tests/01-run.py b/tests/driver_apds99xx/tests-with-config/01-run.py similarity index 80% rename from tests/driver_apds99xx/tests/01-run.py rename to tests/driver_apds99xx/tests-with-config/01-run.py index a8cf62fa38..be7bcf3740 100755 --- a/tests/driver_apds99xx/tests/01-run.py +++ b/tests/driver_apds99xx/tests-with-config/01-run.py @@ -11,11 +11,8 @@ from testrunner import run def testfunc(child): - child.expect('Initializing APDS99XX sensor') - i = child.expect(['[OK]', '[Failed]']) - if i == 1: - print('FAILED') - return + child.expect_exact('Initializing APDS99XX sensor') + child.expect_exact('[OK]') child.expect(r'proximity = \d+ \[cnts\]') child.expect(r'ambient = \d+ \[cnts\]') child.expect([r'red = \d+ \[cnts\], green = \d+ \[cnts\], blue = \d+ \[cnts\]', diff --git a/tests/driver_apds99xx_full/tests/01-run.py b/tests/driver_apds99xx_full/tests-with-config/01-run.py similarity index 79% rename from tests/driver_apds99xx_full/tests/01-run.py rename to tests/driver_apds99xx_full/tests-with-config/01-run.py index fd84d416cf..429b8920b8 100755 --- a/tests/driver_apds99xx_full/tests/01-run.py +++ b/tests/driver_apds99xx_full/tests-with-config/01-run.py @@ -11,11 +11,8 @@ from testrunner import run def testfunc(child): - child.expect('Initializing APDS99XX sensor') - i = child.expect(['[OK]', '[Failed]']) - if i == 1: - print('FAILED') - return + child.expect_exact('Initializing APDS99XX sensor') + child.expect_exact('[OK]') child.expect(r'ambient = \d+ \[cnts\]') child.expect([r'red = \d+ \[cnts\], green = \d+ \[cnts\], blue = \d+ \[cnts\]', r'illuminance = %d [lux]']) diff --git a/tests/driver_at86rf2xx_aes/tests/01-run.py b/tests/driver_at86rf2xx_aes/tests-with-config/01-run.py similarity index 100% rename from tests/driver_at86rf2xx_aes/tests/01-run.py rename to tests/driver_at86rf2xx_aes/tests-with-config/01-run.py diff --git a/tests/driver_bme680/tests/01-run.py b/tests/driver_bme680/tests-with-config/01-run.py similarity index 79% rename from tests/driver_bme680/tests/01-run.py rename to tests/driver_bme680/tests-with-config/01-run.py index 213640011f..de50a83c09 100755 --- a/tests/driver_bme680/tests/01-run.py +++ b/tests/driver_bme680/tests-with-config/01-run.py @@ -11,11 +11,7 @@ from testrunner import run def testfunc(child): - child.expect('Initialize BME680 sensor 0 ... ') - i = child.expect(['[OK]', '[failed]']) - if i == 1: - print('FAILED') - return + child.expect_exact('Initialize BME680 sensor 0 ... OK') child.expect(r'\[bme680\]: dev=0, ' r'T = \d+.\d+ degC, ' r'P = \d+ Pa, ' diff --git a/tests/driver_ds1307/Makefile b/tests/driver_ds1307/Makefile index 2b39618a9a..9f9eb79bca 100644 --- a/tests/driver_ds1307/Makefile +++ b/tests/driver_ds1307/Makefile @@ -1,9 +1,5 @@ include ../Makefile.tests_common -# Blacklist iotlab boards since a different device has the same i2c address -BOARD_BLACKLIST := iotlab-a8-m3 \ - iotlab-m3 - USEMODULE += ds1307 USEMODULE += embunit USEMODULE += xtimer diff --git a/tests/driver_ds1307/tests/01-run.py b/tests/driver_ds1307/tests-with-config/01-run.py similarity index 100% rename from tests/driver_ds1307/tests/01-run.py rename to tests/driver_ds1307/tests-with-config/01-run.py diff --git a/tests/driver_ds3231/Makefile b/tests/driver_ds3231/Makefile index 1abdcebbb0..fb8eb35773 100644 --- a/tests/driver_ds3231/Makefile +++ b/tests/driver_ds3231/Makefile @@ -1,12 +1,5 @@ include ../Makefile.tests_common -# the test needs real hardware to run -TEST_ON_CI_BLACKLIST += all - -# Blacklist iotlab boards since a different device has the same i2c address -BOARD_BLACKLIST := iotlab-a8-m3 \ - iotlab-m3 - USEMODULE += ds3231 USEMODULE += xtimer USEMODULE += shell diff --git a/tests/driver_ds3231/tests/01-run.py b/tests/driver_ds3231/tests-with-config/01-run.py similarity index 100% rename from tests/driver_ds3231/tests/01-run.py rename to tests/driver_ds3231/tests-with-config/01-run.py diff --git a/tests/driver_grove_ledbar/tests/01-run.py b/tests/driver_grove_ledbar/tests-with-config/01-run.py similarity index 100% rename from tests/driver_grove_ledbar/tests/01-run.py rename to tests/driver_grove_ledbar/tests-with-config/01-run.py diff --git a/tests/driver_hd44780/Makefile b/tests/driver_hd44780/Makefile index 669e5be139..b9fb552708 100644 --- a/tests/driver_hd44780/Makefile +++ b/tests/driver_hd44780/Makefile @@ -11,8 +11,4 @@ ifeq (native,$(BOARD)) endif endif -# Fails on esp32 because the driver defines default GPIOs that are used for the -# SPI flash interface. -TEST_ON_CI_BLACKLIST += esp32-wroom-32 - include $(RIOTBASE)/Makefile.include diff --git a/tests/driver_hd44780/tests/01-run.py b/tests/driver_hd44780/tests-with-config/01-run.py similarity index 100% rename from tests/driver_hd44780/tests/01-run.py rename to tests/driver_hd44780/tests-with-config/01-run.py diff --git a/tests/driver_my9221/tests/01-run.py b/tests/driver_my9221/tests-with-config/01-run.py similarity index 100% rename from tests/driver_my9221/tests/01-run.py rename to tests/driver_my9221/tests-with-config/01-run.py diff --git a/tests/driver_si1133/tests/01-run.py b/tests/driver_si1133/tests-with-config/01-run.py similarity index 77% rename from tests/driver_si1133/tests/01-run.py rename to tests/driver_si1133/tests-with-config/01-run.py index 4ef52c72e7..f4cfa52413 100755 --- a/tests/driver_si1133/tests/01-run.py +++ b/tests/driver_si1133/tests-with-config/01-run.py @@ -12,10 +12,7 @@ from testrunner import run def testfunc(child): child.expect_exact('Testing Si1133 in blocking mode:') - i = child.expect([r'.*Result: OK\s', r'.*Result: FAILED (\d+)\s']) - if i == 1: - print('FAILED') - return + child.expect_exact('Result: OK') print('SUCCESS') diff --git a/tests/emcute/tests/01-run.py b/tests/emcute/tests-as-root/01-run.py similarity index 100% rename from tests/emcute/tests/01-run.py rename to tests/emcute/tests-as-root/01-run.py diff --git a/tests/gnrc_dhcpv6_client/tests/01-run.py b/tests/gnrc_dhcpv6_client/tests-as-root/01-run.py similarity index 100% rename from tests/gnrc_dhcpv6_client/tests/01-run.py rename to tests/gnrc_dhcpv6_client/tests-as-root/01-run.py diff --git a/tests/gnrc_dhcpv6_client_6lbr/tests/01-run.py b/tests/gnrc_dhcpv6_client_6lbr/tests-as-root/01-run.py similarity index 100% rename from tests/gnrc_dhcpv6_client_6lbr/tests/01-run.py rename to tests/gnrc_dhcpv6_client_6lbr/tests-as-root/01-run.py diff --git a/tests/gnrc_ipv6_ext/tests/01-run.py b/tests/gnrc_ipv6_ext/tests-as-root/01-run.py similarity index 100% rename from tests/gnrc_ipv6_ext/tests/01-run.py rename to tests/gnrc_ipv6_ext/tests-as-root/01-run.py diff --git a/tests/gnrc_ipv6_ext_frag/tests/01-run.py b/tests/gnrc_ipv6_ext_frag/tests-as-root/01-run.py similarity index 100% rename from tests/gnrc_ipv6_ext_frag/tests/01-run.py rename to tests/gnrc_ipv6_ext_frag/tests-as-root/01-run.py diff --git a/tests/gnrc_ipv6_ext_opt/tests/01-run.py b/tests/gnrc_ipv6_ext_opt/tests-as-root/01-run.py similarity index 100% rename from tests/gnrc_ipv6_ext_opt/tests/01-run.py rename to tests/gnrc_ipv6_ext_opt/tests-as-root/01-run.py diff --git a/tests/gnrc_ipv6_nib_dns/tests/01-run.py b/tests/gnrc_ipv6_nib_dns/tests-as-root/01-run.py similarity index 100% rename from tests/gnrc_ipv6_nib_dns/tests/01-run.py rename to tests/gnrc_ipv6_nib_dns/tests-as-root/01-run.py diff --git a/tests/gnrc_rpl_srh/tests/01-run.py b/tests/gnrc_rpl_srh/tests-as-root/01-run.py similarity index 100% rename from tests/gnrc_rpl_srh/tests/01-run.py rename to tests/gnrc_rpl_srh/tests-as-root/01-run.py diff --git a/tests/gnrc_sock_dns/tests/01-run.py b/tests/gnrc_sock_dns/tests-with-config/01-run.py similarity index 100% rename from tests/gnrc_sock_dns/tests/01-run.py rename to tests/gnrc_sock_dns/tests-with-config/01-run.py diff --git a/tests/gnrc_tcp/tests/01-conn_lifecycle_as_client.py b/tests/gnrc_tcp/tests-as-root/01-conn_lifecycle_as_client.py similarity index 100% rename from tests/gnrc_tcp/tests/01-conn_lifecycle_as_client.py rename to tests/gnrc_tcp/tests-as-root/01-conn_lifecycle_as_client.py diff --git a/tests/gnrc_tcp/tests/02-conn_lifecycle_as_server.py b/tests/gnrc_tcp/tests-as-root/02-conn_lifecycle_as_server.py similarity index 100% rename from tests/gnrc_tcp/tests/02-conn_lifecycle_as_server.py rename to tests/gnrc_tcp/tests-as-root/02-conn_lifecycle_as_server.py diff --git a/tests/gnrc_tcp/tests/03-send_data.py b/tests/gnrc_tcp/tests-as-root/03-send_data.py similarity index 100% rename from tests/gnrc_tcp/tests/03-send_data.py rename to tests/gnrc_tcp/tests-as-root/03-send_data.py diff --git a/tests/gnrc_tcp/tests/04-receive_data.py b/tests/gnrc_tcp/tests-as-root/04-receive_data.py similarity index 100% rename from tests/gnrc_tcp/tests/04-receive_data.py rename to tests/gnrc_tcp/tests-as-root/04-receive_data.py diff --git a/tests/gnrc_tcp/tests/05-garbage-pkts.py b/tests/gnrc_tcp/tests-as-root/05-garbage-pkts.py similarity index 100% rename from tests/gnrc_tcp/tests/05-garbage-pkts.py rename to tests/gnrc_tcp/tests-as-root/05-garbage-pkts.py diff --git a/tests/gnrc_tcp/tests/06-receive_data_closed_conn.py b/tests/gnrc_tcp/tests-as-root/06-receive_data_closed_conn.py similarity index 100% rename from tests/gnrc_tcp/tests/06-receive_data_closed_conn.py rename to tests/gnrc_tcp/tests-as-root/06-receive_data_closed_conn.py diff --git a/tests/gnrc_tcp/tests/07-endpoint_construction.py b/tests/gnrc_tcp/tests-as-root/07-endpoint_construction.py similarity index 100% rename from tests/gnrc_tcp/tests/07-endpoint_construction.py rename to tests/gnrc_tcp/tests-as-root/07-endpoint_construction.py diff --git a/tests/gnrc_tcp/tests/shared_func.py b/tests/gnrc_tcp/tests-as-root/shared_func.py similarity index 100% rename from tests/gnrc_tcp/tests/shared_func.py rename to tests/gnrc_tcp/tests-as-root/shared_func.py diff --git a/tests/pkg_fatfs_vfs/Makefile b/tests/pkg_fatfs_vfs/Makefile index e420b54668..3fb1aaffa1 100644 --- a/tests/pkg_fatfs_vfs/Makefile +++ b/tests/pkg_fatfs_vfs/Makefile @@ -45,10 +45,6 @@ BOARD_WHITELIST := airfy-beacon arduino-due arduino-duemilanove \ TEST_DEPS += image -# The test requires some manual setup to work -# So it cannot currently be run -TEST_ON_CI_BLACKLIST += all - include $(RIOTBASE)/Makefile.include image: diff --git a/tests/pkg_fatfs_vfs/README.md b/tests/pkg_fatfs_vfs/README.md index 9b6a94cf0f..e71a36f89f 100644 --- a/tests/pkg_fatfs_vfs/README.md +++ b/tests/pkg_fatfs_vfs/README.md @@ -21,6 +21,8 @@ FAT) That implies it doesn't show any modifications in RIOT that you perform on your OS and the other way round. So always remember to mount/unmount correctly or your FS will probably get damaged. + make flash test-with-config + # Real Hardware Currently the test defaults to sdcard_spi on real hardware. But generally any @@ -30,3 +32,5 @@ storage device (e.g. your SD-card). To copy the image onto the card you can use something like `make image && dd if=bin/riot_fatfs_disk.img of=/dev/`. After that you can connect the card to your RIOT device and check the test output via terminal. + + make flash test-with-config diff --git a/tests/pkg_fatfs_vfs/tests/01-run.py b/tests/pkg_fatfs_vfs/tests-with-config/01-run.py similarity index 100% rename from tests/pkg_fatfs_vfs/tests/01-run.py rename to tests/pkg_fatfs_vfs/tests-with-config/01-run.py diff --git a/tests/pkg_semtech-loramac/Makefile b/tests/pkg_semtech-loramac/Makefile index ceadd1f564..ad2f10c202 100644 --- a/tests/pkg_semtech-loramac/Makefile +++ b/tests/pkg_semtech-loramac/Makefile @@ -34,7 +34,4 @@ ifneq (,$(filter iotlab%,$(MAKECMDGOALS))) include $(RIOTBASE)/dist/testbed-support/Makefile.iotlab endif -# Can't be run on ci since it requires gateway + lora node -TEST_ON_CI_BLACKLIST = all - include $(RIOTBASE)/Makefile.include diff --git a/tests/pkg_semtech-loramac/tests/01-run.py b/tests/pkg_semtech-loramac/tests-with-config/01-run.py similarity index 100% rename from tests/pkg_semtech-loramac/tests/01-run.py rename to tests/pkg_semtech-loramac/tests-with-config/01-run.py