From 4412b9a4aba7b91a9724525587c43a012ead5e9f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ga=C3=ABtan=20Harter?= Date: Fri, 2 Aug 2019 15:51:52 +0200 Subject: [PATCH 01/19] tests/tests.inc.mk: move tests targets and variables Create a file for setting tests targets and variables. It is a refactoring before adding new commands. --- Makefile.include | 38 ++---------------------------------- makefiles/tests/tests.inc.mk | 37 +++++++++++++++++++++++++++++++++++ 2 files changed, 39 insertions(+), 36 deletions(-) create mode 100644 makefiles/tests/tests.inc.mk diff --git a/Makefile.include b/Makefile.include index 69cc428e57..42ac2099da 100644 --- a/Makefile.include +++ b/Makefile.include @@ -758,42 +758,8 @@ reset: $(call check_cmd,$(RESET),Reset program) $(RESET) $(RESET_FLAGS) -.PHONY: test test/available -TESTS ?= $(foreach file,$(wildcard $(APPDIR)/tests/*[^~]),\ - $(shell test -f $(file) -a -x $(file) && echo $(file))) - -# "make test" calls "make term" which would implicitly build it's dependencies, -# but that increases the time "make test" needs to get ready. That can cause -# problems ("make term" missing some lines) as terminal startup is not properly -# sychronized, but depends on a static timeout (TESTRUNNER_START_DELAY). -# Murdock builds the term dependencies before running "make test" to circumvent -# this. In order to make local builds behave similar, add the term deps here. -# See #11762. -TEST_DEPS += $(TERMDEPS) - -# Export TESTRUNNER_RESET_AFTER_TERM only for the test target. This allows for -# it to be accessed through the environment from python test script. -# This is currently needed only by `examples/%/tests` and should be removed in -# the future since `make reset` after `term` is not a valid synch method across -# all platforms. -$(call target-export-variables,test,TESTRUNNER_RESET_AFTER_TERM) -test: $(TEST_DEPS) - $(Q) for t in $(TESTS); do \ - $$t || exit 1; \ - done - -test/available: - $(Q)test -n "$(strip $(TESTS))" - -# this target only makes sense if an ELFFILE is actually created, thus guard by -# RIOTNOLINK="". -ifeq (,$(RIOTNOLINK)) -test-input-hash: $(TESTS) $(ELFFILE) $(TEST_EXTRA_FILES) - sha1sum $^ > $(BINDIR)/test-input-hash.sha1 -else -test-input-hash: - true -endif +# tests related targets and variables +include $(RIOTMAKE)/tests/tests.inc.mk .PHONY: fuzz fuzz: diff --git a/makefiles/tests/tests.inc.mk b/makefiles/tests/tests.inc.mk new file mode 100644 index 0000000000..cb03717140 --- /dev/null +++ b/makefiles/tests/tests.inc.mk @@ -0,0 +1,37 @@ +# Export TESTRUNNER_RESET_AFTER_TERM only for the test target. This allows for +# it to be accessed through the environment from python test script. +# This is currently needed only by `examples/%/tests` and should be removed in +# the future since `make reset` after `term` is not a valid synch method across +# all platforms. +$(call target-export-variables,test test-as-root test-with-config,TESTRUNNER_RESET_AFTER_TERM) + +.PHONY: test test/available +TESTS ?= $(foreach file,$(wildcard $(APPDIR)/tests/*[^~]),\ + $(shell test -f $(file) -a -x $(file) && echo $(file))) + +# "make test" calls "make term" which would implicitly build it's dependencies, +# but that increases the time "make test" needs to get ready. That can cause +# problems ("make term" missing some lines) as terminal startup is not properly +# sychronized, but depends on a static timeout (TESTRUNNER_START_DELAY). +# Murdock builds the term dependencies before running "make test" to circumvent +# this. In order to make local builds behave similar, add the term deps here. +# See #11762. +TEST_DEPS += $(TERMDEPS) + +test: $(TEST_DEPS) + $(Q) for t in $(TESTS); do \ + $$t || exit 1; \ + done + +test/available: + $(Q)test -n "$(strip $(TESTS))" + +# this target only makes sense if an ELFFILE is actually created, thus guard by +# RIOTNOLINK="". +ifeq (,$(RIOTNOLINK)) +test-input-hash: $(TESTS) $(ELFFILE) $(TEST_EXTRA_FILES) + sha1sum $^ > $(BINDIR)/test-input-hash.sha1 +else +test-input-hash: + true +endif From 54c4536945ba313167b285a4af5f6d4f016652b0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ga=C3=ABtan=20Harter?= Date: Fri, 2 Aug 2019 16:15:52 +0200 Subject: [PATCH 02/19] tests: add 'test-as-root' and 'test-with-config' targets These targets cannot be used in an automated testing workflow without complex configuration or extend rights. - Add new 'test-as-root' target for tests that require to be root or start an external daemon as root - Add new 'test-with-config' target for tests that require a specific configuration to succeed (module configuration or hardware configuration) --- makefiles/tests/tests.inc.mk | 28 +++++++++++++++++++++++++++- tests/README.md | 19 +++++++++++++++++++ 2 files changed, 46 insertions(+), 1 deletion(-) diff --git a/makefiles/tests/tests.inc.mk b/makefiles/tests/tests.inc.mk index cb03717140..c2152ccb21 100644 --- a/makefiles/tests/tests.inc.mk +++ b/makefiles/tests/tests.inc.mk @@ -26,10 +26,36 @@ test: $(TEST_DEPS) test/available: $(Q)test -n "$(strip $(TESTS))" +# Tests that require root privileges +.PHONY: test-as-root test-as-root/available +TESTS_AS_ROOT ?= $(foreach file,$(wildcard $(APPDIR)/tests-as-root/*[^~]),\ + $(shell test -f $(file) -a -x $(file) && echo $(file))) + +test-as-root: $(TEST_DEPS) + $(Q) for t in $(TESTS_AS_ROOT); do \ + sudo -E PYTHONPATH=$(PYTHONPATH) $$t || exit 1; \ + done + +test-as-root/available: + $(Q)test -n "$(strip $(TESTS_AS_ROOT))" + +# Tests that require specific configuration +.PHONY: test-with-config test-with-config/available +TESTS_WITH_CONFIG ?= $(foreach file,$(wildcard $(APPDIR)/tests-with-config/*[^~]),\ + $(shell test -f $(file) -a -x $(file) && echo $(file))) + +test-with-config: $(TEST_DEPS) + $(Q) for t in $(TESTS_WITH_CONFIG); do \ + $$t || exit 1; \ + done + +test-with-config/available: + $(Q)test -n "$(strip $(TESTS_WITH_CONFIG))" + # this target only makes sense if an ELFFILE is actually created, thus guard by # RIOTNOLINK="". ifeq (,$(RIOTNOLINK)) -test-input-hash: $(TESTS) $(ELFFILE) $(TEST_EXTRA_FILES) +test-input-hash: $(TESTS) $(TESTS_WITH_CONFIG) $(TESTS_AS_ROOT) $(ELFFILE) $(TEST_EXTRA_FILES) sha1sum $^ > $(BINDIR)/test-input-hash.sha1 else test-input-hash: diff --git a/tests/README.md b/tests/README.md index f57b7b9eeb..2db0ac5faa 100644 --- a/tests/README.md +++ b/tests/README.md @@ -117,3 +117,22 @@ The expected behavior is verified with the test in `tests/test_tools`. Tests cannot rely on having on all boards and terminal programs: * unbuffered input * allowing sending special characters like `ctrl+c/ctrl+d` + + +Running tests that require a preliminary manual configuration +------------------------------------------------------------- + +Some tests need active monitoring or manual setup steps but still have some +automated scripts. The test automation scripts are defined in the +`/tests-with-config/` folder. +For running them, follow the setup or analysis documentation and use the +`test-with-config` target. + +Running tests that require root privileges +------------------------------------------ + +Some tests require root privileges to launch their automated script. In this +case, the test automation scripts are defined in the +`/tests-as-root/` folder. +For running them, follow the setup or analysis documentation and use the +`test-as-root` target. From 80b9290bb1b112d665cb5f059ae2487d0be9bdc8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ga=C3=ABtan=20Harter?= Date: Fri, 2 Aug 2019 16:14:46 +0200 Subject: [PATCH 03/19] tests/pkg_fatfs_vfs: declare test as test-with-config For running the test on boards, it needs a manual setup. This however declares the "native" test as requiring config too. No need to be blacklisted when using test-with-config. --- tests/pkg_fatfs_vfs/Makefile | 4 ---- tests/pkg_fatfs_vfs/README.md | 4 ++++ tests/pkg_fatfs_vfs/{tests => tests-with-config}/01-run.py | 0 3 files changed, 4 insertions(+), 4 deletions(-) rename tests/pkg_fatfs_vfs/{tests => tests-with-config}/01-run.py (100%) diff --git a/tests/pkg_fatfs_vfs/Makefile b/tests/pkg_fatfs_vfs/Makefile index e420b54668..3fb1aaffa1 100644 --- a/tests/pkg_fatfs_vfs/Makefile +++ b/tests/pkg_fatfs_vfs/Makefile @@ -45,10 +45,6 @@ BOARD_WHITELIST := airfy-beacon arduino-due arduino-duemilanove \ TEST_DEPS += image -# The test requires some manual setup to work -# So it cannot currently be run -TEST_ON_CI_BLACKLIST += all - include $(RIOTBASE)/Makefile.include image: diff --git a/tests/pkg_fatfs_vfs/README.md b/tests/pkg_fatfs_vfs/README.md index 9b6a94cf0f..e71a36f89f 100644 --- a/tests/pkg_fatfs_vfs/README.md +++ b/tests/pkg_fatfs_vfs/README.md @@ -21,6 +21,8 @@ FAT) That implies it doesn't show any modifications in RIOT that you perform on your OS and the other way round. So always remember to mount/unmount correctly or your FS will probably get damaged. + make flash test-with-config + # Real Hardware Currently the test defaults to sdcard_spi on real hardware. But generally any @@ -30,3 +32,5 @@ storage device (e.g. your SD-card). To copy the image onto the card you can use something like `make image && dd if=bin/riot_fatfs_disk.img of=/dev/`. After that you can connect the card to your RIOT device and check the test output via terminal. + + make flash test-with-config diff --git a/tests/pkg_fatfs_vfs/tests/01-run.py b/tests/pkg_fatfs_vfs/tests-with-config/01-run.py similarity index 100% rename from tests/pkg_fatfs_vfs/tests/01-run.py rename to tests/pkg_fatfs_vfs/tests-with-config/01-run.py From 072718b04b072df49cfe716a3c54ad6860f716a1 Mon Sep 17 00:00:00 2001 From: Alexandre Abadie Date: Fri, 9 Oct 2020 17:51:21 +0200 Subject: [PATCH 04/19] examples: tests: move lorawan examples to tests-with-config --- examples/lorawan/Makefile | 3 --- examples/lorawan/{tests => tests-with-config}/01-run.py | 0 tests/pkg_semtech-loramac/Makefile | 3 --- .../pkg_semtech-loramac/{tests => tests-with-config}/01-run.py | 0 4 files changed, 6 deletions(-) rename examples/lorawan/{tests => tests-with-config}/01-run.py (100%) rename tests/pkg_semtech-loramac/{tests => tests-with-config}/01-run.py (100%) diff --git a/examples/lorawan/Makefile b/examples/lorawan/Makefile index b18d388841..76a803707a 100644 --- a/examples/lorawan/Makefile +++ b/examples/lorawan/Makefile @@ -46,7 +46,4 @@ ifneq (,$(filter test,$(MAKECMDGOALS))) DEFAULT_MODULE += test_utils_interactive_sync endif -# Can't be run on ci since it requires gateway + lora node -TEST_ON_CI_BLACKLIST = all - include $(RIOTBASE)/Makefile.include diff --git a/examples/lorawan/tests/01-run.py b/examples/lorawan/tests-with-config/01-run.py similarity index 100% rename from examples/lorawan/tests/01-run.py rename to examples/lorawan/tests-with-config/01-run.py diff --git a/tests/pkg_semtech-loramac/Makefile b/tests/pkg_semtech-loramac/Makefile index ceadd1f564..ad2f10c202 100644 --- a/tests/pkg_semtech-loramac/Makefile +++ b/tests/pkg_semtech-loramac/Makefile @@ -34,7 +34,4 @@ ifneq (,$(filter iotlab%,$(MAKECMDGOALS))) include $(RIOTBASE)/dist/testbed-support/Makefile.iotlab endif -# Can't be run on ci since it requires gateway + lora node -TEST_ON_CI_BLACKLIST = all - include $(RIOTBASE)/Makefile.include diff --git a/tests/pkg_semtech-loramac/tests/01-run.py b/tests/pkg_semtech-loramac/tests-with-config/01-run.py similarity index 100% rename from tests/pkg_semtech-loramac/tests/01-run.py rename to tests/pkg_semtech-loramac/tests-with-config/01-run.py From 87a382aa5e399108043cf65a4476895beffd4e03 Mon Sep 17 00:00:00 2001 From: Alexandre Abadie Date: Fri, 9 Oct 2020 18:04:56 +0200 Subject: [PATCH 05/19] examples/suit_update: move to test-with-config --- examples/suit_update/{tests => tests-with-config}/01-run.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename examples/suit_update/{tests => tests-with-config}/01-run.py (100%) diff --git a/examples/suit_update/tests/01-run.py b/examples/suit_update/tests-with-config/01-run.py similarity index 100% rename from examples/suit_update/tests/01-run.py rename to examples/suit_update/tests-with-config/01-run.py From 9821835e288976e950ede4109eb9e7204730f816 Mon Sep 17 00:00:00 2001 From: Alexandre Abadie Date: Fri, 9 Oct 2020 18:06:06 +0200 Subject: [PATCH 06/19] tests/emcute: move to tests-as-root --- tests/emcute/{tests => tests-as-root}/01-run.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename tests/emcute/{tests => tests-as-root}/01-run.py (100%) diff --git a/tests/emcute/tests/01-run.py b/tests/emcute/tests-as-root/01-run.py similarity index 100% rename from tests/emcute/tests/01-run.py rename to tests/emcute/tests-as-root/01-run.py From 239e7cadfbdc82b7400c5bfd648e175fb56b5fcd Mon Sep 17 00:00:00 2001 From: Alexandre Abadie Date: Fri, 9 Oct 2020 18:07:53 +0200 Subject: [PATCH 07/19] tests/gnrc_*: move to tests-as-root --- tests/gnrc_dhcpv6_client/{tests => tests-as-root}/01-run.py | 0 tests/gnrc_dhcpv6_client_6lbr/{tests => tests-as-root}/01-run.py | 0 tests/gnrc_ipv6_ext/{tests => tests-as-root}/01-run.py | 0 tests/gnrc_ipv6_ext_frag/{tests => tests-as-root}/01-run.py | 0 tests/gnrc_ipv6_ext_opt/{tests => tests-as-root}/01-run.py | 0 tests/gnrc_ipv6_nib_dns/{tests => tests-as-root}/01-run.py | 0 tests/gnrc_rpl_srh/{tests => tests-as-root}/01-run.py | 0 tests/gnrc_sock_dns/{tests => tests-as-root}/01-run.py | 0 .../{tests => tests-as-root}/01-conn_lifecycle_as_client.py | 0 .../{tests => tests-as-root}/02-conn_lifecycle_as_server.py | 0 tests/gnrc_tcp/{tests => tests-as-root}/03-send_data.py | 0 tests/gnrc_tcp/{tests => tests-as-root}/04-receive_data.py | 0 tests/gnrc_tcp/{tests => tests-as-root}/05-garbage-pkts.py | 0 .../{tests => tests-as-root}/06-receive_data_closed_conn.py | 0 .../gnrc_tcp/{tests => tests-as-root}/07-endpoint_construction.py | 0 tests/gnrc_tcp/{tests => tests-as-root}/shared_func.py | 0 16 files changed, 0 insertions(+), 0 deletions(-) rename tests/gnrc_dhcpv6_client/{tests => tests-as-root}/01-run.py (100%) rename tests/gnrc_dhcpv6_client_6lbr/{tests => tests-as-root}/01-run.py (100%) rename tests/gnrc_ipv6_ext/{tests => tests-as-root}/01-run.py (100%) rename tests/gnrc_ipv6_ext_frag/{tests => tests-as-root}/01-run.py (100%) rename tests/gnrc_ipv6_ext_opt/{tests => tests-as-root}/01-run.py (100%) rename tests/gnrc_ipv6_nib_dns/{tests => tests-as-root}/01-run.py (100%) rename tests/gnrc_rpl_srh/{tests => tests-as-root}/01-run.py (100%) rename tests/gnrc_sock_dns/{tests => tests-as-root}/01-run.py (100%) rename tests/gnrc_tcp/{tests => tests-as-root}/01-conn_lifecycle_as_client.py (100%) rename tests/gnrc_tcp/{tests => tests-as-root}/02-conn_lifecycle_as_server.py (100%) rename tests/gnrc_tcp/{tests => tests-as-root}/03-send_data.py (100%) rename tests/gnrc_tcp/{tests => tests-as-root}/04-receive_data.py (100%) rename tests/gnrc_tcp/{tests => tests-as-root}/05-garbage-pkts.py (100%) rename tests/gnrc_tcp/{tests => tests-as-root}/06-receive_data_closed_conn.py (100%) rename tests/gnrc_tcp/{tests => tests-as-root}/07-endpoint_construction.py (100%) rename tests/gnrc_tcp/{tests => tests-as-root}/shared_func.py (100%) diff --git a/tests/gnrc_dhcpv6_client/tests/01-run.py b/tests/gnrc_dhcpv6_client/tests-as-root/01-run.py similarity index 100% rename from tests/gnrc_dhcpv6_client/tests/01-run.py rename to tests/gnrc_dhcpv6_client/tests-as-root/01-run.py diff --git a/tests/gnrc_dhcpv6_client_6lbr/tests/01-run.py b/tests/gnrc_dhcpv6_client_6lbr/tests-as-root/01-run.py similarity index 100% rename from tests/gnrc_dhcpv6_client_6lbr/tests/01-run.py rename to tests/gnrc_dhcpv6_client_6lbr/tests-as-root/01-run.py diff --git a/tests/gnrc_ipv6_ext/tests/01-run.py b/tests/gnrc_ipv6_ext/tests-as-root/01-run.py similarity index 100% rename from tests/gnrc_ipv6_ext/tests/01-run.py rename to tests/gnrc_ipv6_ext/tests-as-root/01-run.py diff --git a/tests/gnrc_ipv6_ext_frag/tests/01-run.py b/tests/gnrc_ipv6_ext_frag/tests-as-root/01-run.py similarity index 100% rename from tests/gnrc_ipv6_ext_frag/tests/01-run.py rename to tests/gnrc_ipv6_ext_frag/tests-as-root/01-run.py diff --git a/tests/gnrc_ipv6_ext_opt/tests/01-run.py b/tests/gnrc_ipv6_ext_opt/tests-as-root/01-run.py similarity index 100% rename from tests/gnrc_ipv6_ext_opt/tests/01-run.py rename to tests/gnrc_ipv6_ext_opt/tests-as-root/01-run.py diff --git a/tests/gnrc_ipv6_nib_dns/tests/01-run.py b/tests/gnrc_ipv6_nib_dns/tests-as-root/01-run.py similarity index 100% rename from tests/gnrc_ipv6_nib_dns/tests/01-run.py rename to tests/gnrc_ipv6_nib_dns/tests-as-root/01-run.py diff --git a/tests/gnrc_rpl_srh/tests/01-run.py b/tests/gnrc_rpl_srh/tests-as-root/01-run.py similarity index 100% rename from tests/gnrc_rpl_srh/tests/01-run.py rename to tests/gnrc_rpl_srh/tests-as-root/01-run.py diff --git a/tests/gnrc_sock_dns/tests/01-run.py b/tests/gnrc_sock_dns/tests-as-root/01-run.py similarity index 100% rename from tests/gnrc_sock_dns/tests/01-run.py rename to tests/gnrc_sock_dns/tests-as-root/01-run.py diff --git a/tests/gnrc_tcp/tests/01-conn_lifecycle_as_client.py b/tests/gnrc_tcp/tests-as-root/01-conn_lifecycle_as_client.py similarity index 100% rename from tests/gnrc_tcp/tests/01-conn_lifecycle_as_client.py rename to tests/gnrc_tcp/tests-as-root/01-conn_lifecycle_as_client.py diff --git a/tests/gnrc_tcp/tests/02-conn_lifecycle_as_server.py b/tests/gnrc_tcp/tests-as-root/02-conn_lifecycle_as_server.py similarity index 100% rename from tests/gnrc_tcp/tests/02-conn_lifecycle_as_server.py rename to tests/gnrc_tcp/tests-as-root/02-conn_lifecycle_as_server.py diff --git a/tests/gnrc_tcp/tests/03-send_data.py b/tests/gnrc_tcp/tests-as-root/03-send_data.py similarity index 100% rename from tests/gnrc_tcp/tests/03-send_data.py rename to tests/gnrc_tcp/tests-as-root/03-send_data.py diff --git a/tests/gnrc_tcp/tests/04-receive_data.py b/tests/gnrc_tcp/tests-as-root/04-receive_data.py similarity index 100% rename from tests/gnrc_tcp/tests/04-receive_data.py rename to tests/gnrc_tcp/tests-as-root/04-receive_data.py diff --git a/tests/gnrc_tcp/tests/05-garbage-pkts.py b/tests/gnrc_tcp/tests-as-root/05-garbage-pkts.py similarity index 100% rename from tests/gnrc_tcp/tests/05-garbage-pkts.py rename to tests/gnrc_tcp/tests-as-root/05-garbage-pkts.py diff --git a/tests/gnrc_tcp/tests/06-receive_data_closed_conn.py b/tests/gnrc_tcp/tests-as-root/06-receive_data_closed_conn.py similarity index 100% rename from tests/gnrc_tcp/tests/06-receive_data_closed_conn.py rename to tests/gnrc_tcp/tests-as-root/06-receive_data_closed_conn.py diff --git a/tests/gnrc_tcp/tests/07-endpoint_construction.py b/tests/gnrc_tcp/tests-as-root/07-endpoint_construction.py similarity index 100% rename from tests/gnrc_tcp/tests/07-endpoint_construction.py rename to tests/gnrc_tcp/tests-as-root/07-endpoint_construction.py diff --git a/tests/gnrc_tcp/tests/shared_func.py b/tests/gnrc_tcp/tests-as-root/shared_func.py similarity index 100% rename from tests/gnrc_tcp/tests/shared_func.py rename to tests/gnrc_tcp/tests-as-root/shared_func.py From e071444af7c9b53dea7df8fdf6544e11906763eb Mon Sep 17 00:00:00 2001 From: Alexandre Abadie Date: Fri, 9 Oct 2020 18:09:23 +0200 Subject: [PATCH 08/19] tests/driver_at86rf2xx_aes: move to test-with-config --- tests/driver_at86rf2xx_aes/{tests => tests-with-config}/01-run.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename tests/driver_at86rf2xx_aes/{tests => tests-with-config}/01-run.py (100%) diff --git a/tests/driver_at86rf2xx_aes/tests/01-run.py b/tests/driver_at86rf2xx_aes/tests-with-config/01-run.py similarity index 100% rename from tests/driver_at86rf2xx_aes/tests/01-run.py rename to tests/driver_at86rf2xx_aes/tests-with-config/01-run.py From 16652e83f165efccb765f9f1fa34aa73f534061e Mon Sep 17 00:00:00 2001 From: Alexandre Abadie Date: Fri, 9 Oct 2020 18:10:39 +0200 Subject: [PATCH 09/19] tests/driver_ds1307: move to test-with-config --- tests/driver_ds1307/Makefile | 4 ---- tests/driver_ds1307/{tests => tests-with-config}/01-run.py | 0 2 files changed, 4 deletions(-) rename tests/driver_ds1307/{tests => tests-with-config}/01-run.py (100%) diff --git a/tests/driver_ds1307/Makefile b/tests/driver_ds1307/Makefile index 2b39618a9a..9f9eb79bca 100644 --- a/tests/driver_ds1307/Makefile +++ b/tests/driver_ds1307/Makefile @@ -1,9 +1,5 @@ include ../Makefile.tests_common -# Blacklist iotlab boards since a different device has the same i2c address -BOARD_BLACKLIST := iotlab-a8-m3 \ - iotlab-m3 - USEMODULE += ds1307 USEMODULE += embunit USEMODULE += xtimer diff --git a/tests/driver_ds1307/tests/01-run.py b/tests/driver_ds1307/tests-with-config/01-run.py similarity index 100% rename from tests/driver_ds1307/tests/01-run.py rename to tests/driver_ds1307/tests-with-config/01-run.py From 60c8e027d0c81d93c00b48d3d5d3b2ec4fabba6a Mon Sep 17 00:00:00 2001 From: Alexandre Abadie Date: Fri, 9 Oct 2020 18:11:25 +0200 Subject: [PATCH 10/19] tests/driver_hd44780: move to test-with-config --- tests/driver_hd44780/Makefile | 4 ---- tests/driver_hd44780/{tests => tests-with-config}/01-run.py | 0 2 files changed, 4 deletions(-) rename tests/driver_hd44780/{tests => tests-with-config}/01-run.py (100%) diff --git a/tests/driver_hd44780/Makefile b/tests/driver_hd44780/Makefile index 669e5be139..b9fb552708 100644 --- a/tests/driver_hd44780/Makefile +++ b/tests/driver_hd44780/Makefile @@ -11,8 +11,4 @@ ifeq (native,$(BOARD)) endif endif -# Fails on esp32 because the driver defines default GPIOs that are used for the -# SPI flash interface. -TEST_ON_CI_BLACKLIST += esp32-wroom-32 - include $(RIOTBASE)/Makefile.include diff --git a/tests/driver_hd44780/tests/01-run.py b/tests/driver_hd44780/tests-with-config/01-run.py similarity index 100% rename from tests/driver_hd44780/tests/01-run.py rename to tests/driver_hd44780/tests-with-config/01-run.py From d0a3f3090b0fb55e5a0da2f9606fddac3507bdda Mon Sep 17 00:00:00 2001 From: Alexandre Abadie Date: Fri, 9 Oct 2020 18:12:00 +0200 Subject: [PATCH 11/19] tests/driver_grove_ledbar: move to test-with-config --- tests/driver_grove_ledbar/{tests => tests-with-config}/01-run.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename tests/driver_grove_ledbar/{tests => tests-with-config}/01-run.py (100%) diff --git a/tests/driver_grove_ledbar/tests/01-run.py b/tests/driver_grove_ledbar/tests-with-config/01-run.py similarity index 100% rename from tests/driver_grove_ledbar/tests/01-run.py rename to tests/driver_grove_ledbar/tests-with-config/01-run.py From aeadbfc80d4091e97d212a19dad0228f26dab642 Mon Sep 17 00:00:00 2001 From: Alexandre Abadie Date: Fri, 9 Oct 2020 18:12:36 +0200 Subject: [PATCH 12/19] tests/driver_my9221: move to test-with-config --- tests/driver_my9221/{tests => tests-with-config}/01-run.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename tests/driver_my9221/{tests => tests-with-config}/01-run.py (100%) diff --git a/tests/driver_my9221/tests/01-run.py b/tests/driver_my9221/tests-with-config/01-run.py similarity index 100% rename from tests/driver_my9221/tests/01-run.py rename to tests/driver_my9221/tests-with-config/01-run.py From ffa28ef7310bf50ac9fb6f1434be616ea786197c Mon Sep 17 00:00:00 2001 From: Alexandre Abadie Date: Thu, 14 Jan 2021 17:29:16 +0100 Subject: [PATCH 13/19] tests/gnrc_sock_dns: move to test-with-config --- .../gnrc_sock_dns/{tests-as-root => tests-with-config}/01-run.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename tests/gnrc_sock_dns/{tests-as-root => tests-with-config}/01-run.py (100%) diff --git a/tests/gnrc_sock_dns/tests-as-root/01-run.py b/tests/gnrc_sock_dns/tests-with-config/01-run.py similarity index 100% rename from tests/gnrc_sock_dns/tests-as-root/01-run.py rename to tests/gnrc_sock_dns/tests-with-config/01-run.py From 76e186571b337087cbc30a6b1a2f7850593bd430 Mon Sep 17 00:00:00 2001 From: Alexandre Abadie Date: Fri, 15 Jan 2021 10:37:47 +0100 Subject: [PATCH 14/19] tests: fix auto use of interactive sync module --- tests/Makefile.tests_common | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/Makefile.tests_common b/tests/Makefile.tests_common index a9a798c3b9..506cc6ef65 100644 --- a/tests/Makefile.tests_common +++ b/tests/Makefile.tests_common @@ -1,6 +1,6 @@ APPLICATION ?= tests_$(notdir $(patsubst %/,%,$(CURDIR))) -ifneq (,$(wildcard $(CURDIR)/tests/.)) +ifneq (,$(wildcard $(CURDIR)/tests*/.)) DEFAULT_MODULE += test_utils_interactive_sync # add interactive test configuration when testing Kconfig ifeq (1,$(TEST_KCONFIG)) From 64998021307c78dbfd13efee95665acf291e2108 Mon Sep 17 00:00:00 2001 From: Alexandre Abadie Date: Fri, 15 Jan 2021 12:26:09 +0100 Subject: [PATCH 15/19] .murdock: adapt run-test for test-with-config target --- .murdock | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/.murdock b/.murdock index 1e2fb2d7e6..b3e7b7e16b 100755 --- a/.murdock +++ b/.murdock @@ -27,6 +27,8 @@ tests/driver_v*"} : ${TEST_KCONFIG_native:="examples/hello-world tests/periph_* tests/xtimer_* tests/ztimer_* tests/driver_ws281x"} +: ${TEST_WITH_CONFIG_SUPPORTED:="examples/suit_update tests/driver_at86rf2xx_aes"} + export RIOT_CI_BUILD=1 export CC_NOCOLOR=1 export STATIC_TESTS=0 @@ -382,7 +384,11 @@ run_test() { BOARD=$board TOOLCHAIN=${toolchain} make -C$appdir flash-only termdeps -j2 # now run the actual test - BOARD=$board TOOLCHAIN=${toolchain} make -C$appdir test + if is_in_list "${appdir}" "${TEST_WITH_CONFIG_SUPPORTED}"; then + BOARD=${board} TOOLCHAIN=${toolchain} make -C${appdir} test-with-config + else + BOARD=$board TOOLCHAIN=${toolchain} make -C$appdir test + fi RES=$? if [ $RES -eq 0 -a -n "$TEST_HASH" ]; then From 5e43172c87b854ea78f7e1a22f9e27c187ed8126 Mon Sep 17 00:00:00 2001 From: Alexandre Abadie Date: Tue, 19 Jan 2021 16:15:36 +0100 Subject: [PATCH 16/19] tests/driver_ds3231: move to test-with-config --- tests/driver_ds3231/Makefile | 7 ------- tests/driver_ds3231/{tests => tests-with-config}/01-run.py | 0 2 files changed, 7 deletions(-) rename tests/driver_ds3231/{tests => tests-with-config}/01-run.py (100%) diff --git a/tests/driver_ds3231/Makefile b/tests/driver_ds3231/Makefile index 1abdcebbb0..fb8eb35773 100644 --- a/tests/driver_ds3231/Makefile +++ b/tests/driver_ds3231/Makefile @@ -1,12 +1,5 @@ include ../Makefile.tests_common -# the test needs real hardware to run -TEST_ON_CI_BLACKLIST += all - -# Blacklist iotlab boards since a different device has the same i2c address -BOARD_BLACKLIST := iotlab-a8-m3 \ - iotlab-m3 - USEMODULE += ds3231 USEMODULE += xtimer USEMODULE += shell diff --git a/tests/driver_ds3231/tests/01-run.py b/tests/driver_ds3231/tests-with-config/01-run.py similarity index 100% rename from tests/driver_ds3231/tests/01-run.py rename to tests/driver_ds3231/tests-with-config/01-run.py From 5ec41f35c7af37834197a9236cd37bbca12a7bc1 Mon Sep 17 00:00:00 2001 From: Alexandre Abadie Date: Tue, 19 Jan 2021 16:18:51 +0100 Subject: [PATCH 17/19] tests/driver_bme680: move to test-with-config If the configuration is correct, the test should succeed, so the test script is adapted --- tests/driver_bme680/{tests => tests-with-config}/01-run.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) rename tests/driver_bme680/{tests => tests-with-config}/01-run.py (79%) diff --git a/tests/driver_bme680/tests/01-run.py b/tests/driver_bme680/tests-with-config/01-run.py similarity index 79% rename from tests/driver_bme680/tests/01-run.py rename to tests/driver_bme680/tests-with-config/01-run.py index 213640011f..de50a83c09 100755 --- a/tests/driver_bme680/tests/01-run.py +++ b/tests/driver_bme680/tests-with-config/01-run.py @@ -11,11 +11,7 @@ from testrunner import run def testfunc(child): - child.expect('Initialize BME680 sensor 0 ... ') - i = child.expect(['[OK]', '[failed]']) - if i == 1: - print('FAILED') - return + child.expect_exact('Initialize BME680 sensor 0 ... OK') child.expect(r'\[bme680\]: dev=0, ' r'T = \d+.\d+ degC, ' r'P = \d+ Pa, ' From f8fb26835ce0cfd44cc68f20683bc668588ccca0 Mon Sep 17 00:00:00 2001 From: Alexandre Abadie Date: Tue, 19 Jan 2021 16:22:14 +0100 Subject: [PATCH 18/19] tests/driver_si1133: move to test-with-config --- tests/driver_si1133/{tests => tests-with-config}/01-run.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) rename tests/driver_si1133/{tests => tests-with-config}/01-run.py (77%) diff --git a/tests/driver_si1133/tests/01-run.py b/tests/driver_si1133/tests-with-config/01-run.py similarity index 77% rename from tests/driver_si1133/tests/01-run.py rename to tests/driver_si1133/tests-with-config/01-run.py index 4ef52c72e7..f4cfa52413 100755 --- a/tests/driver_si1133/tests/01-run.py +++ b/tests/driver_si1133/tests-with-config/01-run.py @@ -12,10 +12,7 @@ from testrunner import run def testfunc(child): child.expect_exact('Testing Si1133 in blocking mode:') - i = child.expect([r'.*Result: OK\s', r'.*Result: FAILED (\d+)\s']) - if i == 1: - print('FAILED') - return + child.expect_exact('Result: OK') print('SUCCESS') From d0f3cefc7e2c52d7352b222e0851f4beec5fe12e Mon Sep 17 00:00:00 2001 From: Alexandre Abadie Date: Tue, 19 Jan 2021 16:24:53 +0100 Subject: [PATCH 19/19] tests/driver_apds99xx*: move to test-with-config --- .../driver_apds99xx/{tests => tests-with-config}/01-run.py | 7 ++----- .../{tests => tests-with-config}/01-run.py | 7 ++----- 2 files changed, 4 insertions(+), 10 deletions(-) rename tests/driver_apds99xx/{tests => tests-with-config}/01-run.py (80%) rename tests/driver_apds99xx_full/{tests => tests-with-config}/01-run.py (79%) diff --git a/tests/driver_apds99xx/tests/01-run.py b/tests/driver_apds99xx/tests-with-config/01-run.py similarity index 80% rename from tests/driver_apds99xx/tests/01-run.py rename to tests/driver_apds99xx/tests-with-config/01-run.py index a8cf62fa38..be7bcf3740 100755 --- a/tests/driver_apds99xx/tests/01-run.py +++ b/tests/driver_apds99xx/tests-with-config/01-run.py @@ -11,11 +11,8 @@ from testrunner import run def testfunc(child): - child.expect('Initializing APDS99XX sensor') - i = child.expect(['[OK]', '[Failed]']) - if i == 1: - print('FAILED') - return + child.expect_exact('Initializing APDS99XX sensor') + child.expect_exact('[OK]') child.expect(r'proximity = \d+ \[cnts\]') child.expect(r'ambient = \d+ \[cnts\]') child.expect([r'red = \d+ \[cnts\], green = \d+ \[cnts\], blue = \d+ \[cnts\]', diff --git a/tests/driver_apds99xx_full/tests/01-run.py b/tests/driver_apds99xx_full/tests-with-config/01-run.py similarity index 79% rename from tests/driver_apds99xx_full/tests/01-run.py rename to tests/driver_apds99xx_full/tests-with-config/01-run.py index fd84d416cf..429b8920b8 100755 --- a/tests/driver_apds99xx_full/tests/01-run.py +++ b/tests/driver_apds99xx_full/tests-with-config/01-run.py @@ -11,11 +11,8 @@ from testrunner import run def testfunc(child): - child.expect('Initializing APDS99XX sensor') - i = child.expect(['[OK]', '[Failed]']) - if i == 1: - print('FAILED') - return + child.expect_exact('Initializing APDS99XX sensor') + child.expect_exact('[OK]') child.expect(r'ambient = \d+ \[cnts\]') child.expect([r'red = \d+ \[cnts\], green = \d+ \[cnts\], blue = \d+ \[cnts\]', r'illuminance = %d [lux]'])