diff --git a/makefiles/murdock.inc.mk b/makefiles/murdock.inc.mk index 9fa08fe8ff..fe08550052 100644 --- a/makefiles/murdock.inc.mk +++ b/makefiles/murdock.inc.mk @@ -15,16 +15,38 @@ test-murdock: "$(BOARD):$(TOOLCHAIN)" \ $(FLASHFILE) -# don't whitelist tests if there's no binary -ifeq (1,$(RIOTNOLINK)) - TEST_ON_CI_WHITELIST:= -endif -# create $(BINDIR)/.test file only if BOARD is in $(TEST_ON_CI_WHITELIST) +# Control running tests on boards during the CI tests. +# +# Testing is enabled if all the following conditions are met: +# +# * the board is whitelisted +# * the board is not blacklisted (by default none) +# * the board has enough memory and the executable is being linked +# +# TEST_ON_CI_WHITELIST and TEST_ON_CI_BLACKLIST can be empty, a board list or 'all' +# +# Prefer blacklisting boards that fail to whitelisting the ones that work. +# It will help tracking what is failing. +# +# Disabling a test in some case must be justified to keep track of the reason. + +TEST_ON_CI_WHITELIST ?= +TEST_ON_CI_BLACKLIST ?= +TEST_ON_BOARD_ENABLED ?= $(filter-out $(TEST_ON_CI_BLACKLIST:all=%),$(filter $(TEST_ON_CI_WHITELIST:all=%),$(BOARD))) + +TEST_ON_CI_ENABLED ?= $(if $(RIOTNOLINK),,$(TEST_ON_BOARD_ENABLED)) + +.PHONY: test-on-ci-enabled +test-on-ci-enabled: + $(Q)test -n "$(TEST_ON_CI_ENABLED)" + + +# create $(BINDIR)/.test file only if the test is enabled for this board .PHONY: $(BINDIR)/.test link: $(BINDIR)/.test $(BINDIR)/.test: $(filter clean, $(MAKECMDGOALS)) -ifneq (,$(filter $(BOARD) all, $(TEST_ON_CI_WHITELIST))) +ifneq (,$(TEST_ON_CI_ENABLED)) $(Q)mkdir -p $(BINDIR) $(Q)touch $@ else diff --git a/tests/xtimer_now64_continuity/Makefile b/tests/xtimer_now64_continuity/Makefile index 7771d93310..c3c678dd66 100644 --- a/tests/xtimer_now64_continuity/Makefile +++ b/tests/xtimer_now64_continuity/Makefile @@ -3,7 +3,8 @@ include ../Makefile.tests_common USEMODULE += fmt USEMODULE += xtimer +TEST_ON_CI_WHITELIST += all # This test randomly fails on `native` so disable it from CI -TEST_ON_CI_WHITELIST += samr21-xpro +TEST_ON_CI_BLACKLIST += native include $(RIOTBASE)/Makefile.include diff --git a/tests/xtimer_usleep/Makefile b/tests/xtimer_usleep/Makefile index 1af1bdee92..48cf81df4e 100644 --- a/tests/xtimer_usleep/Makefile +++ b/tests/xtimer_usleep/Makefile @@ -2,8 +2,9 @@ include ../Makefile.tests_common USEMODULE += xtimer +TEST_ON_CI_WHITELIST += all # This test randomly fails on `native` so disable it from CI -TEST_ON_CI_WHITELIST += samr21-xpro +TEST_ON_CI_BLACKLIST += native # This application uses getchar and thus expects input from stdio USEMODULE += stdin