From eff6816ded0d0e50e78ca153f34f12d3ff672a8c Mon Sep 17 00:00:00 2001 From: Marian Buschsieweke Date: Thu, 27 Mar 2025 00:32:28 +0100 Subject: [PATCH] tests/unittests: allow out-of-tree tests This adds and documents the new `EXTERNAL_UNIT_TEST_DIRS` environment variable that allows including out-of-tree unit tests. The intention is to allow users of `EXTERNAL_MODULE_DIRS` to also provide corresponding unit tests and run them all with a single test app. Co-authored-by: crasbe --- .../creating_application.md | 8 ++++ makefiles/app_dirs.blacklist | 2 + .../build_system/external_unittests/Makefile | 11 +++++ .../build_system/external_unittests/README.md | 16 +++++++ .../tests-out_of_tree/Makefile | 1 + .../tests-out_of_tree/tests-out-of-tree.c | 35 ++++++++++++++++ tests/build_system/external_unittests/main.c | 1 + tests/build_system/external_unittests/map.h | 1 + .../external_unittests/tests-in_tree/Makefile | 1 + .../tests-in_tree/tests-in-tree.c | 35 ++++++++++++++++ .../external_unittests/tests/01-run.py | 18 ++++++++ tests/unittests/Makefile | 42 +++++++++++++++---- tests/unittests/README.md | 11 +++++ 13 files changed, 174 insertions(+), 8 deletions(-) create mode 100644 tests/build_system/external_unittests/Makefile create mode 100644 tests/build_system/external_unittests/README.md create mode 100644 tests/build_system/external_unittests/external_tests_dir/tests-out_of_tree/Makefile create mode 100644 tests/build_system/external_unittests/external_tests_dir/tests-out_of_tree/tests-out-of-tree.c create mode 120000 tests/build_system/external_unittests/main.c create mode 120000 tests/build_system/external_unittests/map.h create mode 100644 tests/build_system/external_unittests/tests-in_tree/Makefile create mode 100644 tests/build_system/external_unittests/tests-in_tree/tests-in-tree.c create mode 100755 tests/build_system/external_unittests/tests/01-run.py diff --git a/doc/guides/advanced_tutorials/creating_application.md b/doc/guides/advanced_tutorials/creating_application.md index 2f650a5988..2dca9b2e12 100644 --- a/doc/guides/advanced_tutorials/creating_application.md +++ b/doc/guides/advanced_tutorials/creating_application.md @@ -343,3 +343,11 @@ APPLICATION = my_app PROJECT_BASE ?= $(CURDIR)/../.. include $(PROJECT_BASE)/Makefile.include ``` + +### Unit Tests for External Modules + +When writing external modules, it can be helpful to make use of the existing +unit test infrastructure in `tests/unittests` to run both upstream unit tests +as well as downstream project-specific unit tests in a single app. For this the +environment variable `EXTERNAL_UNITTEST_DIRS` can be used to list directories +(separated by a space) to additional unit tests. diff --git a/makefiles/app_dirs.blacklist b/makefiles/app_dirs.blacklist index ff3e1055e1..b5b1d30e21 100644 --- a/makefiles/app_dirs.blacklist +++ b/makefiles/app_dirs.blacklist @@ -11,6 +11,8 @@ tests/build_system/external_board_native/external_boards/ tests/build_system/external_module_dirs/external_modules/ tests/build_system/external_pkg_dirs/external_pkgs/ tests/build_system/kconfig/external_modules/ +tests/build_system/external_unittests/external_tests_dir/ +tests/build_system/external_unittests/tests-in_tree tests/build_system/kconfig/external_pkgs/ tests/periph/qdec/boards_modded/ tests/pkg/openwsn_sock_udp/external_modules/ diff --git a/tests/build_system/external_unittests/Makefile b/tests/build_system/external_unittests/Makefile new file mode 100644 index 0000000000..5e57c156dd --- /dev/null +++ b/tests/build_system/external_unittests/Makefile @@ -0,0 +1,11 @@ +# Run this only on native64 / native, as this rather tests the build system +# than the code actually run. + +BOARD_WHITELIST := native native64 + +EXTERNAL_UNITTEST_DIRS := $(CURDIR)/external_tests_dir + +# Build upon tests/unittests: +RIOTBASE ?= $(CURDIR)/../../.. +UNIT_TESTS_DIR := $(CURDIR)/../../unittests +include ../../unittests/Makefile diff --git a/tests/build_system/external_unittests/README.md b/tests/build_system/external_unittests/README.md new file mode 100644 index 0000000000..14548ee98e --- /dev/null +++ b/tests/build_system/external_unittests/README.md @@ -0,0 +1,16 @@ +External Unit Tests +=================== + +This test apps provides two trivial unit tests: One provided internally in the +folder of this unit tests apps and one provided externally via a variable. +Since the test only validates correct operation of the build system, there is +little value in running it on more than one board. For this reason, the app is +limited to be build only for native / native64. + +Implementation Details +---------------------- + +The two tests contain a reference to a variable of the other tests, so that +the app will not link if only if both of the unit tests are available at +link time (or none). In addition, the python test runner will check that two + tests are executed (to rule out that none was available). diff --git a/tests/build_system/external_unittests/external_tests_dir/tests-out_of_tree/Makefile b/tests/build_system/external_unittests/external_tests_dir/tests-out_of_tree/Makefile new file mode 100644 index 0000000000..48422e909a --- /dev/null +++ b/tests/build_system/external_unittests/external_tests_dir/tests-out_of_tree/Makefile @@ -0,0 +1 @@ +include $(RIOTBASE)/Makefile.base diff --git a/tests/build_system/external_unittests/external_tests_dir/tests-out_of_tree/tests-out-of-tree.c b/tests/build_system/external_unittests/external_tests_dir/tests-out_of_tree/tests-out-of-tree.c new file mode 100644 index 0000000000..40de49bae6 --- /dev/null +++ b/tests/build_system/external_unittests/external_tests_dir/tests-out_of_tree/tests-out-of-tree.c @@ -0,0 +1,35 @@ +/* + * Copyright (C) 2025 ML!PA Consulting GmbH + * + * This file is subject to the terms and conditions of the GNU Lesser + * General Public License v2.1. See the file LICENSE in the top level + * directory for more details. + */ + +#include + +#include "embUnit.h" + +const unsigned external_test_was_linked_in = 1; + +static void test_internal_test_was_linked_in(void) +{ + extern const unsigned internal_test_was_linked_in; + TEST_ASSERT_EQUAL_INT(1, internal_test_was_linked_in); +} + +static Test *out_of_tree_tests(void) +{ + EMB_UNIT_TESTFIXTURES(fixtures) { + new_TestFixture(test_internal_test_was_linked_in), + }; + + EMB_UNIT_TESTCALLER(_out_of_tree_tests, NULL, NULL, fixtures); + + return (Test *)&_out_of_tree_tests; +} + +void tests_out_of_tree(void) +{ + TESTS_RUN(out_of_tree_tests()); +} diff --git a/tests/build_system/external_unittests/main.c b/tests/build_system/external_unittests/main.c new file mode 120000 index 0000000000..ac31dda5ae --- /dev/null +++ b/tests/build_system/external_unittests/main.c @@ -0,0 +1 @@ +../../unittests/main.c \ No newline at end of file diff --git a/tests/build_system/external_unittests/map.h b/tests/build_system/external_unittests/map.h new file mode 120000 index 0000000000..c4a835b73c --- /dev/null +++ b/tests/build_system/external_unittests/map.h @@ -0,0 +1 @@ +../../unittests/map.h \ No newline at end of file diff --git a/tests/build_system/external_unittests/tests-in_tree/Makefile b/tests/build_system/external_unittests/tests-in_tree/Makefile new file mode 100644 index 0000000000..48422e909a --- /dev/null +++ b/tests/build_system/external_unittests/tests-in_tree/Makefile @@ -0,0 +1 @@ +include $(RIOTBASE)/Makefile.base diff --git a/tests/build_system/external_unittests/tests-in_tree/tests-in-tree.c b/tests/build_system/external_unittests/tests-in_tree/tests-in-tree.c new file mode 100644 index 0000000000..4eed17a2d6 --- /dev/null +++ b/tests/build_system/external_unittests/tests-in_tree/tests-in-tree.c @@ -0,0 +1,35 @@ +/* + * Copyright (C) 2025 ML!PA Consulting GmbH + * + * This file is subject to the terms and conditions of the GNU Lesser + * General Public License v2.1. See the file LICENSE in the top level + * directory for more details. + */ + +#include + +#include "embUnit.h" + +const unsigned internal_test_was_linked_in = 1; + +static void test_external_test_was_linked_in(void) +{ + extern const unsigned external_test_was_linked_in; + TEST_ASSERT_EQUAL_INT(1, external_test_was_linked_in); +} + +static Test *in_tree_tests(void) +{ + EMB_UNIT_TESTFIXTURES(fixtures) { + new_TestFixture(test_external_test_was_linked_in), + }; + + EMB_UNIT_TESTCALLER(_in_tree_tests, NULL, NULL, fixtures); + + return (Test *)&_in_tree_tests; +} + +void tests_in_tree(void) +{ + TESTS_RUN(in_tree_tests()); +} diff --git a/tests/build_system/external_unittests/tests/01-run.py b/tests/build_system/external_unittests/tests/01-run.py new file mode 100755 index 0000000000..643e26d485 --- /dev/null +++ b/tests/build_system/external_unittests/tests/01-run.py @@ -0,0 +1,18 @@ +#!/usr/bin/env python3 + +# Copyright (C) 2016 Kaspar Schleiser +# +# This file is subject to the terms and conditions of the GNU Lesser +# General Public License v2.1. See the file LICENSE in the top level +# directory for more details. + +import sys +from testrunner import run + + +def testfunc(child): + child.expect_exact("OK (2 tests)") + + +if __name__ == "__main__": + sys.exit(run(testfunc)) diff --git a/tests/unittests/Makefile b/tests/unittests/Makefile index 3899ed4d92..51a82f6bbb 100644 --- a/tests/unittests/Makefile +++ b/tests/unittests/Makefile @@ -1,18 +1,40 @@ DEVELHELP ?= 0 -include ../Makefile.tests_common + +# Allow including this Makefile from other apps, as done by +# tests/build_system/external_unittests +UNIT_TESTS_DIR ?= $(CURDIR) +include $(UNIT_TESTS_DIR)/../Makefile.tests_common USEMODULE += embunit -ifeq (, $(UNIT_TESTS)) +# Search for unit tests both in-tree and, if configured, out-of-tree +UNIT_TEST_SEARCH_DIRS := $(CURDIR) $(EXTERNAL_UNITTEST_DIRS) +# Canonicalize test search dirs: Add a single solidus at the end: +UNIT_TEST_SEARCH_DIRS := $(addsuffix /,$(abspath $(UNIT_TEST_SEARCH_DIRS))) + +ifeq (,$(UNIT_TESTS)) ifeq (, $(filter tests-%, $(MAKECMDGOALS))) - # the $(dir) Makefile function leaves a trailing slash after the directory - # name, therefore we use patsubst instead. - UNIT_TESTS := $(patsubst %/Makefile,%,$(wildcard tests-*/Makefile)) + # collect all unit test makefiles in all unit test search dirs + UNIT_TESTS_MAKEFILES := $(wildcard $(addsuffix tests-*/Makefile,$(UNIT_TEST_SEARCH_DIRS))) + # $(dir) would leave a solidus at the end, using patsub instead + UNIT_TESTS := $(patsubst %/Makefile,%,$(UNIT_TESTS_MAKEFILES)) + # $(notdir) gives the last component in path, despite its name + UNIT_TESTS := $(notdir $(UNIT_TESTS)) else UNIT_TESTS := $(filter tests-%, $(MAKECMDGOALS)) endif endif +# in case unit tests were provided by name, we need to locate them +ifeq (,$(UNIT_TESTS_MAKEFILES)) + # generate "$searchdir/$testname/Makefile" for every possible combination of + # $searchdir and $testname + UNIT_TESTS_MAKEFILES := $(addsuffix /Makefile,$(UNIT_TESTS)) + UNIT_TESTS_MAKEFILES := $(foreach testdir,$(UNIT_TEST_SEARCH_DIRS),$(addprefix $(testdir),$(UNIT_TESTS_MAKEFILES))) + # reduce that list to files that actually exist + UNIT_TESTS_MAKEFILES := $(wildcard $(UNIT_TESTS_MAKEFILES)) +endif + ifeq (llvm,$(TOOLCHAIN)) # the floating point exception bug is more likely to trigger when build # with LLVM, so we just disable LLVM on native as a work around @@ -25,10 +47,14 @@ DISABLE_MODULE += auto_init auto_init_% # initialize stdio over USB. FEATURES_BLACKLIST += highlevel_stdio -# Pull in `Makefile.include`s from the test suites: --include $(UNIT_TESTS:%=$(RIOTBASE)/tests/unittests/%/Makefile.include) +# extract the test dirs from the unit test makefiles +UNIT_TEST_DIRS := $(dir $(UNIT_TESTS_MAKEFILES)) -DIRS += $(UNIT_TESTS) +# each unit test dir needs to be build as module +DIRS += $(UNIT_TEST_DIRS) + +# Pull in `Makefile.include`s from the test suites: +-include $(addsuffix Makefile.include,$(UNIT_TEST_DIRS)) BASELIBS += $(UNIT_TESTS:%=%.module) INCLUDES += -I$(RIOTBASE)/tests/unittests/common diff --git a/tests/unittests/README.md b/tests/unittests/README.md index 5a94490376..3888b8a69c 100644 --- a/tests/unittests/README.md +++ b/tests/unittests/README.md @@ -330,3 +330,14 @@ The following assertion macros are available via *embUnit* + +## Out of Tree Unit Tests + +Export the environment variable `EXTERNAL_UNITTEST_DIRS` that contains a space +separated list of out-of-tree unit tests to also include in the test. The tests +will be treated the exact same way as tests in this folder and must follow the +same naming convention (each folder in `EXTERNAL_UNITTEST_DIRS` should have +`tests-` folders containing the unit tests). + +This feature works best with `EXTERNAL_MODULE_DIRS` that contain the code the +external unit tests should cover.