tests/unittests: allow out-of-tree tests

This adds and documents the new `EXTERNAL_UNIT_TEST_DIRS` environment
variable that allows including out-of-tree unit tests. The intention is
to allow users of `EXTERNAL_MODULE_DIRS` to also provide corresponding
unit tests and run them all with a single test app.

Co-authored-by: crasbe <crasbe@gmail.com>

doc/guides/advanced_tutorials/creating_application.md

@@ -343,3 +343,11 @@ APPLICATION = my_app
 PROJECT_BASE ?= $(CURDIR)/../..
 include $(PROJECT_BASE)/Makefile.include
 ```
+
+### Unit Tests for External Modules
+
+When writing external modules, it can be helpful to make use of the existing
+unit test infrastructure in `tests/unittests` to run both upstream unit tests
+as well as downstream project-specific unit tests in a single app. For this the
+environment variable `EXTERNAL_UNITTEST_DIRS` can be used to list directories
+(separated by a space) to additional unit tests.

makefiles/app_dirs.blacklist

@@ -11,6 +11,8 @@ tests/build_system/external_board_native/external_boards/
 tests/build_system/external_module_dirs/external_modules/
 tests/build_system/external_pkg_dirs/external_pkgs/
 tests/build_system/kconfig/external_modules/
+tests/build_system/external_unittests/external_tests_dir/
+tests/build_system/external_unittests/tests-in_tree
 tests/build_system/kconfig/external_pkgs/
 tests/periph/qdec/boards_modded/
 tests/pkg/openwsn_sock_udp/external_modules/

tests/build_system/external_unittests/Makefile

@@ -0,0 +1,11 @@
+# Run this only on native64 / native, as this rather tests the build system
+# than the code actually run.
+
+BOARD_WHITELIST := native native64
+
+EXTERNAL_UNITTEST_DIRS := $(CURDIR)/external_tests_dir
+
+# Build upon tests/unittests:
+RIOTBASE ?= $(CURDIR)/../../..
+UNIT_TESTS_DIR := $(CURDIR)/../../unittests
+include ../../unittests/Makefile

tests/build_system/external_unittests/README.md

@@ -0,0 +1,16 @@
+External Unit Tests
+===================
+
+This test apps provides two trivial unit tests: One provided internally in the
+folder of this unit tests apps and one provided externally via a variable.
+Since the test only validates correct operation of the build system, there is
+little value in running it on more than one board. For this reason, the app is
+limited to be build only for native / native64.
+
+Implementation Details
+----------------------
+
+The two tests contain a reference to a variable of the other tests, so that
+the app will not link if only if both of the unit tests are available at
+link time (or none). In addition, the python test runner will check that two
+ tests are executed (to rule out that none was available).

tests/build_system/external_unittests/external_tests_dir/tests-out_of_tree/Makefile

@@ -0,0 +1 @@
+include $(RIOTBASE)/Makefile.base

tests/build_system/external_unittests/external_tests_dir/tests-out_of_tree/tests-out-of-tree.c

@@ -0,0 +1,35 @@
+/*
+ * Copyright (C) 2025 ML!PA Consulting GmbH
+ *
+ * This file is subject to the terms and conditions of the GNU Lesser
+ * General Public License v2.1. See the file LICENSE in the top level
+ * directory for more details.
+ */
+
+#include <stdio.h>
+
+#include "embUnit.h"
+
+const unsigned external_test_was_linked_in = 1;
+
+static void test_internal_test_was_linked_in(void)
+{
+    extern const unsigned internal_test_was_linked_in;
+    TEST_ASSERT_EQUAL_INT(1, internal_test_was_linked_in);
+}
+
+static Test *out_of_tree_tests(void)
+{
+    EMB_UNIT_TESTFIXTURES(fixtures) {
+        new_TestFixture(test_internal_test_was_linked_in),
+    };
+
+    EMB_UNIT_TESTCALLER(_out_of_tree_tests, NULL, NULL, fixtures);
+
+    return (Test *)&_out_of_tree_tests;
+}
+
+void tests_out_of_tree(void)
+{
+    TESTS_RUN(out_of_tree_tests());
+}

tests/build_system/external_unittests/main.c

@@ -0,0 +1 @@
+../../unittests/main.c

tests/build_system/external_unittests/map.h

@@ -0,0 +1 @@
+../../unittests/map.h

tests/build_system/external_unittests/tests-in_tree/Makefile

@@ -0,0 +1 @@
+include $(RIOTBASE)/Makefile.base

tests/build_system/external_unittests/tests-in_tree/tests-in-tree.c

@@ -0,0 +1,35 @@
+/*
+ * Copyright (C) 2025 ML!PA Consulting GmbH
+ *
+ * This file is subject to the terms and conditions of the GNU Lesser
+ * General Public License v2.1. See the file LICENSE in the top level
+ * directory for more details.
+ */
+
+#include <stdio.h>
+
+#include "embUnit.h"
+
+const unsigned internal_test_was_linked_in = 1;
+
+static void test_external_test_was_linked_in(void)
+{
+    extern const unsigned external_test_was_linked_in;
+    TEST_ASSERT_EQUAL_INT(1, external_test_was_linked_in);
+}
+
+static Test *in_tree_tests(void)
+{
+    EMB_UNIT_TESTFIXTURES(fixtures) {
+        new_TestFixture(test_external_test_was_linked_in),
+    };
+
+    EMB_UNIT_TESTCALLER(_in_tree_tests, NULL, NULL, fixtures);
+
+    return (Test *)&_in_tree_tests;
+}
+
+void tests_in_tree(void)
+{
+    TESTS_RUN(in_tree_tests());
+}

tests/build_system/external_unittests/tests/01-run.py

@@ -0,0 +1,18 @@
+#!/usr/bin/env python3
+
+# Copyright (C) 2016 Kaspar Schleiser <kaspar@schleiser.de>
+#
+# This file is subject to the terms and conditions of the GNU Lesser
+# General Public License v2.1. See the file LICENSE in the top level
+# directory for more details.
+
+import sys
+from testrunner import run
+
+
+def testfunc(child):
+    child.expect_exact("OK (2 tests)")
+
+
+if __name__ == "__main__":
+    sys.exit(run(testfunc))

tests/unittests/Makefile

@@ -1,18 +1,40 @@
 DEVELHELP ?= 0
-include ../Makefile.tests_common
+
+# Allow including this Makefile from other apps, as done by
+# tests/build_system/external_unittests
+UNIT_TESTS_DIR ?= $(CURDIR)
+include $(UNIT_TESTS_DIR)/../Makefile.tests_common
 
 USEMODULE += embunit
 
+# Search for unit tests both in-tree and, if configured, out-of-tree
+UNIT_TEST_SEARCH_DIRS := $(CURDIR) $(EXTERNAL_UNITTEST_DIRS)
+# Canonicalize test search dirs: Add a single solidus at the end:
+UNIT_TEST_SEARCH_DIRS := $(addsuffix /,$(abspath $(UNIT_TEST_SEARCH_DIRS)))
+
 ifeq (,$(UNIT_TESTS))
   ifeq (, $(filter tests-%, $(MAKECMDGOALS)))
-    # the $(dir) Makefile function leaves a trailing slash after the directory
+    # collect all unit test makefiles in all unit test search dirs
-    # name, therefore we use patsubst instead.
+    UNIT_TESTS_MAKEFILES := $(wildcard $(addsuffix tests-*/Makefile,$(UNIT_TEST_SEARCH_DIRS)))
-    UNIT_TESTS := $(patsubst %/Makefile,%,$(wildcard tests-*/Makefile))
+    # $(dir) would leave a solidus at the end, using patsub instead
+    UNIT_TESTS := $(patsubst %/Makefile,%,$(UNIT_TESTS_MAKEFILES))
+    # $(notdir) gives the last component in path, despite its name
+    UNIT_TESTS := $(notdir $(UNIT_TESTS))
   else
     UNIT_TESTS := $(filter tests-%, $(MAKECMDGOALS))
   endif
 endif
 
+# in case unit tests were provided by name, we need to locate them
+ifeq (,$(UNIT_TESTS_MAKEFILES))
+  # generate "$searchdir/$testname/Makefile" for every possible combination of
+  # $searchdir and $testname
+  UNIT_TESTS_MAKEFILES := $(addsuffix /Makefile,$(UNIT_TESTS))
+  UNIT_TESTS_MAKEFILES := $(foreach testdir,$(UNIT_TEST_SEARCH_DIRS),$(addprefix $(testdir),$(UNIT_TESTS_MAKEFILES)))
+  # reduce that list to files that actually exist
+  UNIT_TESTS_MAKEFILES := $(wildcard $(UNIT_TESTS_MAKEFILES))
+endif
+
 ifeq (llvm,$(TOOLCHAIN))
   # the floating point exception bug is more likely to trigger when build
   # with LLVM, so we just disable LLVM on native as a work around
@@ -25,10 +47,14 @@ DISABLE_MODULE += auto_init auto_init_%
 # initialize stdio over USB.
 FEATURES_BLACKLIST += highlevel_stdio
 
-# Pull in `Makefile.include`s from the test suites:
+# extract the test dirs from the unit test makefiles
--include $(UNIT_TESTS:%=$(RIOTBASE)/tests/unittests/%/Makefile.include)
+UNIT_TEST_DIRS := $(dir $(UNIT_TESTS_MAKEFILES))
 
-DIRS += $(UNIT_TESTS)
+# each unit test dir needs to be build as module
+DIRS += $(UNIT_TEST_DIRS)
+
+# Pull in `Makefile.include`s from the test suites:
+-include $(addsuffix Makefile.include,$(UNIT_TEST_DIRS))
 BASELIBS += $(UNIT_TESTS:%=%.module)
 
 INCLUDES += -I$(RIOTBASE)/tests/unittests/common

tests/unittests/README.md

@@ -330,3 +330,14 @@ The following assertion macros are available via *embUnit*
     </tr>
   </tbody>
 </table>
+
+## Out of Tree Unit Tests
+
+Export the environment variable `EXTERNAL_UNITTEST_DIRS` that contains a space
+separated list of out-of-tree unit tests to also include in the test. The tests
+will be treated the exact same way as tests in this folder and must follow the
+same naming convention (each folder in `EXTERNAL_UNITTEST_DIRS` should have
+`tests-<name>` folders containing the unit tests).
+
+This feature works best with `EXTERNAL_MODULE_DIRS` that contain the code the
+external unit tests should cover.