Merge pull request #17551 from NikLeberg/feature/pkg_paths

buildsystem/pkg: expand packages from USEPKG early as path into PKG_PATHS

Makefile.base

@@ -97,8 +97,8 @@ compile-commands: | $(DIRS:%=COMPILE-COMMANDS--%)
 	$(file >>$(BINDIR)/$(MODULE)/compile_cmds.txt,TARGET_ARCH: $(TARGET_ARCH))
 	$(file >>$(BINDIR)/$(MODULE)/compile_cmds.txt,TARGET_ARCH_LLVM: $(TARGET_ARCH_LLVM))
 
-# include makefile snippets for packages in $(USEPKG) that modify GENSRC:
+# include makefile snippets for packages in $(PKG_PATHS) that modify GENSRC:
--include $(USEPKG:%=$(RIOTPKG)/%/Makefile.gensrc)
+-include $(PKG_PATHS:%=%Makefile.gensrc)
 
 GENOBJC     := $(GENSRC:%.c=%.o)
 OBJC_LTO    := $(SRC:%.c=$(BINDIR)/$(MODULE)/%.o)

Makefile.dep

@@ -20,7 +20,7 @@ include $(RIOTBASE)/drivers/Makefile.dep
 -include $(sort $(USEMODULE:%=$(RIOTBASE)/drivers/%/Makefile.dep))
 
 # pull dependencies from packages
--include $(USEPKG:%=$(RIOTPKG)/%/Makefile.dep)
+-include $(PKG_PATHS:%=%Makefile.dep)
 
 ifneq (,$(filter mpu_stack_guard,$(USEMODULE)))
   FEATURES_REQUIRED += cortexm_mpu

Makefile.include

@@ -427,6 +427,10 @@ ifeq (1,$(TEST_KCONFIG))
   USEMODULE := $(KCONFIG_MODULES)
   KCONFIG_PACKAGES := $(call lowercase,$(patsubst CONFIG_PACKAGE_%,%,$(filter CONFIG_PACKAGE_%,$(.VARIABLES))))
   USEPKG := $(KCONFIG_PACKAGES)
+
+  # Locate used packages in $(RIOTPKG).
+  PKG_PATHS := $(sort $(foreach dir,$(RIOTPKG),\
+    $(foreach pkg,$(USEPKG),$(dir $(wildcard $(dir)/$(pkg)/Makefile)))))
 else
   # always select provided architecture features
   FEATURES_REQUIRED += $(filter arch_%,$(FEATURES_PROVIDED))
@@ -570,8 +574,8 @@ include $(RIOTBASE)/sys/Makefile.include
 # include Makefile.includes of each driver modules if they exist
 -include $(USEMODULE:%=$(RIOTBASE)/drivers/%/Makefile.include)
 
-# include Makefile.includes for packages in $(USEPKG)
+# include Makefile.includes for packages in $(PKG_PATHS)
--include $(USEPKG:%=$(RIOTPKG)/%/Makefile.include)
+-include $(PKG_PATHS:%=%Makefile.include)
 
 # include external modules configuration
 -include $(EXTERNAL_MODULE_PATHS:%=%/Makefile.include)
@@ -771,31 +775,30 @@ endif
 # The `clean` needs to be serialized before everything else.
 all $(BASELIBS) $(ARCHIVES) $(BUILDDEPS) ..in-docker-container: | $(CLEAN)
 
-.PHONY: pkg-prepare pkg-build pkg-build-%
+.PHONY: pkg-prepare pkg-build
 pkg-prepare:
-	-@for i in $(USEPKG) ; do "$(MAKE)" -C $(RIOTPKG)/$$i prepare ; done
+	-@$(foreach dir,$(PKG_PATHS),"$(MAKE)" -C $(dir) prepare $(NEWLINE))
 
-pkg-build: $(USEPKG:%=pkg-build-%)
+pkg-build: $(BUILDDEPS)
-pkg-build-%: $(BUILDDEPS)
+	$(foreach dir,$(PKG_PATHS),$(QQ)"$(MAKE)" -C $(dir) $(NEWLINE))
-	$(QQ)"$(MAKE)" -C $(RIOTPKG)/$*
 
 clean:
 ifndef MAKE_RESTARTS
-	-@for i in $(USEPKG) ; do "$(MAKE)" -C $(RIOTPKG)/$$i clean ; done
+	-@$(foreach dir,$(PKG_PATHS),"$(MAKE)" -C $(dir) clean $(NEWLINE))
 	-@rm -rf $(BINDIR)
 	-@rm -rf $(SCANBUILD_OUTPUTDIR)
 endif
 
 # Remove intermediates, but keep the .elf, .hex and .map etc.
 clean-intermediates:
-	-@for i in $(USEPKG) ; do "$(MAKE)" -C $(RIOTPKG)/$$i distclean ; done
+	-@$(foreach dir,$(PKG_PATHS),"$(MAKE)" -C $(dir) distclean $(NEWLINE))
 	-@rm -rf $(BINDIR)/*.a $(BINDIR)/*/
 
 clean-pkg:
-	-@for i in $(USEPKG) ; do "$(MAKE)" -C $(RIOTPKG)/$$i distclean ; done
+	-@$(foreach dir,$(PKG_PATHS),"$(MAKE)" -C $(dir) distclean $(NEWLINE))
 
 distclean:
-	-@for i in $(USEPKG) ; do "$(MAKE)" -C $(RIOTPKG)/$$i distclean ; done
+	-@$(foreach dir,$(PKG_PATHS),"$(MAKE)" -C $(dir) distclean $(NEWLINE))
 	-@rm -rf $(BINDIRBASE)
 
 # Include PROGRAMMER_FLASH/PROGRAMMER_RESET variables

makefiles/dependency_resolution.inc.mk

@@ -6,6 +6,10 @@
 EXTERNAL_MODULE_PATHS := $(sort $(foreach dir,$(EXTERNAL_MODULE_DIRS),\
   $(foreach mod,$(USEMODULE),$(dir $(wildcard $(dir)/$(mod)/Makefile)))))
 
+# Locate used packages in $(RIOTPKG).
+PKG_PATHS := $(sort $(foreach dir,$(RIOTPKG),\
+  $(foreach pkg,$(USEPKG),$(dir $(wildcard $(dir)/$(pkg)/Makefile)))))
+
 # Back up current state to detect changes
 OLD_STATE := $(USEMODULE) $(USEPKG) $(FEATURES_USED)
 

makefiles/vars.inc.mk

@@ -45,6 +45,7 @@ export BINDIR                # This is the folder where the application should b
 export CARGO_TARGET_DIR      # This is the folder where Rust parts of the application should be built in.
 export BUILD_DIR             # This is the base folder to store common build files and artifacts, e.g. test results.
 export APPDIR                # The base folder containing the application
+export PKG_PATHS             # List of absolute paths where packages of $(USEPKG) can be found
 export PKGDIRBASE            # The base folder for building packages
 
 export PYTHONPATH            # Python default search path for module filesi, with RIOT specific packages

pkg/esp32_sdk/Makefile

@@ -20,11 +20,13 @@ ESP32_SDK_LIBS = $(addprefix $(ESP32_SDK_BUILD_DIR)/, $(ESP32_SDK_COMPONENT_LIBS
 
 all: $(ESP32_SDK_LIBS) $(ESP32_SDK_VER_FILE)
 
-$(ESP32_SDK_BUILD_DIR):
+$(PKG_PREPARED): $(ESP32_SDK_BUILD_DIR) $(ESP32_SDK_VER_FILE)
+
+$(ESP32_SDK_BUILD_DIR): $(PKG_PATCHED)
 	$(Q)mkdir -p $(ESP32_SDK_BUILD_DIR)
 
 # Set the SDK version from the SDK hash/tag. For example "v3.1-51-g913a06a9".
-$(ESP32_SDK_VER_FILE):
+$(ESP32_SDK_VER_FILE): $(PKG_PATCHED) | $(ESP32_SDK_BUILD_DIR)
 	$(Q)echo "#define IDF_VER \"$(ESP32_SDK_VER_CMD)\"" > $@
 
 $(ESP32_SDK_BUILD_DIR)/lib%.a: \

pkg/esp32_sdk_libs/Makefile

@@ -31,7 +31,9 @@ ESP32_SDK_LIBS = $(addprefix $(ESP32_SDK_BUILD_DIR)/, $(ESP32_SDK_COMPONENT_LIBS
 
 all: $(ESP32_SDK_LIBS)
 
-$(ESP32_SDK_BUILD_DIR):
+$(PKG_PREPARED): $(ESP32_SDK_BUILD_DIR)
+
+$(ESP32_SDK_BUILD_DIR): $(PKG_PATCHED)
 	$(Q)mkdir -p $(ESP32_SDK_BUILD_DIR)
 
 $(ESP32_SDK_BUILD_DIR)/lib%.a: \

pkg/esp8266_sdk/Makefile

@@ -20,7 +20,9 @@ ESP_SDK_LIBS = $(addprefix $(ESP8266_SDK_BUILD_DIR)/, $(ESP_SDK_COMPONENT_LIBS))
 
 all: $(ESP_SDK_LIBS) $(ESP8266_SDK_BUILD_DIR)/esp8266_idf_version.h
 
-$(ESP8266_SDK_BUILD_DIR):
+$(PKG_PREPARED): $(ESP8266_SDK_BUILD_DIR)
+
+$(ESP8266_SDK_BUILD_DIR): $(PKG_PATCHED)
 	$(Q)mkdir -p $(ESP8266_SDK_BUILD_DIR)
 
 # Set the SDK version from the SDK hash/tag. For example "v3.1-51-g913a06a9".