build system: Restructure dependency resolution
Goals:
- Untangle dependency resolution and feature checking for better maintainability
- Improve performance of "make info-boards-supported"
Changes:
- Makefile.dep
- Dropped handling of default modules and recursion
- Now only dependencies of the current set of used modules and pkgs are
added
==> External recursion is needed to catch transient dependencies
- Changed Makefile.features:
- Dropped checking of provided features
- Dropped populating FEATURES_USED with provided features that are required
or optional
- Dropped populating FEATURES_MISSING with required but not provided
features
- Dropped adding modules implementing used features to USE_MODULE
==> This now only populates FEATURES_PROVIDED, nothing more
- Added makefiles/features_check.inc.mk:
- This performs the population of FEATURES_USED and FEATURES_MISSING now
- Added makefiles/features_modules.inc.mk:
- This performs now the addition of modules implementing used features
- Added makefiles/dependency_resolution.inc.mk:
- This now performs the recursion required to catch transient dependencies
- Also the feature check is performed recursively to handle also required
and optional features of the transient dependencies
- DEFAULT_MODULES are added repeatedly to allow it to be extended based on
used features and modules
==> This allows modules to have optional dependencies, as these
dependencies can be blacklisted
- Use simply expanded variables instead of recursively expended variables
(`foo := $(bar)` instead `foo = $(bar)`) for internal variables during feature
resolution. This improves performance significantly for
`make info-boards-supported`.
- Reduce dependency resolution steps in `make info-boards-supported`
- Globally resolve dependencies without any features (including arch)
provided
==> This results in the common subset of feature requirements and modules
used
- But for individual boards additional modules might be used on top due
to architecture specific dependencies or optional features
- Boards not supporting this subset of commonly required features are not
supported, so no additional dependency resolution is needed for them
- For each board supporting the common set of requirements a complete
dependency resolution is still needed to also catch architecture specific
hacks
- But this resolution is seeded with the common set of dependencies to
speed this up
This commit is contained in:
Marian Buschsieweke
parent
007ac492ff
commit
7bc15acee2
57
Makefile.dep
57
Makefile.dep
@ -1,7 +1,3 @@
|
||||
#
|
||||
OLD_USEMODULE := $(sort $(USEMODULE))
|
||||
OLD_USEPKG := $(sort $(USEPKG))
|
||||
|
||||
# include board specific application dependencies
|
||||
-include $(APPDIR)/Makefile.board.dep
|
||||
|
||||
@ -1036,18 +1032,6 @@ FEATURES_OPTIONAL += periph_pm
|
||||
# always select provided architecture features
|
||||
FEATURES_REQUIRED += $(filter arch_%,$(FEATURES_PROVIDED))
|
||||
|
||||
# all periph features correspond to a periph submodule
|
||||
# FEATURES_USED is defined in Makefile.features
|
||||
USEMODULE += $(filter periph_%,$(FEATURES_USED))
|
||||
|
||||
# select cpu_check_address pseudomodule if the corresponding feature is used
|
||||
USEMODULE += $(filter cpu_check_address, $(FEATURES_USED))
|
||||
|
||||
# include periph_common if any periph_* driver is used
|
||||
ifneq (,$(filter periph_%, $(USEMODULE)))
|
||||
USEMODULE += periph_common
|
||||
endif
|
||||
|
||||
ifneq (,$(filter ecc_%,$(USEMODULE)))
|
||||
USEMODULE += ecc
|
||||
endif
|
||||
@ -1057,44 +1041,3 @@ ifneq (,$(filter test_utils_interactive_sync,$(USEMODULE)))
|
||||
DEFAULT_MODULE += test_utils_interactive_sync_shell
|
||||
endif
|
||||
endif
|
||||
|
||||
# recursively catch transitive dependencies
|
||||
USEMODULE := $(sort $(USEMODULE))
|
||||
USEPKG := $(sort $(USEPKG))
|
||||
ifneq ($(OLD_USEMODULE) $(OLD_USEPKG),$(USEMODULE) $(USEPKG))
|
||||
include $(RIOTBASE)/Makefile.dep
|
||||
else
|
||||
|
||||
# set all USED periph_% init modules as DEFAULT_MODULE
|
||||
ifneq (,$(filter periph_init, $(USEMODULE)))
|
||||
# To enable adding periph_% modules through the environment we cant use
|
||||
# FEATURES_USED since the MODULE might be added directly as USEMODULE
|
||||
PERIPH_MODULES_NO_INIT = periph_init% periph_common
|
||||
PERIPH_MODULES = $(filter periph_%,$(USEMODULE))
|
||||
PERIPH_INIT_MODULES = $(subst periph_,periph_init_,\
|
||||
$(filter-out $(PERIPH_MODULES_NO_INIT),$(PERIPH_MODULES)))
|
||||
DEFAULT_MODULE += $(PERIPH_INIT_MODULES)
|
||||
endif
|
||||
|
||||
# add periph_init_% modules to USEMODULE unless disabled
|
||||
ifneq (,$(filter periph_init, $(USEMODULE)))
|
||||
USEMODULE += $(filter $(PERIPH_INIT_MODULES),\
|
||||
$(filter-out $(DISABLE_MODULE),$(DEFAULT_MODULE)))
|
||||
endif
|
||||
|
||||
# Add auto_init_% DEFAULT_MODULES. This is done after the recursive cach since
|
||||
# none of these modules can trigger dependency resolution.
|
||||
ifneq (,$(filter auto_init,$(USEMODULE)))
|
||||
USEMODULE += $(filter auto_init_%,$(filter-out $(DISABLE_MODULE),$(DEFAULT_MODULE)))
|
||||
endif
|
||||
|
||||
# Add test_utils_interactive_sync_shell
|
||||
ifneq (,$(filter test_utils_interactive_sync,$(USEMODULE)))
|
||||
USEMODULE += $(filter test_utils_interactive_sync_%, \
|
||||
$(filter-out $(DISABLE_MODULE),$(DEFAULT_MODULE)))
|
||||
endif
|
||||
|
||||
# Sort and remove duplicates
|
||||
DEFAULT_MODULE := $(sort $(DEFAULT_MODULE))
|
||||
USEMODULE := $(sort $(USEMODULE))
|
||||
endif
|
||||
|
||||
@ -1,4 +1,4 @@
|
||||
# Process FEATURES variables
|
||||
# Process provided FEATURES
|
||||
#
|
||||
# The board/board common are responsible for defining the CPU and CPU_MODEL
|
||||
# variables in their Makefile.features.
|
||||
@ -17,56 +17,3 @@ ifeq (,$(CPU))
|
||||
endif
|
||||
|
||||
include $(RIOTCPU)/$(CPU)/Makefile.features
|
||||
|
||||
|
||||
# Resolve FEATURES_ variables
|
||||
# Their value will only be complete after resolving dependencies
|
||||
|
||||
# Features that are only optional and not required at the same time.
|
||||
# The policy is to by default use by features if they are provided by the BSP.
|
||||
FEATURES_OPTIONAL_ONLY = $(sort $(filter-out $(FEATURES_REQUIRED),$(FEATURES_OPTIONAL)))
|
||||
FEATURES_OPTIONAL_USED = $(sort $(filter $(FEATURES_PROVIDED),$(FEATURES_OPTIONAL_ONLY)))
|
||||
# Optional features that will not be used because they are not provided
|
||||
FEATURES_OPTIONAL_MISSING = $(sort $(filter-out $(FEATURES_PROVIDED),$(FEATURES_OPTIONAL_ONLY)))
|
||||
|
||||
# Features that are used without taking "one out of" dependencies into account
|
||||
FEATURES_USED_SO_FAR = $(sort $(FEATURES_REQUIRED) $(FEATURES_OPTIONAL_USED))
|
||||
|
||||
# Features that are provided and not blacklisted
|
||||
FEATURES_USABLE = $(filter-out $(FEATURES_BLACKLIST),$(FEATURES_PROVIDED))
|
||||
|
||||
# Additionally required features due to the "one out of" dependencies
|
||||
# Algorithm explained:
|
||||
# - For each feature list "item" in FEATURES_REQUIRED_ANY:
|
||||
# - Filter the list of already used features by any of the features in "item"
|
||||
# - Filter the list of usable features by any of the features in "item"
|
||||
# - Combine (without reordering) those two lists, and append "item" (with pipes)
|
||||
# - Take the first work out of this combined list
|
||||
# ==> All already used features matching one of the feature in item will be in front of the list (if any)
|
||||
# ==> All usable features matching one of the features in item will come next
|
||||
# ==> The whole list as item with pipes (e.g. "periph_spi|periph_i2c") will be the last item
|
||||
# ==> This will only pull in new features if strictly required
|
||||
FEATURES_REQUIRED_ONE_OUT_OF = $(foreach item,$(FEATURES_REQUIRED_ANY),$(word 1,$(filter $(subst |, ,$(item)),$(FEATURES_USED_SO_FAR) $(FEATURES_USABLE)) $(item)))
|
||||
|
||||
# Features that are required by the application but not provided by the BSP
|
||||
# Having features missing may case the build to fail.
|
||||
FEATURES_MISSING = $(sort $(filter-out $(FEATURES_PROVIDED),$(FEATURES_REQUIRED) $(FEATURES_REQUIRED_ONE_OUT_OF)))
|
||||
|
||||
# Features that are used for an application
|
||||
FEATURES_USED = $(sort $(FEATURES_REQUIRED) $(FEATURES_REQUIRED_ONE_OUT_OF) $(FEATURES_OPTIONAL_USED))
|
||||
|
||||
# Used features that conflict when used together
|
||||
FEATURES_CONFLICTING = $(sort $(foreach conflict,$(FEATURES_CONFLICT),$(call _features_conflicting,$(conflict))))
|
||||
|
||||
# Return conflicting features from the conflict string feature1:feature2
|
||||
# $1: feature1:feature2
|
||||
# Return the list of conflicting features
|
||||
_features_conflicting = $(if $(call _features_used_conflicting,$(subst :, ,$1)),$(subst :, ,$1))
|
||||
# Check if all features from the list are used
|
||||
# $1: list of features that conflict together
|
||||
# Return non empty on error
|
||||
_features_used_conflicting = $(filter $(words $1),$(words $(filter $(FEATURES_USED),$1)))
|
||||
|
||||
# Features that are used by the application but blacklisted by the BSP.
|
||||
# Having blacklisted features may cause the build to fail.
|
||||
FEATURES_USED_BLACKLISTED = $(sort $(filter $(FEATURES_USED), $(FEATURES_BLACKLIST)))
|
||||
|
||||
@ -357,7 +357,7 @@ export CCACHE_CPP2=yes
|
||||
USEMODULE += $(filter-out $(DISABLE_MODULE), $(DEFAULT_MODULE))
|
||||
|
||||
# process dependencies
|
||||
include $(RIOTBASE)/Makefile.dep
|
||||
include $(RIOTMAKE)/dependency_resolution.inc.mk
|
||||
|
||||
ifeq ($(strip $(MCU)),)
|
||||
MCU = $(CPU)
|
||||
|
||||
41
makefiles/dependency_resolution.inc.mk
Normal file
41
makefiles/dependency_resolution.inc.mk
Normal file
@ -0,0 +1,41 @@
|
||||
# Perform a recursive dependency resolution: Include $(RIOTBASE)/Makefile.dep
|
||||
# until no new modules, pkgs, or features are pull in order to catch all
|
||||
# transient dependencies
|
||||
|
||||
# Back up current state to detect changes
|
||||
OLD_STATE := $(USEMODULE) $(USEPKG) $(FEATURES_USED)
|
||||
|
||||
# pull in dependencies of the currently used modules and pkgs
|
||||
include $(RIOTBASE)/Makefile.dep
|
||||
|
||||
# check if required features are provided and update $(FEATURES_USED)
|
||||
include $(RIOTMAKE)/features_check.inc.mk
|
||||
|
||||
# translate used features into used module, where needed
|
||||
include $(RIOTMAKE)/features_modules.inc.mk
|
||||
|
||||
# sort and de-duplicate used modules, pkgs, and features for comparison
|
||||
USEMODULE := $(sort $(USEMODULE))
|
||||
USEPKG := $(sort $(USEPKG))
|
||||
FEATURES_USED := $(sort $(FEATURES_USED))
|
||||
|
||||
NEW_STATE := $(USEMODULE) $(USEPKG) $(FEATURES_USED)
|
||||
|
||||
# If set of used modules, pkgs, and features has changed during last run, run
|
||||
# again to recursively catch transitive dependencies
|
||||
ifneq ($(OLD_STATE),$(NEW_STATE))
|
||||
include $(RIOTMAKE)/dependency_resolution.inc.mk
|
||||
else
|
||||
# If module auto_init is not used, silently disable all of its submodules
|
||||
ifeq (,$(filter auto_init,$(USEMODULE)))
|
||||
DISABLE_MODULE += auto_init_%
|
||||
endif
|
||||
|
||||
# add default modules again, as $(DEFAULT_MODULE) might have been extended
|
||||
# during dependency processing
|
||||
USEMODULE += $(filter-out $(DISABLE_MODULE),$(DEFAULT_MODULE))
|
||||
|
||||
# Sort and de-duplicate used modules and default modules for readability
|
||||
USEMODULE := $(sort $(USEMODULE))
|
||||
DEFAULT_MODULE := $(sort $(DEFAULT_MODULE))
|
||||
endif
|
||||
61
makefiles/features_check.inc.mk
Normal file
61
makefiles/features_check.inc.mk
Normal file
@ -0,0 +1,61 @@
|
||||
# Check if all required FEATURES are provided
|
||||
|
||||
FEATURES_OPTIONAL_ONLY := $(sort $(filter-out $(FEATURES_REQUIRED),$(FEATURES_OPTIONAL)))
|
||||
FEATURES_OPTIONAL_USED := $(sort $(filter $(FEATURES_PROVIDED),$(FEATURES_OPTIONAL_ONLY)))
|
||||
# Optional features that will not be used because they are not provided
|
||||
FEATURES_OPTIONAL_MISSING := $(sort $(filter-out $(FEATURES_PROVIDED),$(FEATURES_OPTIONAL_ONLY)))
|
||||
|
||||
# Features that are used without taking "one out of" dependencies into account
|
||||
FEATURES_USED_SO_FAR := $(sort $(FEATURES_REQUIRED) $(FEATURES_OPTIONAL_USED))
|
||||
|
||||
# Features that are provided and not blacklisted
|
||||
FEATURES_USABLE := $(filter-out $(FEATURES_BLACKLIST),$(FEATURES_PROVIDED))
|
||||
|
||||
# Additionally required features due to the "one out of" dependencies
|
||||
# Algorithm:
|
||||
# - For each list in FEATURES_REQUIRED_ANY as "item":
|
||||
# - Store the intersection of FEATURES_USED_SO_FAR and the features in
|
||||
# "item" in "tmp"
|
||||
# - Append the intersection of FEATURES_USABLE and the features in "item"
|
||||
# to "tmp"
|
||||
# - Append "item" to "tmp" (with pipes between features, e.g.
|
||||
# "periph_spi|periph_i2c")
|
||||
# - Append the first element of "tmp" to FEATURES_REQUIRED_ONE_OUT_OF
|
||||
# ==> If one (or more) already used features is listed in item, this (or
|
||||
# one of these) will be in the front of "tmp" and be taken
|
||||
# ==> If one (or more) usable features is listed in item, this will come
|
||||
# afterwards. If no no feature in item is used so for, one of the
|
||||
# features supported and listed in item will be picked
|
||||
# ==> At the end of the list item itself (with pipes between features) is the
|
||||
# last item in "tmp". If no feature is item is supported or used, this
|
||||
# will be the only item in "tmp" and be picked
|
||||
FEATURES_REQUIRED_ONE_OUT_OF := $(foreach item,\
|
||||
$(FEATURES_REQUIRED_ANY),\
|
||||
$(word 1,\
|
||||
$(filter $(subst |, ,$(item)),\
|
||||
$(FEATURES_USED_SO_FAR) \
|
||||
$(FEATURES_USABLE)) \
|
||||
$(item)))
|
||||
|
||||
# Features that are required by the application but not provided by the BSP
|
||||
# Having features missing may case the build to fail.
|
||||
FEATURES_MISSING := $(sort $(filter-out $(FEATURES_PROVIDED),$(FEATURES_REQUIRED) $(FEATURES_REQUIRED_ONE_OUT_OF)))
|
||||
|
||||
# Features that are used for an application
|
||||
FEATURES_USED := $(sort $(FEATURES_REQUIRED) $(FEATURES_REQUIRED_ONE_OUT_OF) $(FEATURES_OPTIONAL_USED))
|
||||
|
||||
# Used features that conflict when used together
|
||||
FEATURES_CONFLICTING := $(sort $(foreach conflict,$(FEATURES_CONFLICT),$(call _features_conflicting,$(conflict))))
|
||||
|
||||
# Return conflicting features from the conflict string feature1:feature2
|
||||
# $1: feature1:feature2
|
||||
# Return the list of conflicting features
|
||||
_features_conflicting = $(if $(call _features_used_conflicting,$(subst :, ,$1)),$(subst :, ,$1))
|
||||
# Check if all features from the list are used
|
||||
# $1: list of features that conflict together
|
||||
# Return non empty on error
|
||||
_features_used_conflicting = $(filter $(words $1),$(words $(filter $(FEATURES_USED),$1)))
|
||||
|
||||
# Features that are used by the application but blacklisted by the BSP.
|
||||
# Having blacklisted features may cause the build to fail.
|
||||
FEATURES_USED_BLACKLISTED := $(sort $(filter $(FEATURES_USED), $(FEATURES_BLACKLIST)))
|
||||
27
makefiles/features_modules.inc.mk
Normal file
27
makefiles/features_modules.inc.mk
Normal file
@ -0,0 +1,27 @@
|
||||
# Add modules implementing used features
|
||||
#
|
||||
# This is done after the regular dependency resolution in Makefile.dep, as
|
||||
# feature resolution depends on the used modules. As these modules however have
|
||||
# no dependencies(except for periph_common), no dependency resolution is needed.
|
||||
|
||||
PERIPH_FEATURES := $(filter periph_%,$(FEATURES_USED))
|
||||
# all periph features correspond to a periph submodule
|
||||
# FEATURES_USED is defined in Makefile.features
|
||||
USEMODULE += $(PERIPH_FEATURES)
|
||||
|
||||
# Add all USED periph_% init modules unless they are blacklisted
|
||||
ifneq (,$(filter periph_init, $(USEMODULE)))
|
||||
PERIPH_MODULES := $(filter-out periph_init% periph_common,\
|
||||
$(filter periph_%,$(USEMODULE)))
|
||||
# Use simple expansion to avoid USEMODULE referencing itself
|
||||
PERIPH_INIT_MODULES := $(subst periph_,periph_init_,$(PERIPH_MODULES))
|
||||
DEFAULT_MODULE += $(PERIPH_INIT_MODULES)
|
||||
endif
|
||||
|
||||
# select cpu_check_address pseudomodule if the corresponding feature is used
|
||||
USEMODULE += $(filter cpu_check_address, $(FEATURES_USED))
|
||||
|
||||
# include periph_common if any periph_* driver is used
|
||||
ifneq (,$(filter periph_%, $(USEMODULE)))
|
||||
USEMODULE += periph_common
|
||||
endif
|
||||
@ -1,6 +1,16 @@
|
||||
.PHONY: info-buildsizes info-buildsizes-diff info-features-missing \
|
||||
info-boards-features-missing
|
||||
|
||||
# Perform dependency resolution without having included
|
||||
# $(RIOTBASE)/Makefile.features for now. This results in no optional modules and
|
||||
# no architecture specific handling being done. The result will be a subset of
|
||||
# used modules and requirements that is present in for *all* boards, so this can
|
||||
# be cached to speed up individual dependency checks
|
||||
include $(RIOTMAKE)/defaultmodules.inc.mk
|
||||
# add default modules
|
||||
USEMODULE += $(filter-out $(DISABLE_MODULE),$(DEFAULT_MODULE))
|
||||
include $(RIOTMAKE)/dependency_resolution.inc.mk
|
||||
|
||||
BOARDSDIR_GLOBAL := $(BOARDSDIR)
|
||||
USEMODULE_GLOBAL := $(USEMODULE)
|
||||
USEPKG_GLOBAL := $(USEPKG)
|
||||
@ -27,6 +37,7 @@ define board_unsatisfied_features
|
||||
|
||||
# Remove board specific variables set by Makefile.features/Makefile.dep
|
||||
FEATURES_PROVIDED :=
|
||||
FEATURES_USED :=
|
||||
|
||||
# Undefine variables that must not be defined when starting.
|
||||
# Some are sometime set as `?=`
|
||||
@ -40,20 +51,28 @@ define board_unsatisfied_features
|
||||
endif
|
||||
|
||||
include $(RIOTBASE)/Makefile.features
|
||||
|
||||
include $(RIOTMAKE)/defaultmodules.inc.mk
|
||||
USEMODULE += $(filter-out $(DISABLE_MODULE), $(DEFAULT_MODULE))
|
||||
|
||||
include $(RIOTBASE)/Makefile.dep
|
||||
|
||||
# FEATURES_USED must be populated first in this case so that dependency
|
||||
# resolution can take optional features into account during the first pass.
|
||||
# Also: This allows us to skip resolution if already a missing feature is
|
||||
# detected
|
||||
include $(RIOTMAKE)/features_check.inc.mk
|
||||
ifneq (,$$(FEATURES_MISSING))
|
||||
# Skip full dependency resolution, as even without optional modules features
|
||||
# and architecture specific limitations already some features are missing
|
||||
BOARDS_FEATURES_MISSING += "$(1) $$(FEATURES_MISSING)"
|
||||
BOARDS_WITH_MISSING_FEATURES += $(1)
|
||||
endif
|
||||
else
|
||||
include $(RIOTMAKE)/dependency_resolution.inc.mk
|
||||
|
||||
ifneq (,$$(FEATURES_USED_BLACKLISTED))
|
||||
BOARDS_FEATURES_USED_BLACKLISTED += "$(1) $$(FEATURES_USED_BLACKLISTED)"
|
||||
BOARDS_WITH_BLACKLISTED_FEATURES += $(1)
|
||||
ifneq (,$$(FEATURES_MISSING))
|
||||
BOARDS_FEATURES_MISSING += "$(1) $$(FEATURES_MISSING)"
|
||||
BOARDS_WITH_MISSING_FEATURES += $(1)
|
||||
endif
|
||||
|
||||
ifneq (,$$(FEATURES_USED_BLACKLISTED))
|
||||
BOARDS_FEATURES_USED_BLACKLISTED += "$(1) $$(FEATURES_USED_BLACKLISTED)"
|
||||
BOARDS_WITH_BLACKLISTED_FEATURES += $(1)
|
||||
endif
|
||||
endif
|
||||
|
||||
ifneq (,$$(DEPENDENCY_DEBUG))
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user