From a0f25ed47b3855ac5a51a37b481e6891a2b61847 Mon Sep 17 00:00:00 2001 From: cladmi Date: Tue, 4 Sep 2018 14:51:02 +0200 Subject: [PATCH 01/11] makefiles/docker.inc.mk: add DOCKER_RIOTBASE variable It represents the path of RIOTBASE inside the docker container. --- makefiles/docker.inc.mk | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/makefiles/docker.inc.mk b/makefiles/docker.inc.mk index 775549f630..e05ba130da 100644 --- a/makefiles/docker.inc.mk +++ b/makefiles/docker.inc.mk @@ -1,5 +1,6 @@ export DOCKER_IMAGE ?= riot/riotbuild:latest export DOCKER_BUILD_ROOT ?= /data/riotbuild +DOCKER_RIOTBASE ?= $(DOCKER_BUILD_ROOT)/riotbase export DOCKER_FLAGS ?= --rm # List of Docker-enabled make goals export DOCKER_MAKECMDGOALS_POSSIBLE = \ @@ -111,14 +112,14 @@ ETC_LOCALTIME = $(realpath /etc/localtime) @# HACK: Handle directory creation here until it is provided globally $(Q)mkdir -p $(BUILD_DIR) $(DOCKER) run $(DOCKER_FLAGS) -t -u "$$(id -u)" \ - -v '$(RIOTBASE):$(DOCKER_BUILD_ROOT)/riotbase' \ + -v '$(RIOTBASE):$(DOCKER_RIOTBASE)' \ -v '$(BUILD_DIR):$(DOCKER_BUILD_ROOT)/build' \ -v '$(RIOTCPU):$(DOCKER_BUILD_ROOT)/riotcpu' \ -v '$(RIOTBOARD):$(DOCKER_BUILD_ROOT)/riotboard' \ -v '$(RIOTMAKE):$(DOCKER_BUILD_ROOT)/riotmake' \ -v '$(RIOTPROJECT):$(DOCKER_BUILD_ROOT)/riotproject' \ -v '$(ETC_LOCALTIME):/etc/localtime:ro' \ - -e 'RIOTBASE=$(DOCKER_BUILD_ROOT)/riotbase' \ + -e 'RIOTBASE=$(DOCKER_RIOTBASE)' \ -e 'BUILD_DIR=$(DOCKER_BUILD_ROOT)/build' \ -e 'CCACHE_BASEDIR=$(DOCKER_BUILD_ROOT)/riotbase' \ -e 'RIOTCPU=$(DOCKER_BUILD_ROOT)/riotcpu' \ From 378f9fd1448087a7343e3d0297186ee4048e69ac Mon Sep 17 00:00:00 2001 From: cladmi Date: Tue, 4 Sep 2018 15:07:06 +0200 Subject: [PATCH 02/11] makefiles/docker.inc.mk: add DOCKER_APPDIR variable Prepare for when it can have a different value. --- makefiles/docker.inc.mk | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/makefiles/docker.inc.mk b/makefiles/docker.inc.mk index e05ba130da..85220feaa7 100644 --- a/makefiles/docker.inc.mk +++ b/makefiles/docker.inc.mk @@ -84,6 +84,9 @@ DOCKER_OVERRIDE_CMDLINE := $(strip $(DOCKER_OVERRIDE_CMDLINE)) # Overwrite if you want to use `docker` with sudo DOCKER ?= docker + +DOCKER_APPDIR = $(DOCKER_BUILD_ROOT)/riotproject/$(BUILDRELPATH) + # Mounted volumes and exported environment variables # Add GIT_CACHE_DIR if the directory exists @@ -128,5 +131,5 @@ ETC_LOCALTIME = $(realpath /etc/localtime) -e 'RIOTPROJECT=$(DOCKER_BUILD_ROOT)/riotproject' \ $(DOCKER_VOLUMES_AND_ENV) \ $(DOCKER_ENVIRONMENT_CMDLINE) \ - -w '$(DOCKER_BUILD_ROOT)/riotproject/$(BUILDRELPATH)' \ + -w '$(DOCKER_APPDIR)' \ '$(DOCKER_IMAGE)' make $(DOCKER_MAKECMDGOALS) $(DOCKER_OVERRIDE_CMDLINE) From e5871d9deb3d00346317984120733b2d5a421aed Mon Sep 17 00:00:00 2001 From: cladmi Date: Tue, 4 Sep 2018 15:09:28 +0200 Subject: [PATCH 03/11] makefiles/docker.inc.mk: use a variable for all volumes and environment --- makefiles/docker.inc.mk | 40 ++++++++++++++++++++-------------------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/makefiles/docker.inc.mk b/makefiles/docker.inc.mk index 85220feaa7..5cb0f737a0 100644 --- a/makefiles/docker.inc.mk +++ b/makefiles/docker.inc.mk @@ -84,10 +84,29 @@ DOCKER_OVERRIDE_CMDLINE := $(strip $(DOCKER_OVERRIDE_CMDLINE)) # Overwrite if you want to use `docker` with sudo DOCKER ?= docker +# Resolve symlink of /etc/localtime to its real path +# This is a workaround for docker on macOS, for more information see: +# https://github.com/docker/for-mac/issues/2396 +ETC_LOCALTIME = $(realpath /etc/localtime) + DOCKER_APPDIR = $(DOCKER_BUILD_ROOT)/riotproject/$(BUILDRELPATH) -# Mounted volumes and exported environment variables +# Directory mapping in docker and directories environment variable configuration +DOCKER_VOLUMES_AND_ENV += -v '$(RIOTBASE):$(DOCKER_RIOTBASE)' +DOCKER_VOLUMES_AND_ENV += -v '$(BUILD_DIR):$(DOCKER_BUILD_ROOT)/build' +DOCKER_VOLUMES_AND_ENV += -v '$(RIOTCPU):$(DOCKER_BUILD_ROOT)/riotcpu' +DOCKER_VOLUMES_AND_ENV += -v '$(RIOTBOARD):$(DOCKER_BUILD_ROOT)/riotboard' +DOCKER_VOLUMES_AND_ENV += -v '$(RIOTMAKE):$(DOCKER_BUILD_ROOT)/riotmake' +DOCKER_VOLUMES_AND_ENV += -v '$(RIOTPROJECT):$(DOCKER_BUILD_ROOT)/riotproject' +DOCKER_VOLUMES_AND_ENV += -v '$(ETC_LOCALTIME):/etc/localtime:ro' +DOCKER_VOLUMES_AND_ENV += -e 'RIOTBASE=$(DOCKER_RIOTBASE)' +DOCKER_VOLUMES_AND_ENV += -e 'BUILD_DIR=$(DOCKER_BUILD_ROOT)/build' +DOCKER_VOLUMES_AND_ENV += -e 'CCACHE_BASEDIR=$(DOCKER_RIOTBASE)' +DOCKER_VOLUMES_AND_ENV += -e 'RIOTCPU=$(DOCKER_BUILD_ROOT)/riotcpu' +DOCKER_VOLUMES_AND_ENV += -e 'RIOTBOARD=$(DOCKER_BUILD_ROOT)/riotboard' +DOCKER_VOLUMES_AND_ENV += -e 'RIOTMAKE=$(DOCKER_BUILD_ROOT)/riotmake' +DOCKER_VOLUMES_AND_ENV += -e 'RIOTPROJECT=$(DOCKER_BUILD_ROOT)/riotproject' # Add GIT_CACHE_DIR if the directory exists DOCKER_VOLUMES_AND_ENV += $(if $(wildcard $(GIT_CACHE_DIR)),-v $(GIT_CACHE_DIR):$(DOCKER_BUILD_ROOT)/gitcache) @@ -98,11 +117,6 @@ _is_git_worktree = $(shell grep '^gitdir: ' $(RIOTBASE)/.git 2>/dev/null) GIT_WORKTREE_COMMONDIR = $(abspath $(shell git rev-parse --git-common-dir)) DOCKER_VOLUMES_AND_ENV += $(if $(_is_git_worktree),-v $(GIT_WORKTREE_COMMONDIR):$(GIT_WORKTREE_COMMONDIR)) -# Resolve symlink of /etc/localtime to its real path -# This is a workaround for docker on macOS, for more information see: -# https://github.com/docker/for-mac/issues/2396 -ETC_LOCALTIME = $(realpath /etc/localtime) - # This will execute `make $(DOCKER_MAKECMDGOALS)` inside a Docker container. # We do not push the regular $(MAKECMDGOALS) to the container's make command in # order to only perform building inside the container and defer executing any @@ -115,20 +129,6 @@ ETC_LOCALTIME = $(realpath /etc/localtime) @# HACK: Handle directory creation here until it is provided globally $(Q)mkdir -p $(BUILD_DIR) $(DOCKER) run $(DOCKER_FLAGS) -t -u "$$(id -u)" \ - -v '$(RIOTBASE):$(DOCKER_RIOTBASE)' \ - -v '$(BUILD_DIR):$(DOCKER_BUILD_ROOT)/build' \ - -v '$(RIOTCPU):$(DOCKER_BUILD_ROOT)/riotcpu' \ - -v '$(RIOTBOARD):$(DOCKER_BUILD_ROOT)/riotboard' \ - -v '$(RIOTMAKE):$(DOCKER_BUILD_ROOT)/riotmake' \ - -v '$(RIOTPROJECT):$(DOCKER_BUILD_ROOT)/riotproject' \ - -v '$(ETC_LOCALTIME):/etc/localtime:ro' \ - -e 'RIOTBASE=$(DOCKER_RIOTBASE)' \ - -e 'BUILD_DIR=$(DOCKER_BUILD_ROOT)/build' \ - -e 'CCACHE_BASEDIR=$(DOCKER_BUILD_ROOT)/riotbase' \ - -e 'RIOTCPU=$(DOCKER_BUILD_ROOT)/riotcpu' \ - -e 'RIOTBOARD=$(DOCKER_BUILD_ROOT)/riotboard' \ - -e 'RIOTMAKE=$(DOCKER_BUILD_ROOT)/riotmake' \ - -e 'RIOTPROJECT=$(DOCKER_BUILD_ROOT)/riotproject' \ $(DOCKER_VOLUMES_AND_ENV) \ $(DOCKER_ENVIRONMENT_CMDLINE) \ -w '$(DOCKER_APPDIR)' \ From a462c4aeb982da37f92ee85479721c72420eeb79 Mon Sep 17 00:00:00 2001 From: cladmi Date: Tue, 4 Sep 2018 15:11:18 +0200 Subject: [PATCH 04/11] makefiles/docker.inc.mk: change definition order Update definition order for DOCKER_VOLUMES_AND_ENV. * Localtime * Mapping and env related to `RIOTBASE` * Build directories * Project * CPU/BOARD/make --- makefiles/docker.inc.mk | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/makefiles/docker.inc.mk b/makefiles/docker.inc.mk index 5cb0f737a0..6f151a4a7b 100644 --- a/makefiles/docker.inc.mk +++ b/makefiles/docker.inc.mk @@ -93,21 +93,24 @@ ETC_LOCALTIME = $(realpath /etc/localtime) DOCKER_APPDIR = $(DOCKER_BUILD_ROOT)/riotproject/$(BUILDRELPATH) # Directory mapping in docker and directories environment variable configuration -DOCKER_VOLUMES_AND_ENV += -v '$(RIOTBASE):$(DOCKER_RIOTBASE)' -DOCKER_VOLUMES_AND_ENV += -v '$(BUILD_DIR):$(DOCKER_BUILD_ROOT)/build' -DOCKER_VOLUMES_AND_ENV += -v '$(RIOTCPU):$(DOCKER_BUILD_ROOT)/riotcpu' -DOCKER_VOLUMES_AND_ENV += -v '$(RIOTBOARD):$(DOCKER_BUILD_ROOT)/riotboard' -DOCKER_VOLUMES_AND_ENV += -v '$(RIOTMAKE):$(DOCKER_BUILD_ROOT)/riotmake' -DOCKER_VOLUMES_AND_ENV += -v '$(RIOTPROJECT):$(DOCKER_BUILD_ROOT)/riotproject' DOCKER_VOLUMES_AND_ENV += -v '$(ETC_LOCALTIME):/etc/localtime:ro' +DOCKER_VOLUMES_AND_ENV += -v '$(RIOTBASE):$(DOCKER_RIOTBASE)' DOCKER_VOLUMES_AND_ENV += -e 'RIOTBASE=$(DOCKER_RIOTBASE)' -DOCKER_VOLUMES_AND_ENV += -e 'BUILD_DIR=$(DOCKER_BUILD_ROOT)/build' DOCKER_VOLUMES_AND_ENV += -e 'CCACHE_BASEDIR=$(DOCKER_RIOTBASE)' -DOCKER_VOLUMES_AND_ENV += -e 'RIOTCPU=$(DOCKER_BUILD_ROOT)/riotcpu' -DOCKER_VOLUMES_AND_ENV += -e 'RIOTBOARD=$(DOCKER_BUILD_ROOT)/riotboard' -DOCKER_VOLUMES_AND_ENV += -e 'RIOTMAKE=$(DOCKER_BUILD_ROOT)/riotmake' + +DOCKER_VOLUMES_AND_ENV += -v '$(BUILD_DIR):$(DOCKER_BUILD_ROOT)/build' +DOCKER_VOLUMES_AND_ENV += -e 'BUILD_DIR=$(DOCKER_BUILD_ROOT)/build' + +DOCKER_VOLUMES_AND_ENV += -v '$(RIOTPROJECT):$(DOCKER_BUILD_ROOT)/riotproject' DOCKER_VOLUMES_AND_ENV += -e 'RIOTPROJECT=$(DOCKER_BUILD_ROOT)/riotproject' +DOCKER_VOLUMES_AND_ENV += -v '$(RIOTCPU):$(DOCKER_BUILD_ROOT)/riotcpu' +DOCKER_VOLUMES_AND_ENV += -e 'RIOTCPU=$(DOCKER_BUILD_ROOT)/riotcpu' +DOCKER_VOLUMES_AND_ENV += -v '$(RIOTBOARD):$(DOCKER_BUILD_ROOT)/riotboard' +DOCKER_VOLUMES_AND_ENV += -e 'RIOTBOARD=$(DOCKER_BUILD_ROOT)/riotboard' +DOCKER_VOLUMES_AND_ENV += -v '$(RIOTMAKE):$(DOCKER_BUILD_ROOT)/riotmake' +DOCKER_VOLUMES_AND_ENV += -e 'RIOTMAKE=$(DOCKER_BUILD_ROOT)/riotmake' + # Add GIT_CACHE_DIR if the directory exists DOCKER_VOLUMES_AND_ENV += $(if $(wildcard $(GIT_CACHE_DIR)),-v $(GIT_CACHE_DIR):$(DOCKER_BUILD_ROOT)/gitcache) DOCKER_VOLUMES_AND_ENV += $(if $(wildcard $(GIT_CACHE_DIR)),-e GIT_CACHE_DIR=$(DOCKER_BUILD_ROOT)/gitcache) From 8351b13cc365d0b7d05ec9b3a1130f031901d676 Mon Sep 17 00:00:00 2001 From: cladmi Date: Tue, 4 Sep 2018 14:53:51 +0200 Subject: [PATCH 05/11] makefiles/docker.inc.mk: add path_in_docker function Return to which directory in the container this directory should be mapped. --- makefiles/docker.inc.mk | 51 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 51 insertions(+) diff --git a/makefiles/docker.inc.mk b/makefiles/docker.inc.mk index 6f151a4a7b..c97448fbe8 100644 --- a/makefiles/docker.inc.mk +++ b/makefiles/docker.inc.mk @@ -90,6 +90,57 @@ DOCKER ?= docker ETC_LOCALTIME = $(realpath /etc/localtime) +# # # # # # # # # # # # # # # # +# Directory mapping functions # +# # # # # # # # # # # # # # # # + +# Test if a directory is a subdirectory of `RIOTBASE` +# +# dir_is_outside_riotbase +# +# $1 = directory +# Returns: a non empty value if it is True +# +# From env: +# * RIOTBASE +# +# The terminating '/' in patsubst is important to match $1 == $(RIOTBASE) +# It also handles relative directories + +define dir_is_outside_riotbase +$(filter $(abspath $1)/,$(patsubst $(RIOTBASE)/%,%,$(abspath $1)/)) +endef + + +# Mapping of directores inside docker +# +# Return the path of directories from the host within the container +# +# path_in_docker +# +# $1 = directories (can be a list of relative directories) +# $2 = docker remap base directory (defaults to DOCKER_BUILD_ROOT) +# $3 = mapname (defaults to each directory name). +# If provided $1 must only contain one directory. +# Returns: the path the directory would have in docker +# +# For each directory: +# * if inside $(RIOTBASE), returns $(DOCKER_RIOTBASE)/ +# * if outside $(RIOTBASE), returns / +# +# From env: +# * RIOTBASE +# * DOCKER_RIOTBASE +# * DOCKER_BUILD_ROOT + +path_in_docker = $(foreach d,$1,$(strip $(call _dir_path_in_docker,$d,$2,$3))) +define _dir_path_in_docker + $(if $(call dir_is_outside_riotbase,$1),\ + $(if $2,$2,$(DOCKER_BUILD_ROOT))/$(if $3,$3,$(notdir $(abspath $1))),\ + $(patsubst %/,%,$(patsubst $(RIOTBASE)/%,$(DOCKER_RIOTBASE)/%,$(abspath $1)/))) +endef + + DOCKER_APPDIR = $(DOCKER_BUILD_ROOT)/riotproject/$(BUILDRELPATH) # Directory mapping in docker and directories environment variable configuration From e59b77988d3ddc4ec5de7aea61703bc010b5d9b5 Mon Sep 17 00:00:00 2001 From: cladmi Date: Tue, 4 Sep 2018 15:55:06 +0200 Subject: [PATCH 06/11] makefiles/docker.inc.mk: add docker_volume_and_env Add functions to get volume and env arguments for a given directory environment variable. It handles: * variables with multiple directories like EXTERNAL_MODULE_DIRS * relative path * if the 'directories' variable is empty, it will not be exported to docker --- makefiles/docker.inc.mk | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/makefiles/docker.inc.mk b/makefiles/docker.inc.mk index c97448fbe8..d86eb5d3c5 100644 --- a/makefiles/docker.inc.mk +++ b/makefiles/docker.inc.mk @@ -140,6 +140,42 @@ define _dir_path_in_docker $(patsubst %/,%,$(patsubst $(RIOTBASE)/%,$(DOCKER_RIOTBASE)/%,$(abspath $1)/))) endef +# Volume mapping and environment arguments +# +# Docker arguments for mapping directories: +# +# * volume mapping for each directory not in RIOT +# * remap environment variable directories to the docker ones +# +# +# docker_volume_and_env +# docker_volumes_mapping and docker_environ_mapping on different lines +# +# docker_volumes_mapping +# Command line argument for mapping volumes, if it should be mounted +# -v directory:docker_directory +# +# docker_environ_mapping +# Command line argument for mapping environment variables +# -e variable=docker_directory +# +# docker_cmdline_mapping +# Command line argument for mapping environment variables +# variable=docker_directory +# +# Arguments are the same as 'path_in_docker' +# If the 'directories' variable is empty, it will not be exported to docker + +docker_volume_and_env = $(strip $(call _docker_volume_and_env,$1,$2,$3)) +define _docker_volume_and_env + $(call docker_volumes_mapping,$($1),$2,$3) + $(call docker_environ_mapping,$1,$2,$3) +endef +docker_volumes_mapping = $(foreach d,$1,$(call _docker_volume_mapping,$d,$2,$3)) +_docker_volume_mapping = $(if $1,$(if $(call dir_is_outside_riotbase,$1), -v '$(abspath $1):$(call path_in_docker,$1,$2,$3)')) +docker_environ_mapping = $(addprefix -e ,$(call docker_cmdline_mapping,$1,$2,$3)) +docker_cmdline_mapping = $(if $($1),'$1=$(call path_in_docker,$($1),$2,$3)') + DOCKER_APPDIR = $(DOCKER_BUILD_ROOT)/riotproject/$(BUILDRELPATH) From a57f141cf61747ed22fb06f53b247643f476303d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ga=C3=ABtan=20Harter?= Date: Thu, 28 Feb 2019 14:39:36 +0100 Subject: [PATCH 07/11] makefiles/docker.inc.mk: use RIOTPROJECT in the RIOT if possible Use RIOTPROJECT from within the riot repository if it is inside. This means when it is the case to use: * Not mounting the directory to `riotproject` * Use `APPDIR` relative to inside RIOT If it is not inside, do the same as before: * Mount the RIOTPROJECT to `riotproject` * Use `APPDIR` relative to RIOTPROJECT --- makefiles/docker.inc.mk | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/makefiles/docker.inc.mk b/makefiles/docker.inc.mk index d86eb5d3c5..383d11dc15 100644 --- a/makefiles/docker.inc.mk +++ b/makefiles/docker.inc.mk @@ -177,7 +177,10 @@ docker_environ_mapping = $(addprefix -e ,$(call docker_cmdline_mapping,$1,$2,$3) docker_cmdline_mapping = $(if $($1),'$1=$(call path_in_docker,$($1),$2,$3)') -DOCKER_APPDIR = $(DOCKER_BUILD_ROOT)/riotproject/$(BUILDRELPATH) +# Application directory relative to either riotbase or riotproject +DOCKER_RIOTPROJECT = $(call path_in_docker,$(RIOTPROJECT),,riotproject) +DOCKER_APPDIR = $(DOCKER_RIOTPROJECT)/$(BUILDRELPATH) + # Directory mapping in docker and directories environment variable configuration DOCKER_VOLUMES_AND_ENV += -v '$(ETC_LOCALTIME):/etc/localtime:ro' @@ -188,8 +191,7 @@ DOCKER_VOLUMES_AND_ENV += -e 'CCACHE_BASEDIR=$(DOCKER_RIOTBASE)' DOCKER_VOLUMES_AND_ENV += -v '$(BUILD_DIR):$(DOCKER_BUILD_ROOT)/build' DOCKER_VOLUMES_AND_ENV += -e 'BUILD_DIR=$(DOCKER_BUILD_ROOT)/build' -DOCKER_VOLUMES_AND_ENV += -v '$(RIOTPROJECT):$(DOCKER_BUILD_ROOT)/riotproject' -DOCKER_VOLUMES_AND_ENV += -e 'RIOTPROJECT=$(DOCKER_BUILD_ROOT)/riotproject' +DOCKER_VOLUMES_AND_ENV += $(call docker_volume_and_env,RIOTPROJECT,,riotproject) DOCKER_VOLUMES_AND_ENV += -v '$(RIOTCPU):$(DOCKER_BUILD_ROOT)/riotcpu' DOCKER_VOLUMES_AND_ENV += -e 'RIOTCPU=$(DOCKER_BUILD_ROOT)/riotcpu' From 8687d53c822d3e4e4b60d62687f645f31086553c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ga=C3=ABtan=20Harter?= Date: Thu, 28 Feb 2019 14:41:11 +0100 Subject: [PATCH 08/11] makefiles/docker.inc.mk: use BUILD_DIR in RIOT if possible Use BUILD_DIR from within the riot repository if it is inside. --- makefiles/docker.inc.mk | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/makefiles/docker.inc.mk b/makefiles/docker.inc.mk index 383d11dc15..2966989b21 100644 --- a/makefiles/docker.inc.mk +++ b/makefiles/docker.inc.mk @@ -188,8 +188,7 @@ DOCKER_VOLUMES_AND_ENV += -v '$(RIOTBASE):$(DOCKER_RIOTBASE)' DOCKER_VOLUMES_AND_ENV += -e 'RIOTBASE=$(DOCKER_RIOTBASE)' DOCKER_VOLUMES_AND_ENV += -e 'CCACHE_BASEDIR=$(DOCKER_RIOTBASE)' -DOCKER_VOLUMES_AND_ENV += -v '$(BUILD_DIR):$(DOCKER_BUILD_ROOT)/build' -DOCKER_VOLUMES_AND_ENV += -e 'BUILD_DIR=$(DOCKER_BUILD_ROOT)/build' +DOCKER_VOLUMES_AND_ENV += $(call docker_volume_and_env,BUILD_DIR,,build) DOCKER_VOLUMES_AND_ENV += $(call docker_volume_and_env,RIOTPROJECT,,riotproject) From 3c0ecebb2cf58964761be0270b55df328cc724e6 Mon Sep 17 00:00:00 2001 From: cladmi Date: Tue, 4 Sep 2018 15:58:14 +0200 Subject: [PATCH 09/11] makefiles/docker.inc.mk: use more directories in RIOT Use the directories from in RIOT if possible for RIOTCPU/RIOTBOARD/RIOTMAKE. --- makefiles/docker.inc.mk | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/makefiles/docker.inc.mk b/makefiles/docker.inc.mk index 2966989b21..b136c7915a 100644 --- a/makefiles/docker.inc.mk +++ b/makefiles/docker.inc.mk @@ -191,13 +191,9 @@ DOCKER_VOLUMES_AND_ENV += -e 'CCACHE_BASEDIR=$(DOCKER_RIOTBASE)' DOCKER_VOLUMES_AND_ENV += $(call docker_volume_and_env,BUILD_DIR,,build) DOCKER_VOLUMES_AND_ENV += $(call docker_volume_and_env,RIOTPROJECT,,riotproject) - -DOCKER_VOLUMES_AND_ENV += -v '$(RIOTCPU):$(DOCKER_BUILD_ROOT)/riotcpu' -DOCKER_VOLUMES_AND_ENV += -e 'RIOTCPU=$(DOCKER_BUILD_ROOT)/riotcpu' -DOCKER_VOLUMES_AND_ENV += -v '$(RIOTBOARD):$(DOCKER_BUILD_ROOT)/riotboard' -DOCKER_VOLUMES_AND_ENV += -e 'RIOTBOARD=$(DOCKER_BUILD_ROOT)/riotboard' -DOCKER_VOLUMES_AND_ENV += -v '$(RIOTMAKE):$(DOCKER_BUILD_ROOT)/riotmake' -DOCKER_VOLUMES_AND_ENV += -e 'RIOTMAKE=$(DOCKER_BUILD_ROOT)/riotmake' +DOCKER_VOLUMES_AND_ENV += $(call docker_volume_and_env,RIOTCPU,,riotcpu) +DOCKER_VOLUMES_AND_ENV += $(call docker_volume_and_env,RIOTBOARD,,riotboard) +DOCKER_VOLUMES_AND_ENV += $(call docker_volume_and_env,RIOTMAKE,,riotmake) # Add GIT_CACHE_DIR if the directory exists DOCKER_VOLUMES_AND_ENV += $(if $(wildcard $(GIT_CACHE_DIR)),-v $(GIT_CACHE_DIR):$(DOCKER_BUILD_ROOT)/gitcache) From 8b54f87fc4caa67aad95f1f5ca47a5d6f1f0a16c Mon Sep 17 00:00:00 2001 From: cladmi Date: Thu, 18 Oct 2018 01:27:43 +0200 Subject: [PATCH 10/11] makefiles:docker.inc.mk: add a global documentation on directory mapping --- makefiles/docker.inc.mk | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/makefiles/docker.inc.mk b/makefiles/docker.inc.mk index b136c7915a..6205aafdab 100644 --- a/makefiles/docker.inc.mk +++ b/makefiles/docker.inc.mk @@ -93,6 +93,28 @@ ETC_LOCALTIME = $(realpath /etc/localtime) # # # # # # # # # # # # # # # # # Directory mapping functions # # # # # # # # # # # # # # # # # +# +# This part handles mapping and mounting directories variables from the +# host system in the docker container. +# +# In the container all directories are mapped in subdirectories of +# `DOCKER_BUILD_ROOT` (`/data/riotbuild` by default). +# +# +# The RIOT directory `RIOTBASE` is mounted to `DOCKER_RIOTBASE` +# (`DOCKER_BUILD_ROOT/riotbase` by default). +# +# For other directories variables: +# +# * if the directory is contained within the `RIOT` repository, +# the variable is mapped to a path inside `DOCKER_RIOTBASE` in the container. +# +# * if the directory is not contained in the `RIOT` repository, +# the directory must be mounted in the countainer. +# The variable and directory are mapped to a path outside `DOCKER_RIOTBASE`. +# Some variables have hardwritten mapping directories (`RIOTCPU` for example), +# and other have a mapping directory based on their directory name. + # Test if a directory is a subdirectory of `RIOTBASE` # From 3a17ddc43c9024c47d5b5c6d4204c6cab638c2b2 Mon Sep 17 00:00:00 2001 From: cladmi Date: Tue, 4 Sep 2018 15:59:04 +0200 Subject: [PATCH 11/11] makefiles/docker.inc.mk: add support for EXTERNAL_MODULE_DIRS They are remapped to `$(DOCKER_BUILD_ROOT)/external` if they are not inside RIOT (usually the case but not for `tests/external_modul_dirs`). If they are inside 'riotproject' they are currently also remapped to 'external'. The value of `EXTERNAL_MODULE_DIRS` is then enforced by configuring it on the command line as the application should not try to set it anymore. The remapping is done in `external/directory_name` so cannot handle multiple external directories with the same name. --- makefiles/docker.inc.mk | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/makefiles/docker.inc.mk b/makefiles/docker.inc.mk index 6205aafdab..00395ec470 100644 --- a/makefiles/docker.inc.mk +++ b/makefiles/docker.inc.mk @@ -221,6 +221,33 @@ DOCKER_VOLUMES_AND_ENV += $(call docker_volume_and_env,RIOTMAKE,,riotmake) DOCKER_VOLUMES_AND_ENV += $(if $(wildcard $(GIT_CACHE_DIR)),-v $(GIT_CACHE_DIR):$(DOCKER_BUILD_ROOT)/gitcache) DOCKER_VOLUMES_AND_ENV += $(if $(wildcard $(GIT_CACHE_DIR)),-e GIT_CACHE_DIR=$(DOCKER_BUILD_ROOT)/gitcache) +# Remap external module directories. +# +# This remaps directories from EXTERNAL_MODULE_DIRS to subdirectories of +# $(DOCKER_BUILD_ROOT)/external +# +# Remapped directories must all have different basenames +# +# Limitation: If a directory is inside RIOTPROJECT and not in RIOT it is +# remapped anyway instead of loading from inside RIOTPROJECT. +# +# As EXTERNAL_MODULE_DIRS should ignore the 'Makefile' configuration, they must +# be set using command line variable settings to not be modified within docker. +DOCKER_VOLUMES_AND_ENV += $(call docker_volumes_mapping,$(EXTERNAL_MODULE_DIRS),$(DOCKER_BUILD_ROOT)/external,) +DOCKER_OVERRIDE_CMDLINE += $(call docker_cmdline_mapping,EXTERNAL_MODULE_DIRS,$(DOCKER_BUILD_ROOT)/external,) + +# External module directories sanity check: +# +# Detect if there are remapped directories with the same name as it is not handled. +# Having EXTERNAL_MODULE_DIRS = /path/to/dir/name \ +# /another/directory/also/called/name +# would lead to both being mapped to '$(DOCKER_BUILD_ROOT)/external/name' +_mounted_dirs = $(foreach d,$(EXTERNAL_MODULE_DIRS),$(if $(call dir_is_outside_riotbase,$(d)),$(d))) +ifneq ($(words $(sort $(notdir $(_mounted_dirs)))),$(words $(sort $(_mounted_dirs)))) + $(warning Mounted EXTERNAL_MODULE_DIRS: $(_mounted_dirs)) + $(error Mapping EXTERNAL_MODULE_DIRS in docker is not supported for directories with the same name) +endif + # Handle worktree by mounting the git common dir in the same location _is_git_worktree = $(shell grep '^gitdir: ' $(RIOTBASE)/.git 2>/dev/null) GIT_WORKTREE_COMMONDIR = $(abspath $(shell git rev-parse --git-common-dir))