Merge pull request #20472 from maribu/docker/pin/version

makefiles/docker.inc.mk: Pin riotbuild version with BUILD_IN_DOCKER=1

dist/tools/buildsystem_sanity_check/check.sh

@@ -16,6 +16,7 @@
 : "${RIOTBASE:="$(cd "$(dirname "$0")/../../../" || exit; pwd)"}"
 
 : "${RIOTTOOLS:=${RIOTBASE}/dist/tools}"
+: "${RIOTMAKE:=${RIOTBASE}/makefiles}"
 # not running shellcheck with -x in the CI --> disable SC1091
 # shellcheck disable=SC1091
 . "${RIOTTOOLS}"/ci/github_annotate.sh
@@ -380,6 +381,28 @@ check_tests_application_path() {
     find tests/ -type f "${patterns[@]}" | error_with_message "Invalid application path in tests/"
 }
 
+check_pinned_docker_version_is_up_to_date() {
+    local pinned_digest
+    local pinned_repo_digest
+    local upstream_digest
+    local upstream_repo_digest
+    pinned_digest="$(awk '/^DOCKER_TESTED_IMAGE_ID := (.*)$/ { print substr($0, index($0, $3)); exit }' "$RIOTMAKE/docker.inc.mk")"
+    pinned_repo_digest="$(awk '/^DOCKER_TESTED_IMAGE_REPO_DIGEST := (.*)$/ { print substr($0, index($0, $3)); exit }' "$RIOTMAKE/docker.inc.mk")"
+    # not using docker and jq here but a python script to not have to install
+    # more stuff for the static test docker image
+    IFS=' ' read -r upstream_digest upstream_repo_digest <<< "$("$RIOTTOOLS/buildsystem_sanity_check/get_dockerhub_digests.py" "riot/riotbuild")"
+
+    if [ "$pinned_digest" != "$upstream_digest" ]; then
+        git -C "${RIOTBASE}" grep -n '^DOCKER_TESTED_IMAGE_ID :=' "$RIOTMAKE/docker.inc.mk" \
+            | error_with_message  "Update docker image SHA256 to ${upstream_digest}"
+    fi
+
+    if [ "$pinned_repo_digest" != "$upstream_repo_digest" ]; then
+        git -C "${RIOTBASE}" grep -n '^DOCKER_TESTED_IMAGE_REPO_DIGEST :=' "$RIOTMAKE/docker.inc.mk" \
+            | error_with_message  "Update manifest digest to ${upstream_repo_digest}"
+    fi
+}
+
 error_on_input() {
     ! grep ''
 }
@@ -402,6 +425,7 @@ all_checks() {
     check_no_riot_config
     check_stderr_null
     check_tests_application_path
+    check_pinned_docker_version_is_up_to_date
 }
 
 if [[ "${BASH_SOURCE[0]}" == "${0}" ]]; then

dist/tools/buildsystem_sanity_check/get_dockerhub_digests.py

@@ -0,0 +1,91 @@
+#!/usr/bin/env python3
+"""
+Command line utility to get the image sha256 sum and the manifest sha256 sum
+of the latest image at dockerhub.
+"""
+import http.client
+import json
+import sys
+
+
+def get_docker_token(repo):
+    """
+    Get an API access token for the docker registry
+
+    :param repo: the repository the API token should be valid for
+    :type repo: str
+    :return: the access token to use
+    :rtype: str
+    """
+    conn = http.client.HTTPSConnection("auth.docker.io")
+    conn.request("GET",
+                 f"/token?service=registry.docker.io&scope=repository:{repo}:pull",
+                 headers={"Accept": "*/*"})
+    resp = conn.getresponse()
+    if resp.status != 200:
+        raise Exception(f"Tried to get a docker token, but auth.docker.io "
+                        f"replied with {resp.status} {resp.reason}")
+    decoded = json.loads(resp.read())
+    conn.close()
+    return decoded["token"]
+
+
+def get_manifest(repo, tag="latest", token=None):
+    """
+    Get the manifest of the given repo
+
+    :param repo: The repository to get the latest manifest of
+    :type repo: str
+    :param tag: The tag to get the manifest of. (Default: "latest")
+    :type tag: str
+    :param token: The authorization token to use for the given repo.
+                  (Default: get a fresh one.)
+    :type token: str
+    :return: the parsed manifast
+    :rtype: dict
+    """
+    token = get_docker_token(repo) if token is None else token
+    conn = http.client.HTTPSConnection("index.docker.io")
+    hdrs = {
+        "Accept": "application/vnd.docker.distribution.manifest.v2+json",
+        "Authorization": f"Bearer {token}"
+    }
+    conn.request("GET", f"/v2/{repo}/manifests/{tag}", headers=hdrs)
+    resp = conn.getresponse()
+    if resp.status != 200:
+        raise Exception(f"Tried to get a docker manifest, but "
+                        f"index.docker.io replied with {resp.status} "
+                        f"{resp.reason}")
+    repo_digest = resp.getheader("ETag")[len("\"sha256:"):-1]
+    decoded = json.loads(resp.read())
+    conn.close()
+    return (decoded, repo_digest)
+
+
+def get_upstream_digests(repo, tag="latest", token=None):
+    """
+    Get the SHA256 hash of the latest image of the given repo at dockerhub
+    as string of hex digests
+
+    :param repo: The repository to get the hash from
+    :type repo: str
+    :param tag: The tag to get the manifest of. (Default: "latest")
+    :type tag: str
+    :param token: The authorization token to use for the given repo.
+                  (Default: get a fresh one.)
+    :type token: str
+    :return: A 2-tuple of the image digest and the repo digest
+    :rtype: (str, str)
+    """
+    token = get_docker_token(repo) if token is None else token
+    manifest, repo_digest = get_manifest(repo, tag=tag, token=token)
+    digest = manifest["config"]["digest"]
+    return (digest[len("sha256:"):], repo_digest)
+
+
+if __name__ == '__main__':
+    if len(sys.argv) != 2:
+        sys.exit(f"Usage {sys.argv[0]} <REPO_NAME>")
+
+    digest, repo_digest = get_upstream_digests(sys.argv[1])
+    print(f"{digest} {repo_digest}")

makefiles/docker.inc.mk

@@ -1,6 +1,22 @@
-export DOCKER_IMAGE ?= docker.io/riot/riotbuild:latest
+# This *MUST* be updated in lock-step with the riotbuild image in
+# https://github.com/RIOT-OS/riotdocker. The idea is that when checking out
+# a random RIOT merge commit, `make BUILD_IN_DOCKER=1` should always succeed.
+DOCKER_TESTED_IMAGE_ID := f5951bc41dfface6cac869181d703e62cbdd3b7976b0946130a38f2e658000b3
+DOCKER_TESTED_IMAGE_REPO_DIGEST := 75dec511ba26424987a26bdee5ac2f94d5f4928d79b627d1620b9d2391aab3e1
+
+DOCKER_PULL_IDENTIFIER := docker.io/riot/riotbuild@sha256:$(DOCKER_TESTED_IMAGE_REPO_DIGEST)
+DOCKER_IMAGE_DEFAULT := sha256:$(DOCKER_TESTED_IMAGE_ID)
+DOCKER_AUTO_PULL ?= 1
+export DOCKER_IMAGE ?= $(DOCKER_IMAGE_DEFAULT)
 export DOCKER_BUILD_ROOT ?= /data/riotbuild
 DOCKER_RIOTBASE ?= $(DOCKER_BUILD_ROOT)/riotbase
+
+# These targets need to be run before docker can be run
+DEPS_FOR_RUNNING_DOCKER :=
+
+# Overwrite if you want to use `docker` with sudo
+DOCKER ?= docker
+
 # List of Docker-enabled make goals
 export DOCKER_MAKECMDGOALS_POSSIBLE = \
   all \
@@ -19,6 +35,25 @@ else
   export INSIDE_DOCKER := 0
 endif
 
+ifeq (0:1,$(INSIDE_DOCKER):$(BUILD_IN_DOCKER))
+  ifeq ($(DOCKER_IMAGE),$(DOCKER_IMAGE_DEFAULT))
+    IMAGE_PRESENT:=$(shell $(DOCKER) image inspect $(DOCKER_IMAGE) 2>/dev/null >/dev/null && echo 1 || echo 0)
+    ifeq (0,$(IMAGE_PRESENT))
+      $(warning Required docker image $(DOCKER_IMAGE) not installed)
+      ifeq (1,$(DOCKER_AUTO_PULL))
+        $(info Pulling required image automatically. You can disable this with DOCKER_AUTO_PULL=0)
+        DEPS_FOR_RUNNING_DOCKER += docker-pull
+      else
+        $(info Building with latest available riotbuild image. You can pull the correct image automatically with DOCKER_AUTO_PULL=1)
+        # The currently set DOCKER_IMAGE is not locally available, and the
+        # user opted out to automatically pull it. Fall back to the
+        # latest (locally) available riot/riotbuild image instead.
+        export DOCKER_IMAGE := docker.io/riot/riotbuild:latest
+      endif
+    endif
+  endif
+endif
+
 # Default target for building inside a Docker container if nothing was given
 export DOCKER_MAKECMDGOALS ?= all
 # List of all exported environment variables that shall be passed on to the
@@ -115,8 +150,6 @@ DOCKER_OVERRIDE_CMDLINE_AUTO := $(foreach varname,$(DOCKER_ENV_VARS), \
     ))
 DOCKER_OVERRIDE_CMDLINE += $(strip $(DOCKER_OVERRIDE_CMDLINE_AUTO))
 
-# Overwrite if you want to use `docker` with sudo
-DOCKER ?= docker
 _docker_is_podman = $(shell $(DOCKER) --version | grep podman 2>/dev/null)
 
 # Set default run flags:
@@ -189,7 +222,6 @@ define dir_is_outside_riotbase
 $(filter $(abspath $1)/,$(patsubst $(RIOTBASE)/%,%,$(abspath $1)/))
 endef
 
-
 # Mapping of directores inside docker
 #
 # Return the path of directories from the host within the container
@@ -342,6 +374,11 @@ docker_run_make = \
 	-w '$(DOCKER_APPDIR)' '$2' \
 	$(MAKE) $(DOCKER_OVERRIDE_CMDLINE) $4 $1
 
+# This target pulls the docker image required for BUILD_IN_DOCKER
+.PHONY: docker-pull
+docker-pull:
+	$(DOCKER) pull '$(DOCKER_PULL_IDENTIFIER)'
+
 # This will execute `make $(DOCKER_MAKECMDGOALS)` inside a Docker container.
 # We do not push the regular $(MAKECMDGOALS) to the container's make command in
 # order to only perform building inside the container and defer executing any
@@ -349,6 +386,6 @@ docker_run_make = \
 # container.
 # The `flash`, `term`, `debugserver` etc. targets usually require access to
 # hardware which may not be reachable from inside the container.
-..in-docker-container:
+..in-docker-container: $(DEPS_FOR_RUNNING_DOCKER)
 	@$(COLOR_ECHO) '$(COLOR_GREEN)Launching build container using image "$(DOCKER_IMAGE)".$(COLOR_RESET)'
 	$(call docker_run_make,$(DOCKER_MAKECMDGOALS),$(DOCKER_IMAGE),,$(DOCKER_MAKE_ARGS))