dist/tools: add dlcache.sh

.gitignore

@@ -47,3 +47,4 @@ Makefile.local
 .ycm_extra_conf.py
 .ycm_extra_conf.pyc
 __pycache__
+.dlcache

Makefile.include

@@ -16,6 +16,8 @@ APPDIR         ?= $(CURDIR)
 BINDIRBASE     ?= $(APPDIR)/bin
 BINDIR         ?= $(BINDIRBASE)/$(BOARD)
 PKGDIRBASE     ?= $(BINDIRBASE)/pkg/$(BOARD)
+DLCACHE        ?= $(RIOTBASE)/dist/tools/dlcache/dlcache.sh
+DLCACHE_DIR    ?= $(RIOTBASE)/.dlcache
 
 __DIRECTORY_VARIABLES := \
   RIOTBASE \
@@ -30,6 +32,7 @@ __DIRECTORY_VARIABLES := \
   CCACHE_BASEDIR \
   GITCACHE \
   PKGDIRBASE \
+  DLCACHE_DIR \
   #
 
 # Make all paths absolute.
@@ -45,6 +48,7 @@ override APPDIR         := $(abspath $(APPDIR))
 override BINDIRBASE     := $(abspath $(BINDIRBASE))
 override BINDIR         := $(abspath $(BINDIR))
 override PKGDIRBASE     := $(abspath $(PKGDIRBASE))
+override DLCACHE_DIR    := $(abspath $(DLCACHE_DIR))
 
 # Ensure that all directories are set and don't contain spaces.
 ifneq (, $(filter-out 1, $(foreach v,${__DIRECTORY_VARIABLES},$(words ${${v}}))))

dist/tools/dlcache/README.md

@@ -0,0 +1,11 @@
+# Introduction
+
+This script aims to cache http(s)-downloads
+
+## How it works
+
+- if a file with the right name and md5 exists, nothing happens
+- if a file with the right name and md5 is in cache, use that
+- if a file with the right name but wrong md5 is in cache, redownload
+  download to cache
+- after download, check md5, then copy to target

dist/tools/dlcache/dlcache.sh

@@ -0,0 +1,108 @@
+#!/bin/sh
+
+DLCACHE_DIR=${DLCACHE_DIR:-~/.dlcache}
+
+mkdir -p "$DLCACHE_DIR"
+
+_echo() {
+    echo "$@" 1>&2
+}
+
+if [ "$(uname)" = Darwin ]; then
+    _locked() {
+        local lockfile="$1"
+        shift
+
+        while ! shlock -p $$ -f $lockfile; do
+            sleep 0.2
+        done
+
+        $*
+
+        rm $lockfile
+    }
+else
+    _locked() {
+        local lockfile="$1"
+        shift
+
+        (
+        flock -w 600 9 || exit 1
+        $*
+        ) 9>"$lockfile"
+    }
+fi
+
+if [ "$(uname)" = Darwin ]; then
+    MD5="md5 -r"
+else
+    MD5=md5sum
+fi
+
+calcmd5() {
+    local file="$1"
+    local md5="$2"
+
+    local file_md5=$(${MD5} "$file" | cut -d\  -f1)
+
+    test "$md5" = "$file_md5"
+}
+
+downloader() {
+    if [ -n "$(command -v wget)" ]; then
+        wget -nv "$1" -O $2
+    elif [ -n "$(command -v curl)" ]; then
+        curl -L $1 -o $2
+    else
+        _echo "$0: neither wget nor curl available!"
+        return 1
+    fi
+}
+
+download() {
+    local url="$1"
+    local _md5="$2"
+    local basename_url=$(basename ${url})
+    local target="${3:-${basename_url}}"
+
+    [ -f "$target" ] && {
+        # if our target file exists, check it's md5.
+        calcmd5 "$target" "$_md5" && {
+            _echo "$0: target exists, md5 matches."
+            exit 0
+        }
+    }
+
+    local filename="$(basename $url)"
+    [ -f "$DLCACHE_DIR/$filename" ] && {
+        # if the file exists in cache, check it's md5 and possibly remove it.
+        if calcmd5 "$DLCACHE_DIR/$filename" "$_md5"; then
+            _echo "$0: getting \"$url\" from cache"
+        else
+            _echo "$0: \"$DLCACHE_DIR/$filename\" has wrong checksum, re-downloading"
+            rm "$DLCACHE_DIR/$filename"
+        fi
+    }
+
+    [ ! -f "$DLCACHE_DIR/$filename" ] && {
+        _echo "$0: downloading \"$url\"..."
+        downloader "$url" "$DLCACHE_DIR/$filename" || {
+            _echo "$0: error downloading $url to $DLCACHE_DIR/$filename!"
+            exit 1
+        }
+        _echo "$0: done downloading \"$url\""
+    }
+
+    calcmd5 "$DLCACHE_DIR/$filename" "$_md5" || {
+        _echo "$0: checksum mismatch!"
+        exit 1
+    }
+
+    if [ "$target" = "-" ]; then
+        cat "$DLCACHE_DIR/$filename"
+    else
+        cp "$DLCACHE_DIR/$filename" "$target"
+    fi
+}
+
+_locked "$DLCACHE_DIR/$(basename $1).locked" download "$@"

makefiles/vars.inc.mk

@@ -68,6 +68,7 @@ export RESET_FLAGS           # The parameters to supply to RESET.
 export CCACHE_BASEDIR        # ccache basedir, allows multiple riot build
                              # directories to share a ccache directory
 
+export DLCACHE               # directory used to cache http downloads
 export DOWNLOAD_TO_FILE      # Use `$(DOWNLOAD_TO_FILE) $(DESTINATION) $(URL)` to download `$(URL)` to `$(DESTINATION)`.
 export DOWNLOAD_TO_STDOUT    # Use `$(DOWNLOAD_TO_STDOUT) $(URL)` to download `$(URL)` output `$(URL)` to stdout, e.g. to be piped into `tar xz`.
 export UNZIP_HERE            # Use `cd $(SOME_FOLDER) && $(UNZIP_HERE) $(SOME_FILE)` to extract the contents of the zip file `$(SOME_FILE)` into `$(SOME_FOLDER)`.