Merge pull request #9634 from cladmi/pr/make/lazysponge

dist/tools: add lazysponge tool

Makefile.include

@@ -186,6 +186,11 @@ ifeq (,$(UNZIP_HERE))
   endif
 endif
 
+# Tool saving stdin to a file only on content update.
+# It keeps the file timestamp if it would end up the same.
+LAZYSPONGE ?= $(RIOTTOOLS)/lazysponge/lazysponge.py
+LAZYSPONGE_FLAGS ?= $(if $(filter 1,$(QUIET)),,--verbose)
+
 ifeq (, $(APPLICATION))
     $(error An application name must be specified as APPLICATION.)
 endif
@@ -675,7 +680,8 @@ include $(RIOTTOOLS)/desvirt/Makefile.desvirt
 # The script will only touch the file if anything has changed since last time.
 $(RIOTBUILD_CONFIG_HEADER_C): FORCE
 	@mkdir -p '$(dir $@)'
-	$(Q)'$(RIOTTOOLS)/genconfigheader/genconfigheader.sh' '$@' $(CFLAGS_WITH_MACROS)
+	$(Q)'$(RIOTTOOLS)/genconfigheader/genconfigheader.sh' $(CFLAGS_WITH_MACROS) \
+		| '$(LAZYSPONGE)' $(LAZYSPONGE_FLAGS) '$@'
 
 CFLAGS_WITH_MACROS := $(CFLAGS)
 

dist/tools/genconfigheader/genconfigheader.sh

@@ -7,43 +7,18 @@
 # Public License v2.1. See the file LICENSE in the top level directory for more
 # details.
 #
-
-DEBUG=0
-if [ "${QUIET}" != "1" ]; then
-  DEBUG=1
-fi
-
-if [ $# -lt 1 ]; then
-  echo "Usage: $0 <output.h> [CFLAGS]..."
-  echo "Extract all macros from CFLAGS and generate a header file"
-  exit 1
-fi
-OUTPUTFILE="$1"
-shift
-
-MD5SUM=md5sum
-if [ "$(uname -s)" = "Darwin" -o "$(uname -s)" = "FreeBSD" ]; then
-  MD5SUM="md5 -r"
-fi
-
-# atomically update the file
-TMPFILE=
-trap '[ -n "${TMPFILE}" ] && rm -f "${TMPFILE}"' EXIT
-# Create temporary output file
-TMPFILE=$(mktemp ${OUTPUTFILE}.XXXXXX)
-
-if [ -z "${TMPFILE}" ]; then
-  echo "Error creating temporary file, aborting"
-  exit 1
-fi
+# Usage: $0 [CFLAGS]...
+#
+# Extract all macros from CFLAGS and generate a header file format"
+#
 
 # exit on any errors below this line
 set -e
 
-echo "/* DO NOT edit this file, your changes will be overwritten and won't take any effect! */" > "${TMPFILE}"
-echo "/* Generated from CFLAGS: $@ */" >> "${TMPFILE}"
+echo "/* DO NOT edit this file, your changes will be overwritten and won't take any effect! */"
+echo "/* Generated from CFLAGS: $@ */"
 
-[ -n "${LTOFLAGS}" ] && echo "/* LTOFLAGS=${LTOFLAGS} */" >> "${TMPFILE}"
+[ -n "${LTOFLAGS}" ] && echo "/* LTOFLAGS=${LTOFLAGS} */"
 
 for arg in "$@"; do
   case ${arg} in
@@ -54,34 +29,19 @@ for arg in "$@"; do
         # key=value pairs
         key=${d%%=*}
         value=${d#*=}
-        echo "#define $key $value" >> "${TMPFILE}"
+        echo "#define $key $value"
       else
         # simple #define
-        echo "#define $d 1" >> "${TMPFILE}"
+        echo "#define $d 1"
       fi
       ;;
     -U*)
       # Strip leading -U
       d=${arg#-U}
-      echo "#undef $d" >> "${TMPFILE}"
+      echo "#undef $d"
       ;;
     *)
       continue
       ;;
   esac
 done
-
-# Only replace old file if the new file differs. This allows make to check the
-# date of the config header for dependency calculations.
-NEWMD5=$(${MD5SUM} ${TMPFILE} | cut -c -32)
-OLDMD5=$(${MD5SUM} ${OUTPUTFILE} 2>/dev/null | cut -c -32)
-if [ "${NEWMD5}" != "${OLDMD5}" ]; then
-  if [ "${DEBUG}" -eq 1 ]; then echo "Replacing ${OUTPUTFILE} (${NEWMD5} != ${OLDMD5})"; fi
-  # Set mode according to umask
-  chmod +rw "${TMPFILE}"
-  mv -f "${TMPFILE}" "${OUTPUTFILE}"
-else
-  if [ "${DEBUG}" -eq 1 ]; then echo "Keeping old ${OUTPUTFILE}"; fi
-fi
-
-# $TMPFILE will be deleted by the EXIT trap above if it still exists when we exit

dist/tools/lazysponge/lazysponge.py

@@ -0,0 +1,102 @@
+#! /usr/bin/env python3
+
+#
+# Copyright (C) 2018 Gaëtan Harter <gaetan.harter@fu-berlin.de>
+#
+# This file is subject to the terms and conditions of the GNU Lesser
+# General Public License v2.1. See the file LICENSE in the top level
+# directory for more details.
+#
+
+"""
+lazysponge
+
+Adaptation of moreutils `sponge` with added functionnality that it does not
+modify the output file if the content would be unchanged.
+
+Description
+-----------
+
+Reads standard input and writes it to the specified file if its content was
+different.
+
+The file is not changed if the content is the same so modification timestamp is
+unchanged.
+
+Note
+----
+
+It only works with input provided by a `pipe` and not interractive input.
+The reason is that `ctrl+c` would not be handled properly in that case.
+
+Usage
+-----
+
+    usage: lazysponge.py [-h] outfile
+
+    Soak up all input from stdin and write it to <outfile> if it differs from
+    previous content. If the content is the same, file is not modified.
+
+    positional arguments:
+    outfile     Output file
+
+    optional arguments:
+    -h, --help  show this help message and exit
+"""
+
+import os
+import sys
+import argparse
+import hashlib
+
+DESCRIPTION = ('Soak up all input from stdin and write it to <outfile>'
+               ' if it differs from previous content.\n'
+               ' If the content is the same, file is not modified.')
+PARSER = argparse.ArgumentParser(description=DESCRIPTION)
+PARSER.add_argument('outfile', help='Output file')
+PARSER.add_argument('--verbose', '-v', help='Verbose output', default=False,
+                    action='store_true')
+
+
+def _print_hash_debug_info(outfilename, oldbytes, newbytes):
+    """Print debug information on hashs."""
+    oldhash = hashlib.md5(oldbytes).hexdigest() if oldbytes is not None else ''
+    newhash = hashlib.md5(newbytes).hexdigest()
+    if oldbytes == newbytes:
+        msg = 'Keeping old {} ({})'.format(outfilename, oldhash)
+    else:
+        msg = 'Replacing {} ({} != {})'.format(outfilename, oldhash, newhash)
+    print(msg, file=sys.stderr)
+
+
+def main():
+    """Write stdin to given <outfile> if it would change its content."""
+    opts = PARSER.parse_args()
+
+    # No support for 'interactive' input as catching Ctrl+c breaks in 'read'
+    if os.isatty(sys.stdin.fileno()):
+        print('Interactive input not supported. Use piped input',
+              file=sys.stderr)
+        print('  echo message | {}'.format(' '.join(sys.argv)),
+              file=sys.stderr)
+        exit(1)
+
+    try:
+        with open(opts.outfile, 'rb') as outfd:
+            oldbytes = outfd.read()
+    except FileNotFoundError:
+        oldbytes = None
+
+    stdinbytes = sys.stdin.buffer.read()
+    if opts.verbose:
+        _print_hash_debug_info(opts.outfile, oldbytes, stdinbytes)
+
+    if oldbytes == stdinbytes:
+        return
+
+    with open(opts.outfile, 'wb') as outfd:
+        outfd.write(stdinbytes)
+
+
+if __name__ == '__main__':
+    main()

dist/tools/lazysponge/lazysponge_test.py

@@ -0,0 +1,149 @@
+#! /usr/bin/env python3
+
+#
+# Copyright (C) 2018 Gaëtan Harter <gaetan.harter@fu-berlin.de>
+#
+# This file is subject to the terms and conditions of the GNU Lesser
+# General Public License v2.1. See the file LICENSE in the top level
+# directory for more details.
+#
+
+"""Test script for lazysponge."""
+
+import os
+import sys
+import shutil
+import tempfile
+from io import StringIO, BytesIO
+
+import unittest
+from unittest import mock
+
+import lazysponge
+
+
+class TestLazysponge(unittest.TestCase):
+    """Test the lazysponge script.
+
+    Tested using mocks for stdin.
+    """
+
+    def setUp(self):
+        self.isatty_ret = False
+        self.isatty = mock.patch.object(
+            os, 'isatty', lambda _: self.isatty_ret).start()
+
+        self.tmpdir = tempfile.mkdtemp()
+        self.outfile = os.path.join(self.tmpdir, 'outfile')
+
+        self.argv = ['lazysponge', self.outfile]
+        mock.patch.object(sys, 'argv', self.argv).start()
+
+        self.stdin = mock.Mock()
+        self.stdin.fileno.return_value = 0
+        mock.patch.object(sys, 'stdin', self.stdin).start()
+        self.stdin.buffer = BytesIO()
+
+    def tearDown(self):
+        shutil.rmtree(self.tmpdir, ignore_errors=True)
+        mock.patch.stopall()
+
+    def test_write_one_file(self):
+        """Test a simple case where we write one file without quiet output."""
+        first_input = b'First input\n'
+
+        # Write input once
+        self.stdin.buffer.write(first_input)
+        self.stdin.buffer.seek(0)
+        stderr = StringIO()
+        with mock.patch('sys.stderr', stderr):
+            lazysponge.main()
+        self.assertEqual(stderr.getvalue(), '')
+        # no errors
+        os.stat(self.outfile)
+        with open(self.outfile, 'rb') as outfd:
+            self.assertEqual(outfd.read(), first_input)
+
+    def test_write_two_times_and_update(self):
+        """Test writing two times the same output plus a new one."""
+        first_input = b'First input\n'
+        updated_input = b'Second input\n'
+        stderr = StringIO()
+
+        self.argv.append('--verbose')
+
+        # File does not exist
+        with self.assertRaises(OSError):
+            os.stat(self.outfile)
+
+        # Write input once
+        self.stdin.buffer.write(first_input)
+        self.stdin.buffer.seek(0)
+        with mock.patch('sys.stderr', stderr):
+            lazysponge.main()
+        first_stat = os.stat(self.outfile)
+        with open(self.outfile, 'rb') as outfd:
+            self.assertEqual(outfd.read(), first_input)
+        self._truncate(self.stdin.buffer)
+
+        # compare stderr verbose output
+        errmsg = 'Replacing %s ( != 96022020c795ee69653958a3cb4bb083)\n'
+        self.assertEqual(stderr.getvalue(), errmsg % self.outfile)
+        self._truncate(stderr)
+
+        # Re-Write the same input
+        self.stdin.buffer.write(first_input)
+        self.stdin.buffer.seek(0)
+        with mock.patch('sys.stderr', stderr):
+            lazysponge.main()
+        second_stat = os.stat(self.outfile)
+        with open(self.outfile, 'rb') as outfd:
+            self.assertEqual(outfd.read(), first_input)
+        self._truncate(self.stdin.buffer)
+
+        # File has not been modified
+        self.assertEqual(first_stat, second_stat)
+        # compare stderr verbose output
+        errmsg = 'Keeping old %s (96022020c795ee69653958a3cb4bb083)\n'
+        self.assertEqual(stderr.getvalue(), errmsg % self.outfile)
+        self._truncate(stderr)
+
+        # Update with a new input
+        self.stdin.buffer.write(updated_input)
+        self.stdin.buffer.seek(0)
+        with mock.patch('sys.stderr', stderr):
+            lazysponge.main()
+        third_stat = os.stat(self.outfile)
+        with open(self.outfile, 'rb') as outfd:
+            self.assertEqual(outfd.read(), updated_input)
+        self._truncate(self.stdin.buffer)
+
+        # File is newer
+        self.assertGreater(third_stat, second_stat)
+        # compare stderr verbose output
+        errmsg = ('Replacing %s (96022020c795ee69653958a3cb4bb083'
+                  ' != 1015f2c7f2fc3d575b7aeb1e92c0f6bf)\n')
+        self.assertEqual(stderr.getvalue(), errmsg % self.outfile)
+        self._truncate(stderr)
+
+    @staticmethod
+    def _truncate(filefd):
+        filefd.seek(0)
+        filefd.truncate(0)
+
+    def test_no_tty_detection(self):
+        """Test detecting that 'stdin' is not a tty."""
+        self.isatty_ret = True
+        stderr = StringIO()
+
+        with mock.patch('sys.stderr', stderr):
+            with self.assertRaises(SystemExit):
+                lazysponge.main()
+
+        not_a_tty = ('Interactive input not supported. Use piped input\n'
+                     '  echo message | {}\n'.format(' '.join(self.argv)))
+        self.assertEqual(stderr.getvalue(), not_a_tty)
+
+
+if __name__ == '__main__':
+    unittest.main()

makefiles/vars.inc.mk

@@ -88,3 +88,6 @@ export DLCACHE               # directory used to cache http downloads
 export DOWNLOAD_TO_FILE      # Use `$(DOWNLOAD_TO_FILE) $(DESTINATION) $(URL)` to download `$(URL)` to `$(DESTINATION)`.
 export DOWNLOAD_TO_STDOUT    # Use `$(DOWNLOAD_TO_STDOUT) $(URL)` to download `$(URL)` output `$(URL)` to stdout, e.g. to be piped into `tar xz`.
 export UNZIP_HERE            # Use `cd $(SOME_FOLDER) && $(UNZIP_HERE) $(SOME_FILE)` to extract the contents of the zip file `$(SOME_FILE)` into `$(SOME_FOLDER)`.
+
+export LAZYSPONGE            # Command saving stdin to a file only on content update.
+export LAZYSPONGE_FLAGS      # Parameters supplied to LAZYSPONGE.