riot-ci/RIOT
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

dist: compile_test.py refactored

Browse Source
This commit is contained in:
Philipp Rosenkranz 2015-07-16 19:54:11 +02:00
parent 224b03b836
commit e3fc0a3fb5
1 changed files with 136 additions and 115 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

149
dist/tools/compile_test/compile_test.py vendored
View File

@ -2,6 +2,7 @@
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
# Copyright (C) 2014 René Kijewski <rene.kijewski@fu-berlin.de> # Copyright (C) 2014 René Kijewski <rene.kijewski@fu-berlin.de>
# Copyright (C) 2015 Philipp Rosenkranz <philipp.rosenkranz@fu-berlin.de>
# #
# This library is free software; you can redistribute it and/or # This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public # modify it under the terms of the GNU Lesser General Public
@ -28,24 +29,13 @@ from sys import exit, stdout, argv, exc_info
from StringIO import StringIO from StringIO import StringIO
from itertools import tee from itertools import tee
riotbase = environ.get('RIOTBASE') or abspath(join(dirname(abspath(__file__)), '../' * 3)) class Termcolor:
red = '\033[1;31m'
if len(argv) > 1: green = '\033[1;32m'
base_branch = argv[1] yellow = '\033[1;33m'
diff_files = check_output(('git', 'diff', '--name-only', base_branch, 'HEAD')) blue = '\033[1;34m'
diff_files = set(diff_files.split()) purple = '\033[1;35m'
else: end = '\033[0m'
base_branch = ''
null = open(devnull, 'w', 0)
success = []
failed = []
skipped = []
exceptions = []
warnings = []
errors = []
def is_tracked(application_folder): def is_tracked(application_folder):
if not isfile(join(application_folder, 'Makefile')): if not isfile(join(application_folder, 'Makefile')):
@ -71,6 +61,7 @@ def get_results_and_output_from(fd):
yield (' .. '.join(result[:-1]), result[-1], output) yield (' .. '.join(result[:-1]), result[-1], output)
break break
elif line.startswith(results_prefix): elif line.startswith(results_prefix):
read_more_output = False
if prev_results: if prev_results:
yield (' .. '.join(result[:-1]), result[-1], output) yield (' .. '.join(result[:-1]), result[-1], output)
prev_results = True prev_results = True
@ -81,22 +72,23 @@ def get_results_and_output_from(fd):
elif line.startswith(output_prefix): elif line.startswith(output_prefix):
output = StringIO() output = StringIO()
output.write(line) output.write(line)
else: read_more_output = True
elif read_more_output:
output.write(line) output.write(line)
def _get_common_user(common): def _get_common_user(common):
return [f for f in check_output(r'grep -l "{}" cpu/*/Makefile* boards/*/Makefile*'.format(common), return [f for f in check_output(r'grep -l "{}" cpu/*/Makefile* boards/*/Makefile*'.format(common),
shell=True).split() if "common" not in f] shell=True).split() if 'common' not in f]
def _get_boards_from_files(files): def _get_boards_from_files(files):
boards = set() boards = set()
if any("boards/" in s for s in files): if any('boards/' in s for s in files):
for f in files: for f in files:
if "boards/" not in f: if 'boards/' not in f:
continue continue
board = re.sub(r"^boards/([^/]+)/.*$", r"\1", f) board = re.sub(r'^boards/([^/]+)/.*$', r'\1', f)
if "common" in board: if 'common' in board:
boards |= _get_boards_from_files(_get_common_user(board)) boards |= _get_boards_from_files(_get_common_user(board))
else: else:
boards |= { board } boards |= { board }
@ -106,14 +98,14 @@ def _get_boards_from_files(files):
def _get_cpus_from_files(files): def _get_cpus_from_files(files):
cpus = set() cpus = set()
if any("cpu/" in s for s in files): if any('cpu/' in s for s in files):
for f in files: for f in files:
if "cpu/" not in f: if 'cpu/' not in f:
continue continue
cpu = re.sub(r"^cpu/([^/]+)/.*", r"\1", f) cpu = re.sub(r'^cpu/([^/]+)/.*', r'\1', f)
if "common" in cpu: if 'common' in cpu:
cpus |= _get_cpus_from_files(_get_common_user(cpu)) cpus |= _get_cpus_from_files(_get_common_user(cpu))
else: else:
cpus |= { cpu } cpus |= { cpu }
@ -125,8 +117,8 @@ def is_updated(application_folder, subprocess_env):
if base_branch == '': if base_branch == '':
return True return True
if ".travis.yml" in diff_files or \ if '.travis.yml' in diff_files or \
any("dist/" in s for s in diff_files): any('dist/' in s for s in diff_files):
return True return True
boards_changes = set() boards_changes = set()
@ -142,7 +134,7 @@ def is_updated(application_folder, subprocess_env):
app_files = set() app_files = set()
for board in boards_changes: for board in boards_changes:
env = { "BOARD": board } env = { 'BOARD': board }
env.update(subprocess_env) env.update(subprocess_env)
tmp = check_output(('make', 'info-files'), stderr=null, tmp = check_output(('make', 'info-files'), stderr=null,
cwd=application_folder, env=env) cwd=application_folder, env=env)
@ -159,8 +151,10 @@ def is_updated(application_folder, subprocess_env):
except CalledProcessError as e: except CalledProcessError as e:
return True return True
for folder in ('examples', 'tests'): def build_all():
print('Building all applications in: \033[1;34m{}\033[0m'.format(folder)) riotbase = environ.get('RIOTBASE') or abspath(join(dirname(abspath(__file__)), '../' * 3))
for folder in ('examples', 'tests'):
print('Building all applications in: {}'.format(colorize_str(folder, Termcolor.blue)))
applications = listdir(join(riotbase, folder)) applications = listdir(join(riotbase, folder))
applications = filter(lambda app: is_tracked(join(riotbase, folder, app)), applications) applications = filter(lambda app: is_tracked(join(riotbase, folder, app)), applications)
@ -171,11 +165,11 @@ for folder in ('examples', 'tests'):
subprocess_env['BUILDTEST_VERBOSE'] = '1' subprocess_env['BUILDTEST_VERBOSE'] = '1'
for nth, application in enumerate(applications, 1): for nth, application in enumerate(applications, 1):
stdout.write('\tBuilding application: \033[1;34m{}\033[0m ({}/{}) '.format(application, nth, len(applications))) stdout.write('\tBuilding application: {} ({}/{}) '.format(colorize_str(application, Termcolor.blue), nth, len(applications)))
stdout.flush() stdout.flush()
try: try:
if not is_updated(join(riotbase, folder, application), subprocess_env): if not is_updated(join(riotbase, folder, application), subprocess_env):
print("\033[1;33m(skipped)\033[0m") print(colorize_str('(skipped)', Termcolor.yellow))
skipped.append(application) skipped.append(application)
continue continue
subprocess = Popen(('make', 'buildtest'), subprocess = Popen(('make', 'buildtest'),
@ -207,46 +201,73 @@ for folder in ('examples', 'tests'):
subprocess.kill() subprocess.kill()
except: except:
pass pass
if warnings:
print('Warnings:') def colorize_str(string, color):
for application, details in warnings: return '%s%s%s' % (color, string, Termcolor.end)
def print_output_for(buf, name, color):
if buf:
print('%s:' % name)
for application, details in buf:
for outcome, board, output in details: for outcome, board, output in details:
print() print()
print("\033[1;33m%s:%s:\033[0m" % (application, board)) print(colorize_str('%s:%s:' % (application, board), color))
print("%s" % output.getvalue()) print('%s' % output.getvalue())
print()
if errors: def print_outcome(outputListDescription):
print('Errors:')
for application, details in errors:
for outcome, board, output in details:
print() print()
print("\033[1;31m%s:%s:\033[0m" % (application, board)) print('Outcome:')
print("%s" % output.getvalue()) for color, group, name in outputListDescription:
print() applications = group
print('Outcome:')
for color, group in (('3', 'skipped'), ('2', 'success'), ('1', 'failed'), ('4', 'exceptions')):
applications = locals()[group]
if applications: if applications:
print('\t\033[1;3{}m{}\033[0m: {}'.format(color, group, ', '.join(applications))) print('\t{}{}{}: {}'.format(color, name, Termcolor.end, ', '.join(applications)))
stdout.write('Errors: ') def print_num_of_errors_and_warnings():
if errors: stdout.write('Errors: ')
if errors:
num_of_errors = sum(map(lambda x: len(x[1]), errors)) num_of_errors = sum(map(lambda x: len(x[1]), errors))
stdout.write('\033[1;31m%d\033[0m' % num_of_errors) stdout.write('%s' % colorize_str(str(num_of_errors), Termcolor.red))
else: else:
stdout.write('0') stdout.write('0')
stdout.write(' Warnings: ') stdout.write(' Warnings: ')
if warnings: if warnings:
num_of_warnings = sum(map(lambda x: len(x[1]), warnings)) num_of_warnings = sum(map(lambda x: len(x[1]), warnings))
stdout.write('\033[1;33m%d\033[0m' % num_of_warnings) stdout.write('%s' % colorize_str(str(num_of_warnings), Termcolor.yellow))
else: else:
stdout.write('0') stdout.write('0')
stdout.write('\n') stdout.write('\n')
if exceptions: if __name__ == '__main__':
success = []
failed = []
skipped = []
exceptions = []
warnings = []
errors = []
null = open(devnull, 'w', 0)
if len(argv) > 1:
base_branch = argv[1]
diff_files = check_output(('git', 'diff', '--name-only', base_branch, 'HEAD'))
diff_files = set(diff_files.split())
else:
base_branch = ''
build_all()
print_output_for(warnings, 'Warnings', Termcolor.yellow)
print_output_for(errors, 'Errors', Termcolor.red)
outputListDescription = [(Termcolor.yellow, skipped, 'skipped'), (Termcolor.green, success, 'success'),
(Termcolor.red, failed, 'failed'), (Termcolor.blue, exceptions, 'exceptions')]
print_outcome(outputListDescription)
print_num_of_errors_and_warnings()
if exceptions:
exit(2) exit(2)
elif failed: elif failed:
exit(1) exit(1)
else: else:
exit(0) exit(0)