diff --git a/dist/tools/compile_and_test_for_board/compile_and_test_for_board.py b/dist/tools/compile_and_test_for_board/compile_and_test_for_board.py index b76199ef87..fadcca0ac1 100755 --- a/dist/tools/compile_and_test_for_board/compile_and_test_for_board.py +++ b/dist/tools/compile_and_test_for_board/compile_and_test_for_board.py @@ -41,7 +41,7 @@ usage: compile_and_test_for_board.py [-h] [--applications APPLICATIONS] [--flash-targets FLASH_TARGETS] [--test-targets TEST_TARGETS] [--test-available-targets TEST_AVAILABLE_TARGETS] - [--jobs JOBS] + [--report-xml] [--jobs JOBS] riot_directory board [result_directory] positional arguments: @@ -76,6 +76,8 @@ optional arguments: --test-available-targets TEST_AVAILABLE_TARGETS List of make targets to know if a test is present (default: test/available) + --report-xml Output results to report.xml in the result_directory + (default: False) --jobs JOBS, -j JOBS Parallel building (0 means not limit, like '--jobs') (default: None) ``` @@ -90,6 +92,14 @@ import argparse import subprocess import collections +try: + import junit_xml + import io + import time +except ImportError: + junit_xml = None + + LOG_HANDLER = logging.StreamHandler() LOG_HANDLER.setFormatter(logging.Formatter(logging.BASIC_FORMAT)) @@ -219,6 +229,7 @@ class RIOTApplication(): :param riotdir: RIOT repository directory :param appdir: directory of the application, can be relative to riotdir :param resultdir: base directory where to put execution results + :param junit: track application in JUnit XML """ MAKEFLAGS = ('RIOT_CI_BUILD=1', 'CC_NOCOLOR=1', '--no-print-directory') @@ -228,11 +239,21 @@ class RIOTApplication(): TEST_TARGETS = ('test',) TEST_AVAILABLE_TARGETS = ('test/available',) - def __init__(self, board, riotdir, appdir, resultdir): + # pylint: disable=too-many-arguments + def __init__(self, board, riotdir, appdir, resultdir, junit=False): self.board = board self.riotdir = riotdir self.appdir = appdir self.resultdir = os.path.join(resultdir, appdir) + if junit: + if not junit_xml: + raise ImportError("`junit-xml` required for --report-xml") + self.testcase = junit_xml.TestCase(name=self.appdir, + stdout='', stderr='') + self.log_stream = io.StringIO() + logging.basicConfig(stream=self.log_stream) + else: + self.testcase = None self.logger = logging.getLogger('%s.%s' % (board, appdir)) # Currently not handling absolute directories or outside of RIOT @@ -286,6 +307,8 @@ class RIOTApplication(): cmd = ['clean', 'clean-pkg'] self.make(cmd) except subprocess.CalledProcessError as err: + if self.testcase: + self.testcase.stderr += err.output + '\n' self.logger.warning('Got an error during clean, ignore: %r', err) def clean_intermediates(self): @@ -294,6 +317,8 @@ class RIOTApplication(): cmd = ['clean-intermediates'] self.make(cmd) except subprocess.CalledProcessError as err: + if self.testcase: + self.testcase.stderr += err.output + '\n' self.logger.warning('Got an error during clean-intermediates,' ' ignore: %r', err) @@ -303,11 +328,29 @@ class RIOTApplication(): :returns: 0 on success and 1 on error. """ try: + if self.testcase: + self.testcase.timestamp = time.time() self.compilation_and_test(**test_kwargs) - return None + res = None except ErrorInTest as err: self.logger.error('Failed during: %s', err) - return (str(err), err.application.appdir, err.errorfile) + res = (str(err), err.application.appdir, err.errorfile) + if self.testcase: + self.testcase.elapsed_sec = time.time() - self.testcase.timestamp + self.testcase.log = self.log_stream.getvalue() + if not self.testcase.stdout: + self.testcase.stdout = None + if not self.testcase.stderr: + self.testcase.stderr = None + return res + + def _skip(self, skip_reason, skip_reason_details=None, output=None): + if self.testcase: + self.testcase.add_skipped_info( + skip_reason_details if skip_reason_details else skip_reason, + output, + ) + self._write_resultfile('skip', skip_reason) def compilation_and_test(self, clean_after=False, runtest=True, incremental=False, jobs=False, @@ -332,19 +375,25 @@ class RIOTApplication(): # Ignore incompatible APPS if not self.board_is_supported(): create_directory(self.resultdir, clean=True) - self._write_resultfile('skip', 'not_supported') + self._skip('not_supported', 'Board not supported') return if not self.board_has_enough_memory(): create_directory(self.resultdir, clean=True) - self._write_resultfile('skip', 'not_enough_memory') + self._skip( + 'not_enough_memory', + 'Board has not enough memory to carry application', + ) return has_test = self.has_test() if with_test_only and not has_test: create_directory(self.resultdir, clean=True) - self._write_resultfile('skip', 'disabled_has_no_tests') + self._skip( + 'disabled_has_no_tests', + "{} has no tests".format(self.appdir) + ) return # Normal case for supported apps @@ -371,7 +420,10 @@ class RIOTApplication(): if clean_after: self.clean() else: - self._write_resultfile('test', 'skip.no_test') + self._skip( + 'skip.no_test', + "{} has no tests".format(self.appdir) + ) self.logger.info('Success') @@ -421,6 +473,8 @@ class RIOTApplication(): # Do not re-run if success output = self._make_get_previous_output(name) if output is not None: + if self.testcase: + self.testcase.stdout += output + '\n' return output # Run setup-tasks, output is only kept in case of error @@ -437,6 +491,8 @@ class RIOTApplication(): output = self.make(args) if not save_output: output = '' + if self.testcase: + self.testcase.stdout += output + '\n' self._write_resultfile(name, 'success', output) return output except subprocess.CalledProcessError as err: @@ -465,6 +521,14 @@ class RIOTApplication(): self.logger.warning(output) self.logger.error('Error during %s, writing to %s', name, outfile) + if self.testcase: + self.testcase.stderr += err.output + '\n' + if name == "test": + self.testcase.add_failure_info("{} failed".format(err.cmd), + err.output) + else: + self.testcase.add_error_info("{} had an error".format(err.cmd), + err.output) raise ErrorInTest(name, self, outfile) def _write_resultfile(self, name, status, body=''): @@ -623,6 +687,9 @@ PARSER.add_argument('--test-targets', type=list_from_string, PARSER.add_argument('--test-available-targets', type=list_from_string, default=' '.join(RIOTApplication.TEST_AVAILABLE_TARGETS), help='List of make targets to know if a test is present') +PARSER.add_argument('--report-xml', action='store_true', default=False, + help='Output results to report.xml in the ' + 'result_directory') PARSER.add_argument( '--jobs', '-j', type=int, default=None, @@ -665,7 +732,8 @@ def main(args): # List of applications for board applications = [RIOTApplication(board, args.riot_directory, app_dir, - board_result_directory) + board_result_directory, + junit=args.report_xml) for app_dir in app_dirs] # Execute tests @@ -681,6 +749,16 @@ def main(args): summary = _test_failed_summary(errors, relpathstart=board_result_directory) save_failure_summary(board_result_directory, summary) + if args.report_xml: + if not junit_xml: + raise ImportError("`junit-xml` required for --report-xml") + report_file = os.path.join(board_result_directory, "report.xml") + with open(report_file, "w+") as report: + junit_xml.TestSuite.to_file( + report, + [junit_xml.TestSuite('compile_and_test_for_{}'.format(board), + [app.testcase for app in applications])] + ) if num_errors: logger.error('Tests failed: %d', num_errors) print(summary, end='')