diff --git a/tests/bench_ztimer/Makefile b/tests/bench_ztimer/Makefile new file mode 100644 index 0000000000..00d60c8286 --- /dev/null +++ b/tests/bench_ztimer/Makefile @@ -0,0 +1,101 @@ +include ../Makefile.tests_common + +USEMODULE += ztimer_usec ztimer_msec + +# this test uses 1000 timers by default. for boards that boards don't have +# enough memory, reduce that to 100 or 20, unless NUMOF_TIMERS has been overridden. +LOW_MEMORY_BOARDS += \ + airfy-beacon \ + arduino-mega2560 \ + arduino-mkr1000 \ + arduino-mkrfox1200 \ + arduino-mkrwan1300 \ + arduino-mkrzero \ + arduino-nano-33-iot \ + atmega1284p \ + b-l072z-lrwan1 \ + bastwan \ + blackpill \ + blackpill-128kib \ + bluepill \ + bluepill-128kib \ + calliope-mini \ + cc1312-launchpad \ + cc1350-launchpad \ + cc1352-launchpad \ + cc2650-launchpad \ + cc2650stk \ + e104-bt5010a-tb \ + e104-bt5011a-tb \ + derfmega128 \ + feather-m0 \ + feather-m0-lora \ + feather-m0-wifi \ + hifive1 \ + hifive1b \ + i-nucleo-lrwan1 \ + lsn50 \ + maple-mini \ + mega-xplained \ + microbit \ + microduino-corerf \ + msb-430 \ + msb-430h \ + nrf51dongle \ + nrf6310 \ + nucleo-f030r8 \ + nucleo-f042k6 \ + nucleo-f070rb \ + nucleo-f072rb \ + nucleo-f103rb \ + nucleo-f303k8 \ + nucleo-f334r8 \ + nucleo-l031k6 \ + nucleo-l053r8 \ + nucleo-l073rz \ + opencm904 \ + saml10-xpro \ + saml11-xpro \ + seeeduino_xiao \ + sensebox_samd21 \ + serpente \ + sodaq-autonomo \ + sodaq-explorer \ + sodaq-one \ + sodaq-sara-aff \ + sodaq-sara-sff \ + spark-core \ + stm32f0discovery \ + stm32l0538-disco \ + telosb \ + waspmote-pro \ + wemos-zero \ + yarm \ + yunjia-nrf51822 \ + z1 \ + # + +SUPER_LOW_MEMORY_BOARDS += \ + arduino-duemilanove \ + arduino-leonardo \ + arduino-nano \ + arduino-uno \ + atmega328p \ + atmega328p-xplained-mini \ + nucleo-f031k6 \ + stm32f030f4-demo \ + # + +ifneq (, $(filter $(BOARD), $(LOW_MEMORY_BOARDS))) + NUMOF_TIMERS ?= 100 +endif + +ifneq (, $(filter $(BOARD), $(SUPER_LOW_MEMORY_BOARDS))) + NUMOF_TIMERS ?= 12 +endif + +NUMOF_TIMERS ?= 1000 + +CFLAGS += -DNUMOF_TIMERS=$(NUMOF_TIMERS) + +include $(RIOTBASE)/Makefile.include diff --git a/tests/bench_ztimer/Makefile.ci b/tests/bench_ztimer/Makefile.ci new file mode 100644 index 0000000000..4fca3c400b --- /dev/null +++ b/tests/bench_ztimer/Makefile.ci @@ -0,0 +1,17 @@ +BOARD_INSUFFICIENT_MEMORY := \ + arduino-duemilanove \ + arduino-nano \ + arduino-uno \ + atmega328p \ + atmega328p-xplained-mini \ + atxmega-a3bu-xplained \ + bluepill-stm32f030c8 \ + im880b \ + nucleo-l011k4 \ + olimexino-stm32 \ + samd10-xmini \ + slstk3400a \ + stk3200 \ + stm32g0316-disco \ + zigduino \ + # diff --git a/tests/bench_ztimer/README.md b/tests/bench_ztimer/README.md new file mode 100644 index 0000000000..10af334ecb --- /dev/null +++ b/tests/bench_ztimer/README.md @@ -0,0 +1,77 @@ +# Introduction + +This test executes some benchmarks for ztimer's set() / remove() / now() +operations. + +# Details + +This set of benchmarks measures ztimer's list operation efficiency. +Depending on the available memory, the individual benchmarks that are using +multiple timers are run with either 1000 (the default), 100 or 20 timers. +Each benchmark is repeated REPEAT times (default 1000). +As only the operations are benchmarked, it is asserted that no timer ever +actually triggers. + +### set() one + +This repeatedly sets one timer in an otherwise empty list. +All but the first iteration will cause ztimer to implicitly remove the timer +first. +All iterations will cause the underlying periph timer to be updated. + +### remove() one + +This repeatedly removes one timer from the list. In all but the first iteration, +this list will be empty. + + +### set() + remove() one + +This repeatedly first sets, then removes one timer. The list is empty +before and after each iteration. +All iterations will cause the underlying periph timer to be updated. + +### set() many increasing targets + +This adds NUMOF timers with increasing target times. Each iteration will add a +timer at the end of ztimer's timer list. +Only the first iteration will cause the underlying periph timer to be updated. +After this test, the list is populated with NUMOF timers. + +### re-set() first + +This repatedly re-sets the first timer in the list (to the same target time). +All iterations will cause the underlying periph timer to be updated. + +### re-set() middle + +This repatedly re-sets the timer in the middle of the list (to the same target +time). + +### re-set() last + +This repatedly re-sets the last timer in the list (to the same target time). + +### remove() + set() first, middle, last + +Same as the previous three, but does a remove() before set(). + +### remove() many decreasing + +This removes all timers from the list, starting with the last. + +### ztimer_now() + +This simply calls ztimer_now() in a loop. + + +# How to interpret results + +The aim is to measure the time spent in ztimer's list operations. +Lower values are better. +The first/middle/last tests give an idea of the best case / average case / +worst case when running the operation with NUMOF timers. +Note that every set() on an already set timer will trigger an implicit remove(), +thus the timer list has to be iterated twice. +The tests that do a remove() before set() show whether ztimer correctly +identifies an unset timer. diff --git a/tests/bench_ztimer/main.c b/tests/bench_ztimer/main.c new file mode 100644 index 0000000000..ba56c15443 --- /dev/null +++ b/tests/bench_ztimer/main.c @@ -0,0 +1,296 @@ +/* + * Copyright (C) 2019 Kaspar Schleiser + * + * This file is subject to the terms and conditions of the GNU Lesser + * General Public License v2.1. See the file LICENSE in the top level + * directory for more details. + */ + +/** + * @ingroup tests + * @{ + * + * @file + * @brief ztimer set / remove / now benchmark application + * + * @author Kaspar Schleiser + * + * @} + */ + +#include + +#include "test_utils/expect.h" + +#include "msg.h" +#include "thread.h" +#include "ztimer.h" + +#ifndef ZTIMER +#define ZTIMER ZTIMER_MSEC +#endif + +#ifndef NUMOF_TIMERS +#define NUMOF_TIMERS (1000U) +#endif + +#ifndef REPEAT +#define REPEAT (1000U) +#endif + +#ifndef BASE +#define BASE (10000000LU) +#endif + +#ifndef SPREAD +#define SPREAD (10LU) +#endif + +static ztimer_t _timers[NUMOF_TIMERS]; + +/* This variable is set by any timer that actually triggers. As the test is + * only testing set/remove/now operations, timers are not supposed to trigger. + * Thus, after every test there's an 'expect(!_triggers)' + */ +static unsigned _triggers; + +/* + * The test assumes that first, middle and last will always end up in at the + * same index within the timer queue. In order to compensate for the time that + * previous operations take themselves, the interval is corrected. The + * variables "start" and "_base" are used for that. + */ +uint32_t _base; + +static void _callback(void *arg) { + unsigned *triggers = arg; + *triggers += 1; +} + +/* returns the interval for timer 'n' that has to be set in order to insert it + * into position n */ +static uint32_t _timer_val(unsigned n) +{ + return _base + (SPREAD * n); +} + +/* set timer 'n' to its intended position 'n' */ +static void _timer_set(unsigned n) +{ + ztimer_set(ZTIMER, &_timers[n], _timer_val(n)); +} + +/* remove timer 'n' */ +static void _timer_remove(unsigned n) +{ + ztimer_remove(ZTIMER, &_timers[n]); +} + +static void _print_result(const char *desc, unsigned n, uint32_t total) +{ + printf("%30s %8"PRIu32" / %u = %"PRIu32"\n", desc, total, n, total/n); +} + +int main(void) +{ + puts("ztimer benchmark application.\n"); + + unsigned n; + uint32_t before, diff, start; + + /* initializing timer structs */ + for (unsigned int n = 0; n < NUMOF_TIMERS; n++) { + _timers[n].callback = _callback; + _timers[n].arg = &_triggers; + } + + start = ztimer_now(ZTIMER_USEC); + + /* + * test setting one set timer REPEAT times + * + */ + _base = BASE; + before = ztimer_now(ZTIMER_USEC); + for (n = 0; n < REPEAT; n++) { + _timer_set(0); + } + + diff = ztimer_now(ZTIMER_USEC) - before; + + _print_result("set() one", REPEAT, diff); + expect(!_triggers); + + /* + * test removing one unset timer REPEAT times + * + */ + before = ztimer_now(ZTIMER_USEC); + for (n = 0; n < REPEAT; n++) { + _timer_remove(0); + } + + diff = ztimer_now(ZTIMER_USEC) - before; + + _print_result("remove() one", REPEAT, diff); + expect(!_triggers); + + /* + * test setting / removing one timer REPEAT times + * + */ + before = ztimer_now(ZTIMER_USEC); + _base = BASE - (before - start); + for (n = 0; n < REPEAT; n++) { + _timer_set(0); + _timer_remove(0); + } + + diff = ztimer_now(ZTIMER_USEC) - before; + + _print_result("set() + remove() one", REPEAT, diff); + expect(!_triggers); + + /* + * test setting NUMOF_TIMERS timers with increasing targets + * + */ + before = ztimer_now(ZTIMER_USEC); + _base = BASE - (before - start); + for (unsigned int n = 0; n < NUMOF_TIMERS; n++) { + _timer_set(n); + } + + diff = ztimer_now(ZTIMER_USEC) - before; + + _print_result("set() many increasing target", NUMOF_TIMERS, diff); + expect(!_triggers); + + /* + * test re-setting first timer REPEAT times + * + */ + before = ztimer_now(ZTIMER_USEC); + _base = BASE - (before - start); + for (n = 0; n < REPEAT; n++) { + _timer_set(0); + } + + diff = ztimer_now(ZTIMER_USEC) - before; + + _print_result("re-set() first", REPEAT, diff); + expect(!_triggers); + + /* + * test setting middle timer REPEAT times + * + */ + before = ztimer_now(ZTIMER_USEC); + _base = BASE - (before - start); + for (n = 0; n < REPEAT; n++) { + _timer_set(NUMOF_TIMERS/2); + } + + diff = ztimer_now(ZTIMER_USEC) - before; + + _print_result("re-set() middle", REPEAT, diff); + expect(!_triggers); + + /* + * test setting last timer REPEAT times + * + */ + before = ztimer_now(ZTIMER_USEC); + _base = BASE - (before - start); + for (n = 0; n < REPEAT; n++) { + _timer_set(NUMOF_TIMERS - 1); + } + + diff = ztimer_now(ZTIMER_USEC) - before; + + _print_result("re-set() last", REPEAT, diff); + expect(!_triggers); + + /* + * test removing / setting first timer REPEAT times + * + */ + before = ztimer_now(ZTIMER_USEC); + _base = BASE - (before - start); + for (n = 0; n < REPEAT; n++) { + _timer_remove(0); + _timer_set(0); + } + + diff = ztimer_now(ZTIMER_USEC) - before; + + _print_result("remove() + set() first", REPEAT, diff); + expect(!_triggers); + + /* + * test removing / setting middle timer REPEAT times + * + */ + before = ztimer_now(ZTIMER_USEC); + _base = BASE - (before - start); + for (n = 0; n < REPEAT; n++) { + _timer_remove(NUMOF_TIMERS/2); + _timer_set(NUMOF_TIMERS/2); + } + + diff = ztimer_now(ZTIMER_USEC) - before; + + _print_result("remove() + set() middle", REPEAT, diff); + expect(!_triggers); + + /* + * test removing / setting last timer REPEAT times + * + */ + before = ztimer_now(ZTIMER_USEC); + _base = BASE - (before - start); + for (n = 0; n < REPEAT; n++) { + _timer_remove(NUMOF_TIMERS - 1); + _timer_set(NUMOF_TIMERS - 1); + } + + diff = ztimer_now(ZTIMER_USEC) - before; + + _print_result("remove() + set() last", REPEAT, diff); + expect(!_triggers); + + /* + * test removing NUMOF_TIMERS timers (latest first) + * + */ + before = ztimer_now(ZTIMER_USEC); + for (n = 0; n < NUMOF_TIMERS; n++) { + _timer_remove(NUMOF_TIMERS - n - 1); + } + + diff = ztimer_now(ZTIMER_USEC) - before; + + _print_result("remove() many decreasing", NUMOF_TIMERS, diff); + expect(!_triggers); + + /* + * test ztimer_now() + * + */ + before = ztimer_now(ZTIMER_USEC); + n = REPEAT; + while (n--) { + ztimer_now(ZTIMER); + } + + diff = ztimer_now(ZTIMER_USEC) - before; + + _print_result("ztimer_now()", REPEAT, diff); + expect(!_triggers); + + _print_result("sizeof(ztimer_t)", NUMOF_TIMERS, sizeof(_timers)); + + puts("done."); + + return 0; +} diff --git a/tests/bench_ztimer/tests/01-run.py b/tests/bench_ztimer/tests/01-run.py new file mode 100755 index 0000000000..e212476316 --- /dev/null +++ b/tests/bench_ztimer/tests/01-run.py @@ -0,0 +1,22 @@ +#!/usr/bin/env python3 + +# Copyright (C) 2019 Freie Universität Berlin +# +# This file is subject to the terms and conditions of the GNU Lesser +# General Public License v2.1. See the file LICENSE in the top level +# directory for more details. + +import sys +from testrunner import run + + +def testfunc(child): + child.expect_exact("ztimer benchmark application.\r\n") + for i in range(13): + child.expect(r"\s+[\w() _\+]+\s+\d+ / \d+ = \d+\r\n") + + child.expect_exact("done.\r\n") + + +if __name__ == "__main__": + sys.exit(run(testfunc))