diff --git a/makefiles/pseudomodules.inc.mk b/makefiles/pseudomodules.inc.mk index 4f475711b4..90f7856db2 100644 --- a/makefiles/pseudomodules.inc.mk +++ b/makefiles/pseudomodules.inc.mk @@ -4,6 +4,7 @@ PSEUDOMODULES += at_urc_isr_low PSEUDOMODULES += at_urc_isr_medium PSEUDOMODULES += at_urc_isr_highest PSEUDOMODULES += at24c% +PSEUDOMODULES += atomic_utils PSEUDOMODULES += base64url PSEUDOMODULES += can_mbox PSEUDOMODULES += can_pm diff --git a/sys/include/atomic_utils.h b/sys/include/atomic_utils.h new file mode 100644 index 0000000000..318cc4e538 --- /dev/null +++ b/sys/include/atomic_utils.h @@ -0,0 +1,1137 @@ +/* + * Copyright (C) 2020 Otto-von-Guericke-Universität Magdeburg + * + * This file is subject to the terms and conditions of the GNU Lesser General + * Public License v2.1. See the file LICENSE in the top level directory for more + * details. + */ + +/** + * @defgroup sys_atomic_utils Utility functions for atomic access + * @ingroup sys + * + * This modules adds some utility functions to perform atomic accesses. + * + * # Usage + * + * The atomic utilitys allow atomic access to regular variables. + * + * ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~{.c} + * uint32_t global_counter; + * + * void irq_handler(void) + * { + * // No need to use atomic access in IRQ handlers, if other IRQ handlers + * // never touch global_counter: At the beginning and at the end of every + * // ISR a memory barrier is in place, so that at the end of the ISR the + * // memory will be in a state as if all memory accesses within the ISR + * // took place in sequential order. + * // + * // Extra detail only RIOT kernel hackers need to know: If all ISRs + * // accessing the same variable cannot interrupt each other, atomic + * // access is still not needed. (Currently only PendSV on ARM can be + * // interrupted by other IRQs with RIOTs default IRQ priorities. If + * // application developers modifies those, they can be assumed to know + * // what they are doing - or to happily face the consequences otherwise.) + * global_counter++; + * } + * + * void called_by_thread_a(void) { + * if (atomic_load_u32(&global_counter) > THRESHOLD) { + * on_threshold_reached(); + * atomic_store_u32(&global_counter, 0); + * } + * } + * + * void called_by_thread_b(void) { + * atomic_add_u32(&global_counter, 42); + * } + * + * ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + * + * # Motivation + * There are some reasons why these functions might be chosen over the + * [C11 Atomic Operations Library](https://en.cppreference.com/w/c/atomic) in + * some advanced use cases: + * + * - The functions allow mixing of atomic and non-atomic accesses. E.g. while + * IRQs are disabled anyway, even plain accesses cannot be interrupted but + * are often more efficient. + * - On platforms not supporting lock-free access, a library call is generated + * instead. The fallback implementation used here is more efficient in terms + * of both CPU instructions and ROM size. + * - On platforms where some operations can be implemented lock free while + * others can't, at least LLVM will use the library call even for those + * accesses that can be implemented lock-free. This is because without + * assuming how the library call implements atomic access for the other + * functions, mixing library calls and lock free accesses could result in data + * corruption. But this implementation resorts to disabling IRQs when + * lock-free implementations are not possible, which mixes well with lock-free + * accesses. Thus, additional overhead for atomic accesses is only spent where + * needed. + * - In some cases the fallback implementation performs better than the lock + * free implementation. E.g. if a specific platform has an atomic compare and + * swap instruction, this could be used to perform a read-modify-write in a + * loop until the value initially read was not changed in between. Just + * disabling IRQs to implement an atomic read-modify-write operation is likely + * more efficient. C11 atomics will however always use the lock free + * implementation (if such exists), assuming that this is more efficient. + * This assumption was made with desktop class hardware in mind, but is not + * generally true for bare metal targets. These function allow to optimize + * for the actual hardware RIOT is running on. + * - This library provides "semi-atomic" read-modify-write operations, which are + * useful when at most one thread is ever writing to memory. In that case, + * only the write part of the read-modify-write operation needs to be + * performed in an atomic fashion in order for the reading threads to perceive + * atomic updates of the variable. This is significantly cheaper than atomic + * read-modify-write operations for many platforms + * + * # Guarantees + * + * - Every utility function here acts as a barrier for code reordering regarding + * - For the `atomic_*()` family of functions: The whole operation will be done + * in an non-interruptible fashion + * - For the `semi_atomic_*()` family of functions: The write part of the + * operation is done atomically. If at most one thread is ever performing + * changes to a variable using the `semi_atomic_()` functions, those changes + * will appear as if they were atomic to all other threads. + * + * # Porting to new CPUs + * + * At the bare minimum, create an empty `atomic_utils_arch.h` file. This will + * result in the fallback implementations being used. + * + * To expose lock-free atomic operations, add an implementation to the + * `atomic_utils_arch.h` file and disable the fallback implementation by + * defining `HAS_`, where `` is the name + * of the function provided in all upper case. E.g. most platforms will be able + * to provide lock-free reads and writes up to their word size and can expose + * this as follows for GCC: + * + * ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~{.c} + * // All the user header boilerplate + * #define HAS_ATOMIC_LOAD_U8 + * static inline uint8_t atomic_load_u8(const uint8_t *var) + * { + * return __atomic_load_1(var, __ATOMIC_SEQ_CST); + * } + * + * #define HAS_ATOMIC_STORE_U8 + * static inline void atomic_store_u8(uint8_t *dest, uint8_t val) + * { + * __atomic_store_1(dest, val, __ATOMIC_SEQ_CST); + * } + * ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + * + * Note: The `semi_atomic_*()` family of functions is always provided using + * `atomic_*()` functions in the cheapest way possible. + * + * @{ + * + * @file + * @brief API of the utility functions for atomic accesses + * @author Marian Buschsieweke + */ + +#ifndef ATOMIC_UTILS_H +#define ATOMIC_UTILS_H + +#include + +#include "irq.h" +#include "atomic_utils_arch.h" + +#ifdef __cplusplus +extern "C" { +#endif + +/* Declarations and documentation: */ + +#if !defined(HAS_ATOMIC_BIT) || defined(DOXYGEN) +/** + * @name Types used to specify atomic bit access operations + * + * @warning These types are implementation dependent to allow exploiting + * hardware specific features like bit-banding. Use the provided helper + * functions to get or set the bit or destination they refer to. + * + * The motivation of a dedicated type is to allow ahead of time computation of + * e.g. bit-banding addresses, so that actually access can be done in a single + * CPU cycles even if the bit reference was initialized in a separate + * compilation unit and no link time optimization is used. + * + * @{ + */ +/** + * @brief Type specifying a bit in an `uint8_t` + * + * @warning This is an implementation specific type! + */ +typedef struct { + uint8_t *dest; /**< Memory containing the bit to set/clear */ + uint8_t mask; /**< Bitmask used for setting the bit */ +} atomic_bit_u8_t; + +/** + * @brief Type specifying a bit in an `uint16_t` + * + * @warning This is an implementation specific type! + */ +typedef struct { + uint16_t *dest; /**< Memory containing the bit to set/clear */ + uint16_t mask; /**< Bitmask used for setting the bit */ +} atomic_bit_u16_t; + +/** + * @brief Type specifying a bit in an `uint32_t` + * + * @warning This is an implementation specific type! + */ +typedef struct { + uint32_t *dest; /**< Memory containing the bit to set/clear */ + uint32_t mask; /**< Bitmask used for setting the bit */ +} atomic_bit_u32_t; + +/** + * @brief Type specifying a bit in an `uint64_t` + * + * @warning This is an implementation specific type! + */ +typedef struct { + uint64_t *dest; /**< Memory containing the bit to set/clear */ + uint64_t mask; /**< Bitmask used for setting the bit */ +} atomic_bit_u64_t; +/** @} */ +#endif /* HAS_ATOMIC_BIT */ + +/** + * @name Atomic Loads + * @{ + */ +/** + * @brief Load an `uint8_t` atomically + * + * @param[in] var Variable to load atomically + * @return The value stored in @p var + */ +static inline uint8_t atomic_load_u8(const uint8_t *var); +/** + * @brief Load an `uint16_t` atomically + * + * @param[in] var Variable to load atomically + * @return The value stored in @p var + */ +static inline uint16_t atomic_load_u16(const uint16_t *var); +/** + * @brief Load an `uint32_t` atomically + * + * @param[in] var Variable to load atomically + * @return The value stored in @p var + */ +static inline uint32_t atomic_load_u32(const uint32_t *var); +/** + * @brief Load an `uint64_t` atomically + * + * @param[in] var Variable to load atomically + * @return The value stored in @p var + */ +static inline uint64_t atomic_load_u64(const uint64_t *var); +/** @} */ + +/** + * @name Atomic Stores + * @{ + */ +/** + * @brief Store an `uint8_t` atomically + * @param[out] dest Location to atomically write the new value to + * @param[in] val Value to write + */ +static inline void atomic_store_u8(uint8_t *dest, uint8_t val); +/** + * @brief Store an `uint16_t` atomically + * @param[out] dest Location to atomically write the new value to + * @param[in] val Value to write + */ +static inline void atomic_store_u16(uint16_t *dest, uint16_t val); +/** + * @brief Store an `uint32_t` atomically + * @param[out] dest Location to atomically write the new value to + * @param[in] val Value to write + */ +static inline void atomic_store_u32(uint32_t *dest, uint32_t val); +/** + * @brief Store an `uint64_t` atomically + * @param[out] dest Location to atomically write the new value to + * @param[in] val Value to write + */ +static inline void atomic_store_u64(uint64_t *dest, uint64_t val); +/** @} */ + +/** + * @name Atomic In-Place Addition + * @{ + */ +/** + * @brief Atomically add a value onto a given value + * @param[in,out] dest Add @p summand onto this value atomically in-place + * @param[in] summand Value to add onto @p dest + */ +static inline void atomic_fetch_add_u8(uint8_t *dest, uint8_t summand); +/** + * @brief Atomically add a value onto a given value + * @param[in,out] dest Add @p summand onto this value atomically in-place + * @param[in] summand Value to add onto @p dest + */ +static inline void atomic_fetch_add_u16(uint16_t *dest, uint16_t summand); +/** + * @brief Atomically add a value onto a given value + * @param[in,out] dest Add @p summand onto this value atomically in-place + * @param[in] summand Value to add onto @p dest + */ +static inline void atomic_fetch_add_u32(uint32_t *dest, uint32_t summand); +/** + * @brief Atomically add a value onto a given value + * @param[in,out] dest Add @p summand onto this value atomically in-place + * @param[in] summand Value to add onto @p dest + */ +static inline void atomic_fetch_add_u64(uint64_t *dest, uint64_t summand); +/** @} */ + +/** + * @name Atomic In-Place Subtraction + * @{ + */ +/** + * @brief Atomically subtract a value from a given value + * @param[in,out] dest Subtract @p subtrahend from this value + * atomically in-place + * @param[in] subtrahend Value to subtract from @p dest + */ +static inline void atomic_fetch_sub_u8(uint8_t *dest, uint8_t subtrahend); +/** + * @brief Atomically subtract a value from a given value + * @param[in,out] dest Subtract @p subtrahend from this value + * atomically in-place + * @param[in] subtrahend Value to subtract from @p dest + */ +static inline void atomic_fetch_sub_u16(uint16_t *dest, uint16_t subtrahend); +/** + * @brief Atomically subtract a value from a given value + * @param[in,out] dest Subtract @p subtrahend from this value + * atomically in-place + * @param[in] subtrahend Value to subtract from @p dest + */ +static inline void atomic_fetch_sub_u32(uint32_t *dest, uint32_t subtrahend); +/** + * @brief Atomically subtract a value from a given value + * @param[in,out] dest Subtract @p subtrahend from this value + * atomically in-place + * @param[in] subtrahend Value to subtract from @p dest + */ +static inline void atomic_fetch_sub_u64(uint64_t *dest, uint64_t subtrahend); +/** @} */ + +/** + * @name Atomic In-Place Bitwise OR + * @{ + */ +/** + * @brief Atomic version of `*dest |= val` + * @param[in,out] dest Replace this value with the result of + * `*dest | val` + * @param[in] val Value to bitwise or into @p dest in-place + */ +static inline void atomic_fetch_or_u8(uint8_t *dest, uint8_t val); +/** + * @brief Atomic version of `*dest |= val` + * @param[in,out] dest Replace this value with the result of + * `*dest | val` + * @param[in] val Value to bitwise or into @p dest in-place + */ +static inline void atomic_fetch_or_u16(uint16_t *dest, uint16_t val); +/** + * @brief Atomic version of `*dest |= val` + * @param[in,out] dest Replace this value with the result of + * `*dest | val` + * @param[in] val Value to bitwise or into @p dest in-place + */ +static inline void atomic_fetch_or_u32(uint32_t *dest, uint32_t val); +/** + * @brief Atomic version of `*dest |= val` + * @param[in,out] dest Replace this value with the result of + * `*dest | val` + * @param[in] val Value to bitwise or into @p dest in-place + */ +static inline void atomic_fetch_or_u64(uint64_t *dest, uint64_t val); +/** @} */ + +/** + * @name Atomic In-Place Bitwise XOR + * @{ + */ +/** + * @brief Atomic version of `*dest ^= val` + * @param[in,out] dest Replace this value with the result of + * `*dest ^ val` + * @param[in] val Value to bitwise xor into @p dest in-place + */ +static inline void atomic_fetch_xor_u8(uint8_t *dest, uint8_t val); +/** + * @brief Atomic version of `*dest ^= val` + * @param[in,out] dest Replace this value with the result of + * `*dest ^ val` + * @param[in] val Value to bitwise xor into @p dest in-place + */ +static inline void atomic_fetch_xor_u16(uint16_t *dest, uint16_t val); +/** + * @brief Atomic version of `*dest ^= val` + * @param[in,out] dest Replace this value with the result of + * `*dest ^ val` + * @param[in] val Value to bitwise xor into @p dest in-place + */ +static inline void atomic_fetch_xor_u32(uint32_t *dest, uint32_t val); +/** + * @brief Atomic version of `*dest ^= val` + * @param[in,out] dest Replace this value with the result of + * `*dest ^ val` + * @param[in] val Value to bitwise xor into @p dest in-place + */ +static inline void atomic_fetch_xor_u64(uint64_t *dest, uint64_t val); +/** @} */ + +/** + * @name Atomic In-Place Bitwise AND + * @{ + */ +/** + * @brief Atomic version of `*dest &= val` + * @param[in,out] dest Replace this value with the result of + * `*dest & val` + * @param[in] val Value to bitwise and into @p dest in-place + */ +static inline void atomic_fetch_and_u8(uint8_t *dest, uint8_t val); +/** + * @brief Atomic version of `*dest &= val` + * @param[in,out] dest Replace this value with the result of + * `*dest & val` + * @param[in] val Value to bitwise and into @p dest in-place + */ +static inline void atomic_fetch_and_u16(uint16_t *dest, uint16_t val); +/** + * @brief Atomic version of `*dest &= val` + * @param[in,out] dest Replace this value with the result of + * `*dest & val` + * @param[in] val Value to bitwise and into @p dest in-place + */ +static inline void atomic_fetch_and_u32(uint32_t *dest, uint32_t val); +/** + * @brief Atomic version of `*dest &= val` + * @param[in,out] dest Replace this value with the result of + * `*dest & val` + * @param[in] val Value to bitwise and into @p dest in-place + */ +static inline void atomic_fetch_and_u64(uint64_t *dest, uint64_t val); +/** @} */ + +/** + * @name Helper Functions to Handle Atomic Bit References + * @{ + */ +/** + * @brief Create a reference to a bit in an `uint8_t` + * @param[in] dest Memory containing the bit + * @param[in] bit Bit number (`0` refers to the least significant) + */ +static inline atomic_bit_u8_t atomic_bit_u8(uint8_t *dest, uint8_t bit); + +/** + * @brief Create a reference to a bit in an `uint16_t` + * @param[in] dest Memory containing the bit + * @param[in] bit Bit number (`0` refers to the least significant) + */ +static inline atomic_bit_u16_t atomic_bit_u16(uint16_t *dest, uint8_t bit); + +/** + * @brief Create a reference to a bit in an `uint32_t` + * @param[in] dest Memory containing the bit + * @param[in] bit Bit number (`0` refers to the least significant) + */ +static inline atomic_bit_u32_t atomic_bit_u32(uint32_t *dest, uint8_t bit); + +/** + * @brief Create a reference to a bit in an `uint64_t` + * @param[in] dest Memory containing the bit + * @param[in] bit Bit number (`0` refers to the least significant) + */ +static inline atomic_bit_u64_t atomic_bit_u64(uint64_t *dest, uint8_t bit); +/** @} */ + +/** + * @name Atomic Bit Setting + * @{ + */ +/** + * @brief Atomic version of `*dest |= (1 << bit)` + * @param[in,out] bit bit to set + */ +static inline void atomic_set_bit_u8(atomic_bit_u8_t bit); +/** + * @brief Atomic version of `*dest |= (1 << bit)` + * @param[in,out] bit bit to set + */ +static inline void atomic_set_bit_u16(atomic_bit_u16_t bit); +/** + * @brief Atomic version of `*dest |= (1 << bit)` + * @param[in,out] bit bit to set + */ +static inline void atomic_set_bit_u32(atomic_bit_u32_t bit); +/** + * @brief Atomic version of `*dest |= (1 << bit)` + * @param[in,out] bit bit to set + */ +static inline void atomic_set_bit_u64(atomic_bit_u64_t bit); +/** @} */ + +/** + * @name Atomic Bit Clearing + * @{ + */ +/** + * @brief Atomic version of `*dest &= ~(1 << bit)` + * @param[in,out] bit bit to set + */ +static inline void atomic_clear_bit_u8(atomic_bit_u8_t bit); +/** + * @brief Atomic version of `*dest &= ~(1 << bit)` + * @param[in,out] bit bit to set + */ +static inline void atomic_clear_bit_u16(atomic_bit_u16_t bit); +/** + * @brief Atomic version of `*dest &= ~(1 << bit)` + * @param[in,out] bit bit to set + */ +static inline void atomic_clear_bit_u32(atomic_bit_u32_t bit); +/** + * @brief Atomic version of `*dest &= ~(1 << bit)` + * @param[in,out] bit bit to set + */ +static inline void atomic_clear_bit_u64(atomic_bit_u64_t bit); +/** @} */ + +/** + * @name Semi-Atomic In-Place Addition + * @{ + */ +/** + * @brief Semi-atomically add a value onto a given value + * @param[in,out] dest Add @p summand onto this value semi-atomically + * in-place + * @param[in] summand Value to add onto @p dest + */ +static inline void semi_atomic_fetch_add_u8(uint8_t *dest, uint8_t summand); +/** + * @brief Semi-atomically add a value onto a given value + * @param[in,out] dest Add @p summand onto this value semi-atomically + * in-place + * @param[in] summand Value to add onto @p dest + */ +static inline void semi_atomic_fetch_add_u16(uint16_t *dest, uint16_t summand); +/** + * @brief Semi-atomically add a value onto a given value + * @param[in,out] dest Add @p summand onto this value semi-atomically + * in-place + * @param[in] summand Value to add onto @p dest + */ +static inline void semi_atomic_fetch_add_u32(uint32_t *dest, uint32_t summand); +/** + * @brief Semi-atomically add a value onto a given value + * @param[in,out] dest Add @p summand onto this value semi-atomically + * in-place + * @param[in] summand Value to add onto @p dest + */ +static inline void semi_atomic_fetch_add_u64(uint64_t *dest, uint64_t summand); +/** @} */ + +/** + * @name Semi-Atomic In-Place Subtraction + * @{ + */ +/** + * @brief Semi-atomically subtract a value from a given value + * @param[in,out] dest Subtract @p subtrahend from this value + * semi-atomically in-place + * @param[in] subtrahend Value to subtract from @p dest + */ +static inline void semi_atomic_fetch_sub_u8(uint8_t *dest, uint8_t subtrahend); +/** + * @brief Semi-atomically subtract a value from a given value + * @param[in,out] dest Subtract @p subtrahend from this value + * semi-atomically in-place + * @param[in] subtrahend Value to subtract from @p dest + */ +static inline void semi_atomic_fetch_sub_u16(uint16_t *dest, + uint16_t subtrahend); +/** + * @brief Semi-atomically subtract a value from a given value + * @param[in,out] dest Subtract @p subtrahend from this value + * semi-atomically in-place + * @param[in] subtrahend Value to subtract from @p dest + */ +static inline void semi_atomic_fetch_sub_u32(uint32_t *dest, + uint32_t subtrahend); +/** + * @brief Semi-atomically subtract a value from a given value + * @param[in,out] dest Subtract @p subtrahend from this value + * semi-atomically in-place + * @param[in] subtrahend Value to subtract from @p dest + */ +static inline void semi_atomic_fetch_sub_u64(uint64_t *dest, + uint64_t subtrahend); +/** @} */ + +/** + * @name Semi-atomic In-Place Bitwise OR + * @{ + */ +/** + * @brief Semi-atomic version of `*dest |= val` + * @param[in,out] dest Replace this value with the result of + * `*dest | val` + * @param[in] val Value to bitwise or into @p dest in-place + */ +static inline void semi_atomic_fetch_or_u8(uint8_t *dest, uint8_t val); +/** + * @brief Semi-atomic version of `*dest |= val` + * @param[in,out] dest Replace this value with the result of + * `*dest | val` + * @param[in] val Value to bitwise or into @p dest in-place + */ +static inline void semi_atomic_fetch_or_u16(uint16_t *dest, uint16_t val); +/** + * @brief Semi-atomic version of `*dest |= val` + * @param[in,out] dest Replace this value with the result of + * `*dest | val` + * @param[in] val Value to bitwise or into @p dest in-place + */ +static inline void semi_atomic_fetch_or_u32(uint32_t *dest, uint32_t val); +/** + * @brief Semi-atomic version of `*dest |= val` + * @param[in,out] dest Replace this value with the result of + * `*dest | val` + * @param[in] val Value to bitwise or into @p dest in-place + */ +static inline void semi_atomic_fetch_or_u64(uint64_t *dest, uint64_t val); +/** @} */ + +/** + * @name Semi-Atomic In-Place Bitwise XOR + * @{ + */ +/** + * @brief Semi-atomic version of `*dest ^= val` + * @param[in,out] dest Replace this value with the result of + * `*dest ^ val` + * @param[in] val Value to bitwise xor into @p dest in-place + */ +static inline void semi_atomic_fetch_xor_u8(uint8_t *dest, uint8_t val); +/** + * @brief Semi-atomic version of `*dest ^= val` + * @param[in,out] dest Replace this value with the result of + * `*dest ^ val` + * @param[in] val Value to bitwise xor into @p dest in-place + */ +static inline void semi_atomic_fetch_xor_u16(uint16_t *dest, uint16_t val); +/** + * @brief Semi-atomic version of `*dest ^= val` + * @param[in,out] dest Replace this value with the result of + * `*dest ^ val` + * @param[in] val Value to bitwise xor into @p dest in-place + */ +static inline void semi_atomic_fetch_xor_u32(uint32_t *dest, uint32_t val); +/** + * @brief Semi-atomic version of `*dest ^= val` + * @param[in,out] dest Replace this value with the result of + * `*dest ^ val` + * @param[in] val Value to bitwise xor into @p dest in-place + */ +static inline void semi_atomic_fetch_xor_u64(uint64_t *dest, uint64_t val); +/** @} */ + +/** + * @name Semi-Atomic In-Place Bitwise AND + * @{ + */ +/** + * @brief Semi-atomic version of `*dest &= val` + * @param[in,out] dest Replace this value with the result of + * `*dest & val` + * @param[in] val Value to bitwise and into @p dest in-place + */ +static inline void semi_atomic_fetch_and_u8(uint8_t *dest, uint8_t val); +/** + * @brief Semi-atomic version of `*dest &= val` + * @param[in,out] dest Replace this value with the result of + * `*dest & val` + * @param[in] val Value to bitwise and into @p dest in-place + */ +static inline void semi_atomic_fetch_and_u16(uint16_t *dest, uint16_t val); +/** + * @brief Semi-atomic version of `*dest &= val` + * @param[in,out] dest Replace this value with the result of + * `*dest & val` + * @param[in] val Value to bitwise and into @p dest in-place + */ +static inline void semi_atomic_fetch_and_u32(uint32_t *dest, uint32_t val); +/** + * @brief Semi-atomic version of `*dest &= val` + * @param[in,out] dest Replace this value with the result of + * `*dest & val` + * @param[in] val Value to bitwise and into @p dest in-place + */ +static inline void semi_atomic_fetch_and_u64(uint64_t *dest, uint64_t val); +/** @} */ + +/* Fallback implementations of atomic utility functions: */ + +/** + * @brief Concatenate two tokens + */ +#define CONCAT(a, b) a ## b + +/** + * @brief Concatenate four tokens + */ +#define CONCAT4(a, b, c, d) a ## b ## c ## d + +/** + * @brief Generates a static inline function implementing + * `atomic_load_u()` + * + * @param name Name of the variable type, e.g. "u8" + * @param type Variable type, e.g. `uint8_t` + */ +#define ATOMIC_LOAD_IMPL(name, type) \ + static inline type CONCAT(atomic_load_, name)(const type *var) \ + { \ + unsigned state = irq_disable(); \ + /* var can be register allocated, hence the memory barrier of \ + * irq_disable() and irq_restore() may not apply here. Using volatile \ + * ensures that the compiler allocates it in memory and that the \ + * memory access is not optimized out. */ \ + type result = *((const volatile type *)var); \ + irq_restore(state); \ + return result; \ + } + +#ifndef HAS_ATOMIC_LOAD_U8 +ATOMIC_LOAD_IMPL(u8, uint8_t) +#endif +#ifndef HAS_ATOMIC_LOAD_U16 +ATOMIC_LOAD_IMPL(u16, uint16_t) +#endif +#ifndef HAS_ATOMIC_LOAD_U32 +ATOMIC_LOAD_IMPL(u32, uint32_t) +#endif +#ifndef HAS_ATOMIC_LOAD_U64 +ATOMIC_LOAD_IMPL(u64, uint64_t) +#endif + +/** + * @brief Generates a static inline function implementing + * `atomic_store_u()` + * + * @param name Name of the variable type, e.g. "u8" + * @param type Variable type, e.g. `uint8_t` + */ +#define ATOMIC_STORE_IMPL(name, type) \ + static inline void CONCAT(atomic_store_, name)(type *dest, type val) \ + { \ + unsigned state = irq_disable(); \ + /* dest can be register allocated, hence the memory barrier of \ + * irq_disable() and irq_restore() may not apply here. Using volatile \ + * ensures that the compiler allocates it in memory and that the \ + * memory access is not optimized out. */ \ + *((volatile type *)dest) = val; \ + irq_restore(state); \ + } + +#ifndef HAS_ATOMIC_STORE_U8 +ATOMIC_STORE_IMPL(u8, uint8_t) +#endif +#ifndef HAS_ATOMIC_STORE_U16 +ATOMIC_STORE_IMPL(u16, uint16_t) +#endif +#ifndef HAS_ATOMIC_STORE_U32 +ATOMIC_STORE_IMPL(u32, uint32_t) +#endif +#ifndef HAS_ATOMIC_STORE_U64 +ATOMIC_STORE_IMPL(u64, uint64_t) +#endif + +/** + * @brief Generates a static inline function implementing + * `atomic_fecth__u()` + * + * @param opname Name of the operator in @p op, e.g. "and" for `+` + * @param op Operator to implement atomically, e.g. `+` + * @param name Name of the variable type, e.g. "u8" + * @param type Variable type, e.g. `uint8_t` + */ +#define ATOMIC_FETCH_OP_IMPL(opname, op, name, type) \ + static inline void CONCAT4(atomic_fetch_, opname, _, name)(type *dest, \ + type val) \ + { \ + unsigned state = irq_disable(); \ + /* dest can be register allocated, hence the memory barrier of \ + * irq_disable() and irq_restore() may not apply here. Using volatile \ + * ensures that the compiler allocates it in memory and that the \ + * memory access is not optimized out. */ \ + volatile type *tmp = dest; \ + *tmp = *tmp op val; \ + irq_restore(state); \ + } + +#ifndef HAS_ATOMIC_FETCH_ADD_U8 +ATOMIC_FETCH_OP_IMPL(add, +, u8, uint8_t) +#endif +#ifndef HAS_ATOMIC_FETCH_ADD_U16 +ATOMIC_FETCH_OP_IMPL(add, +, u16, uint16_t) +#endif +#ifndef HAS_ATOMIC_FETCH_ADD_U32 +ATOMIC_FETCH_OP_IMPL(add, +, u32, uint32_t) +#endif +#ifndef HAS_ATOMIC_FETCH_ADD_U64 +ATOMIC_FETCH_OP_IMPL(add, +, u64, uint64_t) +#endif + +#ifndef HAS_ATOMIC_FETCH_SUB_U8 +ATOMIC_FETCH_OP_IMPL(sub, -, u8, uint8_t) +#endif +#ifndef HAS_ATOMIC_FETCH_SUB_U16 +ATOMIC_FETCH_OP_IMPL(sub, -, u16, uint16_t) +#endif +#ifndef HAS_ATOMIC_FETCH_SUB_U32 +ATOMIC_FETCH_OP_IMPL(sub, -, u32, uint32_t) +#endif +#ifndef HAS_ATOMIC_FETCH_SUB_U64 +ATOMIC_FETCH_OP_IMPL(sub, -, u64, uint64_t) +#endif + +#ifndef HAS_ATOMIC_FETCH_OR_U8 +ATOMIC_FETCH_OP_IMPL(or, |, u8, uint8_t) +#endif +#ifndef HAS_ATOMIC_FETCH_OR_U16 +ATOMIC_FETCH_OP_IMPL(or, |, u16, uint16_t) +#endif +#ifndef HAS_ATOMIC_FETCH_OR_U32 +ATOMIC_FETCH_OP_IMPL(or, |, u32, uint32_t) +#endif +#ifndef HAS_ATOMIC_FETCH_OR_U64 +ATOMIC_FETCH_OP_IMPL(or, |, u64, uint64_t) +#endif + +#ifndef HAS_ATOMIC_FETCH_XOR_U8 +ATOMIC_FETCH_OP_IMPL(xor, ^, u8, uint8_t) +#endif +#ifndef HAS_ATOMIC_FETCH_XOR_U16 +ATOMIC_FETCH_OP_IMPL(xor, ^, u16, uint16_t) +#endif +#ifndef HAS_ATOMIC_FETCH_XOR_U32 +ATOMIC_FETCH_OP_IMPL(xor, ^, u32, uint32_t) +#endif +#ifndef HAS_ATOMIC_FETCH_XOR_U64 +ATOMIC_FETCH_OP_IMPL(xor, ^, u64, uint64_t) +#endif + +#ifndef HAS_ATOMIC_FETCH_AND_U8 +ATOMIC_FETCH_OP_IMPL(and, &, u8, uint8_t) +#endif +#ifndef HAS_ATOMIC_FETCH_AND_U16 +ATOMIC_FETCH_OP_IMPL(and, &, u16, uint16_t) +#endif +#ifndef HAS_ATOMIC_FETCH_AND_U32 +ATOMIC_FETCH_OP_IMPL(and, &, u32, uint32_t) +#endif +#ifndef HAS_ATOMIC_FETCH_AND_U64 +ATOMIC_FETCH_OP_IMPL(and, &, u64, uint64_t) +#endif + +#ifndef HAS_ATOMIC_BIT +static inline atomic_bit_u8_t atomic_bit_u8(uint8_t *dest, uint8_t bit) +{ + atomic_bit_u8_t result = { .dest = dest, .mask = 1U << bit }; + return result; +} +static inline atomic_bit_u16_t atomic_bit_u16(uint16_t *dest, uint8_t bit) +{ + atomic_bit_u16_t result = { .dest = dest, .mask = 1U << bit }; + return result; +} +static inline atomic_bit_u32_t atomic_bit_u32(uint32_t *dest, uint8_t bit) +{ + atomic_bit_u32_t result = { .dest = dest, .mask = 1UL << bit }; + return result; +} +static inline atomic_bit_u64_t atomic_bit_u64(uint64_t *dest, uint8_t bit) +{ + atomic_bit_u64_t result = { .dest = dest, .mask = 1ULL << bit }; + return result; +} +static inline void atomic_set_bit_u8(atomic_bit_u8_t bit) +{ + atomic_fetch_or_u8(bit.dest, bit.mask); +} +static inline void atomic_set_bit_u16(atomic_bit_u16_t bit) +{ + atomic_fetch_or_u16(bit.dest, bit.mask); +} +static inline void atomic_set_bit_u32(atomic_bit_u32_t bit) +{ + atomic_fetch_or_u32(bit.dest, bit.mask); +} +static inline void atomic_set_bit_u64(atomic_bit_u64_t bit) +{ + atomic_fetch_or_u64(bit.dest, bit.mask); +} +static inline void atomic_clear_bit_u8(atomic_bit_u8_t bit) +{ + atomic_fetch_and_u8(bit.dest, ~bit.mask); +} +static inline void atomic_clear_bit_u16(atomic_bit_u16_t bit) +{ + atomic_fetch_and_u16(bit.dest, ~bit.mask); +} +static inline void atomic_clear_bit_u32(atomic_bit_u32_t bit) +{ + atomic_fetch_and_u32(bit.dest, ~bit.mask); +} +static inline void atomic_clear_bit_u64(atomic_bit_u64_t bit) +{ + atomic_fetch_and_u64(bit.dest, ~bit.mask); +} +#endif + +/* Provide semi_atomic_*() functions on top. + * + * - If atomic_() is provided: Use this for semi_atomic_() as well + * - Else: + * - If matching `atomic_store_u()` is provided: Only make final + * store atomic, as we can avoid touching the IRQ state register that + * way + * - Else: We need to disable and re-enable IRQs anyway, we just use the + * fallback implementation of `atomic_()` for `semi_atomic()` + * as well + */ + +/* FETCH_ADD */ +#if defined(HAS_ATOMIC_FETCH_ADD_U8) || !defined(HAS_ATOMIC_STORE_U8) +static inline void semi_atomic_fetch_add_u8(uint8_t *dest, uint8_t val) { + atomic_fetch_add_u8(dest, val); +} +#else +static inline void semi_atomic_fetch_add_u8(uint8_t *dest, uint8_t val) { + atomic_store_u8(dest, *dest + val); +} +#endif /* HAS_ATOMIC_FETCH_ADD_U8 || !HAS_ATOMIC_STORE_U8 */ + +#if defined(HAS_ATOMIC_FETCH_ADD_U16) || !defined(HAS_ATOMIC_STORE_U16) +static inline void semi_atomic_fetch_add_u16(uint16_t *dest, uint16_t val) { + atomic_fetch_add_u16(dest, val); +} +#else +static inline void semi_atomic_fetch_add_u16(uint16_t *dest, uint16_t val) { + atomic_store_u16(dest, *dest + val); +} +#endif /* HAS_ATOMIC_FETCH_ADD_U16 || !HAS_ATOMIC_STORE_U16 */ + +#if defined(HAS_ATOMIC_FETCH_ADD_U32) || !defined(HAS_ATOMIC_STORE_U32) +static inline void semi_atomic_fetch_add_u32(uint32_t *dest, uint32_t val) { + atomic_fetch_add_u32(dest, val); +} +#else +static inline void semi_atomic_fetch_add_u32(uint32_t *dest, uint32_t val) { + atomic_store_u32(dest, *dest + val); +} +#endif /* HAS_ATOMIC_FETCH_ADD_U32 || !HAS_ATOMIC_STORE_U32 */ + +#if defined(HAS_ATOMIC_FETCH_ADD_U64) || !defined(HAS_ATOMIC_STORE_U64) +static inline void semi_atomic_fetch_add_u64(uint64_t *dest, uint64_t val) { + atomic_fetch_add_u64(dest, val); +} +#else +static inline void semi_atomic_fetch_add_u64(uint64_t *dest, uint64_t val) { + atomic_store_u64(dest, *dest + val); +} +#endif /* HAS_ATOMIC_FETCH_ADD_U32 || !HAS_ATOMIC_STORE_U32 */ + +/* FETCH_SUB */ +#if defined(HAS_ATOMIC_FETCH_SUB_U8) || !defined(HAS_ATOMIC_STORE_U8) +static inline void semi_atomic_fetch_sub_u8(uint8_t *dest, uint8_t val) { + atomic_fetch_sub_u8(dest, val); +} +#else +static inline void semi_atomic_fetch_sub_u8(uint8_t *dest, uint8_t val) { + atomic_store_u8(dest, *dest - val); +} +#endif /* HAS_ATOMIC_FETCH_SUB_U8 || !HAS_ATOMIC_STORE_U8 */ + +#if defined(HAS_ATOMIC_FETCH_SUB_U16) || !defined(HAS_ATOMIC_STORE_U16) +static inline void semi_atomic_fetch_sub_u16(uint16_t *dest, uint16_t val) { + atomic_fetch_sub_u16(dest, val); +} +#else +static inline void semi_atomic_fetch_sub_u16(uint16_t *dest, uint16_t val) { + atomic_store_u16(dest, *dest - val); +} +#endif /* HAS_ATOMIC_FETCH_SUB_U16 || !HAS_ATOMIC_STORE_U16 */ + +#if defined(HAS_ATOMIC_FETCH_SUB_U32) || !defined(HAS_ATOMIC_STORE_U32) +static inline void semi_atomic_fetch_sub_u32(uint32_t *dest, uint32_t val) { + atomic_fetch_sub_u32(dest, val); +} +#else +static inline void semi_atomic_fetch_sub_u32(uint32_t *dest, uint32_t val) { + atomic_store_u32(dest, *dest - val); +} +#endif /* HAS_ATOMIC_FETCH_SUB_U32 || !HAS_ATOMIC_STORE_U64 */ + +#if defined(HAS_ATOMIC_FETCH_SUB_U64) || !defined(HAS_ATOMIC_STORE_U64) +static inline void semi_atomic_fetch_sub_u64(uint64_t *dest, uint64_t val) { + atomic_fetch_sub_u64(dest, val); +} +#else +static inline void semi_atomic_fetch_sub_u64(uint64_t *dest, uint64_t val) { + atomic_store_u64(dest, *dest - val); +} +#endif /* HAS_ATOMIC_FETCH_SUB_U64 || !HAS_ATOMIC_STORE_U64 */ + +/* FETCH_OR */ +#if defined(HAS_ATOMIC_FETCH_OR_U8) || !defined(HAS_ATOMIC_STORE_U8) +static inline void semi_atomic_fetch_or_u8(uint8_t *dest, uint8_t val) { + atomic_fetch_or_u8(dest, val); +} +#else +static inline void semi_atomic_fetch_or_u8(uint8_t *dest, uint8_t val) { + atomic_store_u8(dest, *dest | val); +} +#endif /* HAS_ATOMIC_FETCH_OR_U8 || !HAS_ATOMIC_STORE_U8 */ + +#if defined(HAS_ATOMIC_FETCH_OR_U16) || !defined(HAS_ATOMIC_STORE_U16) +static inline void semi_atomic_fetch_or_u16(uint16_t *dest, uint16_t val) { + atomic_fetch_or_u16(dest, val); +} +#else +static inline void semi_atomic_fetch_or_u16(uint16_t *dest, uint16_t val) { + atomic_store_u16(dest, *dest | val); +} +#endif /* HAS_ATOMIC_FETCH_OR_U16 || !HAS_ATOMIC_STORE_U16 */ + +#if defined(HAS_ATOMIC_FETCH_OR_U32) || !defined(HAS_ATOMIC_STORE_U32) +static inline void semi_atomic_fetch_or_u32(uint32_t *dest, uint32_t val) { + atomic_fetch_or_u32(dest, val); +} +#else +static inline void semi_atomic_fetch_or_u32(uint32_t *dest, uint32_t val) { + atomic_store_u32(dest, *dest | val); +} +#endif /* HAS_ATOMIC_FETCH_OR_U32 || !HAS_ATOMIC_STORE_U32 */ + +#if defined(HAS_ATOMIC_FETCH_OR_U64) || !defined(HAS_ATOMIC_STORE_U64) +static inline void semi_atomic_fetch_or_u64(uint64_t *dest, uint64_t val) { + atomic_fetch_or_u64(dest, val); +} +#else +static inline void semi_atomic_fetch_or_u64(uint64_t *dest, uint64_t val) { + atomic_store_u64(dest, *dest | val); +} +#endif /* HAS_ATOMIC_FETCH_OR_U64 || !HAS_ATOMIC_STORE_U64 */ + +/* FETCH_XOR */ +#if defined(HAS_ATOMIC_FETCH_XOR_U8) || !defined(HAS_ATOMIC_STORE_U8) +static inline void semi_atomic_fetch_xor_u8(uint8_t *dest, uint8_t val) { + atomic_fetch_xor_u8(dest, val); +} +#else +static inline void semi_atomic_fetch_xor_u8(uint8_t *dest, uint8_t val) { + atomic_store_u8(dest, *dest ^ val); +} +#endif /* HAS_ATOMIC_FETCH_XOR_U8 || !HAS_ATOMIC_STORE_U8 */ + +#if defined(HAS_ATOMIC_FETCH_XOR_U16) || !defined(HAS_ATOMIC_STORE_U16) +static inline void semi_atomic_fetch_xor_u16(uint16_t *dest, uint16_t val) { + atomic_fetch_xor_u16(dest, val); +} +#else +static inline void semi_atomic_fetch_xor_u16(uint16_t *dest, uint16_t val) { + atomic_store_u16(dest, *dest ^ val); +} +#endif /* HAS_ATOMIC_FETCH_XOR_U16 || !HAS_ATOMIC_STORE_U16 */ + +#if defined(HAS_ATOMIC_FETCH_XOR_U32) || !defined(HAS_ATOMIC_STORE_U32) +static inline void semi_atomic_fetch_xor_u32(uint32_t *dest, uint32_t val) { + atomic_fetch_xor_u32(dest, val); +} +#else +static inline void semi_atomic_fetch_xor_u32(uint32_t *dest, uint32_t val) { + atomic_store_u32(dest, *dest ^ val); +} +#endif /* HAS_ATOMIC_FETCH_XOR_U32 || !HAS_ATOMIC_STORE_U32 */ + +#if defined(HAS_ATOMIC_FETCH_XOR_U64) || !defined(HAS_ATOMIC_STORE_U64) +static inline void semi_atomic_fetch_xor_u64(uint64_t *dest, uint64_t val) { + atomic_fetch_xor_u64(dest, val); +} +#else +static inline void semi_atomic_fetch_xor_u64(uint64_t *dest, uint64_t val) { + atomic_store_u64(dest, *dest ^ val); +} +#endif /* HAS_ATOMIC_FETCH_XOR_U64 || !HAS_ATOMIC_STORE_U64 */ + +/* FETCH_AND */ +#if defined(HAS_ATOMIC_FETCH_AND_U8) || !defined(HAS_ATOMIC_STORE_U8) +static inline void semi_atomic_fetch_and_u8(uint8_t *dest, uint8_t val) { + atomic_fetch_and_u8(dest, val); +} +#else +static inline void semi_atomic_fetch_and_u8(uint8_t *dest, uint8_t val) { + atomic_store_u8(dest, *dest & val); +} +#endif /* HAS_ATOMIC_FETCH_AND_U8 || !HAS_ATOMIC_STORE_U8 */ + +#if defined(HAS_ATOMIC_FETCH_AND_U16) || !defined(HAS_ATOMIC_STORE_U16) +static inline void semi_atomic_fetch_and_u16(uint16_t *dest, uint16_t val) { + atomic_fetch_and_u16(dest, val); +} +#else +static inline void semi_atomic_fetch_and_u16(uint16_t *dest, uint16_t val) { + atomic_store_u16(dest, *dest & val); +} +#endif /* HAS_ATOMIC_FETCH_AND_U16 || !HAS_ATOMIC_STORE_U16 */ + +#if defined(HAS_ATOMIC_FETCH_AND_U32) || !defined(HAS_ATOMIC_STORE_U32) +static inline void semi_atomic_fetch_and_u32(uint32_t *dest, uint32_t val) { + atomic_fetch_and_u32(dest, val); +} +#else +static inline void semi_atomic_fetch_and_u32(uint32_t *dest, uint32_t val) { + atomic_store_u32(dest, *dest & val); +} +#endif /* HAS_ATOMIC_FETCH_AND_U32 || !HAS_ATOMIC_STORE_U32 */ + +#if defined(HAS_ATOMIC_FETCH_AND_U64) || !defined(HAS_ATOMIC_STORE_U64) +static inline void semi_atomic_fetch_and_u64(uint64_t *dest, uint64_t val) { + atomic_fetch_and_u64(dest, val); +} +#else +static inline void semi_atomic_fetch_and_u64(uint64_t *dest, uint64_t val) { + atomic_store_u64(dest, *dest & val); +} +#endif /* HAS_ATOMIC_FETCH_AND_U64 || !HAS_ATOMIC_STORE_U64 */ + +#ifdef __cplusplus +} +#endif + +#endif /* ATOMIC_UTILS_H */ +/** @} */