diff --git a/cpu/fe310/Kconfig b/cpu/fe310/Kconfig
index 9ba0b39870..25c61c6c64 100644
--- a/cpu/fe310/Kconfig
+++ b/cpu/fe310/Kconfig
@@ -21,6 +21,7 @@ config CPU_FAM_FE310
     select MODULE_PERIPH_PLIC if TEST_KCONFIG
     select MODULE_PERIPH_CORETIMER if MODULE_PERIPH_TIMER && HAS_PERIPH_CORETIMER
     select MODULE_PERIPH_RTT if MODULE_PERIPH_RTC && HAS_PERIPH_RTT
+    select MODULE_ZTIMER_NO_PERIPH_RTT if TEST_KCONFIG && MODULE_ZTIMER
 
 config CPU_MODEL_FE310_G000
     bool
diff --git a/cpu/sam0_common/Kconfig b/cpu/sam0_common/Kconfig
index 02a6150e6d..0436401e66 100644
--- a/cpu/sam0_common/Kconfig
+++ b/cpu/sam0_common/Kconfig
@@ -26,6 +26,7 @@ config CPU_COMMON_SAM0
     select HAS_PERIPH_WDT_CB
     select HAS_PERIPH_WDT_WARNING_PERIOD
     select HAVE_SHARED_PERIPH_RTT_PERIPH_RTC
+    select MODULE_ZTIMER_NO_PERIPH_RTT if TEST_KCONFIG && MODULE_ZTIMER
 
 if CPU_COMMON_SAM0
 
diff --git a/cpu/sam3/Kconfig b/cpu/sam3/Kconfig
index d42f79c963..9f8b0da840 100644
--- a/cpu/sam3/Kconfig
+++ b/cpu/sam3/Kconfig
@@ -12,6 +12,7 @@ config CPU_FAM_SAM3
     select HAS_CPU_SAM3
     select HAS_PERIPH_CPUID
     select HAS_PERIPH_HWRNG
+    select MODULE_ZTIMER_NO_PERIPH_RTT if TEST_KCONFIG && MODULE_ZTIMER
 
 ## CPU Models
 config CPU_MODEL_SAM3X8E
diff --git a/sys/ztimer/Makefile.dep b/sys/ztimer/Makefile.dep
index 1042f47e27..928fe13b10 100644
--- a/sys/ztimer/Makefile.dep
+++ b/sys/ztimer/Makefile.dep
@@ -78,6 +78,17 @@ ifneq (,$(filter ztimer_usec,$(USEMODULE)))
   USEMODULE += ztimer_periph_timer
 endif
 
+# NOTE: select the module here and not in the CPU so that order of inclusion
+# does not have periph_rtt selected earlier that it should be while at the same
+# time avoiding the module 'ztimer-no_periph_rtt' being included unecesarily.
+# The sam0 rtt busy loops for 180us every time an alarm is set or
+# the counter is read, this propagates and leads to timing errors
+# on ztimer_msec that are higher than > +-1msec.
+# The same goes for the fe310 rtt.
+ifneq (,$(filter sam% fe310,$(CPU)))
+  USEMODULE += ztimer_no_periph_rtt
+endif
+
 ifneq (,$(filter ztimer_msec ztimer_sec,$(USEMODULE)))
   USEMODULE += ztimer
   ifeq (,$(filter ztimer_no_periph_rtt,$(USEMODULE)))