diff --git a/cpu/fe310/Kconfig b/cpu/fe310/Kconfig
index 9ba0b39870..25c61c6c64 100644
--- a/cpu/fe310/Kconfig
+++ b/cpu/fe310/Kconfig
@@ -21,6 +21,7 @@ config CPU_FAM_FE310
     select MODULE_PERIPH_PLIC if TEST_KCONFIG
     select MODULE_PERIPH_CORETIMER if MODULE_PERIPH_TIMER && HAS_PERIPH_CORETIMER
     select MODULE_PERIPH_RTT if MODULE_PERIPH_RTC && HAS_PERIPH_RTT
+    select MODULE_ZTIMER_NO_PERIPH_RTT if TEST_KCONFIG && MODULE_ZTIMER
 
 config CPU_MODEL_FE310_G000
     bool
diff --git a/cpu/sam0_common/Kconfig b/cpu/sam0_common/Kconfig
index 02a6150e6d..0436401e66 100644
--- a/cpu/sam0_common/Kconfig
+++ b/cpu/sam0_common/Kconfig
@@ -26,6 +26,7 @@ config CPU_COMMON_SAM0
     select HAS_PERIPH_WDT_CB
     select HAS_PERIPH_WDT_WARNING_PERIOD
     select HAVE_SHARED_PERIPH_RTT_PERIPH_RTC
+    select MODULE_ZTIMER_NO_PERIPH_RTT if TEST_KCONFIG && MODULE_ZTIMER
 
 if CPU_COMMON_SAM0
 
diff --git a/cpu/sam3/Kconfig b/cpu/sam3/Kconfig
index d42f79c963..9f8b0da840 100644
--- a/cpu/sam3/Kconfig
+++ b/cpu/sam3/Kconfig
@@ -12,6 +12,7 @@ config CPU_FAM_SAM3
     select HAS_CPU_SAM3
     select HAS_PERIPH_CPUID
     select HAS_PERIPH_HWRNG
+    select MODULE_ZTIMER_NO_PERIPH_RTT if TEST_KCONFIG && MODULE_ZTIMER
 
 ## CPU Models
 config CPU_MODEL_SAM3X8E
diff --git a/sys/ztimer/Kconfig b/sys/ztimer/Kconfig
index 7f7aaed153..fe92804ca4 100644
--- a/sys/ztimer/Kconfig
+++ b/sys/ztimer/Kconfig
@@ -82,7 +82,7 @@ config MODULE_ZTIMER_SEC
 choice
     bool "Backend"
     depends on MODULE_ZTIMER_SEC
-    default ZTIMER_SEC_BACKEND_RTT
+    default ZTIMER_SEC_BACKEND_RTT if !MODULE_ZTIMER_NO_PERIPH_RTT
 
 config ZTIMER_SEC_BACKEND_TIMER
     bool "Timer"
@@ -91,7 +91,7 @@ config ZTIMER_SEC_BACKEND_TIMER
 config ZTIMER_SEC_BACKEND_RTT
     bool "RTT"
     depends on HAS_PERIPH_RTT
-    select MODULE_ZTIMER_PERIPH_RTT if !MODULE_ZTIMER_NO_PERIPH_RTT
+    select MODULE_ZTIMER_PERIPH_RTT
 
 config ZTIMER_SEC_BACKEND_RTC
     bool "RTC"
diff --git a/sys/ztimer/Makefile.dep b/sys/ztimer/Makefile.dep
index 72a2a3239b..d75d73e91c 100644
--- a/sys/ztimer/Makefile.dep
+++ b/sys/ztimer/Makefile.dep
@@ -77,6 +77,17 @@ ifneq (,$(filter ztimer_usec,$(USEMODULE)))
   USEMODULE += ztimer_periph_timer
 endif
 
+# NOTE: select the module here and not in the CPU so that order of inclusion
+# does not have periph_rtt selected earlier that it should be while at the same
+# time avoiding the module 'ztimer-no_periph_rtt' being included unecesarily.
+# The sam0 rtt busy loops for 180us every time an alarm is set or
+# the counter is read, this propagates and leads to timing errors
+# on ztimer_msec that are higher than > +-1msec.
+# The same goes for the fe310 rtt.
+ifneq (,$(filter sam% fe310,$(CPU)))
+  USEMODULE += ztimer_no_periph_rtt
+endif
+
 ifneq (,$(filter ztimer_msec ztimer_sec,$(USEMODULE)))
   USEMODULE += ztimer
   ifeq (,$(filter ztimer_no_periph_rtt,$(USEMODULE)))
diff --git a/tests/posix_sleep/Makefile b/tests/posix_sleep/Makefile
index 4c534b1372..2d5f6f5dda 100644
--- a/tests/posix_sleep/Makefile
+++ b/tests/posix_sleep/Makefile
@@ -2,12 +2,4 @@ include ../Makefile.tests_common
 
 USEMODULE += posix_sleep
 
-# These CPU families have a non configurable RTT of 1Hz, not enough for
-# ztimer_msec to run on periph_rtt
-ifeq (,$(filter efm32 kinetis,$(CPU)))
-  # Pull-in periph-rtt on boards that provide this feature to switch to
-  # the RTT backend of ztimer
-  FEATURES_OPTIONAL += periph_rtt
-endif
-
 include $(RIOTBASE)/Makefile.include