diff --git a/cpu/esp32/vendor/esp-idf/Makefile b/cpu/esp32/vendor/esp-idf/Makefile
index 31d86057e7..1fa83f7205 100644
--- a/cpu/esp32/vendor/esp-idf/Makefile
+++ b/cpu/esp32/vendor/esp-idf/Makefile
@@ -2,13 +2,16 @@ MODULE=esp_idf
 
 DIRS += driver
 DIRS += esp32
-DIRS += heap
 DIRS += soc
 DIRS += spi_flash
-DIRS += wpa_supplicant
+
+ifneq (,$(filter esp_idf_heap,$(USEMODULE)))
+    DIRS += heap
+endif
 
 ifneq (,$(filter esp_wifi_any,$(USEMODULE)))
     DIRS += nvs_flash
+    DIRS += wpa_supplicant
     INCLUDES += -I$(ESP32_SDK_DIR)/components/smartconfig_ack/include
 endif