From 7b1f083cd8123c9a7e9ce38682d9eb961855d222 Mon Sep 17 00:00:00 2001 From: iosabi Date: Sat, 1 May 2021 15:39:25 +0200 Subject: [PATCH] esp8266: Download Espressif RTOS SDK as a new RIOT PKG RIOT-OS uses part of Espressif ESP8266 RTOS SDK to build support for this CPU. The SDK includes some vendor-provided closed source pre-compiled libraries that we need to modify to adapt to RIOT-OS usage. This library modifications was done once and uploaded to a fork of the vendor repository and was provided as an environment variable. This patch changes two things: 1. It installs the SDK as a RIOT PKG from the new pkg/esp8266_sdk directory instead of requiring the user to download it separately. 2. It performs the library modifications (symbol renames) on the pkg Makefile removing the need to use a fork with the modifications applied and simplifying the SDK update and future modifications. This change sets the SDK package version (git SHA) to the same one that our fork was using as a parent in the vendor repository, meaning that the output libraries are exactly the same as before. Tested with ``` ESP8266_RTOS_SDK_DIR=/dev/null USEMODULE=esp_log_startup make -C tests/shell BOARD=esp8266-esp-12x flash ``` and verified that the program works. The boot message now includes: ``` ESP8266-RTOS-SDK Version v3.1-51-g913a06a9 ``` confirming the SDK version used. `/dev/null` in the test is just to make sure that no evaluation of `ESP8266_RTOS_SDK_DIR` in make is affected by the environment variable value which would be set to the SDK for people who followed the set up instructions before this change. Tested the checkout size: ```bash $ du -hs build/pkg/esp8266_sdk/ 124M build/pkg/esp8266_sdk/ ``` --- cpu/esp8266/Makefile.dep | 3 ++ cpu/esp8266/Makefile.include | 17 --------- cpu/esp8266/doc.txt | 28 ++------------ cpu/esp8266/include/sdk_conf.h | 4 +- .../wpa_supplicant/src/crypto/Makefile | 19 ++++------ cpu/esp_common/Makefile.include | 3 ++ cpu/esp_common/lib_printf.c | 3 ++ dist/tools/packer/scripts/toolchains.sh | 15 +------- pkg/esp8266_sdk/Makefile | 37 +++++++++++++++++++ pkg/esp8266_sdk/Makefile.dep | 2 + pkg/esp8266_sdk/Makefile.include | 23 ++++++++++++ pkg/esp8266_sdk/doc.txt | 6 +++ pkg/esp8266_sdk/symbol_renames.txt | 25 +++++++++++++ 13 files changed, 117 insertions(+), 68 deletions(-) create mode 100644 pkg/esp8266_sdk/Makefile create mode 100644 pkg/esp8266_sdk/Makefile.dep create mode 100644 pkg/esp8266_sdk/Makefile.include create mode 100644 pkg/esp8266_sdk/doc.txt create mode 100644 pkg/esp8266_sdk/symbol_renames.txt diff --git a/cpu/esp8266/Makefile.dep b/cpu/esp8266/Makefile.dep index 906343cc94..4c3bf5e9b8 100644 --- a/cpu/esp8266/Makefile.dep +++ b/cpu/esp8266/Makefile.dep @@ -2,6 +2,9 @@ include $(RIOTCPU)/esp_common/Makefile.dep +# Include ESP8266 RTOS SDK vendor package. +USEPKG += esp8266_sdk + USEMODULE += esp_idf_esp8266 USEMODULE += esp_idf_nvs_flash USEMODULE += esp_idf_spi_flash diff --git a/cpu/esp8266/Makefile.include b/cpu/esp8266/Makefile.include index e58310f446..f95ee52a08 100644 --- a/cpu/esp8266/Makefile.include +++ b/cpu/esp8266/Makefile.include @@ -1,12 +1,3 @@ -# check some environment variables first -ifndef ESP8266_RTOS_SDK_DIR - $(info ESP8266_RTOS_SDK_DIR should be defined as /path/to/sdk directory) - $(info ESP8266_RTOS_SDK_DIR is set by default to /opt/esp/ESP8266-RTOS-SDK) - export ESP8266_RTOS_SDK_DIR=/opt/esp/ESP8266-RTOS-SDK -endif - -ESP_SDK_DIR = $(ESP8266_RTOS_SDK_DIR) - # With the '-Os' option, char arrays have not to be 32-bit word aligned. This # leads to an alignment exception when the address of a char array is assigned # to an 'uint32_t' pointer and the pointer is used for the access. @@ -38,14 +29,6 @@ INCLUDES += -I$(RIOTCPU)/$(CPU)/vendor/esp-idf/esp8266/include INCLUDES += -I$(RIOTCPU)/$(CPU)/vendor/esp-idf/esp8266/include/esp8266 INCLUDES += -I$(RIOTCPU)/$(CPU)/vendor/esp-idf/heap/include INCLUDES += -I$(RIOTCPU)/$(CPU)/vendor/esp-idf/log/include -INCLUDES += -I$(ESP8266_RTOS_SDK_DIR)/components/ -INCLUDES += -I$(ESP8266_RTOS_SDK_DIR)/components/bootloader_support/include/ -INCLUDES += -I$(ESP8266_RTOS_SDK_DIR)/components/esp8266/include -INCLUDES += -I$(ESP8266_RTOS_SDK_DIR)/components/esp8266/include/esp8266 -INCLUDES += -I$(ESP8266_RTOS_SDK_DIR)/components/heap/include -INCLUDES += -I$(ESP8266_RTOS_SDK_DIR)/components/heap/port/esp8266/include -INCLUDES += -I$(ESP8266_RTOS_SDK_DIR)/components/nvs_flash/include -INCLUDES += -I$(ESP8266_RTOS_SDK_DIR)/components/spi_flash/include CFLAGS += -D__ESP_FILE__=__FILE__ diff --git a/cpu/esp8266/doc.txt b/cpu/esp8266/doc.txt index e623517c9a..4bbdd218b6 100644 --- a/cpu/esp8266/doc.txt +++ b/cpu/esp8266/doc.txt @@ -340,31 +340,10 @@ export PATH=$HOME/esp/xtensa-esp8266-elf/bin:$PATH [Back to table of contents](#esp8266_toc) -### Installation of the ESP8266 RTOS SDK{#esp8266_installation_of_esp_idf} +### ESP8266 RTOS SDK{#esp8266_esp_sdk} -To compile RIOT-OS with the ESP8266 RTOS SDK, a modified version of the SDK is -required. This modified version can also be downloaded as -[GIT](https://github.com/gschorcht/RIOT-Xtensa-ESP8266-RTOS-SDK.git) repository. - -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -cd $HOME/esp -git clone https://github.com/gschorcht/RIOT-Xtensa-ESP8266-RTOS-SDK.git ESP8266_RTOS_SDK -cd ESP8266_RTOS_SDK/ -git checkout release/v3.1-for-riot-os-v2 -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -@note -- Please be sure to checkout the correct branch that was used for the -RIOT-OS port. Other versions will not work because they do not have the -necessary changes. -- Since we only use a few header files and some binary libraries, ESP8266 RTOS -SDK does not need to be compiled in any way. - -To use the installed ESP8266 RTOS SDK, set the environment variable -`ESP8266_RTOS_SDK_DIR`. - -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -export ESP8266_RTOS_SDK_DIR=$HOME/esp/ESP8266_RTOS_SDK -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +RIOT-OS uses the ESP8266 RTOS SDK as part of the build. This is downloaded as a +package at build-time and there is no need to install it separately. [Back to table of contents](#esp8266_toc) @@ -414,7 +393,6 @@ correctly to ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ export PATH=/path/to/esp/xtensa-esp8266-elf/bin:$PATH -export ESP8266_RTOS_SDK_DIR=/path/to/esp/ESP8266_RTOS_SDK ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ [Back to table of contents](#esp8266_toc) diff --git a/cpu/esp8266/include/sdk_conf.h b/cpu/esp8266/include/sdk_conf.h index 30d868a3de..0fb8980a72 100644 --- a/cpu/esp8266/include/sdk_conf.h +++ b/cpu/esp8266/include/sdk_conf.h @@ -38,7 +38,9 @@ extern "C" { * * Determined with `git describe --tags` in `$ESP8266_SDK_DIR` */ -#define IDF_VER "v3.1-4-g08c234e" +#if !defined(IDF_VER) || DOXYGEN +#include "esp8266_idf_version.h" +#endif /** * @name Default console configuration diff --git a/cpu/esp8266/vendor/esp-idf/wpa_supplicant/src/crypto/Makefile b/cpu/esp8266/vendor/esp-idf/wpa_supplicant/src/crypto/Makefile index fa033d8e33..4430007f4b 100644 --- a/cpu/esp8266/vendor/esp-idf/wpa_supplicant/src/crypto/Makefile +++ b/cpu/esp8266/vendor/esp-idf/wpa_supplicant/src/crypto/Makefile @@ -4,18 +4,13 @@ include $(RIOTBASE)/Makefile.base # we have to do it in that way to avoid that $(RIOTBASE)/sys/include/crypto # is found first -INCLUDES = -I$(RIOTCPU)/$(CPU)/vendor/esp-idf/wpa_supplicant/include -INCLUDES += -I$(RIOTCPU)/$(CPU)/vendor/esp-idf/wpa_supplicant/port/include -INCLUDES += -I$(ESP8266_RTOS_SDK_DIR)/components/wpa_supplicant/include -INCLUDES += -I$(ESP8266_RTOS_SDK_DIR)/components/wpa_supplicant/port/include +PRE_INCLUDES += -I$(RIOTCPU)/$(CPU)/vendor/esp-idf/wpa_supplicant/include +PRE_INCLUDES += -I$(RIOTCPU)/$(CPU)/vendor/esp-idf/wpa_supplicant/port/include +PRE_INCLUDES += -I$(ESP8266_RTOS_SDK_DIR)/components/wpa_supplicant/include +PRE_INCLUDES += -I$(ESP8266_RTOS_SDK_DIR)/components/wpa_supplicant/port/include CFLAGS += -D__ets__ -DESPRESSIF_USE -DESP_PLATFORM=1 CFLAGS += -Wno-strict-aliasing -include $(RIOTCPU)/$(CPU)/Makefile.include - -INCLUDES += -I$(RIOTBASE)/core/include -INCLUDES += -I$(RIOTBASE)/sys/include -INCLUDES += -I$(RIOTCPU)/$(CPU)/vendor/esp-idf/log/include -INCLUDES += -I$(RIOTCPU)/$(CPU)/include -INCLUDES += -I$(BOARDDIR)/include - +# This converts INCLUDES to a simply evaluated variable instead of a recursively +# one but only for this module. +INCLUDES := $(PRE_INCLUDES) $(INCLUDES) diff --git a/cpu/esp_common/Makefile.include b/cpu/esp_common/Makefile.include index 6d34b99845..228ddf4c81 100644 --- a/cpu/esp_common/Makefile.include +++ b/cpu/esp_common/Makefile.include @@ -81,8 +81,11 @@ CFLAGS += $(if $(findstring dout,$(FLASH_MODE)),-DFLASH_MODE_DOUT=1) ARCHIVES += -lhal -lg -lc LINKFLAGS += $(CFLAGS_OPT) $(CFLAGS_DBG) +ifneq ($(CPU),esp8266) +# esp8266 flags are added by the SDK pkg in pkg/esp8266_sdk LINKFLAGS += -L$(ESP_SDK_DIR)/components/$(CPU) LINKFLAGS += -L$(ESP_SDK_DIR)/components/$(CPU)/lib +endif LINKFLAGS += -nostdlib -Wl,-gc-sections -Wl,-static ifeq (,$(filter esp_idf_heap,$(USEMODULE))) diff --git a/cpu/esp_common/lib_printf.c b/cpu/esp_common/lib_printf.c index e4b0b51bf9..b98c71ac2b 100644 --- a/cpu/esp_common/lib_printf.c +++ b/cpu/esp_common/lib_printf.c @@ -72,6 +72,9 @@ LIB_PRINTF(net80211, LOG_DEBUG, "net80211") LIB_PRINTF(phy, LOG_INFO, "core") LIB_PRINTF(pp, LOG_DEBUG, "pp") LIB_PRINTF(sc, LOG_DEBUG, "smartconfig") +/* The ESP8266 SDK uses smartconfig_printf but the ESP32 one uses sc_printf. */ +int smartconfig_printf(const char *format, ...) + __attribute__((alias("sc_printf"))); LIB_PRINTF(ssc, LOG_DEBUG, "ssc") LIB_PRINTF(wpa, LOG_DEBUG, "wpa") LIB_PRINTF(wpa2, LOG_DEBUG, "wpa") diff --git a/dist/tools/packer/scripts/toolchains.sh b/dist/tools/packer/scripts/toolchains.sh index 1406232ef4..5201baa84f 100644 --- a/dist/tools/packer/scripts/toolchains.sh +++ b/dist/tools/packer/scripts/toolchains.sh @@ -77,7 +77,7 @@ echo "export PATH=\$PATH:/opt/esp/esp-open-sdk/xtensa-lx106-elf/bin" >> /home/${ echo "export ESP8266_SDK_DIR=/opt/esp/esp-open-sdk/sdk" >> /home/${SSH_USERNAME}/.bashrc echo "export ESP8266_NEWLIB_DIR=/opt/esp/newlib-xtensa" >> /home/${SSH_USERNAME}/.bashrc -# Install complete ESP8266 toolchain in /opt/esp (139 MB after cleanup) +# Install complete ESP8266 toolchain in /opt/esp (125 MB after cleanup) # remember https://github.com/RIOT-OS/RIOT/pull/10801 when updating # NOTE: We install the toolchain for the RTOS SDK in parallel in the first # step and remove the old version as soon as the RIOT port for the ESP8266 @@ -88,17 +88,6 @@ echo 'Installing ESP8266 toolchain' >&2 && \ git clone https://github.com/gschorcht/xtensa-esp8266-elf && \ cd xtensa-esp8266-elf && \ git checkout -q 696257c2b43e2a107d3108b2c1ca6d5df3fb1a6f && \ - rm -rf .git && \ - cd /opt/esp && \ - git clone https://github.com/gschorcht/RIOT-Xtensa-ESP8266-RTOS-SDK.git ESP8266_RTOS_SDK && \ - cd ESP8266_RTOS_SDK/ && \ - git checkout -q f074414c0705715a44b8e59d53b03d90b7630382 && \ - rm -rf .git* docs examples make tools && \ - cd components && \ - rm -rf app_update aws_iot bootloader cjson coap espos esp-tls freertos \ - jsmn libsodium log mdns mqtt newlib partition_table pthread \ - smartconfig_ack spiffs ssl tcpip_adapter vfs && \ - find . -name '*.[csS]' -exec rm {} \; + rm -rf .git echo "export PATH=\$PATH:/opt/esp/xtensa-esp8266-elf/bin" >> /home/${SSH_USERNAME}/.bashrc -echo "export ESP8266_RTOS_SDK_DIR=/opt/esp/ESP8266_RTOS_SDK" >> /home/${SSH_USERNAME}/.bashrc diff --git a/pkg/esp8266_sdk/Makefile b/pkg/esp8266_sdk/Makefile new file mode 100644 index 0000000000..1453c63006 --- /dev/null +++ b/pkg/esp8266_sdk/Makefile @@ -0,0 +1,37 @@ +PKG_NAME=esp8266_sdk +PKG_URL=https://github.com/espressif/ESP8266_RTOS_SDK +# This is a version in the v3.1 release branch, between 3.1 and 3.1.1. +PKG_VERSION=913a06a9ac3b2f18009e8fee8f092ca9ffeccd38 +PKG_LICENSE=Apache-2.0 + +include $(RIOTBASE)/pkg/pkg.mk + +# This directory is shared across all apps since there's no need to replicate +# these libraries on each app. +ESP8266_SDK_BUILD_DIR = $(PKG_SOURCE_DIR)/build-libs + +# We need to replace some symbols in the binary libraries shipped with the SDK +# to avoid collisions with RIOT symbols. +ESP_SDK_COMPONENT_LIBS = \ + libcore.a libespnow.a libgcc.a libhal.a libnet80211.a libphy.a libpp.a \ + libsmartconfig.a libssc.a libwpa.a libwps.a + +ESP_SDK_LIBS = $(addprefix $(ESP8266_SDK_BUILD_DIR)/, $(ESP_SDK_COMPONENT_LIBS)) + +all: $(ESP_SDK_LIBS) $(ESP8266_SDK_BUILD_DIR)/esp8266_idf_version.h + +$(ESP8266_SDK_BUILD_DIR): + $(Q)mkdir -p $(ESP8266_SDK_BUILD_DIR) + +# Set the SDK version from the SDK hash/tag. For example "v3.1-51-g913a06a9". +$(ESP8266_SDK_BUILD_DIR)/esp8266_idf_version.h: | $(ESP8266_SDK_BUILD_DIR) + $(Q)echo "#define IDF_VER \"$(shell git -C $(PKG_SOURCE_DIR) describe --tags)\"" \ + > $@ + +$(ESP8266_SDK_BUILD_DIR)/lib%.a: \ + $(PKG_SOURCE_DIR)/components/esp8266/lib/lib%.a | $(ESP8266_SDK_BUILD_DIR) + $(Q)$(OBJCOPY) \ + --redefine-syms $(CURDIR)/symbol_renames.txt \ + --redefine-sym printf=$(patsubst lib%.a,%,$(notdir $@))_printf \ + --redefine-sym ets_printf=$(patsubst lib%.a,%,$(notdir $@))_ets_printf \ + $< $@ diff --git a/pkg/esp8266_sdk/Makefile.dep b/pkg/esp8266_sdk/Makefile.dep new file mode 100644 index 0000000000..1df099a72b --- /dev/null +++ b/pkg/esp8266_sdk/Makefile.dep @@ -0,0 +1,2 @@ +# This package can only be used with the ESP8266 CPU +FEATURES_REQUIRED += arch_esp8266 diff --git a/pkg/esp8266_sdk/Makefile.include b/pkg/esp8266_sdk/Makefile.include new file mode 100644 index 0000000000..b8ca392e49 --- /dev/null +++ b/pkg/esp8266_sdk/Makefile.include @@ -0,0 +1,23 @@ +# Directory with the SDK source checkout. Some modules in the cpu/esp8266 use +# internal parts of the SDK and for that they need access to the +# ESP8266_RTOS_SDK_DIR path. +export ESP8266_RTOS_SDK_DIR = $(PKGDIRBASE)/esp8266_sdk + +# Directory where we built the modified libraries and headers. +ESP8266_SDK_BUILD_DIR ?= $(ESP8266_RTOS_SDK_DIR)/build-libs + +INCLUDES += -I$(ESP8266_RTOS_SDK_DIR)/components/ +INCLUDES += -I$(ESP8266_RTOS_SDK_DIR)/components/bootloader_support/include/ +INCLUDES += -I$(ESP8266_RTOS_SDK_DIR)/components/esp8266/include +INCLUDES += -I$(ESP8266_RTOS_SDK_DIR)/components/esp8266/include/esp8266 +INCLUDES += -I$(ESP8266_RTOS_SDK_DIR)/components/heap/include +INCLUDES += -I$(ESP8266_RTOS_SDK_DIR)/components/heap/port/esp8266/include +INCLUDES += -I$(ESP8266_RTOS_SDK_DIR)/components/nvs_flash/include +INCLUDES += -I$(ESP8266_RTOS_SDK_DIR)/components/spi_flash/include +INCLUDES += -I$(ESP8266_SDK_BUILD_DIR) + +# Modified binary libraries are built here in the Makefile. +LINKFLAGS += -L$(ESP8266_SDK_BUILD_DIR) + +# esp8266_sdk doesn't generate any .a +PSEUDOMODULES += esp8266_sdk diff --git a/pkg/esp8266_sdk/doc.txt b/pkg/esp8266_sdk/doc.txt new file mode 100644 index 0000000000..3f1789906c --- /dev/null +++ b/pkg/esp8266_sdk/doc.txt @@ -0,0 +1,6 @@ +/** + * @defgroup pkg_esp8266_sdk Vendor library for the ESP8266 MCU RTOS support + * @ingroup pkg + * @brief Vendor library for the ESP8266 MCU RTOS support by Espressif + * @see https://github.com/espressif/ESP8266_RTOS_SDK + */ diff --git a/pkg/esp8266_sdk/symbol_renames.txt b/pkg/esp8266_sdk/symbol_renames.txt new file mode 100644 index 0000000000..3cb64722ce --- /dev/null +++ b/pkg/esp8266_sdk/symbol_renames.txt @@ -0,0 +1,25 @@ +# This file is passed to objcopy with --redefine-syms. It contains a list of +# symbols to be renamed in the esp8266 vendor libraries as "old new". Symbols +# not present in the library are ignored and lines starting with # are comments, +# however unfortunately empty lines are not ignored. +# +# Note: printf and ets_printf symbol renames are handled in the pkg Makefile. +# +# libcore.a +rtc_init esp_rtc_init +# +# libcore.a libwpa.a and libespnow.a +aes_decrypt wpa_aes_decrypt +aes_decrypt_deinit wpa_aes_decrypt_deinit +aes_decrypt_init wpa_aes_decrypt_init +aes_encrypt wpa_aes_encrypt +aes_encrypt_deinit wpa_aes_encrypt_deinit +aes_encrypt_init wpa_aes_encrypt_init +aes_unwrap wpa_aes_unwrap +aes_wrap wpa_aes_wrap +hmac_md5 wpa_hmac_md5 +hmac_sha1 wpa_hmac_sha1 +hmac_sha1_vector wpa_hmac_sha1_vector +pbkdf2_sha1 wpa_pbkdf2_sha1 +rc4_skip wpa_rc4_skip +sha1_prf wpa_sha1_prf