esp32 - automate ESP-IDF and toolchain installation

I hope it will work going forward.  I tried to anticipate
some upcoming changes in the esp-idf makefiles that will
land in 3.2, but one never knows.

3 files changed, 75 insertions, 144 deletions

diff --git a/build/esp32/Makefile b/build/esp32/Makefile
index 0c3b3da..a129ae6 100644
--- a/build/esp32/Makefile
+++ b/build/esp32/Makefile
@@ -16,12 +16,12 @@ CC := gcc
 # Change these to reflect the locations of external stuff on your system,
 # either here or on the command line
 XTGCC_PARENT_PATH ?= $(UPDIR)
-XTGCCPATH ?= $(XTGCC_PARENT_PATH)/xtensa-esp32-elf/bin/
-CROSS ?= $(XTGCCPATH)xtensa-esp32-elf-
+# XTGCCPATH ?= $(XTGCC_PARENT_PATH)/xtensa-esp32-elf/bin/
 
 # ESP32 SDK
-IDF_PARENT_PATH ?= $(UPDIR)
-IDF_PATH ?= $(IDF_PARENT_PATH)/esp-idf
+ESP_IDF_PARENT_PATH ?= $(UPDIR)
+ESP_IDF_VERSION ?= v3.1.4
+IDF_PATH ?= $(ESP_IDF_PARENT_PATH)/esp-idf-$(ESP_IDF_VERSION)/
 
 CFORTH_PATH ?= $(abspath $(TOPDIR)/build/esp32)
 
diff --git a/build/esp32/sdk_build/sdkconfig b/build/esp32/sdk_build/sdkconfig
index 7980167..5b20ccb 100644
--- a/build/esp32/sdk_build/sdkconfig
+++ b/build/esp32/sdk_build/sdkconfig
@@ -2,7 +2,6 @@
 # Automatically generated file; DO NOT EDIT.
 # Espressif IoT Development Framework Configuration
 #
-CONFIG_IDF_TARGET="esp32"
 
 #
 # SDK tool configuration
@@ -12,13 +11,6 @@ CONFIG_PYTHON="python"
 CONFIG_MAKE_WARN_UNDEFINED_VARIABLES=y
 
 #
-# Application manager
-#
-CONFIG_APP_COMPILE_TIME_DATE=y
-CONFIG_APP_EXCLUDE_PROJECT_VER_VAR=
-CONFIG_APP_EXCLUDE_PROJECT_NAME_VAR=
-
-#
 # Bootloader config
 #
 CONFIG_LOG_BOOTLOADER_LEVEL_NONE=
@@ -32,8 +24,6 @@ CONFIG_BOOTLOADER_VDDSDIO_BOOST_1_8V=
 CONFIG_BOOTLOADER_VDDSDIO_BOOST_1_9V=y
 CONFIG_BOOTLOADER_FACTORY_RESET=
 CONFIG_BOOTLOADER_APP_TEST=
-CONFIG_BOOTLOADER_WDT_ENABLE=
-CONFIG_APP_ROLLBACK_ENABLE=
 
 #
 # Security features
@@ -113,7 +103,6 @@ CONFIG_STACK_CHECK_STRONG=
 CONFIG_STACK_CHECK_ALL=
 CONFIG_STACK_CHECK=
 CONFIG_WARN_WRITE_STRINGS=
-CONFIG_DISABLE_GCC8_WARNINGS=y
 
 #
 # Component config
@@ -132,19 +121,6 @@ CONFIG_AWS_IOT_SDK=
 # Bluetooth
 #
 CONFIG_BT_ENABLED=y
-
-#
-# Bluetooth controller
-#
-CONFIG_BTDM_CONTROLLER_MODE_BLE_ONLY=
-CONFIG_BTDM_CONTROLLER_MODE_BR_EDR_ONLY=
-CONFIG_BTDM_CONTROLLER_MODE_BTDM=y
-CONFIG_BTDM_CONTROLLER_BLE_MAX_CONN=3
-CONFIG_BTDM_CONTROLLER_BR_EDR_MAX_ACL_CONN=2
-CONFIG_BTDM_CONTROLLER_BR_EDR_MAX_SYNC_CONN=0
-CONFIG_BTDM_CONTROLLER_BLE_MAX_CONN_EFF=3
-CONFIG_BTDM_CONTROLLER_BR_EDR_MAX_ACL_CONN_EFF=2
-CONFIG_BTDM_CONTROLLER_BR_EDR_MAX_SYNC_CONN_EFF=0
 CONFIG_BTDM_CONTROLLER_PINNED_TO_CORE=0
 CONFIG_BTDM_CONTROLLER_HCI_MODE_VHCI=y
 CONFIG_BTDM_CONTROLLER_HCI_MODE_UART_H4=
@@ -153,26 +129,16 @@ CONFIG_BTDM_CONTROLLER_HCI_MODE_UART_H4=
 # MODEM SLEEP Options
 #
 CONFIG_BTDM_CONTROLLER_MODEM_SLEEP=
-CONFIG_BLE_SCAN_DUPLICATE=y
-CONFIG_SCAN_DUPLICATE_BY_DEVICE_ADDR=
-CONFIG_SCAN_DUPLICATE_BY_ADV_DATA=
-CONFIG_SCAN_DUPLICATE_BY_ADV_DATA_AND_DEVICE_ADDR=y
-CONFIG_SCAN_DUPLICATE_TYPE=2
-CONFIG_DUPLICATE_SCAN_CACHE_SIZE=50
-CONFIG_BLE_MESH_SCAN_DUPLICATE_EN=
 CONFIG_BLUEDROID_ENABLED=y
 CONFIG_BLUEDROID_PINNED_TO_CORE=0
 CONFIG_BTC_TASK_STACK_SIZE=3072
-CONFIG_BTU_TASK_STACK_SIZE=4096
 CONFIG_BLUEDROID_MEM_DEBUG=
 CONFIG_CLASSIC_BT_ENABLED=
 CONFIG_GATTS_ENABLE=y
-CONFIG_GATTS_SEND_SERVICE_CHANGE_MANUAL=
-CONFIG_GATTS_SEND_SERVICE_CHANGE_AUTO=y
-CONFIG_GATTS_SEND_SERVICE_CHANGE_MODE=0
 CONFIG_GATTC_ENABLE=y
 CONFIG_GATTC_CACHE_NVS_FLASH=
 CONFIG_BLE_SMP_ENABLE=y
+CONFIG_SMP_SLAVE_CON_PARAMS_UPD_ENABLE=
 CONFIG_BT_STACK_NO_LOG=
 
 #
@@ -350,9 +316,15 @@ CONFIG_BT_ACL_CONNECTIONS=4
 CONFIG_BT_ALLOCATION_FROM_SPIRAM_FIRST=
 CONFIG_BT_BLE_DYNAMIC_ENV_MEMORY=
 CONFIG_BLE_HOST_QUEUE_CONGESTION_CHECK=
+CONFIG_BLE_SCAN_DUPLICATE=y
+CONFIG_DUPLICATE_SCAN_CACHE_SIZE=50
+CONFIG_BLE_MESH_SCAN_DUPLICATE_EN=
+CONFIG_BLE_ADV_REPORT_FLOW_CONTROL_SUPPORTED=y
+CONFIG_BLE_ADV_REPORT_FLOW_CONTROL_NUM=100
+CONFIG_BLE_ADV_REPORT_DISCARD_THRSHOLD=20
 CONFIG_SMP_ENABLE=y
 CONFIG_BLE_ACTIVE_SCAN_REPORT_ADV_SCAN_RSP_INDIVIDUALLY=
-CONFIG_BT_RESERVE_DRAM=0xdb5c
+CONFIG_BT_RESERVE_DRAM=0x10000
 
 #
 # Driver configurations
@@ -365,17 +337,14 @@ CONFIG_ADC_FORCE_XPD_FSM=
 CONFIG_ADC2_DISABLE_DAC=y
 
 #
-# SPI configuration
+# SPI master configuration
 #
 CONFIG_SPI_MASTER_IN_IRAM=
 CONFIG_SPI_MASTER_ISR_IN_IRAM=y
-CONFIG_SPI_SLAVE_IN_IRAM=y
-CONFIG_SPI_SLAVE_ISR_IN_IRAM=y
 
 #
 # ESP32-specific
 #
-CONFIG_IDF_TARGET_ESP32=y
 CONFIG_ESP32_DEFAULT_CPU_FREQ_80=
 CONFIG_ESP32_DEFAULT_CPU_FREQ_160=
 CONFIG_ESP32_DEFAULT_CPU_FREQ_240=y
@@ -385,10 +354,6 @@ CONFIG_MEMMAP_TRACEMEM=
 CONFIG_MEMMAP_TRACEMEM_TWOBANKS=
 CONFIG_ESP32_TRAX=
 CONFIG_TRACEMEM_RESERVE_DRAM=0x0
-
-#
-# Core dump
-#
 CONFIG_ESP32_ENABLE_COREDUMP_TO_FLASH=
 CONFIG_ESP32_ENABLE_COREDUMP_TO_UART=
 CONFIG_ESP32_ENABLE_COREDUMP_TO_NONE=y
@@ -438,15 +403,12 @@ CONFIG_BROWNOUT_DET_LVL_SEL_5=
 CONFIG_BROWNOUT_DET_LVL_SEL_6=
 CONFIG_BROWNOUT_DET_LVL_SEL_7=
 CONFIG_BROWNOUT_DET_LVL=0
-CONFIG_REDUCE_PHY_TX_POWER=y
 CONFIG_ESP32_TIME_SYSCALL_USE_RTC_FRC1=y
 CONFIG_ESP32_TIME_SYSCALL_USE_RTC=
 CONFIG_ESP32_TIME_SYSCALL_USE_FRC1=
 CONFIG_ESP32_TIME_SYSCALL_USE_NONE=
 CONFIG_ESP32_RTC_CLOCK_SOURCE_INTERNAL_RC=y
 CONFIG_ESP32_RTC_CLOCK_SOURCE_EXTERNAL_CRYSTAL=
-CONFIG_ESP32_RTC_CLOCK_SOURCE_EXTERNAL_OSC=
-CONFIG_ESP32_RTC_CLOCK_SOURCE_INTERNAL_8MD256=
 CONFIG_ESP32_RTC_CLK_CAL_CYCLES=1024
 CONFIG_ESP32_DEEP_SLEEP_WAKEUP_DELAY=0
 CONFIG_ESP32_XTAL_FREQ_40=
@@ -457,7 +419,6 @@ CONFIG_DISABLE_BASIC_ROM_CONSOLE=
 CONFIG_ESP_TIMER_PROFILING=
 CONFIG_COMPATIBLE_PRE_V2_1_BOOTLOADERS=
 CONFIG_ESP_ERR_TO_NAME_LOOKUP=y
-CONFIG_ESP32_RTCDATA_IN_FAST_MEM=
 
 #
 # Wi-Fi
@@ -476,7 +437,8 @@ CONFIG_ESP32_WIFI_AMPDU_RX_ENABLED=y
 CONFIG_ESP32_WIFI_RX_BA_WIN=6
 CONFIG_ESP32_WIFI_NVS_ENABLED=y
 CONFIG_ESP32_WIFI_SOFTAP_BEACON_MAX_LEN=752
-CONFIG_ESP32_WIFI_DEBUG_LOG_ENABLE=
+CONFIG_ESP32_WIFI_IRAM_OPT=y
+CONFIG_ESP32_WIFI_MGMT_SBUF_NUM=32
 
 #
 # PHY
@@ -499,33 +461,16 @@ CONFIG_ADC_CAL_EFUSE_VREF_ENABLE=y
 CONFIG_ADC_CAL_LUT_ENABLE=y
 
 #
-# Event Loop Library
-#
-CONFIG_EVENT_LOOP_PROFILING=
-
-#
 # ESP HTTP client
 #
 CONFIG_ESP_HTTP_CLIENT_ENABLE_HTTPS=y
 
 #
-# HTTP Server
-#
-CONFIG_HTTPD_MAX_REQ_HDR_LEN=512
-CONFIG_HTTPD_MAX_URI_LEN=512
-
-#
-# ESP HTTPS OTA
-#
-CONFIG_OTA_ALLOW_HTTP=
-
-#
 # Ethernet
 #
 CONFIG_DMA_RX_BUF_NUM=10
 CONFIG_DMA_TX_BUF_NUM=10
 CONFIG_EMAC_L2_TO_L3_RX_BUF_MODE=
-CONFIG_EMAC_CHECK_LINK_PERIOD_MS=2000
 CONFIG_EMAC_TASK_PRIORITY=20
 CONFIG_EMAC_TASK_STACK_SIZE=3072
 
@@ -563,26 +508,9 @@ CONFIG_FATFS_TIMEOUT_MS=10000
 CONFIG_FATFS_PER_FILE_CACHE=y
 
 #
-# Modbus configuration
-#
-CONFIG_MB_QUEUE_LENGTH=20
-CONFIG_MB_SERIAL_TASK_STACK_SIZE=2048
-CONFIG_MB_SERIAL_BUF_SIZE=256
-CONFIG_MB_SERIAL_TASK_PRIO=10
-CONFIG_MB_CONTROLLER_SLAVE_ID_SUPPORT=
-CONFIG_MB_CONTROLLER_NOTIFY_TIMEOUT=20
-CONFIG_MB_CONTROLLER_NOTIFY_QUEUE_SIZE=20
-CONFIG_MB_CONTROLLER_STACK_SIZE=4096
-CONFIG_MB_EVENT_QUEUE_TIMEOUT=20
-CONFIG_MB_TIMER_PORT_ENABLED=y
-CONFIG_MB_TIMER_GROUP=0
-CONFIG_MB_TIMER_INDEX=0
-
-#
 # FreeRTOS
 #
 CONFIG_FREERTOS_UNICORE=y
-CONFIG_FREERTOS_NO_AFFINITY=0x7FFFFFFF
 CONFIG_FREERTOS_CORETIMER_0=y
 CONFIG_FREERTOS_CORETIMER_1=
 CONFIG_FREERTOS_HZ=1000
@@ -608,7 +536,6 @@ CONFIG_FREERTOS_QUEUE_REGISTRY_SIZE=0
 CONFIG_FREERTOS_USE_TRACE_FACILITY=
 CONFIG_FREERTOS_GENERATE_RUN_TIME_STATS=
 CONFIG_FREERTOS_DEBUG_INTERNALS=
-CONFIG_FREERTOS_TASK_FUNCTION_WRAPPER=y
 
 #
 # Heap memory debugging
@@ -651,7 +578,6 @@ CONFIG_LWIP_ETHARP_TRUST_IP_MAC=
 CONFIG_ESP_GRATUITOUS_ARP=
 CONFIG_TCPIP_RECVMBOX_SIZE=32
 CONFIG_LWIP_DHCP_DOES_ARP_CHECK=y
-CONFIG_LWIP_DHCP_RESTORE_LAST_IP=y
 
 #
 # DHCP server
@@ -686,9 +612,6 @@ CONFIG_TCP_OVERSIZE_DISABLE=
 CONFIG_LWIP_MAX_UDP_PCBS=16
 CONFIG_UDP_RECVMBOX_SIZE=6
 CONFIG_TCPIP_TASK_STACK_SIZE=2560
-CONFIG_TCPIP_TASK_AFFINITY_NO_AFFINITY=y
-CONFIG_TCPIP_TASK_AFFINITY_CPU0=
-CONFIG_TCPIP_TASK_AFFINITY=0x7FFFFFFF
 CONFIG_PPP_SUPPORT=
 
 #
@@ -705,11 +628,7 @@ CONFIG_LWIP_MAX_RAW_PCBS=16
 #
 # mbedTLS
 #
-CONFIG_MBEDTLS_INTERNAL_MEM_ALLOC=y
-CONFIG_MBEDTLS_DEFAULT_MEM_ALLOC=
-CONFIG_MBEDTLS_CUSTOM_MEM_ALLOC=
 CONFIG_MBEDTLS_SSL_MAX_CONTENT_LEN=16384
-CONFIG_MBEDTLS_ASYMMETRIC_CONTENT_LEN=
 CONFIG_MBEDTLS_DEBUG=
 CONFIG_MBEDTLS_HARDWARE_AES=y
 CONFIG_MBEDTLS_HARDWARE_MPI=y
@@ -785,24 +704,9 @@ CONFIG_MBEDTLS_ECP_DP_CURVE25519_ENABLED=y
 CONFIG_MBEDTLS_ECP_NIST_OPTIM=y
 
 #
-# mDNS
-#
-CONFIG_MDNS_MAX_SERVICES=10
-
-#
-# ESP-MQTT Configurations
-#
-CONFIG_MQTT_PROTOCOL_311=y
-CONFIG_MQTT_TRANSPORT_SSL=y
-CONFIG_MQTT_TRANSPORT_WEBSOCKET=y
-CONFIG_MQTT_TRANSPORT_WEBSOCKET_SECURE=y
-CONFIG_MQTT_USE_CUSTOM_CONFIG=
-CONFIG_MQTT_TASK_CORE_SELECTION_ENABLED=
-CONFIG_MQTT_CUSTOM_OUTBOX=
-
-#
 # NVS
 #
+CONFIG_MP_BLOB_SUPPORT=
 
 #
 # OpenSSL
@@ -816,9 +720,6 @@ CONFIG_OPENSSL_ASSERT_EXIT=
 #
 CONFIG_ESP32_PTHREAD_TASK_PRIO_DEFAULT=5
 CONFIG_ESP32_PTHREAD_TASK_STACK_SIZE_DEFAULT=3072
-CONFIG_PTHREAD_STACK_MIN=768
-CONFIG_ESP32_PTHREAD_TASK_CORE_DEFAULT=-1
-CONFIG_ESP32_PTHREAD_TASK_NAME_DEFAULT="pthread"
 
 #
 # SPI Flash driver
@@ -826,6 +727,9 @@ CONFIG_ESP32_PTHREAD_TASK_NAME_DEFAULT="pthread"
 CONFIG_SPI_FLASH_VERIFY_WRITE=
 CONFIG_SPI_FLASH_ENABLE_COUNTERS=
 CONFIG_SPI_FLASH_ROM_DRIVER_PATCH=y
+CONFIG_SPI_FLASH_WRITING_DANGEROUS_REGIONS_ABORTS=
+CONFIG_SPI_FLASH_WRITING_DANGEROUS_REGIONS_FAILS=y
+CONFIG_SPI_FLASH_WRITING_DANGEROUS_REGIONS_ALLOWED=
 
 #
 # SPIffs Example Configuration
@@ -835,25 +739,14 @@ CONFIG_SPIFFS_BASE_ADDR=0x110000
 CONFIG_SPIFFS_SIZE=1048576
 
 #
-# TCP/IP Adapter
+# tcpip adapter
 #
 CONFIG_IP_LOST_TIMER_INTERVAL=120
-CONFIG_TCPIP_LWIP=y
-
-#
-# Unity unit testing library
-#
-CONFIG_UNITY_ENABLE_FLOAT=y
-CONFIG_UNITY_ENABLE_DOUBLE=y
-CONFIG_UNITY_ENABLE_COLOR=
-CONFIG_UNITY_ENABLE_IDF_TEST_RUNNER=y
-CONFIG_UNITY_ENABLE_FIXTURE=
 
 #
 # Virtual file system
 #
 CONFIG_SUPPRESS_SELECT_DEBUG_OUTPUT=y
-CONFIG_SUPPORT_TERMIOS=y
 
 #
 # Wear Levelling
diff --git a/src/app/esp32/targets.mk b/src/app/esp32/targets.mk
index 3439993..233e618 100644
--- a/src/app/esp32/targets.mk
+++ b/src/app/esp32/targets.mk
@@ -12,8 +12,61 @@ APPSRCS = $(wildcard $(APPPATH)/*.fth)
 
 SRC=$(TOPDIR)/src
 
+ESP_IDF_REPO = https://github.com/espressif/esp-idf.git
+ESP_IDF_VERSION ?= v3.1.1
+ESP_IDF_ARCHIVE = esp-idf-$(ESP_IDF_VERSION).zip
+ESP_IDF_URL = https://github.com/espressif/esp-idf/releases/download/$(ESP_IDF_VERSION)/$(ESP_IDF_ARCHIVE)
+
+# This file within esp-idf lists the required toolchain versions
+TOOLVERSIONS = $(IDF_PATH)/tools/toolchain_versions.mk
+
+EXTRACT_TOOLVERSIONS := grep SUPPORTED_TOOLCHAIN $(IDF_PATH)/make/project.mk \
+  | grep := \
+  | sed -e s/SUPPORTED/CURRENT/ -e s/VERSIONS/VERSION/ -e s/crosstool-ng-// -e s/DESC/DESC_SHORT/ >$(TOOLVERSIONS)
+
+$(TOOLVERSIONS):
+	@echo Getting esp-idf
+	(cd $(IDF_PARENT_PATH) \
+	&& wget $(ESP_IDF_URL) \
+	&& echo Unzipping esp-idf \
+	&& unzip -q $(ESP_IDF_ARCHIVE) \
+	&& rm $(ESP_IDF_ARCHIVE) \
+	&& python -m pip install --user -r $(IDF_PATH)/requirements.txt \
+	&& if [ ! -e $(TOOLVERSIONS) ]; then $(EXTRACT_TOOLVERSIONS); fi \
+	)
+
+.PHONY: tv
+tv: $(TOOLVERSIONS)
+
+# If the TOOLVERSIONS file is not present, make will trigger the rule to
+# make it and then restart so that its information can then be used to
+# get the right toolchain
+-include $(TOOLVERSIONS)
+
+XTGCC_VERSION = $(CURRENT_TOOLCHAIN_COMMIT_DESC_SHORT)-$(CURRENT_TOOLCHAIN_GCC_VERSION)
+
+XTGCC_ARCHIVE = xtensa-esp32-elf-linux64-$(XTGCC_VERSION).tar.gz
+XTGCC_DOWNLOAD = https://dl.espressif.com/dl/$(XTGCC_ARCHIVE)
+
+# Inside XTGCC_PARENT_PATH we have subdirectories for different
+# toolchain versions so it is easy to tell if we have the right
+# one, and if not, to automatically fetch it
+# Example subdirectory name: toolchain-1.22.0-80-g6c4433a-5.2.0/
+XTGCC_CONTAINER_PATH ?= $(XTGCC_PARENT_PATH)/toolchain-$(XTGCC_VERSION)
+
+XTGCCPATH ?= $(XTGCC_CONTAINER_PATH)/xtensa-esp32-elf/bin
+$(info path $(XTGCCPATH))
+
+$(XTGCCPATH):
+	(mkdir -p $(XTGCC_CONTAINER_PATH) \
+	&& cd $(XTGCC_CONTAINER_PATH) \
+	&& wget $(XTGCC_DOWNLOAD) \
+	&& tar xvf $(XTGCC_ARCHIVE) \
+	&& rm $(XTGCC_ARCHIVE) \
+	)
+
 # Target compiler definitions
-CROSS ?= /Volumes/case-sensitive/esp-open-sdk/xtensa-lx106-elf/bin/xtensa-lx106-elf-
+CROSS ?= $(XTGCCPATH)/xtensa-esp32-elf-
 TCC=$(CROSS)gcc
 TLD=$(CROSS)ld
 TOBJDUMP=$(CROSS)objdump
@@ -79,20 +132,5 @@ PREFIX += CBP=$(realpath $(TOPDIR)/src)
 
 include $(SRC)/cforth/embed/targets.mk
 
-ESP_IDF_REPO ?= https://github.com/espressif/esp-idf.git
-$(IDF_PATH):
-	(cd $(IDF_PARENT_PATH) \
-	&& git clone --recursive $(ESP_IDF_REPO) \
-	&& python -m pip install --user -r esp-idf/requirements.txt \
-	)
-
-XTGCC_ARCHIVE ?= xtensa-esp32-elf-linux64-1.22.0-80-g6c4433a-5.2.0.tar.gz
-XTGCC_DOWNLOAD ?= https://dl.espressif.com/dl/$(XTGCC_ARCHIVE)
-$(XTGCCPATH):
-	(cd $(XTGCC_PARENT_PATH) \
-	&& wget $(XTGCC_DOWNLOAD) \
-	&& tar xvf $(XTGCC_ARCHIVE) \
-	&& rm $(XTGCC_ARCHIVE) \
-	)
 
-$(PLAT_OBJS): $(IDF_PATH) $(XTGCCPATH)
+$(PLAT_OBJS): $(XTGCCPATH)