ESP-IDF support and generic target platforms (#2303)

* Socket refactor and SSL

* esp-idf temp

* Fixes

* Echo component and noise

* Add noise API transport support

* Updates

* ESP-IDF

* Complete

* Fixes

* Fixes

* Versions update

* New i2c APIs

* Complete i2c refactor

* SPI migration

* Revert ESP Preferences migration, too complex for now

* OTA support

* Remove echo again

* Remove ssl again

* GPIOFlags updates

* Rename esphal and ICACHE_RAM_ATTR

* Make ESP32 arduino compilable again

* Fix GPIO flags

* Complete pin registry refactor and fixes

* Fixes to make test1 compile

* Remove sdkconfig file

* Ignore sdkconfig file

* Fixes in reviewing

* Make test2 compile

* Make test4 compile

* Make test5 compile

* Run clang-format

* Fix lint errors

* Use esp-idf APIs instead of btStart

* Another round of fixes

* Start implementing ESP8266

* Make test3 compile

* Guard esp8266 code

* Lint

* Reformat

* Fixes

* Fixes v2

* more fixes

* ESP-IDF tidy target

* Convert ARDUINO_ARCH_ESPxx

* Update WiFiSignalSensor

* Update time ifdefs

* OTA needs millis from hal

* RestartSwitch needs delay from hal

* ESP-IDF Uart

* Fix OTA blank password

* Allow setting sdkconfig

* Fix idf partitions and allow setting sdkconfig from yaml

* Re-add read/write compat APIs and fix esp8266 uart

* Fix esp8266 store log strings in flash

* Fix ESP32 arduino preferences not initialized

* Update ifdefs

* Change how sdkconfig change is detected

* Add checks to ci-custom and fix them

* Run clang-format

* Add esp-idf clang-tidy target and fix errors

* Fixes from clang-tidy idf round 2

* Fixes from compiling tests with esp-idf

* Run clang-format

* Switch test5.yaml to esp-idf

* Implement ESP8266 Preferences

* Lint

* Re-do PIO package version selection a bit

* Fix arduinoespressif32 package version

* Fix unit tests

* Lint

* Lint fixes

* Fix readv/writev not defined

* Fix graphing component

* Re-add all old options from core/config.py

Co-authored-by: Jesse Hills <3060199+jesserockz@users.noreply.github.com>

esphome/components/esp32_ble/__init__.py

@@ -1,8 +1,10 @@
 import esphome.codegen as cg
 import esphome.config_validation as cv
-from esphome.const import CONF_ID, ESP_PLATFORM_ESP32
+from esphome.const import CONF_ID
+from esphome.core import CORE
+from esphome.components.esp32 import add_idf_sdkconfig_option
 
-ESP_PLATFORMS = [ESP_PLATFORM_ESP32]
+DEPENDENCIES = ["esp32"]
 CODEOWNERS = ["@jesserockz"]
 CONFLICTS_WITH = ["esp32_ble_tracker", "esp32_ble_beacon"]
 
@@ -20,3 +22,6 @@ CONFIG_SCHEMA = cv.Schema(
 async def to_code(config):
     var = cg.new_Pvariable(config[CONF_ID])
     await cg.register_component(var, config)
+
+    if CORE.using_esp_idf:
+        add_idf_sdkconfig_option("CONFIG_BT_ENABLED", True)

esphome/components/esp32_ble/ble.cpp

@@ -1,17 +1,21 @@
-#include "ble.h"
+#ifdef USE_ESP32
 
+#include "ble.h"
 #include "esphome/core/application.h"
 #include "esphome/core/log.h"
 
-#ifdef ARDUINO_ARCH_ESP32
-
 #include <nvs_flash.h>
 #include <freertos/FreeRTOSConfig.h>
 #include <esp_bt_main.h>
 #include <esp_bt.h>
+#include <freertos/FreeRTOS.h>
 #include <freertos/task.h>
 #include <esp_gap_ble_api.h>
 
+#ifdef USE_ARDUINO
+#include <esp32-hal-bt.h>
+#endif
+
 namespace esphome {
 namespace esp32_ble {
 
@@ -52,9 +56,29 @@ bool ESP32BLE::ble_setup_() {
     return false;
   }
 
-  if (!btStart()) {
-    ESP_LOGE(TAG, "btStart failed: %d", esp_bt_controller_get_status());
-    return false;
+  if (esp_bt_controller_get_status() != ESP_BT_CONTROLLER_STATUS_ENABLED) {
+    // start bt controller
+    if (esp_bt_controller_get_status() == ESP_BT_CONTROLLER_STATUS_IDLE) {
+      esp_bt_controller_config_t cfg = BT_CONTROLLER_INIT_CONFIG_DEFAULT();
+      err = esp_bt_controller_init(&cfg);
+      if (err != ESP_OK) {
+        ESP_LOGE(TAG, "esp_bt_controller_init failed: %s", esp_err_to_name(err));
+        return false;
+      }
+      while (esp_bt_controller_get_status() == ESP_BT_CONTROLLER_STATUS_IDLE)
+        ;
+    }
+    if (esp_bt_controller_get_status() == ESP_BT_CONTROLLER_STATUS_INITED) {
+      err = esp_bt_controller_enable(ESP_BT_MODE_BLE);
+      if (err != ESP_OK) {
+        ESP_LOGE(TAG, "esp_bt_controller_enable failed: %s", esp_err_to_name(err));
+        return false;
+      }
+    }
+    if (esp_bt_controller_get_status() != ESP_BT_CONTROLLER_STATUS_ENABLED) {
+      ESP_LOGE(TAG, "esp bt controller enable failed");
+      return false;
+    }
   }
 
   esp_bt_controller_mem_release(ESP_BT_MODE_CLASSIC_BT);

esphome/components/esp32_ble/ble.h

@@ -11,7 +11,7 @@
 #include "esphome/components/esp32_ble_server/ble_server.h"
 #endif
 
-#ifdef ARDUINO_ARCH_ESP32
+#ifdef USE_ESP32
 
 #include <esp_gap_ble_api.h>
 #include <esp_gatts_api.h>

esphome/components/esp32_ble/ble_advertising.cpp

@@ -1,8 +1,11 @@
 #include "ble_advertising.h"
 
-#ifdef ARDUINO_ARCH_ESP32
+#ifdef USE_ESP32
 
 #include "ble_uuid.h"
+#include <cstring>
+#include <cstdio>
+#include "esphome/core/log.h"
 
 namespace esphome {
 namespace esp32_ble {

esphome/components/esp32_ble/ble_advertising.h

@@ -2,7 +2,7 @@
 
 #include <vector>
 
-#ifdef ARDUINO_ARCH_ESP32
+#ifdef USE_ESP32
 
 #include <esp_gap_ble_api.h>
 #include <esp_gatts_api.h>

esphome/components/esp32_ble/ble_uuid.cpp

@@ -1,6 +1,10 @@
 #include "ble_uuid.h"
 
-#ifdef ARDUINO_ARCH_ESP32
+#ifdef USE_ESP32
+
+#include <cstring>
+#include <cstdio>
+#include "esphome/core/log.h"
 
 namespace esphome {
 namespace esp32_ble {

esphome/components/esp32_ble/ble_uuid.h

@@ -1,9 +1,9 @@
 #pragma once
 
 #include "esphome/core/helpers.h"
-#include "esphome/core/esphal.h"
+#include "esphome/core/hal.h"
 
-#ifdef ARDUINO_ARCH_ESP32
+#ifdef USE_ESP32
 
 #include <string>
 #include <esp_bt_defs.h>

esphome/components/esp32_ble/queue.h

@@ -1,15 +1,19 @@
 #pragma once
+
+#ifdef USE_ESP32
+
 #include "esphome/core/component.h"
 #include "esphome/core/helpers.h"
 
 #include <queue>
 #include <mutex>
-
-#ifdef ARDUINO_ARCH_ESP32
+#include <cstring>
 
 #include <esp_gap_ble_api.h>
 #include <esp_gatts_api.h>
 #include <esp_gattc_api.h>
+#include <freertos/FreeRTOS.h>
+#include <freertos/semphr.h>
 
 /*
  * BLE events come in from a separate Task (thread) in the ESP32 stack. Rather