ESP-IDF support and generic target platforms (#2303)

* Socket refactor and SSL

* esp-idf temp

* Fixes

* Echo component and noise

* Add noise API transport support

* Updates

* ESP-IDF

* Complete

* Fixes

* Fixes

* Versions update

* New i2c APIs

* Complete i2c refactor

* SPI migration

* Revert ESP Preferences migration, too complex for now

* OTA support

* Remove echo again

* Remove ssl again

* GPIOFlags updates

* Rename esphal and ICACHE_RAM_ATTR

* Make ESP32 arduino compilable again

* Fix GPIO flags

* Complete pin registry refactor and fixes

* Fixes to make test1 compile

* Remove sdkconfig file

* Ignore sdkconfig file

* Fixes in reviewing

* Make test2 compile

* Make test4 compile

* Make test5 compile

* Run clang-format

* Fix lint errors

* Use esp-idf APIs instead of btStart

* Another round of fixes

* Start implementing ESP8266

* Make test3 compile

* Guard esp8266 code

* Lint

* Reformat

* Fixes

* Fixes v2

* more fixes

* ESP-IDF tidy target

* Convert ARDUINO_ARCH_ESPxx

* Update WiFiSignalSensor

* Update time ifdefs

* OTA needs millis from hal

* RestartSwitch needs delay from hal

* ESP-IDF Uart

* Fix OTA blank password

* Allow setting sdkconfig

* Fix idf partitions and allow setting sdkconfig from yaml

* Re-add read/write compat APIs and fix esp8266 uart

* Fix esp8266 store log strings in flash

* Fix ESP32 arduino preferences not initialized

* Update ifdefs

* Change how sdkconfig change is detected

* Add checks to ci-custom and fix them

* Run clang-format

* Add esp-idf clang-tidy target and fix errors

* Fixes from clang-tidy idf round 2

* Fixes from compiling tests with esp-idf

* Run clang-format

* Switch test5.yaml to esp-idf

* Implement ESP8266 Preferences

* Lint

* Re-do PIO package version selection a bit

* Fix arduinoespressif32 package version

* Fix unit tests

* Lint

* Lint fixes

* Fix readv/writev not defined

* Fix graphing component

* Re-add all old options from core/config.py

Co-authored-by: Jesse Hills <3060199+jesserockz@users.noreply.github.com>

script/ci-custom.py

@@ -408,7 +408,6 @@ ARDUINO_FORBIDDEN_RE = r"[^\w\d](" + r"|".join(ARDUINO_FORBIDDEN) + r")\(.*"
     exclude=[
         "esphome/components/mqtt/custom_mqtt_device.h",
         "esphome/components/sun/sun.cpp",
-        "esphome/core/esphal.*",
     ],
 )
 def lint_no_arduino_framework_functions(fname, match):
@@ -422,6 +421,28 @@ def lint_no_arduino_framework_functions(fname, match):
     )
 
 
+IDF_CONVERSION_FORBIDDEN = {
+    "ARDUINO_ARCH_ESP32": "USE_ESP32",
+    "ARDUINO_ARCH_ESP8266": "USE_ESP8266",
+    "pgm_read_byte": "progmem_read_byte",
+    "ICACHE_RAM_ATTR": "IRAM_ATTR",
+    "esphome/core/esphal.h": "esphome/core/hal.h",
+}
+IDF_CONVERSION_FORBIDDEN_RE = r"(" + r"|".join(IDF_CONVERSION_FORBIDDEN) + r").*"
+
+
+@lint_re_check(
+    IDF_CONVERSION_FORBIDDEN_RE,
+    include=cpp_include,
+)
+def lint_no_removed_in_idf_conversions(fname, match):
+    replacement = IDF_CONVERSION_FORBIDDEN[match.group(1)]
+    return (
+        f"The macro {highlight(match.group(1))} can no longer be used in ESPHome directly. "
+        f"Plese use {highlight(replacement)} instead."
+    )
+
+
 @lint_re_check(
     r"[^\w\d]byte\s+[\w\d]+\s*=",
     include=cpp_include,
@@ -498,6 +519,8 @@ def lint_relative_py_import(fname):
     ],
     exclude=[
         "esphome/components/socket/headers.h",
+        "esphome/components/esp32/core.cpp",
+        "esphome/components/esp8266/core.cpp",
     ],
 )
 def lint_namespace(fname, content):
@@ -575,7 +598,7 @@ def lint_inclusive_language(fname, match):
         "esphome/components/text_sensor/text_sensor.h",
         "esphome/components/climate/climate.h",
         "esphome/core/component.h",
-        "esphome/core/esphal.h",
+        "esphome/core/gpio.h",
         "esphome/core/log.h",
         "tests/custom.h",
     ],

script/clang-tidy

@@ -45,7 +45,11 @@ def clang_options(idedata):
         # pretend we're an Xtensa compiler, which gates some features in the headers
         '-D__XTENSA__',
         # allow to condition code on the presence of clang-tidy
-        '-DCLANG_TIDY'
+        '-DCLANG_TIDY',
+        # (esp-idf) Disable this header because they use asm with registers clang-tidy doesn't know
+        '-D__XTENSA_API_H__',
+        # (esp-idf) Fix __once_callable in some libstdc++ headers
+        '-D_GLIBCXX_HAVE_TLS',
     ]
 
     # copy compiler flags, except those clang doesn't understand.
@@ -57,7 +61,8 @@ def clang_options(idedata):
 
     # add include directories, using -isystem for dependencies to suppress their errors
     for directory in idedata['includes']['toolchain']:
-        cmd.extend(['-isystem', directory])
+        if 'xtensa-esp32s2-elf' not in directory:
+            cmd.extend(['-isystem', directory])
     for directory in sorted(set(idedata['includes']['build'])):
         dependency = "framework-arduino" in directory or "/libdeps/" in directory
         cmd.extend(['-isystem' if dependency else '-I', directory])

script/helpers.py

@@ -112,15 +112,36 @@ def git_ls_files(patterns=None):
     return {s[3].strip(): int(s[0]) for s in lines}
 
 
+IDF_TIDY_SDKCONFIG = """\
+CONFIG_BT_ENABLED=y
+"""
+
+
 def load_idedata(environment):
     platformio_ini = Path(root_path) / "platformio.ini"
     temp_idedata = Path(temp_folder) / f"idedata-{environment}.json"
+    changed = False
     if not platformio_ini.is_file() or not temp_idedata.is_file():
         changed = True
     elif platformio_ini.stat().st_mtime >= temp_idedata.stat().st_mtime:
         changed = True
-    else:
-        changed = False
+
+    if environment == "esp32-idf-tidy":
+        # sdkconfig needs to be written before idedata is run
+        # but the file is also modified by the build process, so
+        # store a temp file to keep track of the
+
+        sdk_internal = Path(temp_folder) / f"{environment}-internal-sdkconfig"
+        sdkconfig = Path(root_path) / f"sdkconfig.{environment}"
+        if (
+            changed
+            or not sdk_internal.is_file()
+            or sdk_internal.read_text() != IDF_TIDY_SDKCONFIG
+        ):
+            changed = True
+            sdkconfig.write_text(IDF_TIDY_SDKCONFIG)
+            sdk_internal.parent.mkdir(exist_ok=True)
+            sdk_internal.write_text(IDF_TIDY_SDKCONFIG)
 
     if not changed:
         return json.loads(temp_idedata.read_text())