Merge pull request #5994 from esphome/bump-2023.12.3

2023.12.3

docker/Dockerfile

@@ -50,7 +50,7 @@ RUN \
           libssl-dev=3.0.11-1~deb12u2 \
           libffi-dev=3.4.4-1 \
           libopenjp2-7=2.5.0-2 \
-          libtiff6=4.5.0-6 \
+          libtiff6=4.5.0-6+deb12u1 \
           cargo=0.66.0+ds1-1 \
           pkg-config=1.8.1-1 \
           gcc-arm-linux-gnueabihf=4:12.2.0-3; \

esphome/components/api/api_connection.cpp

@@ -118,7 +118,9 @@ void APIConnection::loop() {
   this->list_entities_iterator_.advance();
   this->initial_state_iterator_.advance();
 
-  const uint32_t keepalive = 60000;
+  static uint32_t keepalive = 60000;
+  static uint8_t max_ping_retries = 60;
+  static uint16_t ping_retry_interval = 1000;
   const uint32_t now = millis();
   if (this->sent_ping_) {
     // Disconnect if not responded within 2.5*keepalive
@@ -126,10 +128,24 @@ void APIConnection::loop() {
       on_fatal_error();
       ESP_LOGW(TAG, "%s didn't respond to ping request in time. Disconnecting...", this->client_combined_info_.c_str());
     }
-  } else if (now - this->last_traffic_ > keepalive) {
+  } else if (now - this->last_traffic_ > keepalive && now > this->next_ping_retry_) {
     ESP_LOGVV(TAG, "Sending keepalive PING...");
-    this->sent_ping_ = true;
-    this->send_ping_request(PingRequest());
+    this->sent_ping_ = this->send_ping_request(PingRequest());
+    if (!this->sent_ping_) {
+      this->next_ping_retry_ = now + ping_retry_interval;
+      this->ping_retries_++;
+      if (this->ping_retries_ >= max_ping_retries) {
+        on_fatal_error();
+        ESP_LOGE(TAG, "%s: Sending keepalive failed %d time(s). Disconnecting...", this->client_combined_info_.c_str(),
+                 this->ping_retries_);
+      } else if (this->ping_retries_ >= 10) {
+        ESP_LOGW(TAG, "%s: Sending keepalive failed %d time(s), will retry in %d ms",
+                 this->client_combined_info_.c_str(), this->ping_retries_, ping_retry_interval);
+      } else {
+        ESP_LOGD(TAG, "%s: Sending keepalive failed %d time(s), will retry in %d ms",
+                 this->client_combined_info_.c_str(), this->ping_retries_, ping_retry_interval);
+      }
+    }
   }
 
 #ifdef USE_ESP32_CAMERA

esphome/components/api/api_connection.h

@@ -140,6 +140,7 @@ class APIConnection : public APIServerConnection {
   void on_disconnect_response(const DisconnectResponse &value) override;
   void on_ping_response(const PingResponse &value) override {
     // we initiated ping
+    this->ping_retries_ = 0;
     this->sent_ping_ = false;
   }
   void on_home_assistant_state_response(const HomeAssistantStateResponse &msg) override;
@@ -217,6 +218,8 @@ class APIConnection : public APIServerConnection {
   bool state_subscription_{false};
   int log_subscription_{ESPHOME_LOG_LEVEL_NONE};
   uint32_t last_traffic_;
+  uint32_t next_ping_retry_{0};
+  uint8_t ping_retries_{0};
   bool sent_ping_{false};
   bool service_call_subscription_{false};
   bool next_close_ = false;

esphome/components/api/api_pb2.cpp

@@ -3848,6 +3848,7 @@ void ListEntitiesClimateResponse::dump_to(std::string &out) const {
   sprintf(buffer, "%g", this->visual_max_humidity);
   out.append(buffer);
   out.append("\n");
+  out.append("}");
 }
 #endif
 bool ClimateStateResponse::decode_varint(uint32_t field_id, ProtoVarInt value) {
@@ -4015,6 +4016,7 @@ void ClimateStateResponse::dump_to(std::string &out) const {
   sprintf(buffer, "%g", this->target_humidity);
   out.append(buffer);
   out.append("\n");
+  out.append("}");
 }
 #endif
 bool ClimateCommandRequest::decode_varint(uint32_t field_id, ProtoVarInt value) {

esphome/components/esp32/boards.py

@@ -133,6 +133,10 @@ ESP32_BOARD_PINS = {
         "BUTTON": 0,
         "SWITCH": 0,
     },
+    "airm2m_core_esp32c3": {
+        "LED1_BUILTIN": 12,
+        "LED2_BUILTIN": 13,
+    },
     "alksesp32": {
         "A0": 32,
         "A1": 33,

esphome/components/htu21d/htu21d.cpp

@@ -39,45 +39,54 @@ void HTU21DComponent::dump_config() {
   LOG_SENSOR("  ", "Humidity", this->humidity_);
 }
 void HTU21DComponent::update() {
-  uint16_t raw_temperature;
   if (this->write(&HTU21D_REGISTER_TEMPERATURE, 1) != i2c::ERROR_OK) {
     this->status_set_warning();
     return;
   }
-  delay(50);  // NOLINT
-  if (this->read(reinterpret_cast<uint8_t *>(&raw_temperature), 2) != i2c::ERROR_OK) {
-    this->status_set_warning();
-    return;
-  }
-  raw_temperature = i2c::i2ctohs(raw_temperature);
 
-  float temperature = (float(raw_temperature & 0xFFFC)) * 175.72f / 65536.0f - 46.85f;
+  // According to the datasheet sht21 temperature readings can take up to 85ms
+  this->set_timeout(85, [this]() {
+    uint16_t raw_temperature;
+    if (this->read(reinterpret_cast<uint8_t *>(&raw_temperature), 2) != i2c::ERROR_OK) {
+      this->status_set_warning();
+      return;
+    }
+    raw_temperature = i2c::i2ctohs(raw_temperature);
 
-  uint16_t raw_humidity;
-  if (this->write(&HTU21D_REGISTER_HUMIDITY, 1) != i2c::ERROR_OK) {
-    this->status_set_warning();
-    return;
-  }
-  delay(50);  // NOLINT
-  if (this->read(reinterpret_cast<uint8_t *>(&raw_humidity), 2) != i2c::ERROR_OK) {
-    this->status_set_warning();
-    return;
-  }
-  raw_humidity = i2c::i2ctohs(raw_humidity);
+    float temperature = (float(raw_temperature & 0xFFFC)) * 175.72f / 65536.0f - 46.85f;
 
-  float humidity = (float(raw_humidity & 0xFFFC)) * 125.0f / 65536.0f - 6.0f;
+    ESP_LOGD(TAG, "Got Temperature=%.1f°C", temperature);
 
-  int8_t heater_level = this->get_heater_level();
+    if (this->temperature_ != nullptr)
+      this->temperature_->publish_state(temperature);
+    this->status_clear_warning();
 
-  ESP_LOGD(TAG, "Got Temperature=%.1f°C Humidity=%.1f%% Heater Level=%d", temperature, humidity, heater_level);
+    if (this->write(&HTU21D_REGISTER_HUMIDITY, 1) != i2c::ERROR_OK) {
+      this->status_set_warning();
+      return;
+    }
 
-  if (this->temperature_ != nullptr)
-    this->temperature_->publish_state(temperature);
-  if (this->humidity_ != nullptr)
-    this->humidity_->publish_state(humidity);
-  if (this->heater_ != nullptr)
-    this->heater_->publish_state(heater_level);
-  this->status_clear_warning();
+    this->set_timeout(50, [this]() {
+      uint16_t raw_humidity;
+      if (this->read(reinterpret_cast<uint8_t *>(&raw_humidity), 2) != i2c::ERROR_OK) {
+        this->status_set_warning();
+        return;
+      }
+      raw_humidity = i2c::i2ctohs(raw_humidity);
+
+      float humidity = (float(raw_humidity & 0xFFFC)) * 125.0f / 65536.0f - 6.0f;
+
+      int8_t heater_level = this->get_heater_level();
+
+      ESP_LOGD(TAG, "Got Humidity=%.1f%% Heater Level=%d", humidity, heater_level);
+
+      if (this->humidity_ != nullptr)
+        this->humidity_->publish_state(humidity);
+      if (this->heater_ != nullptr)
+        this->heater_->publish_state(heater_level);
+      this->status_clear_warning();
+    });
+  });
 }
 
 bool HTU21DComponent::is_heater_enabled() {

esphome/components/logger/__init__.py

@@ -172,7 +172,9 @@ CONFIG_SCHEMA = cv.All(
                 esp8266=UART0,
                 esp32=UART0,
                 esp32_s2=USB_CDC,
-                esp32_s3=USB_CDC,
+                esp32_s3_idf=USB_SERIAL_JTAG,
+                esp32_c3_idf=USB_SERIAL_JTAG,
+                esp32_s3_arduino=USB_CDC,
                 rp2040=USB_CDC,
                 bk72xx=DEFAULT,
                 rtl87xx=DEFAULT,

esphome/components/logger/logger.cpp

@@ -285,6 +285,7 @@ void Logger::pre_setup() {
 #if defined(USE_ESP32_VARIANT_ESP32S2) || defined(USE_ESP32_VARIANT_ESP32S3)
 #if ARDUINO_USB_CDC_ON_BOOT
         this->hw_serial_ = &Serial;
+        Serial.setTxTimeoutMs(0);  // workaround for 2.0.9 crash when there's no data connection
         Serial.begin(this->baud_rate_);
 #else
         this->hw_serial_ = &Serial;

esphome/config.py

@@ -315,7 +315,11 @@ class LoadValidationStep(ConfigValidationStep):
             return
         result.add_output_path([self.domain], self.domain)
         component = get_component(self.domain)
-        if component.multi_conf_no_default and isinstance(self.conf, core.AutoLoad):
+        if (
+            component is not None
+            and component.multi_conf_no_default
+            and isinstance(self.conf, core.AutoLoad)
+        ):
             self.conf = []
         result[self.domain] = self.conf
         path = [self.domain]

esphome/config_validation.py

@@ -1541,6 +1541,9 @@ class SplitDefault(Optional):
         esp32_s3=vol.UNDEFINED,
         esp32_s3_arduino=vol.UNDEFINED,
         esp32_s3_idf=vol.UNDEFINED,
+        esp32_c3=vol.UNDEFINED,
+        esp32_c3_arduino=vol.UNDEFINED,
+        esp32_c3_idf=vol.UNDEFINED,
         rp2040=vol.UNDEFINED,
         bk72xx=vol.UNDEFINED,
         rtl87xx=vol.UNDEFINED,
@@ -1549,22 +1552,28 @@ class SplitDefault(Optional):
         super().__init__(key)
         self._esp8266_default = vol.default_factory(esp8266)
         self._esp32_arduino_default = vol.default_factory(
-            _get_priority_default(esp32, esp32_arduino)
+            _get_priority_default(esp32_arduino, esp32)
         )
         self._esp32_idf_default = vol.default_factory(
-            _get_priority_default(esp32, esp32_idf)
+            _get_priority_default(esp32_idf, esp32)
         )
         self._esp32_s2_arduino_default = vol.default_factory(
-            _get_priority_default(esp32_s2, esp32, esp32_s2_arduino, esp32_arduino)
+            _get_priority_default(esp32_s2_arduino, esp32_s2, esp32_arduino, esp32)
         )
         self._esp32_s2_idf_default = vol.default_factory(
-            _get_priority_default(esp32_s2, esp32, esp32_s2_idf, esp32_idf)
+            _get_priority_default(esp32_s2_idf, esp32_s2, esp32_idf, esp32)
         )
         self._esp32_s3_arduino_default = vol.default_factory(
-            _get_priority_default(esp32_s3, esp32, esp32_s3_arduino, esp32_arduino)
+            _get_priority_default(esp32_s3_arduino, esp32_s3, esp32_arduino, esp32)
         )
         self._esp32_s3_idf_default = vol.default_factory(
-            _get_priority_default(esp32_s3, esp32, esp32_s3_idf, esp32_idf)
+            _get_priority_default(esp32_s3_idf, esp32_s3, esp32_idf, esp32)
+        )
+        self._esp32_c3_arduino_default = vol.default_factory(
+            _get_priority_default(esp32_c3_arduino, esp32_c3, esp32_arduino, esp32)
+        )
+        self._esp32_c3_idf_default = vol.default_factory(
+            _get_priority_default(esp32_c3_idf, esp32_c3, esp32_idf, esp32)
         )
         self._rp2040_default = vol.default_factory(rp2040)
         self._bk72xx_default = vol.default_factory(bk72xx)
@@ -1580,6 +1589,7 @@ class SplitDefault(Optional):
             from esphome.components.esp32.const import (
                 VARIANT_ESP32S2,
                 VARIANT_ESP32S3,
+                VARIANT_ESP32C3,
             )
 
             variant = get_esp32_variant()
@@ -1593,6 +1603,11 @@ class SplitDefault(Optional):
                     return self._esp32_s3_arduino_default
                 if CORE.using_esp_idf:
                     return self._esp32_s3_idf_default
+            elif variant == VARIANT_ESP32C3:
+                if CORE.using_arduino:
+                    return self._esp32_c3_arduino_default
+                if CORE.using_esp_idf:
+                    return self._esp32_c3_idf_default
             else:
                 if CORE.using_arduino:
                     return self._esp32_arduino_default

esphome/const.py

@@ -1,6 +1,6 @@
 """Constants used by esphome."""
 
-__version__ = "2023.12.0b6"
+__version__ = "2023.12.3"
 
 ALLOWED_NAME_CHARS = "abcdefghijklmnopqrstuvwxyz0123456789-_"
 VALID_SUBSTITUTIONS_CHARACTERS = (

esphome/dashboard/web_server.py

@@ -301,11 +301,16 @@ class EsphomePortCommandWebSocket(EsphomeCommandWebSocket):
         config_file = settings.rel_path(configuration)
         port = json_message["port"]
         if (
-            port == "OTA"
+            port == "OTA"  # pylint: disable=too-many-boolean-expressions
             and (mdns := dashboard.mdns_status)
             and (entry := entries.get(config_file))
+            and entry.loaded_integrations
+            and "api" in entry.loaded_integrations
             and (address := await mdns.async_resolve_host(entry.name))
         ):
+            # Use the IP address if available but only
+            # if the API is loaded and the device is online
+            # since MQTT logging will not work otherwise
             port = address
 
         return [
@@ -792,13 +797,22 @@ class EditRequestHandler(BaseHandler):
         """Get the content of a file."""
         loop = asyncio.get_running_loop()
         filename = settings.rel_path(configuration)
-        content = await loop.run_in_executor(None, self._read_file, filename)
-        self.write(content)
+        content = await loop.run_in_executor(
+            None, self._read_file, filename, configuration
+        )
+        if content is not None:
+            self.write(content)
 
-    def _read_file(self, filename: str) -> bytes:
+    def _read_file(self, filename: str, configuration: str) -> bytes | None:
         """Read a file and return the content as bytes."""
-        with open(file=filename, encoding="utf-8") as f:
-            return f.read()
+        try:
+            with open(file=filename, encoding="utf-8") as f:
+                return f.read()
+        except FileNotFoundError:
+            if configuration in const.SECRETS_FILES:
+                return ""
+            self.set_status(404)
+            return None
 
     def _write_file(self, filename: str, content: bytes) -> None:
         """Write a file with the given content."""

esphome/helpers.py

@@ -357,7 +357,7 @@ def snake_case(value):
     return value.replace(" ", "_").lower()
 
 
-_DISALLOWED_CHARS = re.compile(r"[^a-zA-Z0-9_]")
+_DISALLOWED_CHARS = re.compile(r"[^a-zA-Z0-9-_]")
 
 
 def sanitize(value):

tests/test8.1.yaml

@@ -0,0 +1,78 @@
+# Tests for ESP32-S3 boards - IDf
+---
+wifi:
+  ssid: "ssid"
+
+network:
+  enable_ipv6: true
+
+esp32:
+  board: esp32s3box
+  variant: ESP32S3
+  framework:
+    type: esp-idf
+
+esphome:
+  name: esp32-s3-test
+
+logger:
+
+debug:
+
+psram:
+
+spi:
+  - id: spi_id_1
+    clk_pin:
+      number:  GPIO7
+      allow_other_uses: false
+    mosi_pin: GPIO6
+    interface: any
+
+spi_device:
+  id: spidev
+  data_rate: 2MHz
+  spi_id: spi_id_1
+  mode: 3
+  bit_order: lsb_first
+
+display:
+  - platform: ili9xxx
+    id: displ8
+    model: ili9342
+    cs_pin: GPIO5
+    dc_pin: GPIO4
+    reset_pin:
+      number: GPIO48
+      allow_other_uses: true
+
+i2c:
+  scl: GPIO18
+  sda: GPIO8
+
+touchscreen:
+  - platform: tt21100
+    display: displ8
+    interrupt_pin:
+      number: GPIO3
+      ignore_strapping_warning: true
+      allow_other_uses: false
+    reset_pin:
+      number: GPIO48
+      allow_other_uses: true
+
+binary_sensor:
+  - platform: tt21100
+    name: Home Button
+    index: 1
+
+sensor:
+  - platform: debug
+    free:
+      name: "Heap Free"
+    block:
+      name: "Max Block Free"
+    loop_time:
+      name: "Loop Time"
+    psram:
+      name: "PSRAM Free"

tests/test8.2.yaml

@@ -0,0 +1,75 @@
+# Tests for ESP32-C3 boards - IDf
+---
+wifi:
+  ssid: "ssid"
+
+network:
+  enable_ipv6: true
+
+esp32:
+  board: lolin_c3_mini
+  variant: ESP32C3
+  framework:
+    type: esp-idf
+
+esphome:
+  name: esp32-c3-test
+
+logger:
+
+debug:
+
+psram:
+
+spi:
+  - id: spi_id_1
+    clk_pin:
+      number:  GPIO7
+      allow_other_uses: false
+    mosi_pin: GPIO6
+    interface: any
+
+spi_device:
+  id: spidev
+  data_rate: 2MHz
+  spi_id: spi_id_1
+  mode: 3
+  bit_order: lsb_first
+
+display:
+  - platform: ili9xxx
+    id: displ8
+    model: ili9342
+    cs_pin: GPIO5
+    dc_pin: GPIO4
+    reset_pin:
+      number: GPIO21
+
+i2c:
+  scl: GPIO18
+  sda: GPIO8
+
+touchscreen:
+  - platform: tt21100
+    display: displ8
+    interrupt_pin:
+      number: GPIO3
+      allow_other_uses: false
+    reset_pin:
+      number: GPIO20
+
+binary_sensor:
+  - platform: tt21100
+    name: Home Button
+    index: 1
+
+sensor:
+  - platform: debug
+    free:
+      name: "Heap Free"
+    block:
+      name: "Max Block Free"
+    loop_time:
+      name: "Loop Time"
+    psram:
+      name: "PSRAM Free"

tests/unit_tests/test_helpers.py

@@ -261,6 +261,7 @@ def test_snake_case(text, expected):
         ('!"§$%&/()=?foo_bar', "___________foo_bar"),
         ('foo_!"§$%&/()=?bar', "foo____________bar"),
         ('foo_bar!"§$%&/()=?', "foo_bar___________"),
+        ('foo-bar!"§$%&/()=?', "foo-bar___________"),
     ),
 )
 def test_sanitize(text, expected):