Merge branch 'integration' into memory_api

esphome/components/api/api_connection.cpp

@@ -205,7 +205,8 @@ void APIConnection::loop() {
     // Disconnect if not responded within 2.5*keepalive
     if (now - this->last_traffic_ > KEEPALIVE_DISCONNECT_TIMEOUT) {
       on_fatal_error();
-      ESP_LOGW(TAG, "%s (%s) is unresponsive; disconnecting", this->get_name(), this->get_peername());
+      ESP_LOGW(TAG, "%s (%s) is unresponsive; disconnecting", this->client_info_.name.c_str(),
+               this->client_info_.peername.c_str());
     }
   } else if (now - this->last_traffic_ > KEEPALIVE_TIMEOUT_MS && !this->flags_.remove) {
     // Only send ping if we're not disconnecting
@@ -255,7 +256,7 @@ bool APIConnection::send_disconnect_response(const DisconnectRequest &msg) {
   // remote initiated disconnect_client
   // don't close yet, we still need to send the disconnect response
   // close will happen on next loop
-  ESP_LOGD(TAG, "%s (%s) disconnected", this->get_name(), this->get_peername());
+  ESP_LOGD(TAG, "%s (%s) disconnected", this->client_info_.name.c_str(), this->client_info_.peername.c_str());
   this->flags_.next_close = true;
   DisconnectResponse resp;
   return this->send_message(resp, DisconnectResponse::MESSAGE_TYPE);
@@ -1385,7 +1386,7 @@ void APIConnection::complete_authentication_() {
   }
 
   this->flags_.connection_state = static_cast<uint8_t>(ConnectionState::AUTHENTICATED);
-  ESP_LOGD(TAG, "%s (%s) connected", this->get_name(), this->get_peername());
+  ESP_LOGD(TAG, "%s (%s) connected", this->client_info_.name.c_str(), this->client_info_.peername.c_str());
 #ifdef USE_API_CLIENT_CONNECTED_TRIGGER
   this->parent_->get_client_connected_trigger()->trigger(this->client_info_.name, this->client_info_.peername);
 #endif
@@ -1609,12 +1610,12 @@ bool APIConnection::send_buffer(ProtoWriteBuffer buffer, uint8_t message_type) {
 #ifdef USE_API_PASSWORD
 void APIConnection::on_unauthenticated_access() {
   this->on_fatal_error();
-  ESP_LOGD(TAG, "%s (%s) no authentication", this->get_name(), this->get_peername());
+  ESP_LOGD(TAG, "%s (%s) no authentication", this->client_info_.name.c_str(), this->client_info_.peername.c_str());
 }
 #endif
 void APIConnection::on_no_setup_connection() {
   this->on_fatal_error();
-  ESP_LOGD(TAG, "%s (%s) no connection setup", this->get_name(), this->get_peername());
+  ESP_LOGD(TAG, "%s (%s) no connection setup", this->client_info_.name.c_str(), this->client_info_.peername.c_str());
 }
 void APIConnection::on_fatal_error() {
   this->helper_->close();
@@ -1866,8 +1867,8 @@ void APIConnection::process_state_subscriptions_() {
 #endif  // USE_API_HOMEASSISTANT_STATES
 
 void APIConnection::log_warning_(const LogString *message, APIError err) {
-  ESP_LOGW(TAG, "%s (%s): %s %s errno=%d", this->get_name(), this->get_peername(), LOG_STR_ARG(message),
-           LOG_STR_ARG(api_error_to_logstr(err)), errno);
+  ESP_LOGW(TAG, "%s (%s): %s %s errno=%d", this->client_info_.name.c_str(), this->client_info_.peername.c_str(),
+           LOG_STR_ARG(message), LOG_STR_ARG(api_error_to_logstr(err)), errno);
 }
 
 void APIConnection::log_socket_operation_failed_(APIError err) {

esphome/components/api/api_connection.h

@@ -270,8 +270,8 @@ class APIConnection final : public APIServerConnection {
   bool try_to_clear_buffer(bool log_out_of_space);
   bool send_buffer(ProtoWriteBuffer buffer, uint8_t message_type) override;
 
-  const char *get_name() const { return this->client_info_.name.c_str(); }
-  const char *get_peername() const { return this->client_info_.peername.c_str(); }
+  const std::string &get_name() const { return this->client_info_.name; }
+  const std::string &get_peername() const { return this->client_info_.peername; }
 
  protected:
   // Helper function to handle authentication completion

esphome/components/api/api_server.cpp

@@ -177,7 +177,8 @@ void APIServer::loop() {
     // Network is down - disconnect all clients
     for (auto &client : this->clients_) {
       client->on_fatal_error();
-      ESP_LOGW(TAG, "%s (%s): Network down; disconnect", client->get_name(), client->get_peername());
+      ESP_LOGW(TAG, "%s (%s): Network down; disconnect", client->client_info_.name.c_str(),
+               client->client_info_.peername.c_str());
     }
     // Continue to process and clean up the clients below
   }

esphome/components/hdc1080/hdc1080.cpp

@@ -7,24 +7,20 @@ namespace hdc1080 {
 
 static const char *const TAG = "hdc1080";
 
-static const uint8_t HDC1080_ADDRESS = 0x40;  // 0b1000000 from datasheet
 static const uint8_t HDC1080_CMD_CONFIGURATION = 0x02;
 static const uint8_t HDC1080_CMD_TEMPERATURE = 0x00;
 static const uint8_t HDC1080_CMD_HUMIDITY = 0x01;
 
 void HDC1080Component::setup() {
-  const uint8_t data[2] = {
-      0b00000000,  // resolution 14bit for both humidity and temperature
-      0b00000000   // reserved
-  };
+  const uint8_t config[2] = {0x00, 0x00};  // resolution 14bit for both humidity and temperature
 
-  if (!this->write_bytes(HDC1080_CMD_CONFIGURATION, data, 2)) {
-    // as instruction is same as powerup defaults (for now), interpret as warning if this fails
-    ESP_LOGW(TAG, "HDC1080 initial config instruction error");
-    this->status_set_warning();
+  // if configuration fails - there is a problem
+  if (this->write_register(HDC1080_CMD_CONFIGURATION, config, 2) != i2c::ERROR_OK) {
+    this->mark_failed();
     return;
   }
 }
+
 void HDC1080Component::dump_config() {
   ESP_LOGCONFIG(TAG, "HDC1080:");
   LOG_I2C_DEVICE(this);
@@ -35,39 +31,51 @@ void HDC1080Component::dump_config() {
   LOG_SENSOR("  ", "Temperature", this->temperature_);
   LOG_SENSOR("  ", "Humidity", this->humidity_);
 }
+
 void HDC1080Component::update() {
-  uint16_t raw_temp;
+  // regardless of what sensor/s are defined in yaml configuration
+  // the hdc1080 setup configuration used, requires both temperature and humidity to be read
+
+  this->status_clear_warning();
+
   if (this->write(&HDC1080_CMD_TEMPERATURE, 1) != i2c::ERROR_OK) {
     this->status_set_warning();
     return;
   }
-  delay(20);
-  if (this->read(reinterpret_cast<uint8_t *>(&raw_temp), 2) != i2c::ERROR_OK) {
-    this->status_set_warning();
-    return;
-  }
-  raw_temp = i2c::i2ctohs(raw_temp);
-  float temp = raw_temp * 0.0025177f - 40.0f;  // raw * 2^-16 * 165 - 40
-  this->temperature_->publish_state(temp);
 
-  uint16_t raw_humidity;
-  if (this->write(&HDC1080_CMD_HUMIDITY, 1) != i2c::ERROR_OK) {
-    this->status_set_warning();
-    return;
-  }
-  delay(20);
-  if (this->read(reinterpret_cast<uint8_t *>(&raw_humidity), 2) != i2c::ERROR_OK) {
-    this->status_set_warning();
-    return;
-  }
-  raw_humidity = i2c::i2ctohs(raw_humidity);
-  float humidity = raw_humidity * 0.001525879f;  // raw * 2^-16 * 100
-  this->humidity_->publish_state(humidity);
+  this->set_timeout(20, [this]() {
+    uint16_t raw_temperature;
+    if (this->read(reinterpret_cast<uint8_t *>(&raw_temperature), 2) != i2c::ERROR_OK) {
+      this->status_set_warning();
+      return;
+    }
 
-  ESP_LOGD(TAG, "Got temperature=%.1f°C humidity=%.1f%%", temp, humidity);
-  this->status_clear_warning();
+    if (this->temperature_ != nullptr) {
+      raw_temperature = i2c::i2ctohs(raw_temperature);
+      float temperature = raw_temperature * 0.0025177f - 40.0f;  // raw * 2^-16 * 165 - 40
+      this->temperature_->publish_state(temperature);
+    }
+
+    if (this->write(&HDC1080_CMD_HUMIDITY, 1) != i2c::ERROR_OK) {
+      this->status_set_warning();
+      return;
+    }
+
+    this->set_timeout(20, [this]() {
+      uint16_t raw_humidity;
+      if (this->read(reinterpret_cast<uint8_t *>(&raw_humidity), 2) != i2c::ERROR_OK) {
+        this->status_set_warning();
+        return;
+      }
+
+      if (this->humidity_ != nullptr) {
+        raw_humidity = i2c::i2ctohs(raw_humidity);
+        float humidity = raw_humidity * 0.001525879f;  // raw * 2^-16 * 100
+        this->humidity_->publish_state(humidity);
+      }
+    });
+  });
 }
-float HDC1080Component::get_setup_priority() const { return setup_priority::DATA; }
 
 }  // namespace hdc1080
 }  // namespace esphome

esphome/components/hdc1080/hdc1080.h

@@ -12,13 +12,11 @@ class HDC1080Component : public PollingComponent, public i2c::I2CDevice {
   void set_temperature(sensor::Sensor *temperature) { temperature_ = temperature; }
   void set_humidity(sensor::Sensor *humidity) { humidity_ = humidity; }
 
-  /// Setup the sensor and check for connection.
   void setup() override;
   void dump_config() override;
-  /// Retrieve the latest sensor values. This operation takes approximately 16ms.
   void update() override;
 
-  float get_setup_priority() const override;
+  float get_setup_priority() const override { return setup_priority::DATA; }
 
  protected:
   sensor::Sensor *temperature_{nullptr};

esphome/components/psram/__init__.py

@@ -62,6 +62,11 @@ SPIRAM_SPEEDS = {
 }
 
 
+def supported() -> bool:
+    variant = get_esp32_variant()
+    return variant in SPIRAM_MODES
+
+
 def validate_psram_mode(config):
     esp32_config = fv.full_config.get()[PLATFORM_ESP32]
     if config[CONF_SPEED] == "120MHZ":
@@ -95,7 +100,7 @@ def get_config_schema(config):
     variant = get_esp32_variant()
     speeds = [f"{s}MHZ" for s in SPIRAM_SPEEDS.get(variant, [])]
     if not speeds:
-        return cv.Invalid("PSRAM is not supported on this chip")
+        raise cv.Invalid("PSRAM is not supported on this chip")
     modes = SPIRAM_MODES[variant]
     return cv.Schema(
         {

esphome/components/remote_transmitter/remote_transmitter.cpp

@@ -40,7 +40,13 @@ void RemoteTransmitterComponent::await_target_time_() {
   if (this->target_time_ == 0) {
     this->target_time_ = current_time;
   } else if ((int32_t) (this->target_time_ - current_time) > 0) {
+#if defined(USE_LIBRETINY)
+    // busy loop for libretiny is required (see the comment inside micros() in wiring.c)
+    while ((int32_t) (this->target_time_ - micros()) > 0)
+      ;
+#else
     delayMicroseconds(this->target_time_ - current_time);
+#endif
   }
 }
 

esphome/components/substitutions/jinja.py

@@ -1,9 +1,10 @@
+from ast import literal_eval
 import logging
 import math
 import re
 
 import jinja2 as jinja
-from jinja2.nativetypes import NativeEnvironment
+from jinja2.sandbox import SandboxedEnvironment
 
 TemplateError = jinja.TemplateError
 TemplateSyntaxError = jinja.TemplateSyntaxError
@@ -70,7 +71,7 @@ class Jinja:
     """
 
     def __init__(self, context_vars):
-        self.env = NativeEnvironment(
+        self.env = SandboxedEnvironment(
             trim_blocks=True,
             lstrip_blocks=True,
             block_start_string="<%",
@@ -90,6 +91,15 @@ class Jinja:
             **SAFE_GLOBAL_FUNCTIONS,
         }
 
+    def safe_eval(self, expr):
+        try:
+            result = literal_eval(expr)
+            if not isinstance(result, str):
+                return result
+        except (ValueError, SyntaxError, MemoryError, TypeError):
+            pass
+        return expr
+
     def expand(self, content_str):
         """
         Renders a string that may contain Jinja expressions or statements
@@ -106,7 +116,7 @@ class Jinja:
             override_vars = content_str.upvalues
         try:
             template = self.env.from_string(content_str)
-            result = template.render(override_vars)
+            result = self.safe_eval(template.render(override_vars))
             if isinstance(result, Undefined):
                 # This happens when the expression is simply an undefined variable. Jinja does not
                 # raise an exception, instead we get "Undefined".

esphome/components/voice_assistant/voice_assistant.cpp

@@ -429,8 +429,9 @@ void VoiceAssistant::client_subscription(api::APIConnection *client, bool subscr
 
   if (this->api_client_ != nullptr) {
     ESP_LOGE(TAG, "Multiple API Clients attempting to connect to Voice Assistant");
-    ESP_LOGE(TAG, "Current client: %s (%s)", this->api_client_->get_name(), this->api_client_->get_peername());
-    ESP_LOGE(TAG, "New client: %s (%s)", client->get_name(), client->get_peername());
+    ESP_LOGE(TAG, "Current client: %s (%s)", this->api_client_->get_name().c_str(),
+             this->api_client_->get_peername().c_str());
+    ESP_LOGE(TAG, "New client: %s (%s)", client->get_name().c_str(), client->get_peername().c_str());
     return;
   }
 

esphome/loader.py

@@ -192,7 +192,7 @@ def install_custom_components_meta_finder():
     install_meta_finder(custom_components_dir)
 
 
-def _lookup_module(domain, exception):
+def _lookup_module(domain: str, exception: bool) -> ComponentManifest | None:
     if domain in _COMPONENT_CACHE:
         return _COMPONENT_CACHE[domain]
 
@@ -219,16 +219,16 @@ def _lookup_module(domain, exception):
     return manif
 
 
-def get_component(domain, exception=False):
+def get_component(domain: str, exception: bool = False) -> ComponentManifest | None:
     assert "." not in domain
     return _lookup_module(domain, exception)
 
 
-def get_platform(domain, platform):
+def get_platform(domain: str, platform: str) -> ComponentManifest | None:
     full = f"{platform}.{domain}"
     return _lookup_module(full, False)
 
 
-_COMPONENT_CACHE = {}
+_COMPONENT_CACHE: dict[str, ComponentManifest] = {}
 CORE_COMPONENTS_PATH = (Path(__file__).parent / "components").resolve()
 _COMPONENT_CACHE["esphome"] = ComponentManifest(esphome.core.config)

tests/component_tests/psram/test_psram.py

@@ -0,0 +1,194 @@
+"""Tests for PSRAM component."""
+
+from typing import Any
+
+import pytest
+
+from esphome.components.esp32.const import (
+    KEY_VARIANT,
+    VARIANT_ESP32,
+    VARIANT_ESP32C2,
+    VARIANT_ESP32C3,
+    VARIANT_ESP32C5,
+    VARIANT_ESP32C6,
+    VARIANT_ESP32H2,
+    VARIANT_ESP32P4,
+    VARIANT_ESP32S2,
+    VARIANT_ESP32S3,
+)
+import esphome.config_validation as cv
+from esphome.const import CONF_ESPHOME, PlatformFramework
+from tests.component_tests.types import SetCoreConfigCallable
+
+UNSUPPORTED_PSRAM_VARIANTS = [
+    VARIANT_ESP32C2,
+    VARIANT_ESP32C3,
+    VARIANT_ESP32C5,
+    VARIANT_ESP32C6,
+    VARIANT_ESP32H2,
+]
+
+SUPPORTED_PSRAM_VARIANTS = [
+    VARIANT_ESP32,
+    VARIANT_ESP32S2,
+    VARIANT_ESP32S3,
+    VARIANT_ESP32P4,
+]
+
+
+@pytest.mark.parametrize(
+    ("config", "error_match"),
+    [
+        pytest.param(
+            {},
+            r"PSRAM is not supported on this chip",
+            id="psram_not_supported",
+        ),
+    ],
+)
+@pytest.mark.parametrize("variant", UNSUPPORTED_PSRAM_VARIANTS)
+def test_psram_configuration_errors_unsupported_variants(
+    config: Any,
+    error_match: str,
+    variant: str,
+    set_core_config: SetCoreConfigCallable,
+) -> None:
+    set_core_config(
+        PlatformFramework.ESP32_IDF,
+        platform_data={KEY_VARIANT: variant},
+        full_config={CONF_ESPHOME: {}},
+    )
+    """Test detection of invalid PSRAM configuration on unsupported variants."""
+    from esphome.components.psram import CONFIG_SCHEMA
+
+    with pytest.raises(cv.Invalid, match=error_match):
+        CONFIG_SCHEMA(config)
+
+
+@pytest.mark.parametrize("variant", SUPPORTED_PSRAM_VARIANTS)
+def test_psram_configuration_valid_supported_variants(
+    variant: str,
+    set_core_config: SetCoreConfigCallable,
+) -> None:
+    set_core_config(
+        PlatformFramework.ESP32_IDF,
+        platform_data={KEY_VARIANT: variant},
+        full_config={
+            CONF_ESPHOME: {},
+            "esp32": {
+                "variant": variant,
+                "cpu_frequency": "160MHz",
+                "framework": {"type": "esp-idf"},
+            },
+        },
+    )
+    """Test that PSRAM configuration is valid on supported variants."""
+    from esphome.components.psram import CONFIG_SCHEMA, FINAL_VALIDATE_SCHEMA
+
+    # This should not raise an exception
+    config = CONFIG_SCHEMA({})
+    FINAL_VALIDATE_SCHEMA(config)
+
+
+def _setup_psram_final_validation_test(
+    esp32_config: dict,
+    set_core_config: SetCoreConfigCallable,
+    set_component_config: Any,
+) -> str:
+    """Helper function to set up ESP32 configuration for PSRAM final validation tests."""
+    # Use ESP32S3 for schema validation to allow all options, then override for final validation
+    schema_variant = "ESP32S3"
+    final_variant = esp32_config.get("variant", "ESP32S3")
+    full_esp32_config = {
+        "variant": final_variant,
+        "cpu_frequency": esp32_config.get("cpu_frequency", "240MHz"),
+        "framework": {"type": "esp-idf"},
+    }
+
+    set_core_config(
+        PlatformFramework.ESP32_IDF,
+        platform_data={KEY_VARIANT: schema_variant},
+        full_config={
+            CONF_ESPHOME: {},
+            "esp32": full_esp32_config,
+        },
+    )
+    set_component_config("esp32", full_esp32_config)
+
+    return final_variant
+
+
+@pytest.mark.parametrize(
+    ("config", "esp32_config", "expect_error", "error_match"),
+    [
+        pytest.param(
+            {"speed": "120MHz"},
+            {"cpu_frequency": "160MHz"},
+            True,
+            r"PSRAM 120MHz requires 240MHz CPU frequency",
+            id="120mhz_requires_240mhz_cpu",
+        ),
+        pytest.param(
+            {"mode": "octal"},
+            {"variant": "ESP32"},
+            True,
+            r"Octal PSRAM is only supported on ESP32-S3",
+            id="octal_mode_only_esp32s3",
+        ),
+        pytest.param(
+            {"mode": "quad", "enable_ecc": True},
+            {},
+            True,
+            r"ECC is only available in octal mode",
+            id="ecc_only_in_octal_mode",
+        ),
+        pytest.param(
+            {"speed": "120MHZ"},
+            {"cpu_frequency": "240MHZ"},
+            False,
+            None,
+            id="120mhz_with_240mhz_cpu",
+        ),
+        pytest.param(
+            {"mode": "octal"},
+            {"variant": "ESP32S3"},
+            False,
+            None,
+            id="octal_mode_on_esp32s3",
+        ),
+        pytest.param(
+            {"mode": "octal", "enable_ecc": True},
+            {"variant": "ESP32S3"},
+            False,
+            None,
+            id="ecc_in_octal_mode",
+        ),
+    ],
+)
+def test_psram_final_validation(
+    config: Any,
+    esp32_config: dict,
+    expect_error: bool,
+    error_match: str | None,
+    set_core_config: SetCoreConfigCallable,
+    set_component_config: Any,
+) -> None:
+    """Test PSRAM final validation for both error and valid cases."""
+    from esphome.components.psram import CONFIG_SCHEMA, FINAL_VALIDATE_SCHEMA
+    from esphome.core import CORE
+
+    final_variant = _setup_psram_final_validation_test(
+        esp32_config, set_core_config, set_component_config
+    )
+
+    validated_config = CONFIG_SCHEMA(config)
+
+    # Update CORE variant for final validation
+    CORE.data["esp32"][KEY_VARIANT] = final_variant
+
+    if expect_error:
+        with pytest.raises(cv.Invalid, match=error_match):
+            FINAL_VALIDATE_SCHEMA(validated_config)
+    else:
+        # This should not raise an exception
+        FINAL_VALIDATE_SCHEMA(validated_config)

tests/unit_tests/fixtures/substitutions/00-simple_var.approved.yaml

@@ -5,6 +5,9 @@ substitutions:
   var21: '79'
   value: 33
   values: 44
+  position:
+    x: 79
+    y: 82
 
 esphome:
   name: test
@@ -26,3 +29,7 @@ test_list:
   - Literal $values ${are not substituted}
   - ["list $value", "${is not}", "${substituted}"]
   - {"$dictionary": "$value", "${is not}": "${substituted}"}
+  - |-
+    {{{ "x", "79"}, { "y", "82"}}}
+  - '{{{"AA"}}}'
+  - '"HELLO"'

tests/unit_tests/fixtures/substitutions/00-simple_var.input.yaml

@@ -8,6 +8,9 @@ substitutions:
   var21: "79"
   value: 33
   values: 44
+  position:
+    x: 79
+    y: 82
 
 test_list:
   - "$var1"
@@ -27,3 +30,7 @@ test_list:
   - !literal Literal $values ${are not substituted}
   - !literal ["list $value", "${is not}", "${substituted}"]
   - !literal {"$dictionary": "$value", "${is not}": "${substituted}"}
+  - |- # Test parsing things that look like a python set of sets when rendered:
+    {{{ "x", "${ position.x }"}, { "y", "${ position.y }"}}}
+  - ${ '{{{"AA"}}}' }
+  - ${ '"HELLO"' }