From 36ab68c1ea3260e1f2fe693f5287826b26d8a8ab Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Fri, 10 Oct 2025 17:31:13 -1000 Subject: [PATCH 01/90] [esp32_ble] Partial revert of #10862 - Fix GATT client notifications --- esphome/components/esp32_ble/__init__.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/esphome/components/esp32_ble/__init__.py b/esphome/components/esp32_ble/__init__.py index 15afb22ab8..caa3934707 100644 --- a/esphome/components/esp32_ble/__init__.py +++ b/esphome/components/esp32_ble/__init__.py @@ -332,12 +332,15 @@ def final_validation(config): # Check if BLE Server is needed has_ble_server = "esp32_ble_server" in full_config - add_idf_sdkconfig_option("CONFIG_BT_GATTS_ENABLE", has_ble_server) # Check if BLE Client is needed (via esp32_ble_tracker or esp32_ble_client) has_ble_client = ( "esp32_ble_tracker" in full_config or "esp32_ble_client" in full_config ) + + # ESP-IDF BLE stack requires GATT Server to be enabled when GATT Client is enabled + # This is an internal dependency in the Bluedroid stack (tested up to ESP-IDF 5.5.1) + add_idf_sdkconfig_option("CONFIG_BT_GATTS_ENABLE", has_ble_server or has_ble_client) add_idf_sdkconfig_option("CONFIG_BT_GATTC_ENABLE", has_ble_client) # Handle max_connections: check for deprecated location in esp32_ble_tracker From 3ea929eeb2893668807bbbd9f0bf4ba9f21aa218 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Fri, 10 Oct 2025 17:37:36 -1000 Subject: [PATCH 02/90] adj --- esphome/components/esp32_ble/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/esphome/components/esp32_ble/__init__.py b/esphome/components/esp32_ble/__init__.py index caa3934707..816967135d 100644 --- a/esphome/components/esp32_ble/__init__.py +++ b/esphome/components/esp32_ble/__init__.py @@ -339,7 +339,7 @@ def final_validation(config): ) # ESP-IDF BLE stack requires GATT Server to be enabled when GATT Client is enabled - # This is an internal dependency in the Bluedroid stack (tested up to ESP-IDF 5.5.1) + # This is an internal dependency in the Bluedroid stack (tested ESP-IDF 5.4.2-5.5.1) add_idf_sdkconfig_option("CONFIG_BT_GATTS_ENABLE", has_ble_server or has_ble_client) add_idf_sdkconfig_option("CONFIG_BT_GATTC_ENABLE", has_ble_client) From 632cd929ac2ca98b345a3f397328b575bdcf6b4c Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Fri, 10 Oct 2025 17:38:26 -1000 Subject: [PATCH 03/90] adj --- esphome/components/esp32_ble/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/esphome/components/esp32_ble/__init__.py b/esphome/components/esp32_ble/__init__.py index 816967135d..05ef936baf 100644 --- a/esphome/components/esp32_ble/__init__.py +++ b/esphome/components/esp32_ble/__init__.py @@ -340,6 +340,7 @@ def final_validation(config): # ESP-IDF BLE stack requires GATT Server to be enabled when GATT Client is enabled # This is an internal dependency in the Bluedroid stack (tested ESP-IDF 5.4.2-5.5.1) + # See: https://github.com/espressif/esp-idf/issues/17724 add_idf_sdkconfig_option("CONFIG_BT_GATTS_ENABLE", has_ble_server or has_ble_client) add_idf_sdkconfig_option("CONFIG_BT_GATTC_ENABLE", has_ble_client) From 8a4bd0f21c4a93b49f3ee1295f4a0ddc2ea863ed Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Fri, 10 Oct 2025 19:42:41 -1000 Subject: [PATCH 04/90] [socket] Split LWIP socket classes to reduce memory overhead on ESP8266/RP2040 --- .../components/socket/lwip_raw_tcp_impl.cpp | 190 +++++++++++------- 1 file changed, 113 insertions(+), 77 deletions(-) diff --git a/esphome/components/socket/lwip_raw_tcp_impl.cpp b/esphome/components/socket/lwip_raw_tcp_impl.cpp index 3377682474..2d23cbc5c6 100644 --- a/esphome/components/socket/lwip_raw_tcp_impl.cpp +++ b/esphome/components/socket/lwip_raw_tcp_impl.cpp @@ -40,33 +40,14 @@ class LWIPRawImpl : public Socket { void init() { LWIP_LOG("init(%p)", pcb_); tcp_arg(pcb_, this); - tcp_accept(pcb_, LWIPRawImpl::s_accept_fn); tcp_recv(pcb_, LWIPRawImpl::s_recv_fn); tcp_err(pcb_, LWIPRawImpl::s_err_fn); } std::unique_ptr accept(struct sockaddr *addr, socklen_t *addrlen) override { - if (pcb_ == nullptr) { - errno = EBADF; - return nullptr; - } - if (this->accepted_socket_count_ == 0) { - errno = EWOULDBLOCK; - return nullptr; - } - // Take from front for FIFO ordering - std::unique_ptr sock = std::move(this->accepted_sockets_[0]); - // Shift remaining sockets forward - for (uint8_t i = 1; i < this->accepted_socket_count_; i++) { - this->accepted_sockets_[i - 1] = std::move(this->accepted_sockets_[i]); - } - this->accepted_socket_count_--; - LWIP_LOG("Connection accepted by application, queue size: %d", this->accepted_socket_count_); - if (addr != nullptr) { - sock->getpeername(addr, addrlen); - } - LWIP_LOG("accept(%p)", sock.get()); - return std::unique_ptr(std::move(sock)); + // Non-listening sockets return error + errno = EINVAL; + return nullptr; } int bind(const struct sockaddr *name, socklen_t addrlen) override { if (pcb_ == nullptr) { @@ -292,25 +273,10 @@ class LWIPRawImpl : public Socket { return -1; } int listen(int backlog) override { - if (pcb_ == nullptr) { - errno = EBADF; - return -1; - } - LWIP_LOG("tcp_listen_with_backlog(%p backlog=%d)", pcb_, backlog); - struct tcp_pcb *listen_pcb = tcp_listen_with_backlog(pcb_, backlog); - if (listen_pcb == nullptr) { - tcp_abort(pcb_); - pcb_ = nullptr; - errno = EOPNOTSUPP; - return -1; - } - // tcp_listen reallocates the pcb, replace ours - pcb_ = listen_pcb; - // set callbacks on new pcb - LWIP_LOG("tcp_arg(%p)", pcb_); - tcp_arg(pcb_, this); - tcp_accept(pcb_, LWIPRawImpl::s_accept_fn); - return 0; + // Regular sockets can't be converted to listening - this shouldn't happen + // as listen() should only be called on sockets created for listening + errno = EOPNOTSUPP; + return -1; } ssize_t read(void *buf, size_t len) override { if (pcb_ == nullptr) { @@ -491,29 +457,6 @@ class LWIPRawImpl : public Socket { return 0; } - err_t accept_fn(struct tcp_pcb *newpcb, err_t err) { - LWIP_LOG("accept(newpcb=%p err=%d)", newpcb, err); - if (err != ERR_OK || newpcb == nullptr) { - // "An error code if there has been an error accepting. Only return ERR_ABRT if you have - // called tcp_abort from within the callback function!" - // https://www.nongnu.org/lwip/2_1_x/tcp_8h.html#a00517abce6856d6c82f0efebdafb734d - // nothing to do here, we just don't push it to the queue - return ERR_OK; - } - // Check if we've reached the maximum accept queue size - if (this->accepted_socket_count_ >= MAX_ACCEPTED_SOCKETS) { - LWIP_LOG("Rejecting connection, queue full (%d)", this->accepted_socket_count_); - // Abort the connection when queue is full - tcp_abort(newpcb); - // Must return ERR_ABRT since we called tcp_abort() - return ERR_ABRT; - } - auto sock = make_unique(family_, newpcb); - sock->init(); - this->accepted_sockets_[this->accepted_socket_count_++] = std::move(sock); - LWIP_LOG("Accepted connection, queue size: %d", this->accepted_socket_count_); - return ERR_OK; - } void err_fn(err_t err) { LWIP_LOG("err(err=%d)", err); // "If a connection is aborted because of an error, the application is alerted of this event by @@ -545,11 +488,6 @@ class LWIPRawImpl : public Socket { return ERR_OK; } - static err_t s_accept_fn(void *arg, struct tcp_pcb *newpcb, err_t err) { - LWIPRawImpl *arg_this = reinterpret_cast(arg); - return arg_this->accept_fn(newpcb, err); - } - static void s_err_fn(void *arg, err_t err) { LWIPRawImpl *arg_this = reinterpret_cast(arg); arg_this->err_fn(err); @@ -601,7 +539,107 @@ class LWIPRawImpl : public Socket { return -1; } + // Member ordering optimized to minimize padding on 32-bit systems + // Largest members first (4 bytes), then smaller members (1 byte each) struct tcp_pcb *pcb_; + pbuf *rx_buf_ = nullptr; + size_t rx_buf_offset_ = 0; + bool rx_closed_ = false; + // don't use lwip nodelay flag, it sometimes causes reconnect + // instead use it for determining whether to call lwip_output + bool nodelay_ = false; + sa_family_t family_ = 0; +}; + +// Listening socket class - only allocates accept queue when needed (for bind+listen sockets) +// This saves 16 bytes (12 bytes array + 1 byte count + 3 bytes padding) for regular connected sockets on ESP8266/RP2040 +class LWIPRawListenImpl : public LWIPRawImpl { + public: + LWIPRawListenImpl(sa_family_t family, struct tcp_pcb *pcb) : LWIPRawImpl(family, pcb) {} + + void init() { + LWIP_LOG("init(%p)", pcb_); + tcp_arg(pcb_, this); + tcp_accept(pcb_, LWIPRawListenImpl::s_accept_fn); + tcp_err(pcb_, LWIPRawListenImpl::s_err_fn); + } + + std::unique_ptr accept(struct sockaddr *addr, socklen_t *addrlen) override { + if (pcb_ == nullptr) { + errno = EBADF; + return nullptr; + } + if (accepted_socket_count_ == 0) { + errno = EWOULDBLOCK; + return nullptr; + } + // Take from front for FIFO ordering + std::unique_ptr sock = std::move(accepted_sockets_[0]); + // Shift remaining sockets forward + for (uint8_t i = 1; i < accepted_socket_count_; i++) { + accepted_sockets_[i - 1] = std::move(accepted_sockets_[i]); + } + accepted_socket_count_--; + LWIP_LOG("Connection accepted by application, queue size: %d", accepted_socket_count_); + if (addr != nullptr) { + sock->getpeername(addr, addrlen); + } + LWIP_LOG("accept(%p)", sock.get()); + return std::unique_ptr(std::move(sock)); + } + + int listen(int backlog) override { + if (pcb_ == nullptr) { + errno = EBADF; + return -1; + } + LWIP_LOG("tcp_listen_with_backlog(%p backlog=%d)", pcb_, backlog); + struct tcp_pcb *listen_pcb = tcp_listen_with_backlog(pcb_, backlog); + if (listen_pcb == nullptr) { + tcp_abort(pcb_); + pcb_ = nullptr; + errno = EOPNOTSUPP; + return -1; + } + // tcp_listen reallocates the pcb, replace ours + pcb_ = listen_pcb; + // set callbacks on new pcb + LWIP_LOG("tcp_arg(%p)", pcb_); + tcp_arg(pcb_, this); + tcp_accept(pcb_, LWIPRawListenImpl::s_accept_fn); + return 0; + } + + private: + err_t accept_fn(struct tcp_pcb *newpcb, err_t err) { + LWIP_LOG("accept(newpcb=%p err=%d)", newpcb, err); + if (err != ERR_OK || newpcb == nullptr) { + // "An error code if there has been an error accepting. Only return ERR_ABRT if you have + // called tcp_abort from within the callback function!" + // https://www.nongnu.org/lwip/2_1_x/tcp_8h.html#a00517abce6856d6c82f0efebdafb734d + // nothing to do here, we just don't push it to the queue + return ERR_OK; + } + // Check if we've reached the maximum accept queue size + if (accepted_socket_count_ >= MAX_ACCEPTED_SOCKETS) { + LWIP_LOG("Rejecting connection, queue full (%d)", accepted_socket_count_); + // Abort the connection when queue is full + tcp_abort(newpcb); + // Must return ERR_ABRT since we called tcp_abort() + return ERR_ABRT; + } + auto sock = make_unique(family_, newpcb); + sock->init(); + accepted_sockets_[accepted_socket_count_++] = std::move(sock); + LWIP_LOG("Accepted connection, queue size: %d", accepted_socket_count_); + return ERR_OK; + } + + static err_t s_accept_fn(void *arg, struct tcp_pcb *newpcb, err_t err) { + LWIPRawListenImpl *arg_this = reinterpret_cast(arg); + return arg_this->accept_fn(newpcb, err); + } + // Accept queue - holds incoming connections briefly until the event loop calls accept() // This is NOT a connection pool - just a temporary queue between LWIP callbacks and the main loop // 3 slots is plenty since connections are pulled out quickly by the event loop @@ -613,23 +651,21 @@ class LWIPRawImpl : public Socket { // - std::array<3>: 12 bytes fixed (3 pointers × 4 bytes) // Saves ~44+ bytes RAM per listening socket + avoids ALL heap allocations // Used on ESP8266 and RP2040 (platforms using LWIP_TCP implementation) + // + // By using a separate listening socket class, regular connected sockets save + // 16 bytes (12 bytes array + 1 byte count + 3 bytes padding) of memory overhead on 32-bit systems static constexpr size_t MAX_ACCEPTED_SOCKETS = 3; std::array, MAX_ACCEPTED_SOCKETS> accepted_sockets_; uint8_t accepted_socket_count_ = 0; // Number of sockets currently in queue - bool rx_closed_ = false; - pbuf *rx_buf_ = nullptr; - size_t rx_buf_offset_ = 0; - // don't use lwip nodelay flag, it sometimes causes reconnect - // instead use it for determining whether to call lwip_output - bool nodelay_ = false; - sa_family_t family_ = 0; }; std::unique_ptr socket(int domain, int type, int protocol) { auto *pcb = tcp_new(); if (pcb == nullptr) return nullptr; - auto *sock = new LWIPRawImpl((sa_family_t) domain, pcb); // NOLINT(cppcoreguidelines-owning-memory) + // Create listening socket implementation since user sockets typically bind+listen + // Accepted connections are created directly as LWIPRawImpl in the accept callback + auto *sock = new LWIPRawListenImpl((sa_family_t) domain, pcb); // NOLINT(cppcoreguidelines-owning-memory) sock->init(); return std::unique_ptr{sock}; } From 3f49a61b0320314023ebb182c5ce5b276841d9bf Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Fri, 10 Oct 2025 20:01:16 -1000 Subject: [PATCH 05/90] tweak --- esphome/components/socket/lwip_raw_tcp_impl.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/esphome/components/socket/lwip_raw_tcp_impl.cpp b/esphome/components/socket/lwip_raw_tcp_impl.cpp index 2d23cbc5c6..4dedeffb6a 100644 --- a/esphome/components/socket/lwip_raw_tcp_impl.cpp +++ b/esphome/components/socket/lwip_raw_tcp_impl.cpp @@ -561,7 +561,7 @@ class LWIPRawListenImpl : public LWIPRawImpl { LWIP_LOG("init(%p)", pcb_); tcp_arg(pcb_, this); tcp_accept(pcb_, LWIPRawListenImpl::s_accept_fn); - tcp_err(pcb_, LWIPRawListenImpl::s_err_fn); + tcp_err(pcb_, LWIPRawImpl::s_err_fn); // Use base class error handler } std::unique_ptr accept(struct sockaddr *addr, socklen_t *addrlen) override { From dacead836f219c3004c236cd2dc6aedea45ecd63 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Fri, 10 Oct 2025 20:59:34 -1000 Subject: [PATCH 06/90] [esp32_ble_tracker] Replace std::vector with StaticVector for listeners and clients --- .../components/esp32_ble_tracker/__init__.py | 30 +++++++++++++++++++ .../esp32_ble_tracker/esp32_ble_tracker.cpp | 30 ++++++++++++++++++- .../esp32_ble_tracker/esp32_ble_tracker.h | 10 +++++-- esphome/core/defines.h | 2 ++ 4 files changed, 69 insertions(+), 3 deletions(-) diff --git a/esphome/components/esp32_ble_tracker/__init__.py b/esphome/components/esp32_ble_tracker/__init__.py index 247496ccd9..8c7f3e3930 100644 --- a/esphome/components/esp32_ble_tracker/__init__.py +++ b/esphome/components/esp32_ble_tracker/__init__.py @@ -1,5 +1,6 @@ from __future__ import annotations +from dataclasses import dataclass import logging from esphome import automation @@ -52,9 +53,19 @@ class BLEFeatures(StrEnum): ESP_BT_DEVICE = "ESP_BT_DEVICE" +# Dataclass for registration counts +@dataclass +class RegistrationCounts: + listeners: int = 0 + clients: int = 0 + + # Set to track which features are needed by components _required_features: set[BLEFeatures] = set() +# Track registration counts for StaticVector sizing +_registration_counts = RegistrationCounts() + def register_ble_features(features: set[BLEFeatures]) -> None: """Register BLE features that a component needs. @@ -257,12 +268,14 @@ async def to_code(config): register_ble_features({BLEFeatures.ESP_BT_DEVICE}) for conf in config.get(CONF_ON_BLE_ADVERTISE, []): + _registration_counts.listeners += 1 trigger = cg.new_Pvariable(conf[CONF_TRIGGER_ID], var) if CONF_MAC_ADDRESS in conf: addr_list = [it.as_hex for it in conf[CONF_MAC_ADDRESS]] cg.add(trigger.set_addresses(addr_list)) await automation.build_automation(trigger, [(ESPBTDeviceConstRef, "x")], conf) for conf in config.get(CONF_ON_BLE_SERVICE_DATA_ADVERTISE, []): + _registration_counts.listeners += 1 trigger = cg.new_Pvariable(conf[CONF_TRIGGER_ID], var) if len(conf[CONF_SERVICE_UUID]) == len(bt_uuid16_format): cg.add(trigger.set_service_uuid16(as_hex(conf[CONF_SERVICE_UUID]))) @@ -275,6 +288,7 @@ async def to_code(config): cg.add(trigger.set_address(conf[CONF_MAC_ADDRESS].as_hex)) await automation.build_automation(trigger, [(adv_data_t_const_ref, "x")], conf) for conf in config.get(CONF_ON_BLE_MANUFACTURER_DATA_ADVERTISE, []): + _registration_counts.listeners += 1 trigger = cg.new_Pvariable(conf[CONF_TRIGGER_ID], var) if len(conf[CONF_MANUFACTURER_ID]) == len(bt_uuid16_format): cg.add(trigger.set_manufacturer_uuid16(as_hex(conf[CONF_MANUFACTURER_ID]))) @@ -287,6 +301,7 @@ async def to_code(config): cg.add(trigger.set_address(conf[CONF_MAC_ADDRESS].as_hex)) await automation.build_automation(trigger, [(adv_data_t_const_ref, "x")], conf) for conf in config.get(CONF_ON_SCAN_END, []): + _registration_counts.listeners += 1 trigger = cg.new_Pvariable(conf[CONF_TRIGGER_ID], var) await automation.build_automation(trigger, [], conf) @@ -320,6 +335,17 @@ async def _add_ble_features(): cg.add_define("USE_ESP32_BLE_DEVICE") cg.add_define("USE_ESP32_BLE_UUID") + # Add defines for StaticVector sizing based on registration counts + # Only define if count > 0 to avoid allocating unnecessary memory + if _registration_counts.listeners > 0: + cg.add_define( + "ESPHOME_ESP32_BLE_TRACKER_LISTENER_COUNT", _registration_counts.listeners + ) + if _registration_counts.clients > 0: + cg.add_define( + "ESPHOME_ESP32_BLE_TRACKER_CLIENT_COUNT", _registration_counts.clients + ) + ESP32_BLE_START_SCAN_ACTION_SCHEMA = cv.Schema( { @@ -369,6 +395,7 @@ async def register_ble_device( var: cg.SafeExpType, config: ConfigType ) -> cg.SafeExpType: register_ble_features({BLEFeatures.ESP_BT_DEVICE}) + _registration_counts.listeners += 1 paren = await cg.get_variable(config[CONF_ESP32_BLE_ID]) cg.add(paren.register_listener(var)) return var @@ -376,6 +403,7 @@ async def register_ble_device( async def register_client(var: cg.SafeExpType, config: ConfigType) -> cg.SafeExpType: register_ble_features({BLEFeatures.ESP_BT_DEVICE}) + _registration_counts.clients += 1 paren = await cg.get_variable(config[CONF_ESP32_BLE_ID]) cg.add(paren.register_client(var)) return var @@ -389,6 +417,7 @@ async def register_raw_ble_device( This does NOT register the ESP_BT_DEVICE feature, meaning ESPBTDevice will not be compiled in if this is the only registration method used. """ + _registration_counts.listeners += 1 paren = await cg.get_variable(config[CONF_ESP32_BLE_ID]) cg.add(paren.register_listener(var)) return var @@ -402,6 +431,7 @@ async def register_raw_client( This does NOT register the ESP_BT_DEVICE feature, meaning ESPBTDevice will not be compiled in if this is the only registration method used. """ + _registration_counts.clients += 1 paren = await cg.get_variable(config[CONF_ESP32_BLE_ID]) cg.add(paren.register_client(var)) return var diff --git a/esphome/components/esp32_ble_tracker/esp32_ble_tracker.cpp b/esphome/components/esp32_ble_tracker/esp32_ble_tracker.cpp index 83f59d492e..d07e67825b 100644 --- a/esphome/components/esp32_ble_tracker/esp32_ble_tracker.cpp +++ b/esphome/components/esp32_ble_tracker/esp32_ble_tracker.cpp @@ -74,9 +74,11 @@ void ESP32BLETracker::setup() { [this](ota::OTAState state, float progress, uint8_t error, ota::OTAComponent *comp) { if (state == ota::OTA_STARTED) { this->stop_scan(); +#ifdef ESPHOME_ESP32_BLE_TRACKER_CLIENT_COUNT for (auto *client : this->clients_) { client->disconnect(); } +#endif } }); #endif @@ -206,8 +208,10 @@ void ESP32BLETracker::start_scan_(bool first) { this->set_scanner_state_(ScannerState::STARTING); ESP_LOGD(TAG, "Starting scan, set scanner state to STARTING."); if (!first) { +#ifdef ESPHOME_ESP32_BLE_TRACKER_LISTENER_COUNT for (auto *listener : this->listeners_) listener->on_scan_end(); +#endif } #ifdef USE_ESP32_BLE_DEVICE this->already_discovered_.clear(); @@ -236,20 +240,25 @@ void ESP32BLETracker::start_scan_(bool first) { } void ESP32BLETracker::register_client(ESPBTClient *client) { +#ifdef ESPHOME_ESP32_BLE_TRACKER_CLIENT_COUNT client->app_id = ++this->app_id_; this->clients_.push_back(client); this->recalculate_advertisement_parser_types(); +#endif } void ESP32BLETracker::register_listener(ESPBTDeviceListener *listener) { +#ifdef ESPHOME_ESP32_BLE_TRACKER_LISTENER_COUNT listener->set_parent(this); this->listeners_.push_back(listener); this->recalculate_advertisement_parser_types(); +#endif } void ESP32BLETracker::recalculate_advertisement_parser_types() { this->raw_advertisements_ = false; this->parse_advertisements_ = false; +#ifdef ESPHOME_ESP32_BLE_TRACKER_LISTENER_COUNT for (auto *listener : this->listeners_) { if (listener->get_advertisement_parser_type() == AdvertisementParserType::PARSED_ADVERTISEMENTS) { this->parse_advertisements_ = true; @@ -257,6 +266,8 @@ void ESP32BLETracker::recalculate_advertisement_parser_types() { this->raw_advertisements_ = true; } } +#endif +#ifdef ESPHOME_ESP32_BLE_TRACKER_CLIENT_COUNT for (auto *client : this->clients_) { if (client->get_advertisement_parser_type() == AdvertisementParserType::PARSED_ADVERTISEMENTS) { this->parse_advertisements_ = true; @@ -264,6 +275,7 @@ void ESP32BLETracker::recalculate_advertisement_parser_types() { this->raw_advertisements_ = true; } } +#endif } void ESP32BLETracker::gap_event_handler(esp_gap_ble_cb_event_t event, esp_ble_gap_cb_param_t *param) { @@ -282,10 +294,12 @@ void ESP32BLETracker::gap_event_handler(esp_gap_ble_cb_event_t event, esp_ble_ga default: break; } - // Forward all events to clients (scan results are handled separately via gap_scan_event_handler) + // Forward all events to clients (scan results are handled separately via gap_scan_event_handler) +#ifdef ESPHOME_ESP32_BLE_TRACKER_CLIENT_COUNT for (auto *client : this->clients_) { client->gap_event_handler(event, param); } +#endif } void ESP32BLETracker::gap_scan_event_handler(const BLEScanResult &scan_result) { @@ -348,9 +362,11 @@ void ESP32BLETracker::gap_scan_stop_complete_(const esp_ble_gap_cb_param_t::ble_ void ESP32BLETracker::gattc_event_handler(esp_gattc_cb_event_t event, esp_gatt_if_t gattc_if, esp_ble_gattc_cb_param_t *param) { +#ifdef ESPHOME_ESP32_BLE_TRACKER_CLIENT_COUNT for (auto *client : this->clients_) { client->gattc_event_handler(event, gattc_if, param); } +#endif } void ESP32BLETracker::set_scanner_state_(ScannerState state) { @@ -704,12 +720,16 @@ bool ESPBTDevice::resolve_irk(const uint8_t *irk) const { void ESP32BLETracker::process_scan_result_(const BLEScanResult &scan_result) { // Process raw advertisements if (this->raw_advertisements_) { +#ifdef ESPHOME_ESP32_BLE_TRACKER_LISTENER_COUNT for (auto *listener : this->listeners_) { listener->parse_devices(&scan_result, 1); } +#endif +#ifdef ESPHOME_ESP32_BLE_TRACKER_CLIENT_COUNT for (auto *client : this->clients_) { client->parse_devices(&scan_result, 1); } +#endif } // Process parsed advertisements @@ -719,16 +739,20 @@ void ESP32BLETracker::process_scan_result_(const BLEScanResult &scan_result) { device.parse_scan_rst(scan_result); bool found = false; +#ifdef ESPHOME_ESP32_BLE_TRACKER_LISTENER_COUNT for (auto *listener : this->listeners_) { if (listener->parse_device(device)) found = true; } +#endif +#ifdef ESPHOME_ESP32_BLE_TRACKER_CLIENT_COUNT for (auto *client : this->clients_) { if (client->parse_device(device)) { found = true; } } +#endif if (!found && !this->scan_continuous_) { this->print_bt_device_info(device); @@ -745,8 +769,10 @@ void ESP32BLETracker::cleanup_scan_state_(bool is_stop_complete) { // Reset timeout state machine instead of cancelling scheduler timeout this->scan_timeout_state_ = ScanTimeoutState::INACTIVE; +#ifdef ESPHOME_ESP32_BLE_TRACKER_LISTENER_COUNT for (auto *listener : this->listeners_) listener->on_scan_end(); +#endif this->set_scanner_state_(ScannerState::IDLE); } @@ -770,6 +796,7 @@ void ESP32BLETracker::handle_scanner_failure_() { void ESP32BLETracker::try_promote_discovered_clients_() { // Only promote the first discovered client to avoid multiple simultaneous connections +#ifdef ESPHOME_ESP32_BLE_TRACKER_CLIENT_COUNT for (auto *client : this->clients_) { if (client->state() != ClientState::DISCOVERED) { continue; @@ -791,6 +818,7 @@ void ESP32BLETracker::try_promote_discovered_clients_() { client->connect(); break; } +#endif } const char *ESP32BLETracker::scanner_state_to_string_(ScannerState state) const { diff --git a/esphome/components/esp32_ble_tracker/esp32_ble_tracker.h b/esphome/components/esp32_ble_tracker/esp32_ble_tracker.h index e53c2ac097..f80f3e2670 100644 --- a/esphome/components/esp32_ble_tracker/esp32_ble_tracker.h +++ b/esphome/components/esp32_ble_tracker/esp32_ble_tracker.h @@ -302,6 +302,7 @@ class ESP32BLETracker : public Component, /// Count clients in each state ClientStateCounts count_client_states_() const { ClientStateCounts counts; +#ifdef ESPHOME_ESP32_BLE_TRACKER_CLIENT_COUNT for (auto *client : this->clients_) { switch (client->state()) { case ClientState::DISCONNECTING: @@ -317,12 +318,17 @@ class ESP32BLETracker : public Component, break; } } +#endif return counts; } // Group 1: Large objects (12+ bytes) - vectors and callback manager - std::vector listeners_; - std::vector clients_; +#ifdef ESPHOME_ESP32_BLE_TRACKER_LISTENER_COUNT + StaticVector listeners_; +#endif +#ifdef ESPHOME_ESP32_BLE_TRACKER_CLIENT_COUNT + StaticVector clients_; +#endif CallbackManager scanner_state_callbacks_; #ifdef USE_ESP32_BLE_DEVICE /// Vector of addresses that have already been printed in print_bt_device_info diff --git a/esphome/core/defines.h b/esphome/core/defines.h index 0f1d1bcf28..955d0f987c 100644 --- a/esphome/core/defines.h +++ b/esphome/core/defines.h @@ -175,6 +175,8 @@ #define USE_ESP32_BLE_SERVER_DESCRIPTOR_ON_WRITE #define USE_ESP32_BLE_SERVER_ON_CONNECT #define USE_ESP32_BLE_SERVER_ON_DISCONNECT +#define ESPHOME_ESP32_BLE_TRACKER_LISTENER_COUNT 1 +#define ESPHOME_ESP32_BLE_TRACKER_CLIENT_COUNT 1 #define USE_ESP32_CAMERA_JPEG_ENCODER #define USE_I2C #define USE_IMPROV From 460c41d9b880644efbb97000b44ba808cac0988c Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Sat, 11 Oct 2025 05:53:14 -1000 Subject: [PATCH 07/90] [usb_host] Fix transfer slot exhaustion at high data rates and add configurable max_transfer_requests --- esphome/components/usb_host/__init__.py | 8 +++ esphome/components/usb_host/usb_host.h | 37 ++++++------ .../components/usb_host/usb_host_client.cpp | 56 ++++++------------- esphome/core/defines.h | 1 + 4 files changed, 45 insertions(+), 57 deletions(-) diff --git a/esphome/components/usb_host/__init__.py b/esphome/components/usb_host/__init__.py index de734bf425..42c893d296 100644 --- a/esphome/components/usb_host/__init__.py +++ b/esphome/components/usb_host/__init__.py @@ -20,6 +20,7 @@ USBClient = usb_host_ns.class_("USBClient", Component) CONF_VID = "vid" CONF_PID = "pid" CONF_ENABLE_HUBS = "enable_hubs" +CONF_MAX_TRANSFER_REQUESTS = "max_transfer_requests" def usb_device_schema(cls=USBClient, vid: int = None, pid: [int] = None) -> cv.Schema: @@ -44,6 +45,9 @@ CONFIG_SCHEMA = cv.All( { cv.GenerateID(): cv.declare_id(USBHost), cv.Optional(CONF_ENABLE_HUBS, default=False): cv.boolean, + cv.Optional(CONF_MAX_TRANSFER_REQUESTS, default=16): cv.int_range( + min=1, max=32 + ), cv.Optional(CONF_DEVICES): cv.ensure_list(usb_device_schema()), } ), @@ -62,6 +66,10 @@ async def to_code(config): add_idf_sdkconfig_option("CONFIG_USB_HOST_CONTROL_TRANSFER_MAX_SIZE", 1024) if config.get(CONF_ENABLE_HUBS): add_idf_sdkconfig_option("CONFIG_USB_HOST_HUBS_SUPPORTED", True) + + max_requests = config[CONF_MAX_TRANSFER_REQUESTS] + cg.add_define("USB_HOST_MAX_REQUESTS", max_requests) + var = cg.new_Pvariable(config[CONF_ID]) await cg.register_component(var, config) for device in config.get(CONF_DEVICES) or (): diff --git a/esphome/components/usb_host/usb_host.h b/esphome/components/usb_host/usb_host.h index 4f8d2ec9a8..3ccc49a5a0 100644 --- a/esphome/components/usb_host/usb_host.h +++ b/esphome/components/usb_host/usb_host.h @@ -16,23 +16,25 @@ namespace usb_host { // THREADING MODEL: // This component uses a dedicated USB task for event processing to prevent data loss. -// - USB Task (high priority): Handles USB events, executes transfer callbacks -// - Main Loop Task: Initiates transfers, processes completion events +// - USB Task (high priority): Handles USB events, executes transfer callbacks, releases transfer slots +// - Main Loop Task: Initiates transfers, processes device connect/disconnect events // // Thread-safe communication: // - Lock-free queues for USB task -> main loop events (SPSC pattern) -// - Lock-free TransferRequest pool using atomic bitmask (MCSP pattern) +// - Lock-free TransferRequest pool using atomic bitmask (MCMP pattern - multi-consumer, multi-producer) // // TransferRequest pool access pattern: // - get_trq_() [allocate]: Called from BOTH USB task and main loop threads // * USB task: via USB UART input callbacks that restart transfers immediately // * Main loop: for output transfers and flow-controlled input restarts -// - release_trq() [deallocate]: Called from main loop thread only +// - release_trq() [deallocate]: Called from BOTH USB task and main loop threads +// * USB task: immediately after transfer callback completes (critical for preventing slot exhaustion) +// * Main loop: when transfer submission fails // -// The multi-threaded allocation is intentional for performance: -// - USB task can immediately restart input transfers without context switching +// The multi-threaded allocation/deallocation is intentional for performance: +// - USB task can immediately restart input transfers and release slots without context switching // - Main loop controls backpressure by deciding when to restart after consuming data -// The atomic bitmask ensures thread-safe allocation without mutex blocking. +// The atomic bitmask ensures thread-safe allocation/deallocation without mutex blocking. static const char *const TAG = "usb_host"; @@ -52,8 +54,13 @@ static const uint8_t USB_DIR_IN = 1 << 7; static const uint8_t USB_DIR_OUT = 0; static const size_t SETUP_PACKET_SIZE = 8; -static const size_t MAX_REQUESTS = 16; // maximum number of outstanding requests possible. -static_assert(MAX_REQUESTS <= 16, "MAX_REQUESTS must be <= 16 to fit in uint16_t bitmask"); +static const size_t MAX_REQUESTS = USB_HOST_MAX_REQUESTS; // maximum number of outstanding requests possible. +static_assert(MAX_REQUESTS >= 1 && MAX_REQUESTS <= 32, "MAX_REQUESTS must be between 1 and 32"); + +// Select appropriate bitmask type based on MAX_REQUESTS +// uint16_t for <= 16 requests, uint32_t for 17-32 requests +using trq_bitmask_t = std::conditional<(MAX_REQUESTS <= 16), uint16_t, uint32_t>::type; + static constexpr size_t USB_EVENT_QUEUE_SIZE = 32; // Size of event queue between USB task and main loop static constexpr size_t USB_TASK_STACK_SIZE = 4096; // Stack size for USB task (same as ESP-IDF USB examples) static constexpr UBaseType_t USB_TASK_PRIORITY = 5; // Higher priority than main loop (tskIDLE_PRIORITY + 5) @@ -83,8 +90,6 @@ struct TransferRequest { enum EventType : uint8_t { EVENT_DEVICE_NEW, EVENT_DEVICE_GONE, - EVENT_TRANSFER_COMPLETE, - EVENT_CONTROL_COMPLETE, }; struct UsbEvent { @@ -96,9 +101,6 @@ struct UsbEvent { struct { usb_device_handle_t handle; } device_gone; - struct { - TransferRequest *trq; - } transfer; } data; // Required for EventPool - no cleanup needed for POD types @@ -163,10 +165,9 @@ class USBClient : public Component { uint16_t pid_{}; // Lock-free pool management using atomic bitmask (no dynamic allocation) // Bit i = 1: requests_[i] is in use, Bit i = 0: requests_[i] is available - // Supports multiple concurrent consumers (both threads can allocate) - // Single producer for deallocation (main loop only) - // Limited to 16 slots by uint16_t size (enforced by static_assert) - std::atomic trq_in_use_; + // Supports multiple concurrent consumers and producers (both threads can allocate/deallocate) + // Bitmask type automatically selected: uint16_t for <= 16 slots, uint32_t for 17-32 slots + std::atomic trq_in_use_; TransferRequest requests_[MAX_REQUESTS]{}; }; class USBHost : public Component { diff --git a/esphome/components/usb_host/usb_host_client.cpp b/esphome/components/usb_host/usb_host_client.cpp index b26385a8ef..5c8874861e 100644 --- a/esphome/components/usb_host/usb_host_client.cpp +++ b/esphome/components/usb_host/usb_host_client.cpp @@ -228,12 +228,6 @@ void USBClient::loop() { case EVENT_DEVICE_GONE: this->on_removed(event->data.device_gone.handle); break; - case EVENT_TRANSFER_COMPLETE: - case EVENT_CONTROL_COMPLETE: { - auto *trq = event->data.transfer.trq; - this->release_trq(trq); - break; - } } // Return event to pool for reuse this->event_pool.release(event); @@ -313,25 +307,6 @@ void USBClient::on_removed(usb_device_handle_t handle) { } } -// Helper to queue transfer cleanup to main loop -static void queue_transfer_cleanup(TransferRequest *trq, EventType type) { - auto *client = trq->client; - - // Allocate event from pool - UsbEvent *event = client->event_pool.allocate(); - if (event == nullptr) { - // No events available - increment counter for periodic logging - client->event_queue.increment_dropped_count(); - return; - } - - event->type = type; - event->data.transfer.trq = trq; - - // Push to lock-free queue (always succeeds since pool size == queue size) - client->event_queue.push(event); -} - // CALLBACK CONTEXT: USB task (called from usb_host_client_handle_events in USB task) static void control_callback(const usb_transfer_t *xfer) { auto *trq = static_cast(xfer->context); @@ -346,8 +321,9 @@ static void control_callback(const usb_transfer_t *xfer) { trq->callback(trq->status); } - // Queue cleanup to main loop - queue_transfer_cleanup(trq, EVENT_CONTROL_COMPLETE); + // Release transfer slot immediately in USB task + // The release_trq() uses thread-safe atomic operations + trq->client->release_trq(trq); } // THREAD CONTEXT: Called from both USB task and main loop threads (multi-consumer) @@ -358,20 +334,20 @@ static void control_callback(const usb_transfer_t *xfer) { // This multi-threaded access is intentional for performance - USB task can // immediately restart transfers without waiting for main loop scheduling. TransferRequest *USBClient::get_trq_() { - uint16_t mask = this->trq_in_use_.load(std::memory_order_relaxed); + trq_bitmask_t mask = this->trq_in_use_.load(std::memory_order_relaxed); // Find first available slot (bit = 0) and try to claim it atomically // We use a while loop to allow retrying the same slot after CAS failure size_t i = 0; while (i != MAX_REQUESTS) { - if (mask & (1U << i)) { + if (mask & (static_cast(1) << i)) { // Slot is in use, move to next slot i++; continue; } // Slot i appears available, try to claim it atomically - uint16_t desired = mask | (1U << i); // Set bit i to mark as in-use + trq_bitmask_t desired = mask | (static_cast(1) << i); // Set bit i to mark as in-use if (this->trq_in_use_.compare_exchange_weak(mask, desired, std::memory_order_acquire, std::memory_order_relaxed)) { // Successfully claimed slot i - prepare the TransferRequest @@ -386,7 +362,7 @@ TransferRequest *USBClient::get_trq_() { i = 0; } - ESP_LOGE(TAG, "All %d transfer slots in use", MAX_REQUESTS); + ESP_LOGE(TAG, "All %zu transfer slots in use", MAX_REQUESTS); return nullptr; } void USBClient::disconnect() { @@ -452,8 +428,10 @@ static void transfer_callback(usb_transfer_t *xfer) { trq->callback(trq->status); } - // Queue cleanup to main loop - queue_transfer_cleanup(trq, EVENT_TRANSFER_COMPLETE); + // Release transfer slot immediately in USB task to prevent slot exhaustion + // This is critical for high-throughput transfers (e.g., USB UART at 115200 baud) + // The release_trq() uses thread-safe atomic operations + trq->client->release_trq(trq); } /** * Performs a transfer input operation. @@ -521,12 +499,12 @@ void USBClient::dump_config() { " Product id %04X", this->vid_, this->pid_); } -// THREAD CONTEXT: Only called from main loop thread (single producer for deallocation) -// - Via event processing when handling EVENT_TRANSFER_COMPLETE/EVENT_CONTROL_COMPLETE -// - Directly when transfer submission fails +// THREAD CONTEXT: Called from both USB task and main loop threads +// - USB task: Immediately after transfer callback completes +// - Main loop: When transfer submission fails // // THREAD SAFETY: Lock-free using atomic AND to clear bit -// Single-producer pattern makes this simpler than allocation +// Thread-safe atomic operation allows multi-threaded deallocation void USBClient::release_trq(TransferRequest *trq) { if (trq == nullptr) return; @@ -540,8 +518,8 @@ void USBClient::release_trq(TransferRequest *trq) { // Atomically clear bit i to mark slot as available // fetch_and with inverted bitmask clears the bit atomically - uint16_t bit = 1U << index; - this->trq_in_use_.fetch_and(static_cast(~bit), std::memory_order_release); + trq_bitmask_t bit = static_cast(1) << index; + this->trq_in_use_.fetch_and(static_cast(~bit), std::memory_order_release); } } // namespace usb_host diff --git a/esphome/core/defines.h b/esphome/core/defines.h index 0f1d1bcf28..620f15765e 100644 --- a/esphome/core/defines.h +++ b/esphome/core/defines.h @@ -191,6 +191,7 @@ #define USE_WEBSERVER_PORT 80 // NOLINT #define USE_WEBSERVER_SORTING #define USE_WIFI_11KV_SUPPORT +#define USB_HOST_MAX_REQUESTS 16 #ifdef USE_ARDUINO #define USE_ARDUINO_VERSION_CODE VERSION_CODE(3, 2, 1) From dd6085456a5fba720a96bd77060d10163153ebed Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Sat, 11 Oct 2025 06:00:57 -1000 Subject: [PATCH 08/90] tweak --- esphome/components/usb_host/__init__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/esphome/components/usb_host/__init__.py b/esphome/components/usb_host/__init__.py index 42c893d296..d452e0e9fa 100644 --- a/esphome/components/usb_host/__init__.py +++ b/esphome/components/usb_host/__init__.py @@ -9,6 +9,7 @@ from esphome.components.esp32 import ( import esphome.config_validation as cv from esphome.const import CONF_DEVICES, CONF_ID from esphome.cpp_types import Component +from esphome.types import ConfigType AUTO_LOAD = ["bytebuffer"] CODEOWNERS = ["@clydebarrow"] @@ -62,7 +63,7 @@ async def register_usb_client(config): return var -async def to_code(config): +async def to_code(config: ConfigType) -> None: add_idf_sdkconfig_option("CONFIG_USB_HOST_CONTROL_TRANSFER_MAX_SIZE", 1024) if config.get(CONF_ENABLE_HUBS): add_idf_sdkconfig_option("CONFIG_USB_HOST_HUBS_SUPPORTED", True) From 442a60766d089aec8714ecb5ebcd7b9e06a3ea70 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Sat, 11 Oct 2025 06:02:49 -1000 Subject: [PATCH 09/90] missing defines --- esphome/components/usb_host/usb_host.h | 1 + 1 file changed, 1 insertion(+) diff --git a/esphome/components/usb_host/usb_host.h b/esphome/components/usb_host/usb_host.h index 3ccc49a5a0..036c535bd7 100644 --- a/esphome/components/usb_host/usb_host.h +++ b/esphome/components/usb_host/usb_host.h @@ -2,6 +2,7 @@ // Should not be needed, but it's required to pass CI clang-tidy checks #if defined(USE_ESP32_VARIANT_ESP32S2) || defined(USE_ESP32_VARIANT_ESP32S3) || defined(USE_ESP32_VARIANT_ESP32P4) +#include "esphome/core/defines.h" #include "esphome/core/component.h" #include #include "usb/usb_host.h" From 2796cac9727ac53d555cac2269299384051f7168 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Sat, 11 Oct 2025 06:17:36 -1000 Subject: [PATCH 10/90] compile tests --- tests/components/usb_host/test.esp32-s3-idf.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/components/usb_host/test.esp32-s3-idf.yaml b/tests/components/usb_host/test.esp32-s3-idf.yaml index a2892872e5..5360d1f6ff 100644 --- a/tests/components/usb_host/test.esp32-s3-idf.yaml +++ b/tests/components/usb_host/test.esp32-s3-idf.yaml @@ -1,4 +1,5 @@ usb_host: + max_transfer_requests: 32 # Test uint32_t bitmask path (17-32 requests) devices: - id: device_1 vid: 0x1234 From ec71669bff4f645b968793fa683e411cb7aae253 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Sat, 11 Oct 2025 06:24:35 -1000 Subject: [PATCH 11/90] tweak comments --- esphome/components/usb_host/usb_host_client.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/esphome/components/usb_host/usb_host_client.cpp b/esphome/components/usb_host/usb_host_client.cpp index 5c8874861e..91d7af2898 100644 --- a/esphome/components/usb_host/usb_host_client.cpp +++ b/esphome/components/usb_host/usb_host_client.cpp @@ -428,8 +428,8 @@ static void transfer_callback(usb_transfer_t *xfer) { trq->callback(trq->status); } - // Release transfer slot immediately in USB task to prevent slot exhaustion - // This is critical for high-throughput transfers (e.g., USB UART at 115200 baud) + // Release transfer slot AFTER callback completes to prevent slot exhaustion + // The callback has finished accessing xfer->data_buffer, so it's safe to release // The release_trq() uses thread-safe atomic operations trq->client->release_trq(trq); } From fa69b74e6c10e4fb0bfbd67c11cdca7ef1bd2d18 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Sat, 11 Oct 2025 06:25:44 -1000 Subject: [PATCH 12/90] tweak comments --- esphome/components/usb_host/usb_host_client.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/esphome/components/usb_host/usb_host_client.cpp b/esphome/components/usb_host/usb_host_client.cpp index 91d7af2898..2139ed869a 100644 --- a/esphome/components/usb_host/usb_host_client.cpp +++ b/esphome/components/usb_host/usb_host_client.cpp @@ -429,6 +429,7 @@ static void transfer_callback(usb_transfer_t *xfer) { } // Release transfer slot AFTER callback completes to prevent slot exhaustion + // This is critical for high-throughput transfers (e.g., USB UART at 115200 baud) // The callback has finished accessing xfer->data_buffer, so it's safe to release // The release_trq() uses thread-safe atomic operations trq->client->release_trq(trq); From 6273380407e60d0f197b8b84e6d92c7d4d5e9526 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Sat, 11 Oct 2025 10:51:17 -1000 Subject: [PATCH 13/90] [core] Add make_name_with_suffix helper to optimize string concatenation --- esphome/components/esp32_ble/ble.cpp | 3 +-- .../ethernet/ethernet_component.cpp | 2 +- esphome/components/mqtt/mqtt_client.cpp | 2 +- esphome/components/wifi/wifi_component.cpp | 2 +- esphome/config_validation.py | 7 ++++++ esphome/core/application.h | 6 +++-- esphome/core/config.py | 2 +- esphome/core/helpers.cpp | 22 +++++++++++++++++++ esphome/core/helpers.h | 9 ++++++++ 9 files changed, 47 insertions(+), 8 deletions(-) diff --git a/esphome/components/esp32_ble/ble.cpp b/esphome/components/esp32_ble/ble.cpp index 0c340c55cc..e37b45fe71 100644 --- a/esphome/components/esp32_ble/ble.cpp +++ b/esphome/components/esp32_ble/ble.cpp @@ -213,8 +213,7 @@ bool ESP32BLE::ble_setup_() { if (this->name_.has_value()) { name = this->name_.value(); if (App.is_name_add_mac_suffix_enabled()) { - name += "-"; - name += get_mac_address().substr(6); + name = make_name_with_suffix(name, '-', get_mac_address().substr(6)); } } else { name = App.get_name(); diff --git a/esphome/components/ethernet/ethernet_component.cpp b/esphome/components/ethernet/ethernet_component.cpp index 28043dd969..5f28d6db25 100644 --- a/esphome/components/ethernet/ethernet_component.cpp +++ b/esphome/components/ethernet/ethernet_component.cpp @@ -691,7 +691,7 @@ void EthernetComponent::set_manual_ip(const ManualIP &manual_ip) { this->manual_ std::string EthernetComponent::get_use_address() const { if (this->use_address_.empty()) { - return App.get_name() + ".local"; + return make_name_with_suffix(App.get_name(), '.', "local"); } return this->use_address_; } diff --git a/esphome/components/mqtt/mqtt_client.cpp b/esphome/components/mqtt/mqtt_client.cpp index 7ab6efd1a1..3642ddb38e 100644 --- a/esphome/components/mqtt/mqtt_client.cpp +++ b/esphome/components/mqtt/mqtt_client.cpp @@ -29,7 +29,7 @@ static const char *const TAG = "mqtt"; MQTTClientComponent::MQTTClientComponent() { global_mqtt_client = this; - this->credentials_.client_id = App.get_name() + "-" + get_mac_address(); + this->credentials_.client_id = make_name_with_suffix(App.get_name(), '-', get_mac_address()); } // Connection diff --git a/esphome/components/wifi/wifi_component.cpp b/esphome/components/wifi/wifi_component.cpp index 2e083d4c68..ec8687e927 100644 --- a/esphome/components/wifi/wifi_component.cpp +++ b/esphome/components/wifi/wifi_component.cpp @@ -267,7 +267,7 @@ network::IPAddress WiFiComponent::get_dns_address(int num) { } std::string WiFiComponent::get_use_address() const { if (this->use_address_.empty()) { - return App.get_name() + ".local"; + return make_name_with_suffix(App.get_name(), '.', "local"); } return this->use_address_; } diff --git a/esphome/config_validation.py b/esphome/config_validation.py index 7aaba886e3..2746f574ba 100644 --- a/esphome/config_validation.py +++ b/esphome/config_validation.py @@ -1195,6 +1195,13 @@ def validate_bytes(value): def hostname(value): + """Validate that the value is a valid hostname. + + Maximum length is 63 characters per RFC 1035. + + Note: If this limit is changed, update MAX_NAME_WITH_SUFFIX_SIZE in + esphome/core/application.h to accommodate the new maximum length. + """ value = string(value) if re.match(r"^[a-z0-9-]{1,63}$", value, re.IGNORECASE) is not None: return value diff --git a/esphome/core/application.h b/esphome/core/application.h index 1f22499051..47e902a191 100644 --- a/esphome/core/application.h +++ b/esphome/core/application.h @@ -103,8 +103,10 @@ class Application { this->name_add_mac_suffix_ = name_add_mac_suffix; if (name_add_mac_suffix) { const std::string mac_suffix = get_mac_address().substr(6); - this->name_ = name + "-" + mac_suffix; - this->friendly_name_ = friendly_name.empty() ? "" : friendly_name + " " + mac_suffix; + this->name_ = make_name_with_suffix(name, '-', mac_suffix); + if (!friendly_name.empty()) { + this->friendly_name_ = make_name_with_suffix(friendly_name, ' ', mac_suffix); + } } else { this->name_ = name; this->friendly_name_ = friendly_name; diff --git a/esphome/core/config.py b/esphome/core/config.py index 7bf7f82a8b..8a5876dbcf 100644 --- a/esphome/core/config.py +++ b/esphome/core/config.py @@ -200,7 +200,7 @@ CONFIG_SCHEMA = cv.All( cv.Schema( { cv.Required(CONF_NAME): cv.valid_name, - cv.Optional(CONF_FRIENDLY_NAME, ""): cv.string, + cv.Optional(CONF_FRIENDLY_NAME, ""): cv.All(cv.string, cv.Length(max=120)), cv.Optional(CONF_AREA): validate_area_config, cv.Optional(CONF_COMMENT): cv.string, cv.Required(CONF_BUILD_PATH): cv.string, diff --git a/esphome/core/helpers.cpp b/esphome/core/helpers.cpp index d4f6809776..82dbeddf22 100644 --- a/esphome/core/helpers.cpp +++ b/esphome/core/helpers.cpp @@ -235,6 +235,28 @@ std::string str_sprintf(const char *fmt, ...) { return str; } +// Maximum size for name with suffix: 120 (max friendly name) + 1 (separator) + 6 (MAC suffix) + 1 (null term) +static constexpr size_t MAX_NAME_WITH_SUFFIX_SIZE = 128; + +std::string make_name_with_suffix(const std::string &name, char sep, const std::string &suffix) { + char buffer[MAX_NAME_WITH_SUFFIX_SIZE]; + size_t name_len = name.size(); + size_t suffix_len = suffix.size(); + size_t total_len = name_len + 1 + suffix_len; + + // Silently truncate if needed: prioritize keeping the full suffix + if (total_len >= MAX_NAME_WITH_SUFFIX_SIZE) { + name_len = MAX_NAME_WITH_SUFFIX_SIZE - suffix_len - 2; // -2 for separator and null terminator + total_len = name_len + 1 + suffix_len; + } + + memcpy(buffer, name.c_str(), name_len); + buffer[name_len] = sep; + memcpy(buffer + name_len + 1, suffix.c_str(), suffix_len); + buffer[total_len] = '\0'; + return std::string(buffer, total_len); +} + // Parsing & formatting size_t parse_hex(const char *str, size_t length, uint8_t *data, size_t count) { diff --git a/esphome/core/helpers.h b/esphome/core/helpers.h index e06f2d15ef..f64c14aa85 100644 --- a/esphome/core/helpers.h +++ b/esphome/core/helpers.h @@ -306,6 +306,15 @@ std::string __attribute__((format(printf, 1, 3))) str_snprintf(const char *fmt, /// sprintf-like function returning std::string. std::string __attribute__((format(printf, 1, 2))) str_sprintf(const char *fmt, ...); +/// Concatenate a name with a separator and suffix using an efficient stack-based approach. +/// This avoids multiple heap allocations during string construction. +/// Maximum name length supported is 120 characters for friendly names. +/// @param name The base name string +/// @param sep The separator character (e.g., '-', ' ', or '.') +/// @param suffix The suffix to append (e.g., MAC address suffix or ".local") +/// @return The concatenated string: name + sep + suffix +std::string make_name_with_suffix(const std::string &name, char sep, const std::string &suffix); + ///@} /// @name Parsing & formatting From 1acbb007dd1709e60482f75c113febc71b3ad92e Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Sat, 11 Oct 2025 11:08:47 -1000 Subject: [PATCH 14/90] [ci] Filter out components without tests from CI test jobs (#11134 followup) --- .github/workflows/ci.yml | 6 ++- script/determine-jobs.py | 13 ++++- tests/script/test_determine_jobs.py | 75 ++++++++++++++++++++++++++++- 3 files changed, 89 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4451007da0..f692b1f7d0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -177,6 +177,7 @@ jobs: clang-tidy: ${{ steps.determine.outputs.clang-tidy }} python-linters: ${{ steps.determine.outputs.python-linters }} changed-components: ${{ steps.determine.outputs.changed-components }} + changed-components-with-tests: ${{ steps.determine.outputs.changed-components-with-tests }} component-test-count: ${{ steps.determine.outputs.component-test-count }} steps: - name: Check out code from GitHub @@ -204,6 +205,7 @@ jobs: echo "clang-tidy=$(echo "$output" | jq -r '.clang_tidy')" >> $GITHUB_OUTPUT echo "python-linters=$(echo "$output" | jq -r '.python_linters')" >> $GITHUB_OUTPUT echo "changed-components=$(echo "$output" | jq -c '.changed_components')" >> $GITHUB_OUTPUT + echo "changed-components-with-tests=$(echo "$output" | jq -c '.changed_components_with_tests')" >> $GITHUB_OUTPUT echo "component-test-count=$(echo "$output" | jq -r '.component_test_count')" >> $GITHUB_OUTPUT integration-tests: @@ -367,7 +369,7 @@ jobs: fail-fast: false max-parallel: 2 matrix: - file: ${{ fromJson(needs.determine-jobs.outputs.changed-components) }} + file: ${{ fromJson(needs.determine-jobs.outputs.changed-components-with-tests) }} steps: - name: Cache apt packages uses: awalsh128/cache-apt-pkgs-action@acb598e5ddbc6f68a970c5da0688d2f3a9f04d05 # v1.5.3 @@ -414,7 +416,7 @@ jobs: . venv/bin/activate # Use intelligent splitter that groups components with same bus configs - components='${{ needs.determine-jobs.outputs.changed-components }}' + components='${{ needs.determine-jobs.outputs.changed-components-with-tests }}' echo "Splitting components intelligently..." output=$(python3 script/split_components_for_ci.py --components "$components" --batch-size 40 --output github) diff --git a/script/determine-jobs.py b/script/determine-jobs.py index e26bc29c2f..a078fd8f9b 100755 --- a/script/determine-jobs.py +++ b/script/determine-jobs.py @@ -237,6 +237,16 @@ def main() -> None: result = subprocess.run(cmd, capture_output=True, text=True, check=True) changed_components = parse_list_components_output(result.stdout) + # Filter to only components that have test files + # Components without tests shouldn't generate CI test jobs + tests_dir = Path(root_path) / "tests" / "components" + changed_components_with_tests = [ + component + for component in changed_components + if (component_test_dir := tests_dir / component).exists() + and any(component_test_dir.glob("test.*.yaml")) + ] + # Build output output: dict[str, Any] = { "integration_tests": run_integration, @@ -244,7 +254,8 @@ def main() -> None: "clang_format": run_clang_format, "python_linters": run_python_linters, "changed_components": changed_components, - "component_test_count": len(changed_components), + "changed_components_with_tests": changed_components_with_tests, + "component_test_count": len(changed_components_with_tests), } # Output as JSON diff --git a/tests/script/test_determine_jobs.py b/tests/script/test_determine_jobs.py index 7200afc2ee..5d8746f434 100644 --- a/tests/script/test_determine_jobs.py +++ b/tests/script/test_determine_jobs.py @@ -4,6 +4,7 @@ from collections.abc import Generator import importlib.util import json import os +from pathlib import Path import subprocess import sys from unittest.mock import Mock, call, patch @@ -90,7 +91,13 @@ def test_main_all_tests_should_run( assert output["clang_format"] is True assert output["python_linters"] is True assert output["changed_components"] == ["wifi", "api", "sensor"] - assert output["component_test_count"] == 3 + # changed_components_with_tests will only include components that actually have test files + assert "changed_components_with_tests" in output + assert isinstance(output["changed_components_with_tests"], list) + # component_test_count matches number of components with tests + assert output["component_test_count"] == len( + output["changed_components_with_tests"] + ) def test_main_no_tests_should_run( @@ -125,6 +132,7 @@ def test_main_no_tests_should_run( assert output["clang_format"] is False assert output["python_linters"] is False assert output["changed_components"] == [] + assert output["changed_components_with_tests"] == [] assert output["component_test_count"] == 0 @@ -197,7 +205,13 @@ def test_main_with_branch_argument( assert output["clang_format"] is False assert output["python_linters"] is True assert output["changed_components"] == ["mqtt"] - assert output["component_test_count"] == 1 + # changed_components_with_tests will only include components that actually have test files + assert "changed_components_with_tests" in output + assert isinstance(output["changed_components_with_tests"], list) + # component_test_count matches number of components with tests + assert output["component_test_count"] == len( + output["changed_components_with_tests"] + ) def test_should_run_integration_tests( @@ -377,3 +391,60 @@ def test_should_run_clang_format_with_branch() -> None: mock_changed.return_value = [] determine_jobs.should_run_clang_format("release") mock_changed.assert_called_once_with("release") + + +def test_main_filters_components_without_tests( + mock_should_run_integration_tests: Mock, + mock_should_run_clang_tidy: Mock, + mock_should_run_clang_format: Mock, + mock_should_run_python_linters: Mock, + mock_subprocess_run: Mock, + capsys: pytest.CaptureFixture[str], + tmp_path: Path, +) -> None: + """Test that components without test files are filtered out.""" + mock_should_run_integration_tests.return_value = False + mock_should_run_clang_tidy.return_value = False + mock_should_run_clang_format.return_value = False + mock_should_run_python_linters.return_value = False + + # Mock list-components.py output with 3 components + # wifi: has tests, sensor: has tests, airthings_ble: no tests + mock_result = Mock() + mock_result.stdout = "wifi\nsensor\nairthings_ble\n" + mock_subprocess_run.return_value = mock_result + + # Create test directory structure + tests_dir = tmp_path / "tests" / "components" + + # wifi has tests + wifi_dir = tests_dir / "wifi" + wifi_dir.mkdir(parents=True) + (wifi_dir / "test.esp32.yaml").write_text("test: config") + + # sensor has tests + sensor_dir = tests_dir / "sensor" + sensor_dir.mkdir(parents=True) + (sensor_dir / "test.esp8266.yaml").write_text("test: config") + + # airthings_ble exists but has no test files + airthings_dir = tests_dir / "airthings_ble" + airthings_dir.mkdir(parents=True) + + # Mock root_path to use tmp_path + with ( + patch.object(determine_jobs, "root_path", str(tmp_path)), + patch("sys.argv", ["determine-jobs.py"]), + ): + determine_jobs.main() + + # Check output + captured = capsys.readouterr() + output = json.loads(captured.out) + + # changed_components should have all components + assert set(output["changed_components"]) == {"wifi", "sensor", "airthings_ble"} + # changed_components_with_tests should only have components with test files + assert set(output["changed_components_with_tests"]) == {"wifi", "sensor"} + # component_test_count should be based on components with tests + assert output["component_test_count"] == 2 From 5e1848854e6590d095e13341a82af6cca5b3f997 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Sat, 11 Oct 2025 11:25:19 -1000 Subject: [PATCH 15/90] tweak for bot --- esphome/core/helpers.cpp | 3 +++ script/determine-jobs.py | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/esphome/core/helpers.cpp b/esphome/core/helpers.cpp index 82dbeddf22..0b6c203c01 100644 --- a/esphome/core/helpers.cpp +++ b/esphome/core/helpers.cpp @@ -246,6 +246,9 @@ std::string make_name_with_suffix(const std::string &name, char sep, const std:: // Silently truncate if needed: prioritize keeping the full suffix if (total_len >= MAX_NAME_WITH_SUFFIX_SIZE) { + // NOTE: This calculation could underflow if suffix_len >= MAX_NAME_WITH_SUFFIX_SIZE - 2, + // but this is safe because this helper is only called with small suffixes: + // MAC suffixes (6-12 bytes), ".local" (6 bytes), etc. name_len = MAX_NAME_WITH_SUFFIX_SIZE - suffix_len - 2; // -2 for separator and null terminator total_len = name_len + 1 + suffix_len; } diff --git a/script/determine-jobs.py b/script/determine-jobs.py index a078fd8f9b..1601496877 100755 --- a/script/determine-jobs.py +++ b/script/determine-jobs.py @@ -244,7 +244,7 @@ def main() -> None: component for component in changed_components if (component_test_dir := tests_dir / component).exists() - and any(component_test_dir.glob("test.*.yaml")) + and next(component_test_dir.glob("test.*.yaml"), None) is not None ] # Build output From 0c8c99dbf857faba37a4c5e11977e3745e374f08 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Sat, 11 Oct 2025 12:27:39 -1000 Subject: [PATCH 16/90] [mdns] Conditionally store services to reduce RAM usage by 200-464 bytes --- esphome/components/mdns/__init__.py | 19 ++++++++++++++++++- esphome/components/mdns/mdns_component.cpp | 17 +++++++++++------ esphome/components/mdns/mdns_component.h | 6 +++++- esphome/components/mdns/mdns_esp32.cpp | 5 +++-- esphome/components/mdns/mdns_esp8266.cpp | 5 +++-- esphome/components/mdns/mdns_libretiny.cpp | 5 +++-- esphome/components/mdns/mdns_rp2040.cpp | 5 +++-- esphome/components/openthread/__init__.py | 5 ++++- esphome/core/defines.h | 1 + 9 files changed, 51 insertions(+), 17 deletions(-) diff --git a/esphome/components/mdns/__init__.py b/esphome/components/mdns/__init__.py index 14e0420ef5..c6a9ee1a0c 100644 --- a/esphome/components/mdns/__init__.py +++ b/esphome/components/mdns/__init__.py @@ -1,6 +1,6 @@ import esphome.codegen as cg from esphome.components.esp32 import add_idf_component -from esphome.config_helpers import filter_source_files_from_platform +from esphome.config_helpers import filter_source_files_from_platform, get_logger_level import esphome.config_validation as cv from esphome.const import ( CONF_DISABLED, @@ -125,6 +125,17 @@ def mdns_service( ) +def enable_mdns_storage(): + """Enable persistent storage of mDNS services in the MDNSComponent. + + Called by external components (like OpenThread) that need access to + services after setup() completes via get_services(). + + Public API for external components. Do not remove. + """ + cg.add_define("USE_MDNS_STORE_SERVICES") + + @coroutine_with_priority(CoroPriority.NETWORK_SERVICES) async def to_code(config): if config[CONF_DISABLED] is True: @@ -150,6 +161,8 @@ async def to_code(config): if config[CONF_SERVICES]: cg.add_define("USE_MDNS_EXTRA_SERVICES") + # Extra services need to be stored persistently + enable_mdns_storage() # Ensure at least 1 service (fallback service) cg.add_define("MDNS_SERVICE_COUNT", max(1, service_count)) @@ -171,6 +184,10 @@ async def to_code(config): # Ensure at least 1 to avoid zero-size array cg.add_define("MDNS_DYNAMIC_TXT_COUNT", max(1, dynamic_txt_count)) + # Enable storage if verbose logging is enabled (for dump_config) + if get_logger_level() in ("VERBOSE", "VERY_VERBOSE"): + enable_mdns_storage() + var = cg.new_Pvariable(config[CONF_ID]) await cg.register_component(var, config) diff --git a/esphome/components/mdns/mdns_component.cpp b/esphome/components/mdns/mdns_component.cpp index 9cb664c3c3..fea3ced99f 100644 --- a/esphome/components/mdns/mdns_component.cpp +++ b/esphome/components/mdns/mdns_component.cpp @@ -36,7 +36,7 @@ MDNS_STATIC_CONST_CHAR(SERVICE_TCP, "_tcp"); // Wrap build-time defines into flash storage MDNS_STATIC_CONST_CHAR(VALUE_VERSION, ESPHOME_VERSION); -void MDNSComponent::compile_records_() { +void MDNSComponent::compile_records_(StaticVector &services) { this->hostname_ = App.get_name(); // IMPORTANT: The #ifdef blocks below must match COMPONENTS_WITH_MDNS_SERVICES @@ -53,7 +53,7 @@ void MDNSComponent::compile_records_() { MDNS_STATIC_CONST_CHAR(VALUE_BOARD, ESPHOME_BOARD); if (api::global_api_server != nullptr) { - auto &service = this->services_.emplace_next(); + auto &service = services.emplace_next(); service.service_type = MDNS_STR(SERVICE_ESPHOMELIB); service.proto = MDNS_STR(SERVICE_TCP); service.port = api::global_api_server->get_port(); @@ -146,7 +146,7 @@ void MDNSComponent::compile_records_() { #ifdef USE_PROMETHEUS MDNS_STATIC_CONST_CHAR(SERVICE_PROMETHEUS, "_prometheus-http"); - auto &prom_service = this->services_.emplace_next(); + auto &prom_service = services.emplace_next(); prom_service.service_type = MDNS_STR(SERVICE_PROMETHEUS); prom_service.proto = MDNS_STR(SERVICE_TCP); prom_service.port = USE_WEBSERVER_PORT; @@ -155,7 +155,7 @@ void MDNSComponent::compile_records_() { #ifdef USE_WEBSERVER MDNS_STATIC_CONST_CHAR(SERVICE_HTTP, "_http"); - auto &web_service = this->services_.emplace_next(); + auto &web_service = services.emplace_next(); web_service.service_type = MDNS_STR(SERVICE_HTTP); web_service.proto = MDNS_STR(SERVICE_TCP); web_service.port = USE_WEBSERVER_PORT; @@ -167,12 +167,17 @@ void MDNSComponent::compile_records_() { // Publish "http" service if not using native API or any other services // This is just to have *some* mDNS service so that .local resolution works - auto &fallback_service = this->services_.emplace_next(); + auto &fallback_service = services.emplace_next(); fallback_service.service_type = MDNS_STR(SERVICE_HTTP); fallback_service.proto = MDNS_STR(SERVICE_TCP); fallback_service.port = USE_WEBSERVER_PORT; fallback_service.txt_records.push_back({MDNS_STR(TXT_VERSION), MDNS_STR(VALUE_VERSION)}); #endif + +#ifdef USE_MDNS_STORE_SERVICES + // Copy to member variable if storage is enabled (verbose logging, OpenThread, or extra services) + this->services_ = services; +#endif } void MDNSComponent::dump_config() { @@ -180,7 +185,7 @@ void MDNSComponent::dump_config() { "mDNS:\n" " Hostname: %s", this->hostname_.c_str()); -#if ESPHOME_LOG_LEVEL >= ESPHOME_LOG_LEVEL_VERBOSE +#ifdef USE_MDNS_STORE_SERVICES ESP_LOGV(TAG, " Services:"); for (const auto &service : this->services_) { ESP_LOGV(TAG, " - %s, %s, %d", MDNS_STR_ARG(service.service_type), MDNS_STR_ARG(service.proto), diff --git a/esphome/components/mdns/mdns_component.h b/esphome/components/mdns/mdns_component.h index 141e42d976..62476e9504 100644 --- a/esphome/components/mdns/mdns_component.h +++ b/esphome/components/mdns/mdns_component.h @@ -55,7 +55,9 @@ class MDNSComponent : public Component { void add_extra_service(MDNSService service) { this->services_.emplace_next() = std::move(service); } #endif +#ifdef USE_MDNS_STORE_SERVICES const StaticVector &get_services() const { return this->services_; } +#endif void on_shutdown() override; @@ -71,9 +73,11 @@ class MDNSComponent : public Component { StaticVector dynamic_txt_values_; protected: +#ifdef USE_MDNS_STORE_SERVICES StaticVector services_{}; +#endif std::string hostname_; - void compile_records_(); + void compile_records_(StaticVector &services); }; } // namespace mdns diff --git a/esphome/components/mdns/mdns_esp32.cpp b/esphome/components/mdns/mdns_esp32.cpp index e77c0b9b05..da47be7dbc 100644 --- a/esphome/components/mdns/mdns_esp32.cpp +++ b/esphome/components/mdns/mdns_esp32.cpp @@ -12,7 +12,8 @@ namespace mdns { static const char *const TAG = "mdns"; void MDNSComponent::setup() { - this->compile_records_(); + StaticVector services; + this->compile_records_(services); esp_err_t err = mdns_init(); if (err != ESP_OK) { @@ -24,7 +25,7 @@ void MDNSComponent::setup() { mdns_hostname_set(this->hostname_.c_str()); mdns_instance_name_set(this->hostname_.c_str()); - for (const auto &service : this->services_) { + for (const auto &service : services) { std::vector txt_records; for (const auto &record : service.txt_records) { mdns_txt_item_t it{}; diff --git a/esphome/components/mdns/mdns_esp8266.cpp b/esphome/components/mdns/mdns_esp8266.cpp index f3779042ed..06503742db 100644 --- a/esphome/components/mdns/mdns_esp8266.cpp +++ b/esphome/components/mdns/mdns_esp8266.cpp @@ -12,11 +12,12 @@ namespace esphome { namespace mdns { void MDNSComponent::setup() { - this->compile_records_(); + StaticVector services; + this->compile_records_(services); MDNS.begin(this->hostname_.c_str()); - for (const auto &service : this->services_) { + for (const auto &service : services) { // Strip the leading underscore from the proto and service_type. While it is // part of the wire protocol to have an underscore, and for example ESP-IDF // expects the underscore to be there, the ESP8266 implementation always adds diff --git a/esphome/components/mdns/mdns_libretiny.cpp b/esphome/components/mdns/mdns_libretiny.cpp index 5540bf361a..a959482ff6 100644 --- a/esphome/components/mdns/mdns_libretiny.cpp +++ b/esphome/components/mdns/mdns_libretiny.cpp @@ -12,11 +12,12 @@ namespace esphome { namespace mdns { void MDNSComponent::setup() { - this->compile_records_(); + StaticVector services; + this->compile_records_(services); MDNS.begin(this->hostname_.c_str()); - for (const auto &service : this->services_) { + for (const auto &service : services) { // Strip the leading underscore from the proto and service_type. While it is // part of the wire protocol to have an underscore, and for example ESP-IDF // expects the underscore to be there, the ESP8266 implementation always adds diff --git a/esphome/components/mdns/mdns_rp2040.cpp b/esphome/components/mdns/mdns_rp2040.cpp index 5ad006f5d4..9dfb05bda9 100644 --- a/esphome/components/mdns/mdns_rp2040.cpp +++ b/esphome/components/mdns/mdns_rp2040.cpp @@ -12,11 +12,12 @@ namespace esphome { namespace mdns { void MDNSComponent::setup() { - this->compile_records_(); + StaticVector services; + this->compile_records_(services); MDNS.begin(this->hostname_.c_str()); - for (const auto &service : this->services_) { + for (const auto &service : services) { // Strip the leading underscore from the proto and service_type. While it is // part of the wire protocol to have an underscore, and for example ESP-IDF // expects the underscore to be there, the ESP8266 implementation always adds diff --git a/esphome/components/openthread/__init__.py b/esphome/components/openthread/__init__.py index 2f085ebaae..3fac497c3d 100644 --- a/esphome/components/openthread/__init__.py +++ b/esphome/components/openthread/__init__.py @@ -5,7 +5,7 @@ from esphome.components.esp32 import ( add_idf_sdkconfig_option, only_on_variant, ) -from esphome.components.mdns import MDNSComponent +from esphome.components.mdns import MDNSComponent, enable_mdns_storage import esphome.config_validation as cv from esphome.const import CONF_CHANNEL, CONF_ENABLE_IPV6, CONF_ID import esphome.final_validate as fv @@ -141,6 +141,9 @@ FINAL_VALIDATE_SCHEMA = _final_validate async def to_code(config): cg.add_define("USE_OPENTHREAD") + # OpenThread SRP needs access to mDNS services after setup + enable_mdns_storage() + ot = cg.new_Pvariable(config[CONF_ID]) await cg.register_component(ot, config) diff --git a/esphome/core/defines.h b/esphome/core/defines.h index 955d0f987c..aa2c95306a 100644 --- a/esphome/core/defines.h +++ b/esphome/core/defines.h @@ -83,6 +83,7 @@ #define USE_LVGL_TILEVIEW #define USE_LVGL_TOUCHSCREEN #define USE_MDNS +#define USE_MDNS_STORE_SERVICES #define MDNS_SERVICE_COUNT 3 #define MDNS_DYNAMIC_TXT_COUNT 3 #define USE_MEDIA_PLAYER From 0975dbfb011e18f44f2368d7fc9b0a47908484a3 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Sat, 11 Oct 2025 12:38:12 -1000 Subject: [PATCH 17/90] cleanup --- esphome/components/mdns/mdns_host.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/esphome/components/mdns/mdns_host.cpp b/esphome/components/mdns/mdns_host.cpp index 78767ed136..f645d8d068 100644 --- a/esphome/components/mdns/mdns_host.cpp +++ b/esphome/components/mdns/mdns_host.cpp @@ -9,7 +9,9 @@ namespace esphome { namespace mdns { -void MDNSComponent::setup() { this->compile_records_(); } +void MDNSComponent::setup() { + // Host platform doesn't have actual mDNS implementation +} void MDNSComponent::on_shutdown() {} From ff6191cfd4c5128f073f7e339fa749c08c96e00b Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Sat, 11 Oct 2025 12:55:03 -1000 Subject: [PATCH 18/90] [esp32_improv] Fix state not transitioning to PROVISIONED when WiFi configured via captive portal --- .../esp32_improv/esp32_improv_component.cpp | 53 ++++++++++++------- .../esp32_improv/esp32_improv_component.h | 1 + 2 files changed, 35 insertions(+), 19 deletions(-) diff --git a/esphome/components/esp32_improv/esp32_improv_component.cpp b/esphome/components/esp32_improv/esp32_improv_component.cpp index f773083890..ed709da89c 100644 --- a/esphome/components/esp32_improv/esp32_improv_component.cpp +++ b/esphome/components/esp32_improv/esp32_improv_component.cpp @@ -40,6 +40,9 @@ void ESP32ImprovComponent::setup() { #endif global_ble_server->on_disconnect([this](uint16_t conn_id) { this->set_error_(improv::ERROR_NONE); }); + // Listen for WiFi connections to detect when provisioning happens via captive portal or other means + wifi::global_wifi_component->get_connect_trigger()->add_callback([this]() { this->on_wifi_connected_(); }); + // Start with loop disabled - will be enabled by start() when needed this->disable_loop(); } @@ -161,25 +164,7 @@ void ESP32ImprovComponent::loop() { case improv::STATE_PROVISIONING: { this->set_status_indicator_state_((now % 200) < 100); if (wifi::global_wifi_component->is_connected()) { - wifi::global_wifi_component->save_wifi_sta(this->connecting_sta_.get_ssid(), - this->connecting_sta_.get_password()); - this->connecting_sta_ = {}; - this->cancel_timeout("wifi-connect-timeout"); - this->set_state_(improv::STATE_PROVISIONED); - - std::vector urls = {ESPHOME_MY_LINK}; -#ifdef USE_WEBSERVER - for (auto &ip : wifi::global_wifi_component->wifi_sta_ip_addresses()) { - if (ip.is_ip4()) { - std::string webserver_url = "http://" + ip.str() + ":" + to_string(USE_WEBSERVER_PORT); - urls.push_back(webserver_url); - break; - } - } -#endif - std::vector data = improv::build_rpc_response(improv::WIFI_SETTINGS, urls); - this->send_response_(data); - this->stop(); + this->on_wifi_connected_(); } break; } @@ -392,6 +377,36 @@ void ESP32ImprovComponent::on_wifi_connect_timeout_() { wifi::global_wifi_component->clear_sta(); } +void ESP32ImprovComponent::on_wifi_connected_() { + // Handle WiFi connection, whether from Improv provisioning or external (e.g., captive portal) + if (this->state_ == improv::STATE_PROVISIONING) { + // WiFi provisioned via Improv - save credentials and send response + wifi::global_wifi_component->save_wifi_sta(this->connecting_sta_.get_ssid(), this->connecting_sta_.get_password()); + this->connecting_sta_ = {}; + this->cancel_timeout("wifi-connect-timeout"); + + std::vector urls = {ESPHOME_MY_LINK}; +#ifdef USE_WEBSERVER + for (auto &ip : wifi::global_wifi_component->wifi_sta_ip_addresses()) { + if (ip.is_ip4()) { + std::string webserver_url = "http://" + ip.str() + ":" + to_string(USE_WEBSERVER_PORT); + urls.push_back(webserver_url); + break; + } + } +#endif + std::vector data = improv::build_rpc_response(improv::WIFI_SETTINGS, urls); + this->send_response_(data); + } else if (this->is_active() && this->state_ != improv::STATE_PROVISIONED) { + // WiFi provisioned externally (e.g., captive portal) - just transition to provisioned + ESP_LOGD(TAG, "WiFi provisioned externally, transitioning to provisioned state"); + } + + // Common actions for both cases + this->set_state_(improv::STATE_PROVISIONED); + this->stop(); +} + void ESP32ImprovComponent::advertise_service_data_() { uint8_t service_data[IMPROV_SERVICE_DATA_SIZE] = {}; service_data[0] = IMPROV_PROTOCOL_ID_1; // PR diff --git a/esphome/components/esp32_improv/esp32_improv_component.h b/esphome/components/esp32_improv/esp32_improv_component.h index eb07e09dce..39c3483b2a 100644 --- a/esphome/components/esp32_improv/esp32_improv_component.h +++ b/esphome/components/esp32_improv/esp32_improv_component.h @@ -111,6 +111,7 @@ class ESP32ImprovComponent : public Component { void send_response_(std::vector &response); void process_incoming_data_(); void on_wifi_connect_timeout_(); + void on_wifi_connected_(); bool check_identify_(); void advertise_service_data_(); #if ESPHOME_LOG_LEVEL >= ESPHOME_LOG_LEVEL_DEBUG From a193d5b40e3518f79ae6a246bd9bad43537b634d Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Sat, 11 Oct 2025 12:56:28 -1000 Subject: [PATCH 19/90] [esp32_improv] Fix state not transitioning to PROVISIONED when WiFi configured via captive portal --- esphome/components/esp32_improv/esp32_improv_component.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/esphome/components/esp32_improv/esp32_improv_component.cpp b/esphome/components/esp32_improv/esp32_improv_component.cpp index ed709da89c..5060a0759a 100644 --- a/esphome/components/esp32_improv/esp32_improv_component.cpp +++ b/esphome/components/esp32_improv/esp32_improv_component.cpp @@ -399,7 +399,7 @@ void ESP32ImprovComponent::on_wifi_connected_() { this->send_response_(data); } else if (this->is_active() && this->state_ != improv::STATE_PROVISIONED) { // WiFi provisioned externally (e.g., captive portal) - just transition to provisioned - ESP_LOGD(TAG, "WiFi provisioned externally, transitioning to provisioned state"); + ESP_LOGD(TAG, "WiFi provisioned externally"); } // Common actions for both cases From c63902781b891f6434a4be3f2ac7a24f7d997b40 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Sat, 11 Oct 2025 12:57:13 -1000 Subject: [PATCH 20/90] [esp32_improv] Fix state not transitioning to PROVISIONED when WiFi configured via captive portal --- esphome/components/esp32_improv/esp32_improv_component.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/esphome/components/esp32_improv/esp32_improv_component.cpp b/esphome/components/esp32_improv/esp32_improv_component.cpp index 5060a0759a..49ec5e8ab9 100644 --- a/esphome/components/esp32_improv/esp32_improv_component.cpp +++ b/esphome/components/esp32_improv/esp32_improv_component.cpp @@ -402,7 +402,6 @@ void ESP32ImprovComponent::on_wifi_connected_() { ESP_LOGD(TAG, "WiFi provisioned externally"); } - // Common actions for both cases this->set_state_(improv::STATE_PROVISIONED); this->stop(); } From 5a0184cb35ef5e1b198ebbe2784abf2aad7abce4 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Sat, 11 Oct 2025 13:01:19 -1000 Subject: [PATCH 21/90] [esp32_improv] Fix state not transitioning to PROVISIONED when WiFi configured via captive portal --- esphome/components/esp32_improv/esp32_improv_component.cpp | 3 --- 1 file changed, 3 deletions(-) diff --git a/esphome/components/esp32_improv/esp32_improv_component.cpp b/esphome/components/esp32_improv/esp32_improv_component.cpp index 49ec5e8ab9..5c32f82abb 100644 --- a/esphome/components/esp32_improv/esp32_improv_component.cpp +++ b/esphome/components/esp32_improv/esp32_improv_component.cpp @@ -163,9 +163,6 @@ void ESP32ImprovComponent::loop() { } case improv::STATE_PROVISIONING: { this->set_status_indicator_state_((now % 200) < 100); - if (wifi::global_wifi_component->is_connected()) { - this->on_wifi_connected_(); - } break; } case improv::STATE_PROVISIONED: { From 678a93cc56edca3efc4fc7add5e50f42463260b7 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Sat, 11 Oct 2025 13:08:10 -1000 Subject: [PATCH 22/90] fix --- .../esp32_improv/esp32_improv_component.cpp | 12 +++++++++--- .../components/esp32_improv/esp32_improv_component.h | 1 + 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/esphome/components/esp32_improv/esp32_improv_component.cpp b/esphome/components/esp32_improv/esp32_improv_component.cpp index 5c32f82abb..b3258aedac 100644 --- a/esphome/components/esp32_improv/esp32_improv_component.cpp +++ b/esphome/components/esp32_improv/esp32_improv_component.cpp @@ -40,9 +40,6 @@ void ESP32ImprovComponent::setup() { #endif global_ble_server->on_disconnect([this](uint16_t conn_id) { this->set_error_(improv::ERROR_NONE); }); - // Listen for WiFi connections to detect when provisioning happens via captive portal or other means - wifi::global_wifi_component->get_connect_trigger()->add_callback([this]() { this->on_wifi_connected_(); }); - // Start with loop disabled - will be enabled by start() when needed this->disable_loop(); } @@ -146,6 +143,7 @@ void ESP32ImprovComponent::loop() { #else this->set_state_(improv::STATE_AUTHORIZED); #endif + this->check_wifi_connection_(); break; } case improv::STATE_AUTHORIZED: { @@ -159,10 +157,12 @@ void ESP32ImprovComponent::loop() { if (!this->check_identify_()) { this->set_status_indicator_state_((now % 1000) < 500); } + this->check_wifi_connection_(); break; } case improv::STATE_PROVISIONING: { this->set_status_indicator_state_((now % 200) < 100); + this->check_wifi_connection_(); break; } case improv::STATE_PROVISIONED: { @@ -374,6 +374,12 @@ void ESP32ImprovComponent::on_wifi_connect_timeout_() { wifi::global_wifi_component->clear_sta(); } +void ESP32ImprovComponent::check_wifi_connection_() { + if (wifi::global_wifi_component->is_connected()) { + this->on_wifi_connected_(); + } +} + void ESP32ImprovComponent::on_wifi_connected_() { // Handle WiFi connection, whether from Improv provisioning or external (e.g., captive portal) if (this->state_ == improv::STATE_PROVISIONING) { diff --git a/esphome/components/esp32_improv/esp32_improv_component.h b/esphome/components/esp32_improv/esp32_improv_component.h index 39c3483b2a..da670f54bc 100644 --- a/esphome/components/esp32_improv/esp32_improv_component.h +++ b/esphome/components/esp32_improv/esp32_improv_component.h @@ -112,6 +112,7 @@ class ESP32ImprovComponent : public Component { void process_incoming_data_(); void on_wifi_connect_timeout_(); void on_wifi_connected_(); + void check_wifi_connection_(); bool check_identify_(); void advertise_service_data_(); #if ESPHOME_LOG_LEVEL >= ESPHOME_LOG_LEVEL_DEBUG From 3758b4c8015406e06d367c3afa8067715d7fa422 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Sat, 11 Oct 2025 13:45:22 -1000 Subject: [PATCH 23/90] preen --- .../components/esp32_improv/esp32_improv_component.cpp | 9 ++------- esphome/components/esp32_improv/esp32_improv_component.h | 1 - 2 files changed, 2 insertions(+), 8 deletions(-) diff --git a/esphome/components/esp32_improv/esp32_improv_component.cpp b/esphome/components/esp32_improv/esp32_improv_component.cpp index b3258aedac..d83caf931b 100644 --- a/esphome/components/esp32_improv/esp32_improv_component.cpp +++ b/esphome/components/esp32_improv/esp32_improv_component.cpp @@ -375,15 +375,11 @@ void ESP32ImprovComponent::on_wifi_connect_timeout_() { } void ESP32ImprovComponent::check_wifi_connection_() { - if (wifi::global_wifi_component->is_connected()) { - this->on_wifi_connected_(); + if (!wifi::global_wifi_component->is_connected()) { + return; } -} -void ESP32ImprovComponent::on_wifi_connected_() { - // Handle WiFi connection, whether from Improv provisioning or external (e.g., captive portal) if (this->state_ == improv::STATE_PROVISIONING) { - // WiFi provisioned via Improv - save credentials and send response wifi::global_wifi_component->save_wifi_sta(this->connecting_sta_.get_ssid(), this->connecting_sta_.get_password()); this->connecting_sta_ = {}; this->cancel_timeout("wifi-connect-timeout"); @@ -401,7 +397,6 @@ void ESP32ImprovComponent::on_wifi_connected_() { std::vector data = improv::build_rpc_response(improv::WIFI_SETTINGS, urls); this->send_response_(data); } else if (this->is_active() && this->state_ != improv::STATE_PROVISIONED) { - // WiFi provisioned externally (e.g., captive portal) - just transition to provisioned ESP_LOGD(TAG, "WiFi provisioned externally"); } diff --git a/esphome/components/esp32_improv/esp32_improv_component.h b/esphome/components/esp32_improv/esp32_improv_component.h index da670f54bc..6782430ffe 100644 --- a/esphome/components/esp32_improv/esp32_improv_component.h +++ b/esphome/components/esp32_improv/esp32_improv_component.h @@ -111,7 +111,6 @@ class ESP32ImprovComponent : public Component { void send_response_(std::vector &response); void process_incoming_data_(); void on_wifi_connect_timeout_(); - void on_wifi_connected_(); void check_wifi_connection_(); bool check_identify_(); void advertise_service_data_(); From 9f20c48a24e4adbada8c7106c8700da433cb59e3 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Sat, 11 Oct 2025 15:23:51 -1000 Subject: [PATCH 24/90] adjust --- esphome/components/esp32_ble/ble.cpp | 6 +++++- esphome/components/ethernet/ethernet_component.cpp | 4 +++- esphome/components/mqtt/mqtt_client.cpp | 3 ++- esphome/components/wifi/wifi_component.cpp | 4 +++- esphome/core/application.h | 10 +++++++--- esphome/core/helpers.cpp | 11 ++++++++--- esphome/core/helpers.h | 5 +++-- 7 files changed, 31 insertions(+), 12 deletions(-) diff --git a/esphome/components/esp32_ble/ble.cpp b/esphome/components/esp32_ble/ble.cpp index e37b45fe71..7072a485a1 100644 --- a/esphome/components/esp32_ble/ble.cpp +++ b/esphome/components/esp32_ble/ble.cpp @@ -213,7 +213,11 @@ bool ESP32BLE::ble_setup_() { if (this->name_.has_value()) { name = this->name_.value(); if (App.is_name_add_mac_suffix_enabled()) { - name = make_name_with_suffix(name, '-', get_mac_address().substr(6)); + // MAC address suffix length (last 6 characters of 12-char MAC address string) + constexpr size_t MAC_ADDRESS_SUFFIX_LEN = 6; + const std::string mac_addr = get_mac_address(); + const char *mac_suffix_ptr = mac_addr.c_str() + MAC_ADDRESS_SUFFIX_LEN; + name = make_name_with_suffix(name, '-', mac_suffix_ptr, MAC_ADDRESS_SUFFIX_LEN); } } else { name = App.get_name(); diff --git a/esphome/components/ethernet/ethernet_component.cpp b/esphome/components/ethernet/ethernet_component.cpp index 5f28d6db25..8257e37b52 100644 --- a/esphome/components/ethernet/ethernet_component.cpp +++ b/esphome/components/ethernet/ethernet_component.cpp @@ -691,7 +691,9 @@ void EthernetComponent::set_manual_ip(const ManualIP &manual_ip) { this->manual_ std::string EthernetComponent::get_use_address() const { if (this->use_address_.empty()) { - return make_name_with_suffix(App.get_name(), '.', "local"); + // ".local" suffix length for mDNS hostnames + constexpr size_t MDNS_LOCAL_SUFFIX_LEN = 5; + return make_name_with_suffix(App.get_name(), '.', "local", MDNS_LOCAL_SUFFIX_LEN); } return this->use_address_; } diff --git a/esphome/components/mqtt/mqtt_client.cpp b/esphome/components/mqtt/mqtt_client.cpp index 3642ddb38e..16f54ab8a0 100644 --- a/esphome/components/mqtt/mqtt_client.cpp +++ b/esphome/components/mqtt/mqtt_client.cpp @@ -29,7 +29,8 @@ static const char *const TAG = "mqtt"; MQTTClientComponent::MQTTClientComponent() { global_mqtt_client = this; - this->credentials_.client_id = make_name_with_suffix(App.get_name(), '-', get_mac_address()); + const std::string mac_addr = get_mac_address(); + this->credentials_.client_id = make_name_with_suffix(App.get_name(), '-', mac_addr.c_str(), mac_addr.size()); } // Connection diff --git a/esphome/components/wifi/wifi_component.cpp b/esphome/components/wifi/wifi_component.cpp index ec8687e927..db9b1d43fa 100644 --- a/esphome/components/wifi/wifi_component.cpp +++ b/esphome/components/wifi/wifi_component.cpp @@ -267,7 +267,9 @@ network::IPAddress WiFiComponent::get_dns_address(int num) { } std::string WiFiComponent::get_use_address() const { if (this->use_address_.empty()) { - return make_name_with_suffix(App.get_name(), '.', "local"); + // ".local" suffix length for mDNS hostnames + constexpr size_t MDNS_LOCAL_SUFFIX_LEN = 5; + return make_name_with_suffix(App.get_name(), '.', "local", MDNS_LOCAL_SUFFIX_LEN); } return this->use_address_; } diff --git a/esphome/core/application.h b/esphome/core/application.h index 47e902a191..646c2376bb 100644 --- a/esphome/core/application.h +++ b/esphome/core/application.h @@ -102,10 +102,14 @@ class Application { arch_init(); this->name_add_mac_suffix_ = name_add_mac_suffix; if (name_add_mac_suffix) { - const std::string mac_suffix = get_mac_address().substr(6); - this->name_ = make_name_with_suffix(name, '-', mac_suffix); + // MAC address suffix length (last 6 characters of 12-char MAC address string) + constexpr size_t MAC_ADDRESS_SUFFIX_LEN = 6; + const std::string mac_addr = get_mac_address(); + // Use pointer + offset to avoid substr() allocation + const char *mac_suffix_ptr = mac_addr.c_str() + MAC_ADDRESS_SUFFIX_LEN; + this->name_ = make_name_with_suffix(name, '-', mac_suffix_ptr, MAC_ADDRESS_SUFFIX_LEN); if (!friendly_name.empty()) { - this->friendly_name_ = make_name_with_suffix(friendly_name, ' ', mac_suffix); + this->friendly_name_ = make_name_with_suffix(friendly_name, ' ', mac_suffix_ptr, MAC_ADDRESS_SUFFIX_LEN); } } else { this->name_ = name; diff --git a/esphome/core/helpers.cpp b/esphome/core/helpers.cpp index 0b6c203c01..cabd9ffd16 100644 --- a/esphome/core/helpers.cpp +++ b/esphome/core/helpers.cpp @@ -237,11 +237,16 @@ std::string str_sprintf(const char *fmt, ...) { // Maximum size for name with suffix: 120 (max friendly name) + 1 (separator) + 6 (MAC suffix) + 1 (null term) static constexpr size_t MAX_NAME_WITH_SUFFIX_SIZE = 128; +// MAC address suffix length (last 6 characters of 12-char MAC address string) +static constexpr size_t MAC_ADDRESS_SUFFIX_LEN = 6; +// Full MAC address string length (lowercase hex without separators) +static constexpr size_t MAC_ADDRESS_LEN = 12; +// ".local" suffix length for mDNS hostnames +static constexpr size_t MDNS_LOCAL_SUFFIX_LEN = 5; -std::string make_name_with_suffix(const std::string &name, char sep, const std::string &suffix) { +std::string make_name_with_suffix(const std::string &name, char sep, const char *suffix_ptr, size_t suffix_len) { char buffer[MAX_NAME_WITH_SUFFIX_SIZE]; size_t name_len = name.size(); - size_t suffix_len = suffix.size(); size_t total_len = name_len + 1 + suffix_len; // Silently truncate if needed: prioritize keeping the full suffix @@ -255,7 +260,7 @@ std::string make_name_with_suffix(const std::string &name, char sep, const std:: memcpy(buffer, name.c_str(), name_len); buffer[name_len] = sep; - memcpy(buffer + name_len + 1, suffix.c_str(), suffix_len); + memcpy(buffer + name_len + 1, suffix_ptr, suffix_len); buffer[total_len] = '\0'; return std::string(buffer, total_len); } diff --git a/esphome/core/helpers.h b/esphome/core/helpers.h index f64c14aa85..adce18408e 100644 --- a/esphome/core/helpers.h +++ b/esphome/core/helpers.h @@ -311,9 +311,10 @@ std::string __attribute__((format(printf, 1, 2))) str_sprintf(const char *fmt, . /// Maximum name length supported is 120 characters for friendly names. /// @param name The base name string /// @param sep The separator character (e.g., '-', ' ', or '.') -/// @param suffix The suffix to append (e.g., MAC address suffix or ".local") +/// @param suffix_ptr Pointer to the suffix characters +/// @param suffix_len Length of the suffix /// @return The concatenated string: name + sep + suffix -std::string make_name_with_suffix(const std::string &name, char sep, const std::string &suffix); +std::string make_name_with_suffix(const std::string &name, char sep, const char *suffix_ptr, size_t suffix_len); ///@} From 21c2c6e7825190d93eb3ea3aba6d529fcea40090 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Sat, 11 Oct 2025 15:33:19 -1000 Subject: [PATCH 25/90] Update esphome/config_validation.py Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- esphome/config_validation.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/esphome/config_validation.py b/esphome/config_validation.py index 2746f574ba..ebfedf2017 100644 --- a/esphome/config_validation.py +++ b/esphome/config_validation.py @@ -1200,7 +1200,7 @@ def hostname(value): Maximum length is 63 characters per RFC 1035. Note: If this limit is changed, update MAX_NAME_WITH_SUFFIX_SIZE in - esphome/core/application.h to accommodate the new maximum length. + esphome/core/helpers.cpp to accommodate the new maximum length. """ value = string(value) if re.match(r"^[a-z0-9-]{1,63}$", value, re.IGNORECASE) is not None: From 5fe319fcc56217c38e47b206b00784691bba1b79 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Sat, 11 Oct 2025 15:33:22 -1000 Subject: [PATCH 26/90] preen --- esphome/core/helpers.cpp | 6 ------ 1 file changed, 6 deletions(-) diff --git a/esphome/core/helpers.cpp b/esphome/core/helpers.cpp index cabd9ffd16..2d8122699b 100644 --- a/esphome/core/helpers.cpp +++ b/esphome/core/helpers.cpp @@ -237,12 +237,6 @@ std::string str_sprintf(const char *fmt, ...) { // Maximum size for name with suffix: 120 (max friendly name) + 1 (separator) + 6 (MAC suffix) + 1 (null term) static constexpr size_t MAX_NAME_WITH_SUFFIX_SIZE = 128; -// MAC address suffix length (last 6 characters of 12-char MAC address string) -static constexpr size_t MAC_ADDRESS_SUFFIX_LEN = 6; -// Full MAC address string length (lowercase hex without separators) -static constexpr size_t MAC_ADDRESS_LEN = 12; -// ".local" suffix length for mDNS hostnames -static constexpr size_t MDNS_LOCAL_SUFFIX_LEN = 5; std::string make_name_with_suffix(const std::string &name, char sep, const char *suffix_ptr, size_t suffix_len) { char buffer[MAX_NAME_WITH_SUFFIX_SIZE]; From e69013317d8c6f64e1d7bb5677e353d68730272c Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Sat, 11 Oct 2025 15:33:46 -1000 Subject: [PATCH 27/90] Update esphome/core/helpers.cpp Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- esphome/core/helpers.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/esphome/core/helpers.cpp b/esphome/core/helpers.cpp index 2d8122699b..fb8b220b2f 100644 --- a/esphome/core/helpers.cpp +++ b/esphome/core/helpers.cpp @@ -247,7 +247,7 @@ std::string make_name_with_suffix(const std::string &name, char sep, const char if (total_len >= MAX_NAME_WITH_SUFFIX_SIZE) { // NOTE: This calculation could underflow if suffix_len >= MAX_NAME_WITH_SUFFIX_SIZE - 2, // but this is safe because this helper is only called with small suffixes: - // MAC suffixes (6-12 bytes), ".local" (6 bytes), etc. + // MAC suffixes (6-12 bytes), ".local" (5 bytes), etc. name_len = MAX_NAME_WITH_SUFFIX_SIZE - suffix_len - 2; // -2 for separator and null terminator total_len = name_len + 1 + suffix_len; } From 153f01ef773c31bedf0350bd2440247906fea120 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Sat, 11 Oct 2025 15:34:15 -1000 Subject: [PATCH 28/90] preen --- script/determine-jobs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/script/determine-jobs.py b/script/determine-jobs.py index 1601496877..a078fd8f9b 100755 --- a/script/determine-jobs.py +++ b/script/determine-jobs.py @@ -244,7 +244,7 @@ def main() -> None: component for component in changed_components if (component_test_dir := tests_dir / component).exists() - and next(component_test_dir.glob("test.*.yaml"), None) is not None + and any(component_test_dir.glob("test.*.yaml")) ] # Build output From b0c20d7adb8df3cf61d5292c09233bc295af3b4d Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Sat, 11 Oct 2025 16:54:11 -1000 Subject: [PATCH 29/90] [core] Optimize looping_components_ with FixedVector to save flash --- esphome/core/application.cpp | 4 ++-- esphome/core/application.h | 2 +- esphome/core/helpers.h | 44 ++++++++++++++++++++++++++++++++++++ 3 files changed, 47 insertions(+), 3 deletions(-) diff --git a/esphome/core/application.cpp b/esphome/core/application.cpp index 1be193bb7e..c745aa0ae5 100644 --- a/esphome/core/application.cpp +++ b/esphome/core/application.cpp @@ -340,8 +340,8 @@ void Application::calculate_looping_components_() { } } - // Pre-reserve vector to avoid reallocations - this->looping_components_.reserve(total_looping); + // Initialize FixedVector with exact size - no reallocation possible + this->looping_components_.init(total_looping); // Add all components with loop override that aren't already LOOP_DONE // Some components (like logger) may call disable_loop() during initialization diff --git a/esphome/core/application.h b/esphome/core/application.h index 1f22499051..b0f9c23191 100644 --- a/esphome/core/application.h +++ b/esphome/core/application.h @@ -472,7 +472,7 @@ class Application { // - When a component is enabled, it's swapped with the first inactive component // and active_end_ is incremented // - This eliminates branch mispredictions from flag checking in the hot loop - std::vector looping_components_{}; + FixedVector looping_components_{}; #ifdef USE_SOCKET_SELECT_SUPPORT std::vector socket_fds_; // Vector of all monitored socket file descriptors #endif diff --git a/esphome/core/helpers.h b/esphome/core/helpers.h index e06f2d15ef..864fd92ffc 100644 --- a/esphome/core/helpers.h +++ b/esphome/core/helpers.h @@ -159,6 +159,50 @@ template class StaticVector { const_reverse_iterator rend() const { return const_reverse_iterator(begin()); } }; +/// Fixed-capacity vector - allocates once at runtime, never reallocates +/// This avoids std::vector template overhead (_M_realloc_insert, _M_default_append) +/// when size is known at initialization but not at compile time +template class FixedVector { + private: + T *data_{nullptr}; + size_t size_{0}; + size_t capacity_{0}; + + public: + FixedVector() = default; + + ~FixedVector() { + if (data_ != nullptr) { + delete[] data_; + } + } + + // Disable copy to avoid accidental copies + FixedVector(const FixedVector &) = delete; + FixedVector &operator=(const FixedVector &) = delete; + + // Allocate capacity - can only be called once on empty vector + void init(size_t n) { + if (data_ == nullptr && n > 0) { + data_ = new T[n]; + capacity_ = n; + size_ = 0; + } + } + + // Add element (no reallocation - must have initialized capacity) + void push_back(const T &value) { + if (size_ < capacity_) { + data_[size_++] = value; + } + } + + size_t size() const { return size_; } + + T &operator[](size_t i) { return data_[i]; } + const T &operator[](size_t i) const { return data_[i]; } +}; + ///@} /// @name Mathematics From fdc9ea285dcd57de91f764c2802dd9cc8f7c3e8b Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Sat, 11 Oct 2025 17:30:30 -1000 Subject: [PATCH 30/90] [http_request] Pass parameters by const reference to reduce flash usage --- esphome/components/http_request/http_request.h | 4 ++-- esphome/components/http_request/http_request_arduino.cpp | 5 +++-- esphome/components/http_request/http_request_arduino.h | 4 ++-- esphome/components/http_request/http_request_host.cpp | 5 +++-- esphome/components/http_request/http_request_host.h | 4 ++-- esphome/components/http_request/http_request_idf.cpp | 5 +++-- esphome/components/http_request/http_request_idf.h | 4 ++-- 7 files changed, 17 insertions(+), 14 deletions(-) diff --git a/esphome/components/http_request/http_request.h b/esphome/components/http_request/http_request.h index 95515f731a..bb14cc6f51 100644 --- a/esphome/components/http_request/http_request.h +++ b/esphome/components/http_request/http_request.h @@ -167,8 +167,8 @@ class HttpRequestComponent : public Component { } protected: - virtual std::shared_ptr perform(std::string url, std::string method, std::string body, - std::list
request_headers, + virtual std::shared_ptr perform(const std::string &url, const std::string &method, + const std::string &body, const std::list
&request_headers, std::set collect_headers) = 0; const char *useragent_{nullptr}; bool follow_redirects_{}; diff --git a/esphome/components/http_request/http_request_arduino.cpp b/esphome/components/http_request/http_request_arduino.cpp index c009b33c2d..dfdbbd3fab 100644 --- a/esphome/components/http_request/http_request_arduino.cpp +++ b/esphome/components/http_request/http_request_arduino.cpp @@ -14,8 +14,9 @@ namespace http_request { static const char *const TAG = "http_request.arduino"; -std::shared_ptr HttpRequestArduino::perform(std::string url, std::string method, std::string body, - std::list
request_headers, +std::shared_ptr HttpRequestArduino::perform(const std::string &url, const std::string &method, + const std::string &body, + const std::list
&request_headers, std::set collect_headers) { if (!network::is_connected()) { this->status_momentary_error("failed", 1000); diff --git a/esphome/components/http_request/http_request_arduino.h b/esphome/components/http_request/http_request_arduino.h index 44744f8c78..c8208c74d8 100644 --- a/esphome/components/http_request/http_request_arduino.h +++ b/esphome/components/http_request/http_request_arduino.h @@ -31,8 +31,8 @@ class HttpContainerArduino : public HttpContainer { class HttpRequestArduino : public HttpRequestComponent { protected: - std::shared_ptr perform(std::string url, std::string method, std::string body, - std::list
request_headers, + std::shared_ptr perform(const std::string &url, const std::string &method, const std::string &body, + const std::list
&request_headers, std::set collect_headers) override; }; diff --git a/esphome/components/http_request/http_request_host.cpp b/esphome/components/http_request/http_request_host.cpp index 0b4c998a40..c20ea552b7 100644 --- a/esphome/components/http_request/http_request_host.cpp +++ b/esphome/components/http_request/http_request_host.cpp @@ -17,8 +17,9 @@ namespace http_request { static const char *const TAG = "http_request.host"; -std::shared_ptr HttpRequestHost::perform(std::string url, std::string method, std::string body, - std::list
request_headers, +std::shared_ptr HttpRequestHost::perform(const std::string &url, const std::string &method, + const std::string &body, + const std::list
&request_headers, std::set response_headers) { if (!network::is_connected()) { this->status_momentary_error("failed", 1000); diff --git a/esphome/components/http_request/http_request_host.h b/esphome/components/http_request/http_request_host.h index bbeed87f70..fdd72e7ea5 100644 --- a/esphome/components/http_request/http_request_host.h +++ b/esphome/components/http_request/http_request_host.h @@ -18,8 +18,8 @@ class HttpContainerHost : public HttpContainer { class HttpRequestHost : public HttpRequestComponent { public: - std::shared_ptr perform(std::string url, std::string method, std::string body, - std::list
request_headers, + std::shared_ptr perform(const std::string &url, const std::string &method, const std::string &body, + const std::list
&request_headers, std::set response_headers) override; void set_ca_path(const char *ca_path) { this->ca_path_ = ca_path; } diff --git a/esphome/components/http_request/http_request_idf.cpp b/esphome/components/http_request/http_request_idf.cpp index 89a0891b03..a91c0bfc25 100644 --- a/esphome/components/http_request/http_request_idf.cpp +++ b/esphome/components/http_request/http_request_idf.cpp @@ -52,8 +52,9 @@ esp_err_t HttpRequestIDF::http_event_handler(esp_http_client_event_t *evt) { return ESP_OK; } -std::shared_ptr HttpRequestIDF::perform(std::string url, std::string method, std::string body, - std::list
request_headers, +std::shared_ptr HttpRequestIDF::perform(const std::string &url, const std::string &method, + const std::string &body, + const std::list
&request_headers, std::set collect_headers) { if (!network::is_connected()) { this->status_momentary_error("failed", 1000); diff --git a/esphome/components/http_request/http_request_idf.h b/esphome/components/http_request/http_request_idf.h index 5c5b784853..90dee0be68 100644 --- a/esphome/components/http_request/http_request_idf.h +++ b/esphome/components/http_request/http_request_idf.h @@ -37,8 +37,8 @@ class HttpRequestIDF : public HttpRequestComponent { void set_buffer_size_tx(uint16_t buffer_size_tx) { this->buffer_size_tx_ = buffer_size_tx; } protected: - std::shared_ptr perform(std::string url, std::string method, std::string body, - std::list
request_headers, + std::shared_ptr perform(const std::string &url, const std::string &method, const std::string &body, + const std::list
&request_headers, std::set collect_headers) override; // if zero ESP-IDF will use DEFAULT_HTTP_BUF_SIZE uint16_t buffer_size_rx_{}; From 2ff3e7fb2bd017dddd187065078c21126f959cbe Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Sat, 11 Oct 2025 17:34:51 -1000 Subject: [PATCH 31/90] add comments for bot --- esphome/core/helpers.h | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/esphome/core/helpers.h b/esphome/core/helpers.h index 864fd92ffc..3782875dcf 100644 --- a/esphome/core/helpers.h +++ b/esphome/core/helpers.h @@ -190,7 +190,9 @@ template class FixedVector { } } - // Add element (no reallocation - must have initialized capacity) + /// Add element without bounds checking + /// Caller must ensure sufficient capacity was allocated via init() + /// Silently ignores pushes beyond capacity to avoid runtime overhead void push_back(const T &value) { if (size_ < capacity_) { data_[size_++] = value; @@ -199,6 +201,8 @@ template class FixedVector { size_t size() const { return size_; } + /// Access element without bounds checking (matches std::vector behavior) + /// Caller must ensure index is valid (i < size()) T &operator[](size_t i) { return data_[i]; } const T &operator[](size_t i) const { return data_[i]; } }; From 4c00861760ce5fd22b7ac82584ba9dd20f1e23ee Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Sat, 11 Oct 2025 17:35:31 -1000 Subject: [PATCH 32/90] add comments for bot --- esphome/core/helpers.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/esphome/core/helpers.h b/esphome/core/helpers.h index 3782875dcf..b5a0a1c8ac 100644 --- a/esphome/core/helpers.h +++ b/esphome/core/helpers.h @@ -192,7 +192,7 @@ template class FixedVector { /// Add element without bounds checking /// Caller must ensure sufficient capacity was allocated via init() - /// Silently ignores pushes beyond capacity to avoid runtime overhead + /// Silently ignores pushes beyond capacity (no exception or assertion) void push_back(const T &value) { if (size_ < capacity_) { data_[size_++] = value; From 66c8c045f21be353f3e68f0056639d3fdea5fcf6 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Sat, 11 Oct 2025 21:23:02 -1000 Subject: [PATCH 33/90] [ota] Increase handshake timeout to 20s now that auth is non-blocking --- esphome/components/esphome/ota/ota_esphome.cpp | 2 +- esphome/espota2.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/esphome/components/esphome/ota/ota_esphome.cpp b/esphome/components/esphome/ota/ota_esphome.cpp index b65bfc5ab8..569268ea15 100644 --- a/esphome/components/esphome/ota/ota_esphome.cpp +++ b/esphome/components/esphome/ota/ota_esphome.cpp @@ -29,7 +29,7 @@ namespace esphome { static const char *const TAG = "esphome.ota"; static constexpr uint16_t OTA_BLOCK_SIZE = 8192; static constexpr size_t OTA_BUFFER_SIZE = 1024; // buffer size for OTA data transfer -static constexpr uint32_t OTA_SOCKET_TIMEOUT_HANDSHAKE = 10000; // milliseconds for initial handshake +static constexpr uint32_t OTA_SOCKET_TIMEOUT_HANDSHAKE = 20000; // milliseconds for initial handshake static constexpr uint32_t OTA_SOCKET_TIMEOUT_DATA = 90000; // milliseconds for data transfer #ifdef USE_OTA_PASSWORD diff --git a/esphome/espota2.py b/esphome/espota2.py index 2712d00127..17a1da8235 100644 --- a/esphome/espota2.py +++ b/esphome/espota2.py @@ -410,7 +410,7 @@ def run_ota_impl_( af, socktype, _, _, sa = r _LOGGER.info("Connecting to %s port %s...", sa[0], sa[1]) sock = socket.socket(af, socktype) - sock.settimeout(10.0) + sock.settimeout(20.0) try: sock.connect(sa) except OSError as err: From 072662c395c27a36b3720f75c753956f79e0550f Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Sat, 11 Oct 2025 21:26:13 -1000 Subject: [PATCH 34/90] timeout --- tests/unit_tests/test_espota2.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit_tests/test_espota2.py b/tests/unit_tests/test_espota2.py index bd1a6bde81..52c72291d6 100644 --- a/tests/unit_tests/test_espota2.py +++ b/tests/unit_tests/test_espota2.py @@ -493,7 +493,7 @@ def test_run_ota_impl_successful( assert result_host == "192.168.1.100" # Verify socket was configured correctly - mock_socket.settimeout.assert_called_with(10.0) + mock_socket.settimeout.assert_called_with(20.0) mock_socket.connect.assert_called_once_with(("192.168.1.100", 3232)) mock_socket.close.assert_called_once() From ddc7a15302b0a0d622bd2a1b78e47b3e07d3a95e Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Sat, 11 Oct 2025 22:02:59 -1000 Subject: [PATCH 35/90] [wifi] Fix missed string literal in flash on ESP8266 --- esphome/components/wifi/wifi_component.cpp | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/esphome/components/wifi/wifi_component.cpp b/esphome/components/wifi/wifi_component.cpp index 2e083d4c68..71ee4271ba 100644 --- a/esphome/components/wifi/wifi_component.cpp +++ b/esphome/components/wifi/wifi_component.cpp @@ -576,8 +576,9 @@ __attribute__((noinline)) static void log_scan_result(const WiFiScanResult &res) format_mac_addr_upper(bssid.data(), bssid_s); if (res.get_matches()) { - ESP_LOGI(TAG, "- '%s' %s" LOG_SECRET("(%s) ") "%s", res.get_ssid().c_str(), res.get_is_hidden() ? "(HIDDEN) " : "", - bssid_s, LOG_STR_ARG(get_signal_bars(res.get_rssi()))); + ESP_LOGI(TAG, "- '%s' %s" LOG_SECRET("(%s) ") "%s", res.get_ssid().c_str(), + res.get_is_hidden() ? LOG_STR_LITERAL("(HIDDEN) ") : LOG_STR_LITERAL(""), bssid_s, + LOG_STR_ARG(get_signal_bars(res.get_rssi()))); ESP_LOGD(TAG, " Channel: %u\n" " RSSI: %d dB", From 2a94463ac1d3f86250efdbf6e25838ce4377ac22 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Sun, 12 Oct 2025 11:07:01 -1000 Subject: [PATCH 36/90] [esp32_ble] Replace handler vectors with StaticVector for 2KB memory savings --- esphome/components/esp32_ble/__init__.py | 78 +++++++++++++++++++ esphome/components/esp32_ble/ble.cpp | 22 +++++- esphome/components/esp32_ble/ble.h | 44 ++++++++--- .../components/esp32_ble_beacon/__init__.py | 2 +- .../components/esp32_ble_server/__init__.py | 4 +- .../components/esp32_ble_tracker/__init__.py | 8 +- esphome/core/defines.h | 5 ++ 7 files changed, 141 insertions(+), 22 deletions(-) diff --git a/esphome/components/esp32_ble/__init__.py b/esphome/components/esp32_ble/__init__.py index 1c142ca7bd..d2ce4716f4 100644 --- a/esphome/components/esp32_ble/__init__.py +++ b/esphome/components/esp32_ble/__init__.py @@ -1,4 +1,5 @@ from collections.abc import Callable, MutableMapping +from dataclasses import dataclass from enum import Enum import logging import re @@ -111,6 +112,58 @@ class BTLoggers(Enum): _required_loggers: set[BTLoggers] = set() +# Dataclass for handler registration counts +@dataclass +class HandlerCounts: + gap_event: int = 0 + gap_scan_event: int = 0 + gattc_event: int = 0 + gatts_event: int = 0 + ble_status_event: int = 0 + + +# Track handler registration counts for StaticVector sizing +_handler_counts = HandlerCounts() + + +def register_gap_event_handler(parent_var: cg.MockObj, handler_var: cg.MockObj) -> None: + """Register a GAP event handler and track the count.""" + _handler_counts.gap_event += 1 + cg.add(parent_var.register_gap_event_handler(handler_var)) + + +def register_gap_scan_event_handler( + parent_var: cg.MockObj, handler_var: cg.MockObj +) -> None: + """Register a GAP scan event handler and track the count.""" + _handler_counts.gap_scan_event += 1 + cg.add(parent_var.register_gap_scan_event_handler(handler_var)) + + +def register_gattc_event_handler( + parent_var: cg.MockObj, handler_var: cg.MockObj +) -> None: + """Register a GATTc event handler and track the count.""" + _handler_counts.gattc_event += 1 + cg.add(parent_var.register_gattc_event_handler(handler_var)) + + +def register_gatts_event_handler( + parent_var: cg.MockObj, handler_var: cg.MockObj +) -> None: + """Register a GATTs event handler and track the count.""" + _handler_counts.gatts_event += 1 + cg.add(parent_var.register_gatts_event_handler(handler_var)) + + +def register_ble_status_event_handler( + parent_var: cg.MockObj, handler_var: cg.MockObj +) -> None: + """Register a BLE status event handler and track the count.""" + _handler_counts.ble_status_event += 1 + cg.add(parent_var.register_ble_status_event_handler(handler_var)) + + def register_bt_logger(*loggers: BTLoggers) -> None: """Register Bluetooth logger categories that a component needs. @@ -368,6 +421,31 @@ def final_validation(config): # For newer chips (C3/S3/etc), different configs are used automatically add_idf_sdkconfig_option("CONFIG_BTDM_CTRL_BLE_MAX_CONN", max_connections) + # Add defines for StaticVector sizing based on handler registration counts + # Only define if count > 0 to avoid allocating unnecessary memory + if _handler_counts.gap_event > 0: + cg.add_define( + "ESPHOME_ESP32_BLE_GAP_EVENT_HANDLER_COUNT", _handler_counts.gap_event + ) + if _handler_counts.gap_scan_event > 0: + cg.add_define( + "ESPHOME_ESP32_BLE_GAP_SCAN_EVENT_HANDLER_COUNT", + _handler_counts.gap_scan_event, + ) + if _handler_counts.gattc_event > 0: + cg.add_define( + "ESPHOME_ESP32_BLE_GATTC_EVENT_HANDLER_COUNT", _handler_counts.gattc_event + ) + if _handler_counts.gatts_event > 0: + cg.add_define( + "ESPHOME_ESP32_BLE_GATTS_EVENT_HANDLER_COUNT", _handler_counts.gatts_event + ) + if _handler_counts.ble_status_event > 0: + cg.add_define( + "ESPHOME_ESP32_BLE_BLE_STATUS_EVENT_HANDLER_COUNT", + _handler_counts.ble_status_event, + ) + return config diff --git a/esphome/components/esp32_ble/ble.cpp b/esphome/components/esp32_ble/ble.cpp index 0c340c55cc..ca119f8836 100644 --- a/esphome/components/esp32_ble/ble.cpp +++ b/esphome/components/esp32_ble/ble.cpp @@ -181,6 +181,7 @@ bool ESP32BLE::ble_setup_() { return false; } +#ifdef ESPHOME_ESP32_BLE_GAP_EVENT_HANDLER_COUNT if (!this->gap_event_handlers_.empty()) { err = esp_ble_gap_register_callback(ESP32BLE::gap_event_handler); if (err != ESP_OK) { @@ -188,8 +189,9 @@ bool ESP32BLE::ble_setup_() { return false; } } +#endif -#ifdef USE_ESP32_BLE_SERVER +#if defined(USE_ESP32_BLE_SERVER) && defined(ESPHOME_ESP32_BLE_GATTS_EVENT_HANDLER_COUNT) if (!this->gatts_event_handlers_.empty()) { err = esp_ble_gatts_register_callback(ESP32BLE::gatts_event_handler); if (err != ESP_OK) { @@ -199,7 +201,7 @@ bool ESP32BLE::ble_setup_() { } #endif -#ifdef USE_ESP32_BLE_CLIENT +#if defined(USE_ESP32_BLE_CLIENT) && defined(ESPHOME_ESP32_BLE_GATTC_EVENT_HANDLER_COUNT) if (!this->gattc_event_handlers_.empty()) { err = esp_ble_gattc_register_callback(ESP32BLE::gattc_event_handler); if (err != ESP_OK) { @@ -299,9 +301,11 @@ void ESP32BLE::loop() { case BLE_COMPONENT_STATE_DISABLE: { ESP_LOGD(TAG, "Disabling"); +#ifdef ESPHOME_ESP32_BLE_BLE_STATUS_EVENT_HANDLER_COUNT for (auto *ble_event_handler : this->ble_status_event_handlers_) { ble_event_handler->ble_before_disabled_event_handler(); } +#endif if (!ble_dismantle_()) { ESP_LOGE(TAG, "Could not be dismantled"); @@ -331,7 +335,7 @@ void ESP32BLE::loop() { BLEEvent *ble_event = this->ble_events_.pop(); while (ble_event != nullptr) { switch (ble_event->type_) { -#ifdef USE_ESP32_BLE_SERVER +#if defined(USE_ESP32_BLE_SERVER) && defined(ESPHOME_ESP32_BLE_GATTS_EVENT_HANDLER_COUNT) case BLEEvent::GATTS: { esp_gatts_cb_event_t event = ble_event->event_.gatts.gatts_event; esp_gatt_if_t gatts_if = ble_event->event_.gatts.gatts_if; @@ -343,7 +347,7 @@ void ESP32BLE::loop() { break; } #endif -#ifdef USE_ESP32_BLE_CLIENT +#if defined(USE_ESP32_BLE_CLIENT) && defined(ESPHOME_ESP32_BLE_GATTC_EVENT_HANDLER_COUNT) case BLEEvent::GATTC: { esp_gattc_cb_event_t event = ble_event->event_.gattc.gattc_event; esp_gatt_if_t gattc_if = ble_event->event_.gattc.gattc_if; @@ -359,10 +363,12 @@ void ESP32BLE::loop() { esp_gap_ble_cb_event_t gap_event = ble_event->event_.gap.gap_event; switch (gap_event) { case ESP_GAP_BLE_SCAN_RESULT_EVT: +#ifdef ESPHOME_ESP32_BLE_GAP_SCAN_EVENT_HANDLER_COUNT // Use the new scan event handler - no memcpy! for (auto *scan_handler : this->gap_scan_event_handlers_) { scan_handler->gap_scan_event_handler(ble_event->scan_result()); } +#endif break; // Scan complete events @@ -374,10 +380,12 @@ void ESP32BLE::loop() { // This is verified at compile-time by static_assert checks in ble_event.h // The struct already contains our copy of the status (copied in BLEEvent constructor) ESP_LOGV(TAG, "gap_event_handler - %d", gap_event); +#ifdef ESPHOME_ESP32_BLE_GAP_EVENT_HANDLER_COUNT for (auto *gap_handler : this->gap_event_handlers_) { gap_handler->gap_event_handler( gap_event, reinterpret_cast(&ble_event->event_.gap.scan_complete)); } +#endif break; // Advertising complete events @@ -388,19 +396,23 @@ void ESP32BLE::loop() { case ESP_GAP_BLE_ADV_STOP_COMPLETE_EVT: // All advertising complete events have the same structure with just status ESP_LOGV(TAG, "gap_event_handler - %d", gap_event); +#ifdef ESPHOME_ESP32_BLE_GAP_EVENT_HANDLER_COUNT for (auto *gap_handler : this->gap_event_handlers_) { gap_handler->gap_event_handler( gap_event, reinterpret_cast(&ble_event->event_.gap.adv_complete)); } +#endif break; // RSSI complete event case ESP_GAP_BLE_READ_RSSI_COMPLETE_EVT: ESP_LOGV(TAG, "gap_event_handler - %d", gap_event); +#ifdef ESPHOME_ESP32_BLE_GAP_EVENT_HANDLER_COUNT for (auto *gap_handler : this->gap_event_handlers_) { gap_handler->gap_event_handler( gap_event, reinterpret_cast(&ble_event->event_.gap.read_rssi_complete)); } +#endif break; // Security events @@ -410,10 +422,12 @@ void ESP32BLE::loop() { case ESP_GAP_BLE_PASSKEY_REQ_EVT: case ESP_GAP_BLE_NC_REQ_EVT: ESP_LOGV(TAG, "gap_event_handler - %d", gap_event); +#ifdef ESPHOME_ESP32_BLE_GAP_EVENT_HANDLER_COUNT for (auto *gap_handler : this->gap_event_handlers_) { gap_handler->gap_event_handler( gap_event, reinterpret_cast(&ble_event->event_.gap.security)); } +#endif break; default: diff --git a/esphome/components/esp32_ble/ble.h b/esphome/components/esp32_ble/ble.h index 1aa3bc86ef..617047372b 100644 --- a/esphome/components/esp32_ble/ble.h +++ b/esphome/components/esp32_ble/ble.h @@ -125,18 +125,34 @@ class ESP32BLE : public Component { void advertising_register_raw_advertisement_callback(std::function &&callback); #endif - void register_gap_event_handler(GAPEventHandler *handler) { this->gap_event_handlers_.push_back(handler); } + void register_gap_event_handler(GAPEventHandler *handler) { +#ifdef ESPHOME_ESP32_BLE_GAP_EVENT_HANDLER_COUNT + this->gap_event_handlers_.push_back(handler); +#endif + } void register_gap_scan_event_handler(GAPScanEventHandler *handler) { +#ifdef ESPHOME_ESP32_BLE_GAP_SCAN_EVENT_HANDLER_COUNT this->gap_scan_event_handlers_.push_back(handler); +#endif } #ifdef USE_ESP32_BLE_CLIENT - void register_gattc_event_handler(GATTcEventHandler *handler) { this->gattc_event_handlers_.push_back(handler); } + void register_gattc_event_handler(GATTcEventHandler *handler) { +#ifdef ESPHOME_ESP32_BLE_GATTC_EVENT_HANDLER_COUNT + this->gattc_event_handlers_.push_back(handler); +#endif + } #endif #ifdef USE_ESP32_BLE_SERVER - void register_gatts_event_handler(GATTsEventHandler *handler) { this->gatts_event_handlers_.push_back(handler); } + void register_gatts_event_handler(GATTsEventHandler *handler) { +#ifdef ESPHOME_ESP32_BLE_GATTS_EVENT_HANDLER_COUNT + this->gatts_event_handlers_.push_back(handler); +#endif + } #endif void register_ble_status_event_handler(BLEStatusEventHandler *handler) { +#ifdef ESPHOME_ESP32_BLE_BLE_STATUS_EVENT_HANDLER_COUNT this->ble_status_event_handlers_.push_back(handler); +#endif } void set_enable_on_boot(bool enable_on_boot) { this->enable_on_boot_ = enable_on_boot; } @@ -159,16 +175,22 @@ class ESP32BLE : public Component { private: template friend void enqueue_ble_event(Args... args); - // Vectors (12 bytes each on 32-bit, naturally aligned to 4 bytes) - std::vector gap_event_handlers_; - std::vector gap_scan_event_handlers_; -#ifdef USE_ESP32_BLE_CLIENT - std::vector gattc_event_handlers_; + // Handler vectors - use StaticVector when counts are known at compile time +#ifdef ESPHOME_ESP32_BLE_GAP_EVENT_HANDLER_COUNT + StaticVector gap_event_handlers_; #endif -#ifdef USE_ESP32_BLE_SERVER - std::vector gatts_event_handlers_; +#ifdef ESPHOME_ESP32_BLE_GAP_SCAN_EVENT_HANDLER_COUNT + StaticVector gap_scan_event_handlers_; +#endif +#if defined(USE_ESP32_BLE_CLIENT) && defined(ESPHOME_ESP32_BLE_GATTC_EVENT_HANDLER_COUNT) + StaticVector gattc_event_handlers_; +#endif +#if defined(USE_ESP32_BLE_SERVER) && defined(ESPHOME_ESP32_BLE_GATTS_EVENT_HANDLER_COUNT) + StaticVector gatts_event_handlers_; +#endif +#ifdef ESPHOME_ESP32_BLE_BLE_STATUS_EVENT_HANDLER_COUNT + StaticVector ble_status_event_handlers_; #endif - std::vector ble_status_event_handlers_; // Large objects (size depends on template parameters, but typically aligned to 4 bytes) esphome::LockFreeQueue ble_events_; diff --git a/esphome/components/esp32_ble_beacon/__init__.py b/esphome/components/esp32_ble_beacon/__init__.py index 794f5637a4..ba5ae4331c 100644 --- a/esphome/components/esp32_ble_beacon/__init__.py +++ b/esphome/components/esp32_ble_beacon/__init__.py @@ -74,7 +74,7 @@ async def to_code(config): var = cg.new_Pvariable(config[CONF_ID], uuid_arr) parent = await cg.get_variable(config[esp32_ble.CONF_BLE_ID]) - cg.add(parent.register_gap_event_handler(var)) + esp32_ble.register_gap_event_handler(parent, var) await cg.register_component(var, config) cg.add(var.set_major(config[CONF_MAJOR])) diff --git a/esphome/components/esp32_ble_server/__init__.py b/esphome/components/esp32_ble_server/__init__.py index 10fa09fcc3..55310f3275 100644 --- a/esphome/components/esp32_ble_server/__init__.py +++ b/esphome/components/esp32_ble_server/__init__.py @@ -546,8 +546,8 @@ async def to_code(config): await cg.register_component(var, config) parent = await cg.get_variable(config[esp32_ble.CONF_BLE_ID]) - cg.add(parent.register_gatts_event_handler(var)) - cg.add(parent.register_ble_status_event_handler(var)) + esp32_ble.register_gatts_event_handler(parent, var) + esp32_ble.register_ble_status_event_handler(parent, var) cg.add(var.set_parent(parent)) cg.add(parent.advertising_set_appearance(config[CONF_APPEARANCE])) if CONF_MANUFACTURER_DATA in config: diff --git a/esphome/components/esp32_ble_tracker/__init__.py b/esphome/components/esp32_ble_tracker/__init__.py index 8c7f3e3930..5910be67af 100644 --- a/esphome/components/esp32_ble_tracker/__init__.py +++ b/esphome/components/esp32_ble_tracker/__init__.py @@ -246,10 +246,10 @@ async def to_code(config): await cg.register_component(var, config) parent = await cg.get_variable(config[esp32_ble.CONF_BLE_ID]) - cg.add(parent.register_gap_event_handler(var)) - cg.add(parent.register_gap_scan_event_handler(var)) - cg.add(parent.register_gattc_event_handler(var)) - cg.add(parent.register_ble_status_event_handler(var)) + esp32_ble.register_gap_event_handler(parent, var) + esp32_ble.register_gap_scan_event_handler(parent, var) + esp32_ble.register_gattc_event_handler(parent, var) + esp32_ble.register_ble_status_event_handler(parent, var) cg.add(var.set_parent(parent)) params = config[CONF_SCAN_PARAMETERS] diff --git a/esphome/core/defines.h b/esphome/core/defines.h index ae44e16624..614689caa6 100644 --- a/esphome/core/defines.h +++ b/esphome/core/defines.h @@ -177,6 +177,11 @@ #define USE_ESP32_BLE_SERVER_ON_DISCONNECT #define ESPHOME_ESP32_BLE_TRACKER_LISTENER_COUNT 1 #define ESPHOME_ESP32_BLE_TRACKER_CLIENT_COUNT 1 +#define ESPHOME_ESP32_BLE_GAP_EVENT_HANDLER_COUNT 2 +#define ESPHOME_ESP32_BLE_GAP_SCAN_EVENT_HANDLER_COUNT 1 +#define ESPHOME_ESP32_BLE_GATTC_EVENT_HANDLER_COUNT 1 +#define ESPHOME_ESP32_BLE_GATTS_EVENT_HANDLER_COUNT 1 +#define ESPHOME_ESP32_BLE_BLE_STATUS_EVENT_HANDLER_COUNT 2 #define USE_ESP32_CAMERA_JPEG_ENCODER #define USE_I2C #define USE_IMPROV From 6f2c7c0e5d922fdfad8483b85b53cf15e9e6c131 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Sun, 12 Oct 2025 11:13:14 -1000 Subject: [PATCH 37/90] fixes --- esphome/components/esp32_ble/__init__.py | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/esphome/components/esp32_ble/__init__.py b/esphome/components/esp32_ble/__init__.py index d2ce4716f4..9d2e3b6004 100644 --- a/esphome/components/esp32_ble/__init__.py +++ b/esphome/components/esp32_ble/__init__.py @@ -17,7 +17,7 @@ from esphome.const import ( CONF_NAME, CONF_NAME_ADD_MAC_SUFFIX, ) -from esphome.core import CORE, TimePeriod +from esphome.core import CORE, CoroPriority, TimePeriod, coroutine_with_priority import esphome.final_validate as fv DEPENDENCIES = ["esp32"] @@ -421,6 +421,16 @@ def final_validation(config): # For newer chips (C3/S3/etc), different configs are used automatically add_idf_sdkconfig_option("CONFIG_BTDM_CTRL_BLE_MAX_CONN", max_connections) + return config + + +FINAL_VALIDATE_SCHEMA = final_validation + + +# This needs to be run as a job with very low priority so that all components have +# a chance to register their handlers before the counts are added to defines. +@coroutine_with_priority(CoroPriority.FINAL) +async def _add_ble_handler_defines(): # Add defines for StaticVector sizing based on handler registration counts # Only define if count > 0 to avoid allocating unnecessary memory if _handler_counts.gap_event > 0: @@ -446,11 +456,6 @@ def final_validation(config): _handler_counts.ble_status_event, ) - return config - - -FINAL_VALIDATE_SCHEMA = final_validation - async def to_code(config): var = cg.new_Pvariable(config[CONF_ID]) @@ -506,6 +511,9 @@ async def to_code(config): cg.add_define("USE_ESP32_BLE_ADVERTISING") cg.add_define("USE_ESP32_BLE_UUID") + # Schedule the handler defines to be added after all components register + CORE.add_job(_add_ble_handler_defines) + @automation.register_condition("ble.enabled", BLEEnabledCondition, cv.Schema({})) async def ble_enabled_to_code(config, condition_id, template_arg, args): From 26ebfa490690ec0e59dd1ee90965634e3ee8ce12 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Sun, 12 Oct 2025 11:19:58 -1000 Subject: [PATCH 38/90] cleaner --- esphome/components/esp32_ble/ble.cpp | 30 +++++++++++----------------- esphome/components/esp32_ble/ble.h | 26 ++++++++---------------- 2 files changed, 20 insertions(+), 36 deletions(-) diff --git a/esphome/components/esp32_ble/ble.cpp b/esphome/components/esp32_ble/ble.cpp index ca119f8836..79a2190ee9 100644 --- a/esphome/components/esp32_ble/ble.cpp +++ b/esphome/components/esp32_ble/ble.cpp @@ -182,32 +182,26 @@ bool ESP32BLE::ble_setup_() { } #ifdef ESPHOME_ESP32_BLE_GAP_EVENT_HANDLER_COUNT - if (!this->gap_event_handlers_.empty()) { - err = esp_ble_gap_register_callback(ESP32BLE::gap_event_handler); - if (err != ESP_OK) { - ESP_LOGE(TAG, "esp_ble_gap_register_callback failed: %d", err); - return false; - } + err = esp_ble_gap_register_callback(ESP32BLE::gap_event_handler); + if (err != ESP_OK) { + ESP_LOGE(TAG, "esp_ble_gap_register_callback failed: %d", err); + return false; } #endif #if defined(USE_ESP32_BLE_SERVER) && defined(ESPHOME_ESP32_BLE_GATTS_EVENT_HANDLER_COUNT) - if (!this->gatts_event_handlers_.empty()) { - err = esp_ble_gatts_register_callback(ESP32BLE::gatts_event_handler); - if (err != ESP_OK) { - ESP_LOGE(TAG, "esp_ble_gatts_register_callback failed: %d", err); - return false; - } + err = esp_ble_gatts_register_callback(ESP32BLE::gatts_event_handler); + if (err != ESP_OK) { + ESP_LOGE(TAG, "esp_ble_gatts_register_callback failed: %d", err); + return false; } #endif #if defined(USE_ESP32_BLE_CLIENT) && defined(ESPHOME_ESP32_BLE_GATTC_EVENT_HANDLER_COUNT) - if (!this->gattc_event_handlers_.empty()) { - err = esp_ble_gattc_register_callback(ESP32BLE::gattc_event_handler); - if (err != ESP_OK) { - ESP_LOGE(TAG, "esp_ble_gattc_register_callback failed: %d", err); - return false; - } + err = esp_ble_gattc_register_callback(ESP32BLE::gattc_event_handler); + if (err != ESP_OK) { + ESP_LOGE(TAG, "esp_ble_gattc_register_callback failed: %d", err); + return false; } #endif diff --git a/esphome/components/esp32_ble/ble.h b/esphome/components/esp32_ble/ble.h index 617047372b..dc973f0e82 100644 --- a/esphome/components/esp32_ble/ble.h +++ b/esphome/components/esp32_ble/ble.h @@ -125,35 +125,25 @@ class ESP32BLE : public Component { void advertising_register_raw_advertisement_callback(std::function &&callback); #endif - void register_gap_event_handler(GAPEventHandler *handler) { #ifdef ESPHOME_ESP32_BLE_GAP_EVENT_HANDLER_COUNT - this->gap_event_handlers_.push_back(handler); + void register_gap_event_handler(GAPEventHandler *handler) { this->gap_event_handlers_.push_back(handler); } #endif - } - void register_gap_scan_event_handler(GAPScanEventHandler *handler) { #ifdef ESPHOME_ESP32_BLE_GAP_SCAN_EVENT_HANDLER_COUNT + void register_gap_scan_event_handler(GAPScanEventHandler *handler) { this->gap_scan_event_handlers_.push_back(handler); -#endif - } -#ifdef USE_ESP32_BLE_CLIENT - void register_gattc_event_handler(GATTcEventHandler *handler) { -#ifdef ESPHOME_ESP32_BLE_GATTC_EVENT_HANDLER_COUNT - this->gattc_event_handlers_.push_back(handler); -#endif } #endif -#ifdef USE_ESP32_BLE_SERVER - void register_gatts_event_handler(GATTsEventHandler *handler) { -#ifdef ESPHOME_ESP32_BLE_GATTS_EVENT_HANDLER_COUNT - this->gatts_event_handlers_.push_back(handler); +#if defined(USE_ESP32_BLE_CLIENT) && defined(ESPHOME_ESP32_BLE_GATTC_EVENT_HANDLER_COUNT) + void register_gattc_event_handler(GATTcEventHandler *handler) { this->gattc_event_handlers_.push_back(handler); } #endif - } +#if defined(USE_ESP32_BLE_SERVER) && defined(ESPHOME_ESP32_BLE_GATTS_EVENT_HANDLER_COUNT) + void register_gatts_event_handler(GATTsEventHandler *handler) { this->gatts_event_handlers_.push_back(handler); } #endif - void register_ble_status_event_handler(BLEStatusEventHandler *handler) { #ifdef ESPHOME_ESP32_BLE_BLE_STATUS_EVENT_HANDLER_COUNT + void register_ble_status_event_handler(BLEStatusEventHandler *handler) { this->ble_status_event_handlers_.push_back(handler); -#endif } +#endif void set_enable_on_boot(bool enable_on_boot) { this->enable_on_boot_ = enable_on_boot; } protected: From 7c8f8e282d77c98a8581a2e91aae890e1c7b3acb Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Sun, 12 Oct 2025 12:52:38 -1000 Subject: [PATCH 39/90] Fix log retrieval with FQDN when mDNS is disabled --- esphome/__main__.py | 15 +++++++----- tests/unit_tests/test_main.py | 45 ++++++++++++++++++++++++++++------- 2 files changed, 46 insertions(+), 14 deletions(-) diff --git a/esphome/__main__.py b/esphome/__main__.py index be4551f6b5..8e0c475525 100644 --- a/esphome/__main__.py +++ b/esphome/__main__.py @@ -268,8 +268,10 @@ def has_ip_address() -> bool: def has_resolvable_address() -> bool: - """Check if CORE.address is resolvable (via mDNS or is an IP address).""" - return has_mdns() or has_ip_address() + """Check if CORE.address is resolvable (via mDNS, DNS, or is an IP address).""" + # Any address (IP, mDNS hostname, or regular DNS hostname) is resolvable + # The resolve_ip_address() function in helpers.py handles all types via AsyncResolver + return CORE.address is not None def mqtt_get_ip(config: ConfigType, username: str, password: str, client_id: str): @@ -578,11 +580,12 @@ def show_logs(config: ConfigType, args: ArgsProtocol, devices: list[str]) -> int if has_api(): addresses_to_use: list[str] | None = None - if port_type == "NETWORK" and (has_mdns() or is_ip_address(port)): + if port_type == "NETWORK": + # Network addresses (IPs, mDNS names, or regular DNS hostnames) can be used + # The resolve_ip_address() function in helpers.py handles all types addresses_to_use = devices - elif port_type in ("NETWORK", "MQTT", "MQTTIP") and has_mqtt_ip_lookup(): - # Only use MQTT IP lookup if the first condition didn't match - # (for MQTT/MQTTIP types, or for NETWORK when mdns/ip check fails) + elif port_type in ("MQTT", "MQTTIP") and has_mqtt_ip_lookup(): + # Use MQTT IP lookup for MQTT/MQTTIP types addresses_to_use = mqtt_get_ip( config, args.username, args.password, args.client_id ) diff --git a/tests/unit_tests/test_main.py b/tests/unit_tests/test_main.py index e35378145a..becf911fa3 100644 --- a/tests/unit_tests/test_main.py +++ b/tests/unit_tests/test_main.py @@ -1203,6 +1203,31 @@ def test_show_logs_api( ) +@patch("esphome.components.api.client.run_logs") +def test_show_logs_api_with_fqdn_mdns_disabled( + mock_run_logs: Mock, +) -> None: + """Test show_logs with API using FQDN when mDNS is disabled.""" + setup_core( + config={ + "logger": {}, + CONF_API: {}, + CONF_MDNS: {CONF_DISABLED: True}, + }, + platform=PLATFORM_ESP32, + ) + mock_run_logs.return_value = 0 + + args = MockArgs() + devices = ["device.example.com"] + + result = show_logs(CORE.config, args, devices) + + assert result == 0 + # Should use the FQDN directly, not try MQTT lookup + mock_run_logs.assert_called_once_with(CORE.config, ["device.example.com"]) + + @patch("esphome.components.api.client.run_logs") def test_show_logs_api_with_mqtt_fallback( mock_run_logs: Mock, @@ -1222,7 +1247,7 @@ def test_show_logs_api_with_mqtt_fallback( mock_mqtt_get_ip.return_value = ["192.168.1.200"] args = MockArgs(username="user", password="pass", client_id="client") - devices = ["device.local"] + devices = ["MQTTIP"] result = show_logs(CORE.config, args, devices) @@ -1487,27 +1512,31 @@ def test_mqtt_get_ip() -> None: def test_has_resolvable_address() -> None: """Test has_resolvable_address function.""" - # Test with mDNS enabled and hostname address + # Test with mDNS enabled and .local hostname address setup_core(config={}, address="esphome-device.local") assert has_resolvable_address() is True - # Test with mDNS disabled and hostname address + # Test with mDNS disabled and .local hostname address (still resolvable via DNS) setup_core( config={CONF_MDNS: {CONF_DISABLED: True}}, address="esphome-device.local" ) - assert has_resolvable_address() is False + assert has_resolvable_address() is True - # Test with IP address (mDNS doesn't matter) + # Test with mDNS disabled and regular DNS hostname (resolvable) + setup_core(config={CONF_MDNS: {CONF_DISABLED: True}}, address="device.example.com") + assert has_resolvable_address() is True + + # Test with IP address (always resolvable, mDNS doesn't matter) setup_core(config={}, address="192.168.1.100") assert has_resolvable_address() is True - # Test with IP address and mDNS disabled + # Test with IP address and mDNS disabled (still resolvable) setup_core(config={CONF_MDNS: {CONF_DISABLED: True}}, address="192.168.1.100") assert has_resolvable_address() is True - # Test with no address but mDNS enabled (can still resolve mDNS names) + # Test with no address setup_core(config={}, address=None) - assert has_resolvable_address() is True + assert has_resolvable_address() is False # Test with no address and mDNS disabled setup_core(config={CONF_MDNS: {CONF_DISABLED: True}}, address=None) From a9fd0a3b26eed1059aa82390b0addbc00e1d916d Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Sun, 12 Oct 2025 18:21:14 -1000 Subject: [PATCH 40/90] fixed_vector, bluetooth services --- esphome/components/api/api.proto | 4 +- esphome/components/api/api_options.proto | 6 +++ esphome/components/api/api_pb2.h | 4 +- esphome/components/api/proto.h | 26 +++++++++-- .../bluetooth_proxy/bluetooth_connection.cpp | 8 ++-- esphome/core/helpers.h | 46 +++++++++++++++++++ script/api_protobuf/api_protobuf.py | 4 ++ 7 files changed, 85 insertions(+), 13 deletions(-) diff --git a/esphome/components/api/api.proto b/esphome/components/api/api.proto index 87f477799d..9b714d00f1 100644 --- a/esphome/components/api/api.proto +++ b/esphome/components/api/api.proto @@ -1519,7 +1519,7 @@ message BluetoothGATTCharacteristic { repeated uint64 uuid = 1 [(fixed_array_size) = 2, (fixed_array_skip_zero) = true]; uint32 handle = 2; uint32 properties = 3; - repeated BluetoothGATTDescriptor descriptors = 4; + repeated BluetoothGATTDescriptor descriptors = 4 [(fixed_vector) = true]; // New field for efficient UUID (v1.12+) // Only one of uuid or short_uuid will be set. @@ -1531,7 +1531,7 @@ message BluetoothGATTCharacteristic { message BluetoothGATTService { repeated uint64 uuid = 1 [(fixed_array_size) = 2, (fixed_array_skip_zero) = true]; uint32 handle = 2; - repeated BluetoothGATTCharacteristic characteristics = 3; + repeated BluetoothGATTCharacteristic characteristics = 3 [(fixed_vector) = true]; // New field for efficient UUID (v1.12+) // Only one of uuid or short_uuid will be set. diff --git a/esphome/components/api/api_options.proto b/esphome/components/api/api_options.proto index 633f39b552..ead8ac0bbc 100644 --- a/esphome/components/api/api_options.proto +++ b/esphome/components/api/api_options.proto @@ -64,4 +64,10 @@ extend google.protobuf.FieldOptions { // This is typically done through methods returning const T& or special accessor // methods like get_options() or supported_modes_for_api_(). optional string container_pointer = 50001; + + // fixed_vector: Use FixedVector instead of std::vector for repeated fields + // When set, the repeated field will use FixedVector which requires calling + // init(size) before adding elements. This eliminates std::vector template overhead + // and is ideal when the exact size is known before populating the array. + optional bool fixed_vector = 50013 [default=false]; } diff --git a/esphome/components/api/api_pb2.h b/esphome/components/api/api_pb2.h index d9e68ece9b..1798458393 100644 --- a/esphome/components/api/api_pb2.h +++ b/esphome/components/api/api_pb2.h @@ -1923,7 +1923,7 @@ class BluetoothGATTCharacteristic final : public ProtoMessage { std::array uuid{}; uint32_t handle{0}; uint32_t properties{0}; - std::vector descriptors{}; + FixedVector descriptors{}; uint32_t short_uuid{0}; void encode(ProtoWriteBuffer buffer) const override; void calculate_size(ProtoSize &size) const override; @@ -1937,7 +1937,7 @@ class BluetoothGATTService final : public ProtoMessage { public: std::array uuid{}; uint32_t handle{0}; - std::vector characteristics{}; + FixedVector characteristics{}; uint32_t short_uuid{0}; void encode(ProtoWriteBuffer buffer) const override; void calculate_size(ProtoSize &size) const override; diff --git a/esphome/components/api/proto.h b/esphome/components/api/proto.h index 9d780692ec..a6a09bf7c5 100644 --- a/esphome/components/api/proto.h +++ b/esphome/components/api/proto.h @@ -749,13 +749,29 @@ class ProtoSize { template inline void add_repeated_message(uint32_t field_id_size, const std::vector &messages) { // Skip if the vector is empty - if (messages.empty()) { - return; + if (!messages.empty()) { + // Use the force version for all messages in the repeated field + for (const auto &message : messages) { + add_message_object_force(field_id_size, message); + } } + } - // Use the force version for all messages in the repeated field - for (const auto &message : messages) { - add_message_object_force(field_id_size, message); + /** + * @brief Calculates and adds the sizes of all messages in a repeated field to the total message size (FixedVector + * version) + * + * @tparam MessageType The type of the nested messages in the FixedVector + * @param messages FixedVector of message objects + */ + template + inline void add_repeated_message(uint32_t field_id_size, const FixedVector &messages) { + // Skip if the fixed vector is empty + if (!messages.empty()) { + // Use the force version for all messages in the repeated field + for (const auto &message : messages) { + add_message_object_force(field_id_size, message); + } } } }; diff --git a/esphome/components/bluetooth_proxy/bluetooth_connection.cpp b/esphome/components/bluetooth_proxy/bluetooth_connection.cpp index cde82fbfb0..6f172b0bcf 100644 --- a/esphome/components/bluetooth_proxy/bluetooth_connection.cpp +++ b/esphome/components/bluetooth_proxy/bluetooth_connection.cpp @@ -230,8 +230,8 @@ void BluetoothConnection::send_service_for_discovery_() { service_resp.handle = service_result.start_handle; if (total_char_count > 0) { - // Reserve space and process characteristics - service_resp.characteristics.reserve(total_char_count); + // Initialize FixedVector with exact count and process characteristics + service_resp.characteristics.init(total_char_count); uint16_t char_offset = 0; esp_gattc_char_elem_t char_result; while (true) { // characteristics @@ -275,8 +275,8 @@ void BluetoothConnection::send_service_for_discovery_() { continue; } - // Reserve space and process descriptors - characteristic_resp.descriptors.reserve(total_desc_count); + // Initialize FixedVector with exact count and process descriptors + characteristic_resp.descriptors.init(total_desc_count); uint16_t desc_offset = 0; esp_gattc_descr_elem_t desc_result; while (true) { // descriptors diff --git a/esphome/core/helpers.h b/esphome/core/helpers.h index b5a0a1c8ac..2bdfcb4e2a 100644 --- a/esphome/core/helpers.h +++ b/esphome/core/helpers.h @@ -181,6 +181,31 @@ template class FixedVector { FixedVector(const FixedVector &) = delete; FixedVector &operator=(const FixedVector &) = delete; + // Enable move semantics for use in containers + FixedVector(FixedVector &&other) noexcept : data_(other.data_), size_(other.size_), capacity_(other.capacity_) { + other.data_ = nullptr; + other.size_ = 0; + other.capacity_ = 0; + } + + FixedVector &operator=(FixedVector &&other) noexcept { + if (this != &other) { + // Delete our current data + if (data_ != nullptr) { + delete[] data_; + } + // Take ownership of other's data + data_ = other.data_; + size_ = other.size_; + capacity_ = other.capacity_; + // Leave other in valid empty state + other.data_ = nullptr; + other.size_ = 0; + other.capacity_ = 0; + } + return *this; + } + // Allocate capacity - can only be called once on empty vector void init(size_t n) { if (data_ == nullptr && n > 0) { @@ -199,12 +224,33 @@ template class FixedVector { } } + /// Construct element in place and return reference + /// Caller must ensure sufficient capacity was allocated via init() + T &emplace_back() { + if (size_ < capacity_) { + return data_[size_++]; + } + // Should never happen with proper init() - return last element to avoid crash + return data_[capacity_ - 1]; + } + + /// Access last element + T &back() { return data_[size_ - 1]; } + const T &back() const { return data_[size_ - 1]; } + size_t size() const { return size_; } + bool empty() const { return size_ == 0; } /// Access element without bounds checking (matches std::vector behavior) /// Caller must ensure index is valid (i < size()) T &operator[](size_t i) { return data_[i]; } const T &operator[](size_t i) const { return data_[i]; } + + /// Iterators for range-based for loops + T *begin() { return data_; } + T *end() { return data_ + size_; } + const T *begin() const { return data_; } + const T *end() const { return data_ + size_; } }; ///@} diff --git a/script/api_protobuf/api_protobuf.py b/script/api_protobuf/api_protobuf.py index 487c187372..9a55f1d136 100755 --- a/script/api_protobuf/api_protobuf.py +++ b/script/api_protobuf/api_protobuf.py @@ -1415,6 +1415,8 @@ class RepeatedTypeInfo(TypeInfo): # Check if this is a pointer field by looking for container_pointer option self._container_type = get_field_opt(field, pb.container_pointer, "") self._use_pointer = bool(self._container_type) + # Check if this should use FixedVector instead of std::vector + self._use_fixed_vector = get_field_opt(field, pb.fixed_vector, False) # For repeated fields, we need to get the base type info # but we can't call create_field_type_info as it would cause recursion @@ -1438,6 +1440,8 @@ class RepeatedTypeInfo(TypeInfo): if "<" in self._container_type and ">" in self._container_type: return f"const {self._container_type}*" return f"const {self._container_type}<{self._ti.cpp_type}>*" + if self._use_fixed_vector: + return f"FixedVector<{self._ti.cpp_type}>" return f"std::vector<{self._ti.cpp_type}>" @property From 347501d895cb42d1a15633d4b2fd684ae6a2af6a Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Sun, 12 Oct 2025 19:39:55 -1000 Subject: [PATCH 41/90] wifi fixed vector --- esphome/components/wifi/wifi_component.cpp | 2 +- esphome/components/wifi/wifi_component.h | 4 +- .../wifi/wifi_component_esp8266.cpp | 8 ++++ .../wifi/wifi_component_esp_idf.cpp | 5 +- .../wifi/wifi_component_libretiny.cpp | 2 +- esphome/core/helpers.h | 46 +++++++++++++++---- 6 files changed, 51 insertions(+), 16 deletions(-) diff --git a/esphome/components/wifi/wifi_component.cpp b/esphome/components/wifi/wifi_component.cpp index 2e083d4c68..1bb2674ad7 100644 --- a/esphome/components/wifi/wifi_component.cpp +++ b/esphome/components/wifi/wifi_component.cpp @@ -552,7 +552,7 @@ void WiFiComponent::start_scanning() { // Using insertion sort instead of std::stable_sort saves flash memory // by avoiding template instantiations (std::rotate, std::stable_sort, lambdas) // IMPORTANT: This sort is stable (preserves relative order of equal elements) -static void insertion_sort_scan_results(std::vector &results) { +static void insertion_sort_scan_results(FixedVector &results) { const size_t size = results.size(); for (size_t i = 1; i < size; i++) { // Make a copy to avoid issues with move semantics during comparison diff --git a/esphome/components/wifi/wifi_component.h b/esphome/components/wifi/wifi_component.h index ee62ec1a69..e1c3d6df88 100644 --- a/esphome/components/wifi/wifi_component.h +++ b/esphome/components/wifi/wifi_component.h @@ -278,7 +278,7 @@ class WiFiComponent : public Component { std::string get_use_address() const; void set_use_address(const std::string &use_address); - const std::vector &get_scan_result() const { return scan_result_; } + const FixedVector &get_scan_result() const { return scan_result_; } network::IPAddress wifi_soft_ap_ip(); @@ -385,7 +385,7 @@ class WiFiComponent : public Component { std::string use_address_; std::vector sta_; std::vector sta_priorities_; - std::vector scan_result_; + FixedVector scan_result_; WiFiAP selected_ap_; WiFiAP ap_; optional output_power_; diff --git a/esphome/components/wifi/wifi_component_esp8266.cpp b/esphome/components/wifi/wifi_component_esp8266.cpp index 3b3b4b139c..59909b2cb5 100644 --- a/esphome/components/wifi/wifi_component_esp8266.cpp +++ b/esphome/components/wifi/wifi_component_esp8266.cpp @@ -696,7 +696,15 @@ void WiFiComponent::wifi_scan_done_callback_(void *arg, STATUS status) { this->retry_connect(); return; } + + // Count the number of results first auto *head = reinterpret_cast(arg); + size_t count = 0; + for (bss_info *it = head; it != nullptr; it = STAILQ_NEXT(it, next)) { + count++; + } + + this->scan_result_.init(count); for (bss_info *it = head; it != nullptr; it = STAILQ_NEXT(it, next)) { WiFiScanResult res({it->bssid[0], it->bssid[1], it->bssid[2], it->bssid[3], it->bssid[4], it->bssid[5]}, std::string(reinterpret_cast(it->ssid), it->ssid_len), it->channel, it->rssi, diff --git a/esphome/components/wifi/wifi_component_esp_idf.cpp b/esphome/components/wifi/wifi_component_esp_idf.cpp index ccec800205..4c719ef4c3 100644 --- a/esphome/components/wifi/wifi_component_esp_idf.cpp +++ b/esphome/components/wifi/wifi_component_esp_idf.cpp @@ -763,8 +763,9 @@ void WiFiComponent::wifi_process_event_(IDFWiFiEvent *data) { const auto &it = data->data.sta_scan_done; ESP_LOGV(TAG, "Scan done: status=%" PRIu32 " number=%u scan_id=%u", it.status, it.number, it.scan_id); - scan_result_.clear(); this->scan_done_ = true; + scan_result_.clear(); + if (it.status != 0) { // scan error return; @@ -784,7 +785,7 @@ void WiFiComponent::wifi_process_event_(IDFWiFiEvent *data) { } records.resize(number); - scan_result_.reserve(number); + scan_result_.init(number); for (int i = 0; i < number; i++) { auto &record = records[i]; bssid_t bssid; diff --git a/esphome/components/wifi/wifi_component_libretiny.cpp b/esphome/components/wifi/wifi_component_libretiny.cpp index b15f710150..cb179d9022 100644 --- a/esphome/components/wifi/wifi_component_libretiny.cpp +++ b/esphome/components/wifi/wifi_component_libretiny.cpp @@ -411,7 +411,7 @@ void WiFiComponent::wifi_scan_done_callback_() { if (num < 0) return; - this->scan_result_.reserve(static_cast(num)); + this->scan_result_.init(static_cast(num)); for (int i = 0; i < num; i++) { String ssid = WiFi.SSID(i); wifi_auth_mode_t authmode = WiFi.encryptionType(i); diff --git a/esphome/core/helpers.h b/esphome/core/helpers.h index b5a0a1c8ac..12e921e3be 100644 --- a/esphome/core/helpers.h +++ b/esphome/core/helpers.h @@ -168,34 +168,54 @@ template class FixedVector { size_t size_{0}; size_t capacity_{0}; + // Helper to destroy elements and free memory + void cleanup_() { + if (data_ != nullptr) { + // Manually destroy all elements + for (size_t i = 0; i < size_; i++) { + data_[i].~T(); + } + // Free raw memory + ::operator delete(data_); + } + } + public: FixedVector() = default; - ~FixedVector() { - if (data_ != nullptr) { - delete[] data_; - } - } + ~FixedVector() { cleanup_(); } // Disable copy to avoid accidental copies FixedVector(const FixedVector &) = delete; FixedVector &operator=(const FixedVector &) = delete; - // Allocate capacity - can only be called once on empty vector + // Allocate capacity - can be called multiple times to reinit void init(size_t n) { - if (data_ == nullptr && n > 0) { - data_ = new T[n]; + cleanup_(); + data_ = nullptr; + capacity_ = 0; + size_ = 0; + if (n > 0) { + // Allocate raw memory without calling constructors + data_ = static_cast(::operator new(n * sizeof(T))); capacity_ = n; - size_ = 0; } } + // Clear the vector (reset size to 0, keep capacity) + void clear() { size_ = 0; } + + // Check if vector is empty + bool empty() const { return size_ == 0; } + /// Add element without bounds checking /// Caller must ensure sufficient capacity was allocated via init() /// Silently ignores pushes beyond capacity (no exception or assertion) void push_back(const T &value) { if (size_ < capacity_) { - data_[size_++] = value; + // Use placement new to construct the object in pre-allocated memory + new (&data_[size_]) T(value); + size_++; } } @@ -205,6 +225,12 @@ template class FixedVector { /// Caller must ensure index is valid (i < size()) T &operator[](size_t i) { return data_[i]; } const T &operator[](size_t i) const { return data_[i]; } + + // Iterator support for range-based for loops + T *begin() { return data_; } + T *end() { return data_ + size_; } + const T *begin() const { return data_; } + const T *end() const { return data_ + size_; } }; ///@} From 6f3a9966983806b52c0730995c01d478336b2264 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Sun, 12 Oct 2025 20:12:34 -1000 Subject: [PATCH 42/90] [wifi] Free scan results memory after successful connection --- esphome/components/wifi/__init__.py | 21 +++++++++++++++++++++ esphome/components/wifi/wifi_component.cpp | 6 ++++++ esphome/components/wifi/wifi_component.h | 2 ++ esphome/components/wifi_info/text_sensor.py | 4 +++- 4 files changed, 32 insertions(+), 1 deletion(-) diff --git a/esphome/components/wifi/__init__.py b/esphome/components/wifi/__init__.py index a784123006..286e66a06b 100644 --- a/esphome/components/wifi/__init__.py +++ b/esphome/components/wifi/__init__.py @@ -468,6 +468,27 @@ async def wifi_disable_to_code(config, action_id, template_arg, args): return cg.new_Pvariable(action_id, template_arg) +_FLAGS = {"keep_scan_results": False} + + +def request_wifi_scan_results(): + """Request that WiFi scan results be kept in memory after connection. + + Components that need access to scan results after WiFi is connected should + call this function during their code generation. This prevents the WiFi component from + freeing scan result memory after successful connection. + """ + _FLAGS["keep_scan_results"] = True + + +@coroutine_with_priority(CoroPriority.FINAL) +async def final_step(): + """Final code generation step to configure scan result retention.""" + if _FLAGS["keep_scan_results"]: + wifi_var = cg.MockObj(id="global_wifi_component", base="wifi::WiFiComponent *") + cg.add(wifi_var.set_keep_scan_results(True)) + + @automation.register_action( "wifi.configure", WiFiConfigureAction, diff --git a/esphome/components/wifi/wifi_component.cpp b/esphome/components/wifi/wifi_component.cpp index 71ee4271ba..0a30b82aaf 100644 --- a/esphome/components/wifi/wifi_component.cpp +++ b/esphome/components/wifi/wifi_component.cpp @@ -716,6 +716,12 @@ void WiFiComponent::check_connecting_finished() { this->state_ = WIFI_COMPONENT_STATE_STA_CONNECTED; this->num_retried_ = 0; + // Free scan results memory unless a component needs them + if (!this->keep_scan_results_) { + this->scan_result_.clear(); + this->scan_result_.shrink_to_fit(); + } + if (this->fast_connect_) { this->save_fast_connect_settings_(); } diff --git a/esphome/components/wifi/wifi_component.h b/esphome/components/wifi/wifi_component.h index ee62ec1a69..c0b63e1858 100644 --- a/esphome/components/wifi/wifi_component.h +++ b/esphome/components/wifi/wifi_component.h @@ -316,6 +316,7 @@ class WiFiComponent : public Component { int8_t wifi_rssi(); void set_enable_on_boot(bool enable_on_boot) { this->enable_on_boot_ = enable_on_boot; } + void set_keep_scan_results(bool keep_scan_results) { this->keep_scan_results_ = keep_scan_results; } Trigger<> *get_connect_trigger() const { return this->connect_trigger_; }; Trigger<> *get_disconnect_trigger() const { return this->disconnect_trigger_; }; @@ -424,6 +425,7 @@ class WiFiComponent : public Component { #endif bool enable_on_boot_; bool got_ipv4_address_{false}; + bool keep_scan_results_{false}; // Pointers at the end (naturally aligned) Trigger<> *connect_trigger_{new Trigger<>()}; diff --git a/esphome/components/wifi_info/text_sensor.py b/esphome/components/wifi_info/text_sensor.py index 4ceb73a695..ac1c1bee05 100644 --- a/esphome/components/wifi_info/text_sensor.py +++ b/esphome/components/wifi_info/text_sensor.py @@ -1,5 +1,5 @@ import esphome.codegen as cg -from esphome.components import text_sensor +from esphome.components import text_sensor, wifi import esphome.config_validation as cv from esphome.const import ( CONF_BSSID, @@ -77,6 +77,8 @@ async def to_code(config): await setup_conf(config, CONF_SSID) await setup_conf(config, CONF_BSSID) await setup_conf(config, CONF_MAC_ADDRESS) + if CONF_SCAN_RESULTS in config: + wifi.request_wifi_scan_results() await setup_conf(config, CONF_SCAN_RESULTS) await setup_conf(config, CONF_DNS_ADDRESS) if conf := config.get(CONF_IP_ADDRESS): From 4d55c8f309283919b655847d53dbfbe76107ba3a Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Sun, 12 Oct 2025 20:20:17 -1000 Subject: [PATCH 43/90] preen --- esphome/components/wifi/__init__.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/esphome/components/wifi/__init__.py b/esphome/components/wifi/__init__.py index 286e66a06b..9a7c4c1713 100644 --- a/esphome/components/wifi/__init__.py +++ b/esphome/components/wifi/__init__.py @@ -447,6 +447,8 @@ async def to_code(config): var.get_disconnect_trigger(), [], on_disconnect_config ) + CORE.add_job(final_step) + @automation.register_condition("wifi.connected", WiFiConnectedCondition, cv.Schema({})) async def wifi_connected_to_code(config, condition_id, template_arg, args): From d191d1e99a3336dad8608e36bae52741b0ed91f1 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Sun, 12 Oct 2025 20:21:56 -1000 Subject: [PATCH 44/90] preen --- esphome/components/wifi/__init__.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/esphome/components/wifi/__init__.py b/esphome/components/wifi/__init__.py index 9a7c4c1713..ae85e40357 100644 --- a/esphome/components/wifi/__init__.py +++ b/esphome/components/wifi/__init__.py @@ -487,8 +487,9 @@ def request_wifi_scan_results(): async def final_step(): """Final code generation step to configure scan result retention.""" if _FLAGS["keep_scan_results"]: - wifi_var = cg.MockObj(id="global_wifi_component", base="wifi::WiFiComponent *") - cg.add(wifi_var.set_keep_scan_results(True)) + cg.add( + cg.RawExpression("wifi::global_wifi_component->set_keep_scan_results(true)") + ) @automation.register_action( From dd09897a1d37491c99f2449f6bd652c74f96d0f5 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Sun, 12 Oct 2025 20:46:36 -1000 Subject: [PATCH 45/90] Update esphome/components/wifi_info/text_sensor.py Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- esphome/components/wifi_info/text_sensor.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/esphome/components/wifi_info/text_sensor.py b/esphome/components/wifi_info/text_sensor.py index ac1c1bee05..a4da582c55 100644 --- a/esphome/components/wifi_info/text_sensor.py +++ b/esphome/components/wifi_info/text_sensor.py @@ -78,8 +78,8 @@ async def to_code(config): await setup_conf(config, CONF_BSSID) await setup_conf(config, CONF_MAC_ADDRESS) if CONF_SCAN_RESULTS in config: + await setup_conf(config, CONF_SCAN_RESULTS) wifi.request_wifi_scan_results() - await setup_conf(config, CONF_SCAN_RESULTS) await setup_conf(config, CONF_DNS_ADDRESS) if conf := config.get(CONF_IP_ADDRESS): wifi_info = await text_sensor.new_text_sensor(config[CONF_IP_ADDRESS]) From 22370c0ad178ad9931cf4d54197a9edf52f04759 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Mon, 13 Oct 2025 14:03:08 -1000 Subject: [PATCH 46/90] merge --- esphome/core/helpers.h | 42 +++++++++++++++++++++++++++++++----------- 1 file changed, 31 insertions(+), 11 deletions(-) diff --git a/esphome/core/helpers.h b/esphome/core/helpers.h index 80ce21af57..084ad28882 100644 --- a/esphome/core/helpers.h +++ b/esphome/core/helpers.h @@ -168,14 +168,22 @@ template class FixedVector { size_t size_{0}; size_t capacity_{0}; + // Helper to destroy elements and free memory + void cleanup_() { + if (data_ != nullptr) { + // Manually destroy all elements + for (size_t i = 0; i < size_; i++) { + data_[i].~T(); + } + // Free raw memory + ::operator delete(data_); + } + } + public: FixedVector() = default; - ~FixedVector() { - if (data_ != nullptr) { - delete[] data_; - } - } + ~FixedVector() { cleanup_(); } // Enable move semantics for use in containers FixedVector(FixedVector &&other) noexcept : data_(other.data_), size_(other.size_), capacity_(other.capacity_) { @@ -202,21 +210,33 @@ template class FixedVector { return *this; } - // Allocate capacity - can only be called once on empty vector + // Allocate capacity - can be called multiple times to reinit void init(size_t n) { - if (data_ == nullptr && n > 0) { - data_ = new T[n]; + cleanup_(); + data_ = nullptr; + capacity_ = 0; + size_ = 0; + if (n > 0) { + // Allocate raw memory without calling constructors + data_ = static_cast(::operator new(n * sizeof(T))); capacity_ = n; - size_ = 0; } } + // Clear the vector (reset size to 0, keep capacity) + void clear() { size_ = 0; } + + // Check if vector is empty + bool empty() const { return size_ == 0; } + /// Add element without bounds checking /// Caller must ensure sufficient capacity was allocated via init() /// Silently ignores pushes beyond capacity (no exception or assertion) void push_back(const T &value) { if (size_ < capacity_) { - data_[size_++] = value; + // Use placement new to construct the object in pre-allocated memory + new (&data_[size_]) T(value); + size_++; } } @@ -242,7 +262,7 @@ template class FixedVector { T &operator[](size_t i) { return data_[i]; } const T &operator[](size_t i) const { return data_[i]; } - /// Iterators for range-based for loops + // Iterator support for range-based for loops T *begin() { return data_; } T *end() { return data_ + size_; } const T *begin() const { return data_; } From fbef9b126403ca49999b589ba5670fbe980b7c9e Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Mon, 13 Oct 2025 14:03:59 -1000 Subject: [PATCH 47/90] revert --- .../components/bluetooth_proxy/bluetooth_connection.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/esphome/components/bluetooth_proxy/bluetooth_connection.cpp b/esphome/components/bluetooth_proxy/bluetooth_connection.cpp index 6f172b0bcf..cde82fbfb0 100644 --- a/esphome/components/bluetooth_proxy/bluetooth_connection.cpp +++ b/esphome/components/bluetooth_proxy/bluetooth_connection.cpp @@ -230,8 +230,8 @@ void BluetoothConnection::send_service_for_discovery_() { service_resp.handle = service_result.start_handle; if (total_char_count > 0) { - // Initialize FixedVector with exact count and process characteristics - service_resp.characteristics.init(total_char_count); + // Reserve space and process characteristics + service_resp.characteristics.reserve(total_char_count); uint16_t char_offset = 0; esp_gattc_char_elem_t char_result; while (true) { // characteristics @@ -275,8 +275,8 @@ void BluetoothConnection::send_service_for_discovery_() { continue; } - // Initialize FixedVector with exact count and process descriptors - characteristic_resp.descriptors.init(total_desc_count); + // Reserve space and process descriptors + characteristic_resp.descriptors.reserve(total_desc_count); uint16_t desc_offset = 0; esp_gattc_descr_elem_t desc_result; while (true) { // descriptors From ddf6e0a7b61ac09db7bfdafc17362cf4804b189f Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Mon, 13 Oct 2025 14:04:15 -1000 Subject: [PATCH 48/90] revert --- esphome/components/api/api.proto | 4 ++-- esphome/components/api/api_options.proto | 6 ------ esphome/components/api/api_pb2.h | 4 ++-- esphome/components/api/proto.h | 26 +++++------------------- 4 files changed, 9 insertions(+), 31 deletions(-) diff --git a/esphome/components/api/api.proto b/esphome/components/api/api.proto index 9b714d00f1..87f477799d 100644 --- a/esphome/components/api/api.proto +++ b/esphome/components/api/api.proto @@ -1519,7 +1519,7 @@ message BluetoothGATTCharacteristic { repeated uint64 uuid = 1 [(fixed_array_size) = 2, (fixed_array_skip_zero) = true]; uint32 handle = 2; uint32 properties = 3; - repeated BluetoothGATTDescriptor descriptors = 4 [(fixed_vector) = true]; + repeated BluetoothGATTDescriptor descriptors = 4; // New field for efficient UUID (v1.12+) // Only one of uuid or short_uuid will be set. @@ -1531,7 +1531,7 @@ message BluetoothGATTCharacteristic { message BluetoothGATTService { repeated uint64 uuid = 1 [(fixed_array_size) = 2, (fixed_array_skip_zero) = true]; uint32 handle = 2; - repeated BluetoothGATTCharacteristic characteristics = 3 [(fixed_vector) = true]; + repeated BluetoothGATTCharacteristic characteristics = 3; // New field for efficient UUID (v1.12+) // Only one of uuid or short_uuid will be set. diff --git a/esphome/components/api/api_options.proto b/esphome/components/api/api_options.proto index ead8ac0bbc..633f39b552 100644 --- a/esphome/components/api/api_options.proto +++ b/esphome/components/api/api_options.proto @@ -64,10 +64,4 @@ extend google.protobuf.FieldOptions { // This is typically done through methods returning const T& or special accessor // methods like get_options() or supported_modes_for_api_(). optional string container_pointer = 50001; - - // fixed_vector: Use FixedVector instead of std::vector for repeated fields - // When set, the repeated field will use FixedVector which requires calling - // init(size) before adding elements. This eliminates std::vector template overhead - // and is ideal when the exact size is known before populating the array. - optional bool fixed_vector = 50013 [default=false]; } diff --git a/esphome/components/api/api_pb2.h b/esphome/components/api/api_pb2.h index 1798458393..d9e68ece9b 100644 --- a/esphome/components/api/api_pb2.h +++ b/esphome/components/api/api_pb2.h @@ -1923,7 +1923,7 @@ class BluetoothGATTCharacteristic final : public ProtoMessage { std::array uuid{}; uint32_t handle{0}; uint32_t properties{0}; - FixedVector descriptors{}; + std::vector descriptors{}; uint32_t short_uuid{0}; void encode(ProtoWriteBuffer buffer) const override; void calculate_size(ProtoSize &size) const override; @@ -1937,7 +1937,7 @@ class BluetoothGATTService final : public ProtoMessage { public: std::array uuid{}; uint32_t handle{0}; - FixedVector characteristics{}; + std::vector characteristics{}; uint32_t short_uuid{0}; void encode(ProtoWriteBuffer buffer) const override; void calculate_size(ProtoSize &size) const override; diff --git a/esphome/components/api/proto.h b/esphome/components/api/proto.h index a6a09bf7c5..9d780692ec 100644 --- a/esphome/components/api/proto.h +++ b/esphome/components/api/proto.h @@ -749,29 +749,13 @@ class ProtoSize { template inline void add_repeated_message(uint32_t field_id_size, const std::vector &messages) { // Skip if the vector is empty - if (!messages.empty()) { - // Use the force version for all messages in the repeated field - for (const auto &message : messages) { - add_message_object_force(field_id_size, message); - } + if (messages.empty()) { + return; } - } - /** - * @brief Calculates and adds the sizes of all messages in a repeated field to the total message size (FixedVector - * version) - * - * @tparam MessageType The type of the nested messages in the FixedVector - * @param messages FixedVector of message objects - */ - template - inline void add_repeated_message(uint32_t field_id_size, const FixedVector &messages) { - // Skip if the fixed vector is empty - if (!messages.empty()) { - // Use the force version for all messages in the repeated field - for (const auto &message : messages) { - add_message_object_force(field_id_size, message); - } + // Use the force version for all messages in the repeated field + for (const auto &message : messages) { + add_message_object_force(field_id_size, message); } } }; From d5234e335709e63eb022240c98c9c885994b247f Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Mon, 13 Oct 2025 14:04:39 -1000 Subject: [PATCH 49/90] merge --- esphome/components/wifi/wifi_component_esp_idf.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/esphome/components/wifi/wifi_component_esp_idf.cpp b/esphome/components/wifi/wifi_component_esp_idf.cpp index 4c719ef4c3..e45b873e8d 100644 --- a/esphome/components/wifi/wifi_component_esp_idf.cpp +++ b/esphome/components/wifi/wifi_component_esp_idf.cpp @@ -763,8 +763,8 @@ void WiFiComponent::wifi_process_event_(IDFWiFiEvent *data) { const auto &it = data->data.sta_scan_done; ESP_LOGV(TAG, "Scan done: status=%" PRIu32 " number=%u scan_id=%u", it.status, it.number, it.scan_id); - this->scan_done_ = true; scan_result_.clear(); + this->scan_done_ = true; if (it.status != 0) { // scan error From ce46f1630859861534f91ed15e03a3ae5c335855 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Mon, 13 Oct 2025 14:05:19 -1000 Subject: [PATCH 50/90] merge --- esphome/components/wifi/wifi_component_esp_idf.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/esphome/components/wifi/wifi_component_esp_idf.cpp b/esphome/components/wifi/wifi_component_esp_idf.cpp index e45b873e8d..951f5803a6 100644 --- a/esphome/components/wifi/wifi_component_esp_idf.cpp +++ b/esphome/components/wifi/wifi_component_esp_idf.cpp @@ -765,7 +765,6 @@ void WiFiComponent::wifi_process_event_(IDFWiFiEvent *data) { scan_result_.clear(); this->scan_done_ = true; - if (it.status != 0) { // scan error return; From 7792a115c26d93b7d18cb112008457a30dfad2ea Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Mon, 13 Oct 2025 14:05:31 -1000 Subject: [PATCH 51/90] merge --- script/api_protobuf/api_protobuf.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/script/api_protobuf/api_protobuf.py b/script/api_protobuf/api_protobuf.py index 9a55f1d136..487c187372 100755 --- a/script/api_protobuf/api_protobuf.py +++ b/script/api_protobuf/api_protobuf.py @@ -1415,8 +1415,6 @@ class RepeatedTypeInfo(TypeInfo): # Check if this is a pointer field by looking for container_pointer option self._container_type = get_field_opt(field, pb.container_pointer, "") self._use_pointer = bool(self._container_type) - # Check if this should use FixedVector instead of std::vector - self._use_fixed_vector = get_field_opt(field, pb.fixed_vector, False) # For repeated fields, we need to get the base type info # but we can't call create_field_type_info as it would cause recursion @@ -1440,8 +1438,6 @@ class RepeatedTypeInfo(TypeInfo): if "<" in self._container_type and ">" in self._container_type: return f"const {self._container_type}*" return f"const {self._container_type}<{self._ti.cpp_type}>*" - if self._use_fixed_vector: - return f"FixedVector<{self._ti.cpp_type}>" return f"std::vector<{self._ti.cpp_type}>" @property From bb2f568f3d4971b536a6c5a59bb25bb59fc45365 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Mon, 13 Oct 2025 14:07:52 -1000 Subject: [PATCH 52/90] merge --- esphome/core/helpers.h | 3 --- 1 file changed, 3 deletions(-) diff --git a/esphome/core/helpers.h b/esphome/core/helpers.h index 084ad28882..6b04af3c7c 100644 --- a/esphome/core/helpers.h +++ b/esphome/core/helpers.h @@ -226,9 +226,6 @@ template class FixedVector { // Clear the vector (reset size to 0, keep capacity) void clear() { size_ = 0; } - // Check if vector is empty - bool empty() const { return size_ == 0; } - /// Add element without bounds checking /// Caller must ensure sufficient capacity was allocated via init() /// Silently ignores pushes beyond capacity (no exception or assertion) From c9a1664398e8a4a9c8f9a3c38d5e67dadf5c6303 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Mon, 13 Oct 2025 14:08:27 -1000 Subject: [PATCH 53/90] merge --- esphome/core/helpers.h | 3 --- 1 file changed, 3 deletions(-) diff --git a/esphome/core/helpers.h b/esphome/core/helpers.h index 084ad28882..6b04af3c7c 100644 --- a/esphome/core/helpers.h +++ b/esphome/core/helpers.h @@ -226,9 +226,6 @@ template class FixedVector { // Clear the vector (reset size to 0, keep capacity) void clear() { size_ = 0; } - // Check if vector is empty - bool empty() const { return size_ == 0; } - /// Add element without bounds checking /// Caller must ensure sufficient capacity was allocated via init() /// Silently ignores pushes beyond capacity (no exception or assertion) From b878aa0270c2faaedee25342d1a15b69c43dbad6 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Mon, 13 Oct 2025 14:09:44 -1000 Subject: [PATCH 54/90] fix --- esphome/core/helpers.h | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/esphome/core/helpers.h b/esphome/core/helpers.h index 6b04af3c7c..a3c0447a8d 100644 --- a/esphome/core/helpers.h +++ b/esphome/core/helpers.h @@ -226,6 +226,14 @@ template class FixedVector { // Clear the vector (reset size to 0, keep capacity) void clear() { size_ = 0; } + // Shrink capacity to fit current size (frees all memory) + void shrink_to_fit() { + cleanup_(); + data_ = nullptr; + capacity_ = 0; + size_ = 0; + } + /// Add element without bounds checking /// Caller must ensure sufficient capacity was allocated via init() /// Silently ignores pushes beyond capacity (no exception or assertion) From de10d781259fb73b605fed83d5c14812997234b8 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Mon, 13 Oct 2025 14:10:41 -1000 Subject: [PATCH 55/90] dry --- esphome/core/helpers.h | 27 ++++++++++++--------------- 1 file changed, 12 insertions(+), 15 deletions(-) diff --git a/esphome/core/helpers.h b/esphome/core/helpers.h index a3c0447a8d..e838c82f3e 100644 --- a/esphome/core/helpers.h +++ b/esphome/core/helpers.h @@ -180,6 +180,13 @@ template class FixedVector { } } + // Helper to reset pointers after cleanup + void reset_() { + data_ = nullptr; + capacity_ = 0; + size_ = 0; + } + public: FixedVector() = default; @@ -187,25 +194,19 @@ template class FixedVector { // Enable move semantics for use in containers FixedVector(FixedVector &&other) noexcept : data_(other.data_), size_(other.size_), capacity_(other.capacity_) { - other.data_ = nullptr; - other.size_ = 0; - other.capacity_ = 0; + other.reset_(); } FixedVector &operator=(FixedVector &&other) noexcept { if (this != &other) { // Delete our current data - if (data_ != nullptr) { - delete[] data_; - } + cleanup_(); // Take ownership of other's data data_ = other.data_; size_ = other.size_; capacity_ = other.capacity_; // Leave other in valid empty state - other.data_ = nullptr; - other.size_ = 0; - other.capacity_ = 0; + other.reset_(); } return *this; } @@ -213,9 +214,7 @@ template class FixedVector { // Allocate capacity - can be called multiple times to reinit void init(size_t n) { cleanup_(); - data_ = nullptr; - capacity_ = 0; - size_ = 0; + reset_(); if (n > 0) { // Allocate raw memory without calling constructors data_ = static_cast(::operator new(n * sizeof(T))); @@ -229,9 +228,7 @@ template class FixedVector { // Shrink capacity to fit current size (frees all memory) void shrink_to_fit() { cleanup_(); - data_ = nullptr; - capacity_ = 0; - size_ = 0; + reset_(); } /// Add element without bounds checking From 453ab0adb8ec9f6da7af1d84590e1a2d309711e7 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Mon, 13 Oct 2025 14:10:56 -1000 Subject: [PATCH 56/90] backmerge --- esphome/core/helpers.h | 29 +++++++++++++++++------------ 1 file changed, 17 insertions(+), 12 deletions(-) diff --git a/esphome/core/helpers.h b/esphome/core/helpers.h index 6b04af3c7c..e838c82f3e 100644 --- a/esphome/core/helpers.h +++ b/esphome/core/helpers.h @@ -180,6 +180,13 @@ template class FixedVector { } } + // Helper to reset pointers after cleanup + void reset_() { + data_ = nullptr; + capacity_ = 0; + size_ = 0; + } + public: FixedVector() = default; @@ -187,25 +194,19 @@ template class FixedVector { // Enable move semantics for use in containers FixedVector(FixedVector &&other) noexcept : data_(other.data_), size_(other.size_), capacity_(other.capacity_) { - other.data_ = nullptr; - other.size_ = 0; - other.capacity_ = 0; + other.reset_(); } FixedVector &operator=(FixedVector &&other) noexcept { if (this != &other) { // Delete our current data - if (data_ != nullptr) { - delete[] data_; - } + cleanup_(); // Take ownership of other's data data_ = other.data_; size_ = other.size_; capacity_ = other.capacity_; // Leave other in valid empty state - other.data_ = nullptr; - other.size_ = 0; - other.capacity_ = 0; + other.reset_(); } return *this; } @@ -213,9 +214,7 @@ template class FixedVector { // Allocate capacity - can be called multiple times to reinit void init(size_t n) { cleanup_(); - data_ = nullptr; - capacity_ = 0; - size_ = 0; + reset_(); if (n > 0) { // Allocate raw memory without calling constructors data_ = static_cast(::operator new(n * sizeof(T))); @@ -226,6 +225,12 @@ template class FixedVector { // Clear the vector (reset size to 0, keep capacity) void clear() { size_ = 0; } + // Shrink capacity to fit current size (frees all memory) + void shrink_to_fit() { + cleanup_(); + reset_(); + } + /// Add element without bounds checking /// Caller must ensure sufficient capacity was allocated via init() /// Silently ignores pushes beyond capacity (no exception or assertion) From 7b5a86e4df1a90f2521b283d7ec8a17c5c5e2359 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Mon, 13 Oct 2025 14:15:37 -1000 Subject: [PATCH 57/90] fixes --- esphome/components/wifi/wifi_component.cpp | 2 +- esphome/components/wifi/wifi_component.h | 12 ++++++++++-- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/esphome/components/wifi/wifi_component.cpp b/esphome/components/wifi/wifi_component.cpp index 3cdc034be6..8308421220 100644 --- a/esphome/components/wifi/wifi_component.cpp +++ b/esphome/components/wifi/wifi_component.cpp @@ -554,7 +554,7 @@ void WiFiComponent::start_scanning() { // Using insertion sort instead of std::stable_sort saves flash memory // by avoiding template instantiations (std::rotate, std::stable_sort, lambdas) // IMPORTANT: This sort is stable (preserves relative order of equal elements) -static void insertion_sort_scan_results(FixedVector &results) { +template static void insertion_sort_scan_results(VectorType &results) { const size_t size = results.size(); for (size_t i = 1; i < size; i++) { // Make a copy to avoid issues with move semantics during comparison diff --git a/esphome/components/wifi/wifi_component.h b/esphome/components/wifi/wifi_component.h index e1c3d6df88..bbb4649027 100644 --- a/esphome/components/wifi/wifi_component.h +++ b/esphome/components/wifi/wifi_component.h @@ -121,6 +121,14 @@ struct EAPAuth { using bssid_t = std::array; +// Use std::vector for RP2040 since scan count is unknown (callback-based) +// Use FixedVector for other platforms where count is queried first +#ifdef USE_RP2040 +template using wifi_scan_vector_t = std::vector; +#else +template using wifi_scan_vector_t = FixedVector; +#endif + class WiFiAP { public: void set_ssid(const std::string &ssid); @@ -278,7 +286,7 @@ class WiFiComponent : public Component { std::string get_use_address() const; void set_use_address(const std::string &use_address); - const FixedVector &get_scan_result() const { return scan_result_; } + const wifi_scan_vector_t &get_scan_result() const { return scan_result_; } network::IPAddress wifi_soft_ap_ip(); @@ -385,7 +393,7 @@ class WiFiComponent : public Component { std::string use_address_; std::vector sta_; std::vector sta_priorities_; - FixedVector scan_result_; + wifi_scan_vector_t scan_result_; WiFiAP selected_ap_; WiFiAP ap_; optional output_power_; From 10724f411b04fc3a8e40c6a8c075e0af2e0c2326 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Mon, 13 Oct 2025 14:32:17 -1000 Subject: [PATCH 58/90] [network] Optimize get_use_address() to return const reference instead of copy --- esphome/components/ethernet/ethernet_component.cpp | 11 +++-------- esphome/components/ethernet/ethernet_component.h | 2 +- esphome/components/network/util.cpp | 7 +++++-- esphome/components/network/util.h | 2 +- esphome/components/wifi/wifi_component.cpp | 11 +++-------- esphome/components/wifi/wifi_component.h | 2 +- 6 files changed, 14 insertions(+), 21 deletions(-) diff --git a/esphome/components/ethernet/ethernet_component.cpp b/esphome/components/ethernet/ethernet_component.cpp index 13adab8815..24b6e8154b 100644 --- a/esphome/components/ethernet/ethernet_component.cpp +++ b/esphome/components/ethernet/ethernet_component.cpp @@ -689,14 +689,9 @@ void EthernetComponent::add_phy_register(PHYRegister register_value) { this->phy void EthernetComponent::set_type(EthernetType type) { this->type_ = type; } void EthernetComponent::set_manual_ip(const ManualIP &manual_ip) { this->manual_ip_ = manual_ip; } -std::string EthernetComponent::get_use_address() const { - if (this->use_address_.empty()) { - // ".local" suffix length for mDNS hostnames - constexpr size_t mdns_local_suffix_len = 5; - return make_name_with_suffix(App.get_name(), '.', "local", mdns_local_suffix_len); - } - return this->use_address_; -} +// set_use_address() is guaranteed to be called during component setup by Python code generation, +// so use_address_ will always be valid when get_use_address() is called - no fallback needed. +const std::string &EthernetComponent::get_use_address() const { return this->use_address_; } void EthernetComponent::set_use_address(const std::string &use_address) { this->use_address_ = use_address; } diff --git a/esphome/components/ethernet/ethernet_component.h b/esphome/components/ethernet/ethernet_component.h index 6b4e342df5..d5dda3e3ae 100644 --- a/esphome/components/ethernet/ethernet_component.h +++ b/esphome/components/ethernet/ethernet_component.h @@ -88,7 +88,7 @@ class EthernetComponent : public Component { network::IPAddresses get_ip_addresses(); network::IPAddress get_dns_address(uint8_t num); - std::string get_use_address() const; + const std::string &get_use_address() const; void set_use_address(const std::string &use_address); void get_eth_mac_address_raw(uint8_t *mac); std::string get_eth_mac_address_pretty(); diff --git a/esphome/components/network/util.cpp b/esphome/components/network/util.cpp index bf76aefc30..0f28e2d3fb 100644 --- a/esphome/components/network/util.cpp +++ b/esphome/components/network/util.cpp @@ -85,7 +85,7 @@ network::IPAddresses get_ip_addresses() { return {}; } -std::string get_use_address() { +const std::string &get_use_address() { #ifdef USE_ETHERNET if (ethernet::global_eth_component != nullptr) return ethernet::global_eth_component->get_use_address(); @@ -100,7 +100,10 @@ std::string get_use_address() { if (wifi::global_wifi_component != nullptr) return wifi::global_wifi_component->get_use_address(); #endif - return ""; +#if !defined(USE_ETHERNET) && !defined(USE_MODEM) && !defined(USE_WIFI) + static const std::string empty; + return empty; +#endif } } // namespace network diff --git a/esphome/components/network/util.h b/esphome/components/network/util.h index b518696e68..b4a92f8bee 100644 --- a/esphome/components/network/util.h +++ b/esphome/components/network/util.h @@ -12,7 +12,7 @@ bool is_connected(); /// Return whether the network is disabled (only wifi for now) bool is_disabled(); /// Get the active network hostname -std::string get_use_address(); +const std::string &get_use_address(); IPAddresses get_ip_addresses(); } // namespace network diff --git a/esphome/components/wifi/wifi_component.cpp b/esphome/components/wifi/wifi_component.cpp index 0f9f879181..aa197b36e5 100644 --- a/esphome/components/wifi/wifi_component.cpp +++ b/esphome/components/wifi/wifi_component.cpp @@ -265,14 +265,9 @@ network::IPAddress WiFiComponent::get_dns_address(int num) { return this->wifi_dns_ip_(num); return {}; } -std::string WiFiComponent::get_use_address() const { - if (this->use_address_.empty()) { - // ".local" suffix length for mDNS hostnames - constexpr size_t mdns_local_suffix_len = 5; - return make_name_with_suffix(App.get_name(), '.', "local", mdns_local_suffix_len); - } - return this->use_address_; -} +// set_use_address() is guaranteed to be called during component setup by Python code generation, +// so use_address_ will always be valid when get_use_address() is called - no fallback needed. +const std::string &WiFiComponent::get_use_address() const { return this->use_address_; } void WiFiComponent::set_use_address(const std::string &use_address) { this->use_address_ = use_address; } #ifdef USE_WIFI_AP diff --git a/esphome/components/wifi/wifi_component.h b/esphome/components/wifi/wifi_component.h index ee62ec1a69..0e0f89d2c1 100644 --- a/esphome/components/wifi/wifi_component.h +++ b/esphome/components/wifi/wifi_component.h @@ -275,7 +275,7 @@ class WiFiComponent : public Component { network::IPAddress get_dns_address(int num); network::IPAddresses get_ip_addresses(); - std::string get_use_address() const; + const std::string &get_use_address() const; void set_use_address(const std::string &use_address); const std::vector &get_scan_result() const { return scan_result_; } From 2881f32b08549f34c09cfe37b14e71bc984a6008 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Mon, 13 Oct 2025 14:37:10 -1000 Subject: [PATCH 59/90] [network] Optimize get_use_address() to return const reference instead of copy --- esphome/components/network/util.cpp | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/esphome/components/network/util.cpp b/esphome/components/network/util.cpp index 0f28e2d3fb..12939f8f84 100644 --- a/esphome/components/network/util.cpp +++ b/esphome/components/network/util.cpp @@ -86,21 +86,21 @@ network::IPAddresses get_ip_addresses() { } const std::string &get_use_address() { + // Global component pointers are guaranteed to be set by component constructors when USE_* is defined #ifdef USE_ETHERNET - if (ethernet::global_eth_component != nullptr) - return ethernet::global_eth_component->get_use_address(); + return ethernet::global_eth_component->get_use_address(); #endif #ifdef USE_MODEM - if (modem::global_modem_component != nullptr) - return modem::global_modem_component->get_use_address(); + return modem::global_modem_component->get_use_address(); #endif #ifdef USE_WIFI - if (wifi::global_wifi_component != nullptr) - return wifi::global_wifi_component->get_use_address(); + return wifi::global_wifi_component->get_use_address(); #endif + #if !defined(USE_ETHERNET) && !defined(USE_MODEM) && !defined(USE_WIFI) + // Fallback when no network component is defined (shouldn't happen with USE_NETWORK defined) static const std::string empty; return empty; #endif From fa830cfd399384979992b21bf0d225c1a66e6793 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Mon, 13 Oct 2025 14:50:55 -1000 Subject: [PATCH 60/90] fix --- esphome/components/improv_serial/improv_serial_component.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/esphome/components/improv_serial/improv_serial_component.cpp b/esphome/components/improv_serial/improv_serial_component.cpp index 528a155a7f..28245dcfdf 100644 --- a/esphome/components/improv_serial/improv_serial_component.cpp +++ b/esphome/components/improv_serial/improv_serial_component.cpp @@ -218,7 +218,7 @@ bool ImprovSerialComponent::parse_improv_payload_(improv::ImprovCommand &command } case improv::GET_WIFI_NETWORKS: { std::vector networks; - auto results = wifi::global_wifi_component->get_scan_result(); + const auto &results = wifi::global_wifi_component->get_scan_result(); for (auto &scan : results) { if (scan.get_is_hidden()) continue; From e17cdffc78cb879bb7eac15e0e2b11ac66d8118b Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Mon, 13 Oct 2025 15:04:40 -1000 Subject: [PATCH 61/90] merge --- esphome/core/helpers.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/esphome/core/helpers.h b/esphome/core/helpers.h index e838c82f3e..4dcd44a574 100644 --- a/esphome/core/helpers.h +++ b/esphome/core/helpers.h @@ -217,6 +217,8 @@ template class FixedVector { reset_(); if (n > 0) { // Allocate raw memory without calling constructors + // sizeof(T) is correct here - when T is a pointer type, we want the pointer size + // NOLINTNEXTLINE(bugprone-sizeof-expression) data_ = static_cast(::operator new(n * sizeof(T))); capacity_ = n; } From d5ba16f13a57add5db95a600e8bb6d09a1493236 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Mon, 13 Oct 2025 15:22:52 -1000 Subject: [PATCH 62/90] merge --- esphome/core/helpers.h | 39 +++++++++++++++++++++++++++++++-------- 1 file changed, 31 insertions(+), 8 deletions(-) diff --git a/esphome/core/helpers.h b/esphome/core/helpers.h index 4dcd44a574..6d7ae564e8 100644 --- a/esphome/core/helpers.h +++ b/esphome/core/helpers.h @@ -224,8 +224,14 @@ template class FixedVector { } } - // Clear the vector (reset size to 0, keep capacity) - void clear() { size_ = 0; } + // Clear the vector (destroy all elements, reset size to 0, keep capacity) + void clear() { + // Manually destroy all elements + for (size_t i = 0; i < size_; i++) { + data_[i].~T(); + } + size_ = 0; + } // Shrink capacity to fit current size (frees all memory) void shrink_to_fit() { @@ -244,17 +250,34 @@ template class FixedVector { } } - /// Construct element in place and return reference + /// Add element by move without bounds checking /// Caller must ensure sufficient capacity was allocated via init() - T &emplace_back() { + /// Silently ignores pushes beyond capacity (no exception or assertion) + void push_back(T &&value) { if (size_ < capacity_) { - return data_[size_++]; + // Use placement new to move-construct the object in pre-allocated memory + new (&data_[size_]) T(std::move(value)); + size_++; } - // Should never happen with proper init() - return last element to avoid crash - return data_[capacity_ - 1]; } - /// Access last element + /// Emplace element without bounds checking - constructs in-place + /// Caller must ensure sufficient capacity was allocated via init() + /// Returns reference to the newly constructed element + /// Silently ignores emplaces beyond capacity (returns reference to last element) + T &emplace_back() { + if (size_ < capacity_) { + // Use placement new to default-construct the object in pre-allocated memory + new (&data_[size_]) T(); + size_++; + return data_[size_ - 1]; + } + // Beyond capacity - return reference to last element to avoid crash + return data_[size_ - 1]; + } + + /// Access last element (no bounds checking - matches std::vector behavior) + /// Caller must ensure vector is not empty (size() > 0) T &back() { return data_[size_ - 1]; } const T &back() const { return data_[size_ - 1]; } From 9775274007f13edc65f6a3230daa464abbf7fc64 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Mon, 13 Oct 2025 15:25:47 -1000 Subject: [PATCH 63/90] preen --- esphome/core/helpers.h | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/esphome/core/helpers.h b/esphome/core/helpers.h index 6d7ae564e8..349ed663ad 100644 --- a/esphome/core/helpers.h +++ b/esphome/core/helpers.h @@ -168,13 +168,17 @@ template class FixedVector { size_t size_{0}; size_t capacity_{0}; + // Helper to destroy all elements without freeing memory + void destroy_elements_() { + for (size_t i = 0; i < size_; i++) { + data_[i].~T(); + } + } + // Helper to destroy elements and free memory void cleanup_() { if (data_ != nullptr) { - // Manually destroy all elements - for (size_t i = 0; i < size_; i++) { - data_[i].~T(); - } + destroy_elements_(); // Free raw memory ::operator delete(data_); } @@ -226,10 +230,7 @@ template class FixedVector { // Clear the vector (destroy all elements, reset size to 0, keep capacity) void clear() { - // Manually destroy all elements - for (size_t i = 0; i < size_; i++) { - data_[i].~T(); - } + destroy_elements_(); size_ = 0; } From 2626a851fbf1e7feb85c1661373941359bd72f08 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Mon, 13 Oct 2025 15:30:18 -1000 Subject: [PATCH 64/90] cleanup --- esphome/core/helpers.h | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/esphome/core/helpers.h b/esphome/core/helpers.h index 349ed663ad..3ca62a68cb 100644 --- a/esphome/core/helpers.h +++ b/esphome/core/helpers.h @@ -170,8 +170,11 @@ template class FixedVector { // Helper to destroy all elements without freeing memory void destroy_elements_() { - for (size_t i = 0; i < size_; i++) { - data_[i].~T(); + // Only call destructors for non-trivially destructible types + if constexpr (!std::is_trivially_destructible::value) { + for (size_t i = 0; i < size_; i++) { + data_[i].~T(); + } } } @@ -221,7 +224,7 @@ template class FixedVector { reset_(); if (n > 0) { // Allocate raw memory without calling constructors - // sizeof(T) is correct here - when T is a pointer type, we want the pointer size + // sizeof(T) is correct here for any type T (value types, pointers, etc.) // NOLINTNEXTLINE(bugprone-sizeof-expression) data_ = static_cast(::operator new(n * sizeof(T))); capacity_ = n; @@ -265,15 +268,11 @@ template class FixedVector { /// Emplace element without bounds checking - constructs in-place /// Caller must ensure sufficient capacity was allocated via init() /// Returns reference to the newly constructed element - /// Silently ignores emplaces beyond capacity (returns reference to last element) + /// NOTE: Caller MUST ensure size_ < capacity_ before calling T &emplace_back() { - if (size_ < capacity_) { - // Use placement new to default-construct the object in pre-allocated memory - new (&data_[size_]) T(); - size_++; - return data_[size_ - 1]; - } - // Beyond capacity - return reference to last element to avoid crash + // Use placement new to default-construct the object in pre-allocated memory + new (&data_[size_]) T(); + size_++; return data_[size_ - 1]; } From 6b8d5be528f77bb5e0f5e17a2acec1a35594d8ab Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Mon, 13 Oct 2025 15:41:17 -1000 Subject: [PATCH 65/90] Update esphome/components/network/util.cpp Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- esphome/components/network/util.cpp | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/esphome/components/network/util.cpp b/esphome/components/network/util.cpp index 12939f8f84..d29bf8e6bd 100644 --- a/esphome/components/network/util.cpp +++ b/esphome/components/network/util.cpp @@ -100,9 +100,7 @@ const std::string &get_use_address() { #endif #if !defined(USE_ETHERNET) && !defined(USE_MODEM) && !defined(USE_WIFI) - // Fallback when no network component is defined (shouldn't happen with USE_NETWORK defined) - static const std::string empty; - return empty; + static_assert(false, "At least one of USE_ETHERNET, USE_MODEM, or USE_WIFI must be defined when USE_NETWORK is set."); #endif } From c5076e69f0955567f406692ee29ed5503e25c0d0 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Mon, 13 Oct 2025 15:52:00 -1000 Subject: [PATCH 66/90] host platform --- esphome/components/network/util.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/esphome/components/network/util.cpp b/esphome/components/network/util.cpp index d29bf8e6bd..27ad9448a4 100644 --- a/esphome/components/network/util.cpp +++ b/esphome/components/network/util.cpp @@ -100,7 +100,9 @@ const std::string &get_use_address() { #endif #if !defined(USE_ETHERNET) && !defined(USE_MODEM) && !defined(USE_WIFI) - static_assert(false, "At least one of USE_ETHERNET, USE_MODEM, or USE_WIFI must be defined when USE_NETWORK is set."); + // Fallback when no network component is defined (e.g., host platform) + static const std::string empty; + return empty; #endif } From 5c30c1b6916920a406157aeff74d9ad60b981850 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Mon, 13 Oct 2025 17:49:07 -1000 Subject: [PATCH 67/90] core.data --- esphome/components/wifi/__init__.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/esphome/components/wifi/__init__.py b/esphome/components/wifi/__init__.py index ae85e40357..ad5698519b 100644 --- a/esphome/components/wifi/__init__.py +++ b/esphome/components/wifi/__init__.py @@ -470,7 +470,7 @@ async def wifi_disable_to_code(config, action_id, template_arg, args): return cg.new_Pvariable(action_id, template_arg) -_FLAGS = {"keep_scan_results": False} +KEEP_SCAN_RESULTS_KEY = "wifi_keep_scan_results" def request_wifi_scan_results(): @@ -480,13 +480,13 @@ def request_wifi_scan_results(): call this function during their code generation. This prevents the WiFi component from freeing scan result memory after successful connection. """ - _FLAGS["keep_scan_results"] = True + CORE.data[KEEP_SCAN_RESULTS_KEY] = True @coroutine_with_priority(CoroPriority.FINAL) async def final_step(): """Final code generation step to configure scan result retention.""" - if _FLAGS["keep_scan_results"]: + if CORE.data.get(KEEP_SCAN_RESULTS_KEY, False): cg.add( cg.RawExpression("wifi::global_wifi_component->set_keep_scan_results(true)") ) From 5bdd6dac9721307b9b3636122fb63a0353b91564 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Mon, 13 Oct 2025 18:02:28 -1000 Subject: [PATCH 68/90] [esp32_ble_tracker] Refactor to use CORE.data instead of module-level globals --- .../components/esp32_ble_tracker/__init__.py | 50 ++++++++++++------- 1 file changed, 32 insertions(+), 18 deletions(-) diff --git a/esphome/components/esp32_ble_tracker/__init__.py b/esphome/components/esp32_ble_tracker/__init__.py index 8c7f3e3930..d9963d7391 100644 --- a/esphome/components/esp32_ble_tracker/__init__.py +++ b/esphome/components/esp32_ble_tracker/__init__.py @@ -60,11 +60,21 @@ class RegistrationCounts: clients: int = 0 -# Set to track which features are needed by components -_required_features: set[BLEFeatures] = set() +# CORE.data keys for state management +ESP32_BLE_TRACKER_REQUIRED_FEATURES_KEY = "esp32_ble_tracker_required_features" +ESP32_BLE_TRACKER_REGISTRATION_COUNTS_KEY = "esp32_ble_tracker_registration_counts" -# Track registration counts for StaticVector sizing -_registration_counts = RegistrationCounts() + +def _get_required_features() -> set[BLEFeatures]: + """Get the set of required BLE features from CORE.data.""" + return CORE.data.setdefault(ESP32_BLE_TRACKER_REQUIRED_FEATURES_KEY, set()) + + +def _get_registration_counts() -> RegistrationCounts: + """Get the registration counts from CORE.data.""" + return CORE.data.setdefault( + ESP32_BLE_TRACKER_REGISTRATION_COUNTS_KEY, RegistrationCounts() + ) def register_ble_features(features: set[BLEFeatures]) -> None: @@ -73,7 +83,7 @@ def register_ble_features(features: set[BLEFeatures]) -> None: Args: features: Set of BLEFeatures enum members """ - _required_features.update(features) + _get_required_features().update(features) esp32_ble_tracker_ns = cg.esphome_ns.namespace("esp32_ble_tracker") @@ -267,15 +277,17 @@ async def to_code(config): ): register_ble_features({BLEFeatures.ESP_BT_DEVICE}) + registration_counts = _get_registration_counts() + for conf in config.get(CONF_ON_BLE_ADVERTISE, []): - _registration_counts.listeners += 1 + registration_counts.listeners += 1 trigger = cg.new_Pvariable(conf[CONF_TRIGGER_ID], var) if CONF_MAC_ADDRESS in conf: addr_list = [it.as_hex for it in conf[CONF_MAC_ADDRESS]] cg.add(trigger.set_addresses(addr_list)) await automation.build_automation(trigger, [(ESPBTDeviceConstRef, "x")], conf) for conf in config.get(CONF_ON_BLE_SERVICE_DATA_ADVERTISE, []): - _registration_counts.listeners += 1 + registration_counts.listeners += 1 trigger = cg.new_Pvariable(conf[CONF_TRIGGER_ID], var) if len(conf[CONF_SERVICE_UUID]) == len(bt_uuid16_format): cg.add(trigger.set_service_uuid16(as_hex(conf[CONF_SERVICE_UUID]))) @@ -288,7 +300,7 @@ async def to_code(config): cg.add(trigger.set_address(conf[CONF_MAC_ADDRESS].as_hex)) await automation.build_automation(trigger, [(adv_data_t_const_ref, "x")], conf) for conf in config.get(CONF_ON_BLE_MANUFACTURER_DATA_ADVERTISE, []): - _registration_counts.listeners += 1 + registration_counts.listeners += 1 trigger = cg.new_Pvariable(conf[CONF_TRIGGER_ID], var) if len(conf[CONF_MANUFACTURER_ID]) == len(bt_uuid16_format): cg.add(trigger.set_manufacturer_uuid16(as_hex(conf[CONF_MANUFACTURER_ID]))) @@ -301,7 +313,7 @@ async def to_code(config): cg.add(trigger.set_address(conf[CONF_MAC_ADDRESS].as_hex)) await automation.build_automation(trigger, [(adv_data_t_const_ref, "x")], conf) for conf in config.get(CONF_ON_SCAN_END, []): - _registration_counts.listeners += 1 + registration_counts.listeners += 1 trigger = cg.new_Pvariable(conf[CONF_TRIGGER_ID], var) await automation.build_automation(trigger, [], conf) @@ -331,19 +343,21 @@ async def to_code(config): @coroutine_with_priority(CoroPriority.FINAL) async def _add_ble_features(): # Add feature-specific defines based on what's needed - if BLEFeatures.ESP_BT_DEVICE in _required_features: + required_features = _get_required_features() + if BLEFeatures.ESP_BT_DEVICE in required_features: cg.add_define("USE_ESP32_BLE_DEVICE") cg.add_define("USE_ESP32_BLE_UUID") # Add defines for StaticVector sizing based on registration counts # Only define if count > 0 to avoid allocating unnecessary memory - if _registration_counts.listeners > 0: + registration_counts = _get_registration_counts() + if registration_counts.listeners > 0: cg.add_define( - "ESPHOME_ESP32_BLE_TRACKER_LISTENER_COUNT", _registration_counts.listeners + "ESPHOME_ESP32_BLE_TRACKER_LISTENER_COUNT", registration_counts.listeners ) - if _registration_counts.clients > 0: + if registration_counts.clients > 0: cg.add_define( - "ESPHOME_ESP32_BLE_TRACKER_CLIENT_COUNT", _registration_counts.clients + "ESPHOME_ESP32_BLE_TRACKER_CLIENT_COUNT", registration_counts.clients ) @@ -395,7 +409,7 @@ async def register_ble_device( var: cg.SafeExpType, config: ConfigType ) -> cg.SafeExpType: register_ble_features({BLEFeatures.ESP_BT_DEVICE}) - _registration_counts.listeners += 1 + _get_registration_counts().listeners += 1 paren = await cg.get_variable(config[CONF_ESP32_BLE_ID]) cg.add(paren.register_listener(var)) return var @@ -403,7 +417,7 @@ async def register_ble_device( async def register_client(var: cg.SafeExpType, config: ConfigType) -> cg.SafeExpType: register_ble_features({BLEFeatures.ESP_BT_DEVICE}) - _registration_counts.clients += 1 + _get_registration_counts().clients += 1 paren = await cg.get_variable(config[CONF_ESP32_BLE_ID]) cg.add(paren.register_client(var)) return var @@ -417,7 +431,7 @@ async def register_raw_ble_device( This does NOT register the ESP_BT_DEVICE feature, meaning ESPBTDevice will not be compiled in if this is the only registration method used. """ - _registration_counts.listeners += 1 + _get_registration_counts().listeners += 1 paren = await cg.get_variable(config[CONF_ESP32_BLE_ID]) cg.add(paren.register_listener(var)) return var @@ -431,7 +445,7 @@ async def register_raw_client( This does NOT register the ESP_BT_DEVICE feature, meaning ESPBTDevice will not be compiled in if this is the only registration method used. """ - _registration_counts.clients += 1 + _get_registration_counts().clients += 1 paren = await cg.get_variable(config[CONF_ESP32_BLE_ID]) cg.add(paren.register_client(var)) return var From dd0699305e1d4a68228941780b6e1bfab92fc217 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Mon, 13 Oct 2025 18:08:52 -1000 Subject: [PATCH 69/90] [esp32_ble] Refactor to use CORE.data instead of module-level globals --- esphome/components/esp32_ble/__init__.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/esphome/components/esp32_ble/__init__.py b/esphome/components/esp32_ble/__init__.py index 1c142ca7bd..4e001b35c6 100644 --- a/esphome/components/esp32_ble/__init__.py +++ b/esphome/components/esp32_ble/__init__.py @@ -107,8 +107,13 @@ class BTLoggers(Enum): """ESP32 WiFi provisioning over Bluetooth""" -# Set to track which loggers are needed by components -_required_loggers: set[BTLoggers] = set() +# Key for storing required loggers in CORE.data +ESP32_BLE_REQUIRED_LOGGERS_KEY = "esp32_ble_required_loggers" + + +def _get_required_loggers() -> set[BTLoggers]: + """Get the set of required Bluetooth loggers from CORE.data.""" + return CORE.data.setdefault(ESP32_BLE_REQUIRED_LOGGERS_KEY, set()) def register_bt_logger(*loggers: BTLoggers) -> None: @@ -117,12 +122,13 @@ def register_bt_logger(*loggers: BTLoggers) -> None: Args: *loggers: One or more BTLoggers enum members """ + required_loggers = _get_required_loggers() for logger in loggers: if not isinstance(logger, BTLoggers): raise TypeError( f"Logger must be a BTLoggers enum member, got {type(logger)}" ) - _required_loggers.add(logger) + required_loggers.add(logger) CONF_BLE_ID = "ble_id" @@ -396,8 +402,9 @@ async def to_code(config): # Apply logger settings if log disabling is enabled if config.get(CONF_DISABLE_BT_LOGS, False): # Disable all Bluetooth loggers that are not required + required_loggers = _get_required_loggers() for logger in BTLoggers: - if logger not in _required_loggers: + if logger not in required_loggers: add_idf_sdkconfig_option(f"{logger.value}_NONE", True) # Set BLE connection establishment timeout to match aioesphomeapi/bleak-retry-connector From 18d5fd160a90f4536009a31b37a7d36e7de1fdf5 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Mon, 13 Oct 2025 18:11:34 -1000 Subject: [PATCH 70/90] [i2s_audio] Refactor to use CORE.data instead of module-level globals --- esphome/components/i2s_audio/__init__.py | 24 +++++++++++++++++------- 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/esphome/components/i2s_audio/__init__.py b/esphome/components/i2s_audio/__init__.py index 8ceff26d84..907429ee0e 100644 --- a/esphome/components/i2s_audio/__init__.py +++ b/esphome/components/i2s_audio/__init__.py @@ -143,7 +143,18 @@ def validate_mclk_divisible_by_3(config): return config -_use_legacy_driver = None +# Key for storing legacy driver setting in CORE.data +I2S_USE_LEGACY_DRIVER_KEY = "i2s_use_legacy_driver" + + +def _get_use_legacy_driver(): + """Get the legacy driver setting from CORE.data.""" + return CORE.data.get(I2S_USE_LEGACY_DRIVER_KEY) + + +def _set_use_legacy_driver(value: bool) -> None: + """Set the legacy driver setting in CORE.data.""" + CORE.data[I2S_USE_LEGACY_DRIVER_KEY] = value def i2s_audio_component_schema( @@ -209,17 +220,15 @@ async def register_i2s_audio_component(var, config): def validate_use_legacy(value): - global _use_legacy_driver # noqa: PLW0603 if CONF_USE_LEGACY in value: - if (_use_legacy_driver is not None) and ( - _use_legacy_driver != value[CONF_USE_LEGACY] - ): + existing_value = _get_use_legacy_driver() + if (existing_value is not None) and (existing_value != value[CONF_USE_LEGACY]): raise cv.Invalid( f"All i2s_audio components must set {CONF_USE_LEGACY} to the same value." ) if (not value[CONF_USE_LEGACY]) and (CORE.using_arduino): raise cv.Invalid("Arduino supports only the legacy i2s driver") - _use_legacy_driver = value[CONF_USE_LEGACY] + _set_use_legacy_driver(value[CONF_USE_LEGACY]) return value @@ -249,7 +258,8 @@ def _final_validate(_): def use_legacy(): - return not (CORE.using_esp_idf and not _use_legacy_driver) + legacy_driver = _get_use_legacy_driver() + return not (CORE.using_esp_idf and not legacy_driver) FINAL_VALIDATE_SCHEMA = _final_validate From 0f43f4cbbf0e7b5ce38436cbc6ee17454aa3ae8d Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Mon, 13 Oct 2025 18:26:45 -1000 Subject: [PATCH 71/90] [docs] Add embedded systems optimization and state management best practices to CLAUDE.md --- .ai/instructions.md | 64 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 64 insertions(+) diff --git a/.ai/instructions.md b/.ai/instructions.md index d2e173472a..ab382c61e8 100644 --- a/.ai/instructions.md +++ b/.ai/instructions.md @@ -221,6 +221,70 @@ This document provides essential context for AI models interacting with this pro * **Component Development:** Keep dependencies minimal, provide clear error messages, and write comprehensive docstrings and tests. * **Code Generation:** Generate minimal and efficient C++ code. Validate all user inputs thoroughly. Support multiple platform variations. * **Configuration Design:** Aim for simplicity with sensible defaults, while allowing for advanced customization. + * **Embedded Systems Optimization:** ESPHome targets resource-constrained microcontrollers. Be mindful of flash size and RAM usage. + + **STL Container Guidelines:** + + ESPHome runs on embedded systems with limited resources. Choose containers carefully: + + 1. **Compile-time-known sizes:** Use `std::array` instead of `std::vector` when size is known at compile time. + ```cpp + // Bad - generates STL realloc code + std::vector values; + + // Good - no dynamic allocation + std::array values; + ``` + Use `cg.add_define("MAX_VALUES", count)` to set the size from Python configuration. + + **For byte buffers:** Avoid `std::vector` unless the buffer needs to grow. Use `std::unique_ptr` instead. + ```cpp + // Bad - STL overhead for simple byte buffer + std::vector buffer; + buffer.resize(256); + + // Good - minimal overhead, single allocation + std::unique_ptr buffer = std::make_unique(256); + // Or if size is constant: + std::array buffer; + ``` + + 2. **Small datasets (1-16 elements):** Use `std::vector` or `std::array` with simple structs instead of `std::map`/`std::set`/`std::unordered_map`. + ```cpp + // Bad - 2KB+ overhead for red-black tree/hash table + std::map small_lookup; + std::unordered_map tiny_map; + + // Good - simple struct with linear search (std::vector is fine) + struct LookupEntry { + const char *key; + int value; + }; + std::vector small_lookup = { + {"key1", 10}, + {"key2", 20}, + {"key3", 30}, + }; + // Or std::array if size is compile-time constant: + // std::array small_lookup = {{ ... }}; + ``` + Linear search on small datasets (1-16 elements) is faster than hashing/tree overhead. `std::vector` with simple structs is perfectly fine - it's the heavy containers (`map`, `set`, `unordered_map`) that should be avoided for small datasets. + + 3. **Detection:** Look for these patterns in compiler output: + - Large code sections with STL symbols (vector, map, set) + - `alloc`, `realloc`, `dealloc` in symbol names + - Red-black tree code (`rb_tree`, `_Rb_tree`) + - Hash table infrastructure (`unordered_map`, `hash`) + + **When to optimize:** + - Core components (API, network, logger) + - Widely-used components (mdns, wifi, ble) + - Components causing flash size complaints + + **When not to optimize:** + - Single-use niche components + - Code where readability matters more than bytes + - Already using appropriate containers * **Security:** Be mindful of security when making changes to the API, web server, or any other network-related code. Do not hardcode secrets or keys. From 541c697a42e5ff46434a3f4fdcf4e8e1cb6c6777 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Mon, 13 Oct 2025 18:52:49 -1000 Subject: [PATCH 72/90] [mdns] Use FixedVector for txt_records to reduce flash usage --- esphome/components/mdns/mdns_component.cpp | 4 ++-- esphome/components/mdns/mdns_component.h | 2 +- esphome/core/helpers.h | 9 +++++++++ 3 files changed, 12 insertions(+), 3 deletions(-) diff --git a/esphome/components/mdns/mdns_component.cpp b/esphome/components/mdns/mdns_component.cpp index fea3ced99f..ef585db51b 100644 --- a/esphome/components/mdns/mdns_component.cpp +++ b/esphome/components/mdns/mdns_component.cpp @@ -83,7 +83,7 @@ void MDNSComponent::compile_records_(StaticVector port; - std::vector txt_records; + FixedVector txt_records; }; class MDNSComponent : public Component { diff --git a/esphome/core/helpers.h b/esphome/core/helpers.h index e352c9c415..b94826629f 100644 --- a/esphome/core/helpers.h +++ b/esphome/core/helpers.h @@ -197,6 +197,15 @@ template class FixedVector { public: FixedVector() = default; + /// Constructor from initializer list - allocates exact size needed + /// This enables brace initialization: FixedVector v = {1, 2, 3}; + FixedVector(std::initializer_list init) { + init(init.size()); + for (const auto &item : init) { + push_back(item); + } + } + ~FixedVector() { cleanup_(); } // Disable copy operations (avoid accidental expensive copies) From ac35c97a44a339dd6e69bcc503a5aa9edec5e156 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Mon, 13 Oct 2025 18:59:46 -1000 Subject: [PATCH 73/90] we need copy now --- esphome/core/helpers.h | 27 ++++++++++++++++++++++++--- 1 file changed, 24 insertions(+), 3 deletions(-) diff --git a/esphome/core/helpers.h b/esphome/core/helpers.h index b94826629f..c0e73b70e0 100644 --- a/esphome/core/helpers.h +++ b/esphome/core/helpers.h @@ -208,9 +208,30 @@ template class FixedVector { ~FixedVector() { cleanup_(); } - // Disable copy operations (avoid accidental expensive copies) - FixedVector(const FixedVector &) = delete; - FixedVector &operator=(const FixedVector &) = delete; + // Copy constructor - performs deep copy + FixedVector(const FixedVector &other) { + if (other.size_ > 0) { + init(other.size_); + for (size_t i = 0; i < other.size_; i++) { + push_back(other.data_[i]); + } + } + } + + // Copy assignment operator - performs deep copy + FixedVector &operator=(const FixedVector &other) { + if (this != &other) { + cleanup_(); + reset_(); + if (other.size_ > 0) { + init(other.size_); + for (size_t i = 0; i < other.size_; i++) { + push_back(other.data_[i]); + } + } + } + return *this; + } // Enable move semantics (allows use in move-only containers like std::vector) FixedVector(FixedVector &&other) noexcept : data_(other.data_), size_(other.size_), capacity_(other.capacity_) { From 45014db02796aad16e103f649f30f955baa6c242 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Mon, 13 Oct 2025 19:05:26 -1000 Subject: [PATCH 74/90] preen --- esphome/components/mdns/mdns_component.cpp | 4 +-- esphome/core/helpers.h | 29 +++------------------- 2 files changed, 6 insertions(+), 27 deletions(-) diff --git a/esphome/components/mdns/mdns_component.cpp b/esphome/components/mdns/mdns_component.cpp index ef585db51b..7b36fce229 100644 --- a/esphome/components/mdns/mdns_component.cpp +++ b/esphome/components/mdns/mdns_component.cpp @@ -175,8 +175,8 @@ void MDNSComponent::compile_records_(StaticVectorservices_ = services; + // Move to member variable if storage is enabled (verbose logging, OpenThread, or extra services) + this->services_ = std::move(services); #endif } diff --git a/esphome/core/helpers.h b/esphome/core/helpers.h index c0e73b70e0..a9c0427917 100644 --- a/esphome/core/helpers.h +++ b/esphome/core/helpers.h @@ -208,32 +208,11 @@ template class FixedVector { ~FixedVector() { cleanup_(); } - // Copy constructor - performs deep copy - FixedVector(const FixedVector &other) { - if (other.size_ > 0) { - init(other.size_); - for (size_t i = 0; i < other.size_; i++) { - push_back(other.data_[i]); - } - } - } + // Disable copy operations - use std::move() to transfer ownership + FixedVector(const FixedVector &) = delete; + FixedVector &operator=(const FixedVector &) = delete; - // Copy assignment operator - performs deep copy - FixedVector &operator=(const FixedVector &other) { - if (this != &other) { - cleanup_(); - reset_(); - if (other.size_ > 0) { - init(other.size_); - for (size_t i = 0; i < other.size_; i++) { - push_back(other.data_[i]); - } - } - } - return *this; - } - - // Enable move semantics (allows use in move-only containers like std::vector) + // Enable move semantics FixedVector(FixedVector &&other) noexcept : data_(other.data_), size_(other.size_), capacity_(other.capacity_) { other.reset_(); } From fc30326e603834154703b66e8a373bc2cf1ad498 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Mon, 13 Oct 2025 19:06:02 -1000 Subject: [PATCH 75/90] preen --- esphome/core/helpers.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/esphome/core/helpers.h b/esphome/core/helpers.h index a9c0427917..b94826629f 100644 --- a/esphome/core/helpers.h +++ b/esphome/core/helpers.h @@ -208,11 +208,11 @@ template class FixedVector { ~FixedVector() { cleanup_(); } - // Disable copy operations - use std::move() to transfer ownership + // Disable copy operations (avoid accidental expensive copies) FixedVector(const FixedVector &) = delete; FixedVector &operator=(const FixedVector &) = delete; - // Enable move semantics + // Enable move semantics (allows use in move-only containers like std::vector) FixedVector(FixedVector &&other) noexcept : data_(other.data_), size_(other.size_), capacity_(other.capacity_) { other.reset_(); } From 24a7426a2a90f12f9a1a511422358b91fb2caba0 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Mon, 13 Oct 2025 19:15:39 -1000 Subject: [PATCH 76/90] rename to fix shadow --- esphome/core/helpers.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/esphome/core/helpers.h b/esphome/core/helpers.h index b94826629f..75ca600292 100644 --- a/esphome/core/helpers.h +++ b/esphome/core/helpers.h @@ -199,9 +199,9 @@ template class FixedVector { /// Constructor from initializer list - allocates exact size needed /// This enables brace initialization: FixedVector v = {1, 2, 3}; - FixedVector(std::initializer_list init) { - init(init.size()); - for (const auto &item : init) { + FixedVector(std::initializer_list init_list) { + init(init_list.size()); + for (const auto &item : init_list) { push_back(item); } } From 7492d7a437bb3f747b6321282853903a43882ad5 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Mon, 13 Oct 2025 19:27:33 -1000 Subject: [PATCH 77/90] [api] Convert HomeassistantActionRequest vectors to FixedVector for flash savings --- esphome/components/api/api.proto | 6 ++-- esphome/components/api/api_pb2.h | 6 ++-- esphome/components/api/custom_api_device.h | 8 ++--- .../components/api/homeassistant_service.h | 31 ++++++++----------- .../number/homeassistant_number.cpp | 7 ++--- .../switch/homeassistant_switch.cpp | 4 +-- 6 files changed, 28 insertions(+), 34 deletions(-) diff --git a/esphome/components/api/api.proto b/esphome/components/api/api.proto index 9b714d00f1..34864c5ce8 100644 --- a/esphome/components/api/api.proto +++ b/esphome/components/api/api.proto @@ -776,9 +776,9 @@ message HomeassistantActionRequest { option (ifdef) = "USE_API_HOMEASSISTANT_SERVICES"; string service = 1; - repeated HomeassistantServiceMap data = 2; - repeated HomeassistantServiceMap data_template = 3; - repeated HomeassistantServiceMap variables = 4; + repeated HomeassistantServiceMap data = 2 [(fixed_vector) = true]; + repeated HomeassistantServiceMap data_template = 3 [(fixed_vector) = true]; + repeated HomeassistantServiceMap variables = 4 [(fixed_vector) = true]; bool is_event = 5; uint32 call_id = 6 [(field_ifdef) = "USE_API_HOMEASSISTANT_ACTION_RESPONSES"]; bool wants_response = 7 [(field_ifdef) = "USE_API_HOMEASSISTANT_ACTION_RESPONSES_JSON"]; diff --git a/esphome/components/api/api_pb2.h b/esphome/components/api/api_pb2.h index 1798458393..7d6b31ca3c 100644 --- a/esphome/components/api/api_pb2.h +++ b/esphome/components/api/api_pb2.h @@ -1110,9 +1110,9 @@ class HomeassistantActionRequest final : public ProtoMessage { #endif StringRef service_ref_{}; void set_service(const StringRef &ref) { this->service_ref_ = ref; } - std::vector data{}; - std::vector data_template{}; - std::vector variables{}; + FixedVector data{}; + FixedVector data_template{}; + FixedVector variables{}; bool is_event{false}; #ifdef USE_API_HOMEASSISTANT_ACTION_RESPONSES uint32_t call_id{0}; diff --git a/esphome/components/api/custom_api_device.h b/esphome/components/api/custom_api_device.h index 0c6e49d6ca..711eba2444 100644 --- a/esphome/components/api/custom_api_device.h +++ b/esphome/components/api/custom_api_device.h @@ -201,9 +201,9 @@ class CustomAPIDevice { void call_homeassistant_service(const std::string &service_name, const std::map &data) { HomeassistantActionRequest resp; resp.set_service(StringRef(service_name)); + resp.data.init(data.size()); for (auto &it : data) { - resp.data.emplace_back(); - auto &kv = resp.data.back(); + auto &kv = resp.data.emplace_back(); kv.set_key(StringRef(it.first)); kv.value = it.second; } @@ -244,9 +244,9 @@ class CustomAPIDevice { HomeassistantActionRequest resp; resp.set_service(StringRef(service_name)); resp.is_event = true; + resp.data.init(data.size()); for (auto &it : data) { - resp.data.emplace_back(); - auto &kv = resp.data.back(); + auto &kv = resp.data.emplace_back(); kv.set_key(StringRef(it.first)); kv.value = it.second; } diff --git a/esphome/components/api/homeassistant_service.h b/esphome/components/api/homeassistant_service.h index 730024f7b7..b75bca6f71 100644 --- a/esphome/components/api/homeassistant_service.h +++ b/esphome/components/api/homeassistant_service.h @@ -122,29 +122,24 @@ template class HomeAssistantServiceCallAction : public Action *get_error_trigger() const { return this->error_trigger_; } #endif // USE_API_HOMEASSISTANT_ACTION_RESPONSES + template + static void populate_service_map_(VectorType &dest, SourceType &source, Ts... x) { + dest.init(source.size()); + for (auto &it : source) { + auto &kv = dest.emplace_back(); + kv.set_key(StringRef(it.key)); + kv.value = it.value.value(x...); + } + } + void play(Ts... x) override { HomeassistantActionRequest resp; std::string service_value = this->service_.value(x...); resp.set_service(StringRef(service_value)); resp.is_event = this->flags_.is_event; - for (auto &it : this->data_) { - resp.data.emplace_back(); - auto &kv = resp.data.back(); - kv.set_key(StringRef(it.key)); - kv.value = it.value.value(x...); - } - for (auto &it : this->data_template_) { - resp.data_template.emplace_back(); - auto &kv = resp.data_template.back(); - kv.set_key(StringRef(it.key)); - kv.value = it.value.value(x...); - } - for (auto &it : this->variables_) { - resp.variables.emplace_back(); - auto &kv = resp.variables.back(); - kv.set_key(StringRef(it.key)); - kv.value = it.value.value(x...); - } + populate_service_map_(resp.data, this->data_, x...); + populate_service_map_(resp.data_template, this->data_template_, x...); + populate_service_map_(resp.variables, this->variables_, x...); #ifdef USE_API_HOMEASSISTANT_ACTION_RESPONSES if (this->flags_.wants_status) { diff --git a/esphome/components/homeassistant/number/homeassistant_number.cpp b/esphome/components/homeassistant/number/homeassistant_number.cpp index c9fb006568..9963f3431d 100644 --- a/esphome/components/homeassistant/number/homeassistant_number.cpp +++ b/esphome/components/homeassistant/number/homeassistant_number.cpp @@ -90,13 +90,12 @@ void HomeassistantNumber::control(float value) { api::HomeassistantActionRequest resp; resp.set_service(SERVICE_NAME); - resp.data.emplace_back(); - auto &entity_id = resp.data.back(); + resp.data.init(2); + auto &entity_id = resp.data.emplace_back(); entity_id.set_key(ENTITY_ID_KEY); entity_id.value = this->entity_id_; - resp.data.emplace_back(); - auto &entity_value = resp.data.back(); + auto &entity_value = resp.data.emplace_back(); entity_value.set_key(VALUE_KEY); entity_value.value = to_string(value); diff --git a/esphome/components/homeassistant/switch/homeassistant_switch.cpp b/esphome/components/homeassistant/switch/homeassistant_switch.cpp index 8feec26fe6..27d3705fc2 100644 --- a/esphome/components/homeassistant/switch/homeassistant_switch.cpp +++ b/esphome/components/homeassistant/switch/homeassistant_switch.cpp @@ -51,8 +51,8 @@ void HomeassistantSwitch::write_state(bool state) { resp.set_service(SERVICE_OFF); } - resp.data.emplace_back(); - auto &entity_id_kv = resp.data.back(); + resp.data.init(1); + auto &entity_id_kv = resp.data.emplace_back(); entity_id_kv.set_key(ENTITY_ID_KEY); entity_id_kv.value = this->entity_id_; From 43d8386c4abbdec49409b68a17da67de98a1f507 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Mon, 13 Oct 2025 19:31:13 -1000 Subject: [PATCH 78/90] tidy --- esphome/components/api/homeassistant_service.h | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/esphome/components/api/homeassistant_service.h b/esphome/components/api/homeassistant_service.h index b75bca6f71..b24a6470fe 100644 --- a/esphome/components/api/homeassistant_service.h +++ b/esphome/components/api/homeassistant_service.h @@ -123,7 +123,7 @@ template class HomeAssistantServiceCallAction : public Action - static void populate_service_map_(VectorType &dest, SourceType &source, Ts... x) { + static void populate_service_map(VectorType &dest, SourceType &source, Ts... x) { dest.init(source.size()); for (auto &it : source) { auto &kv = dest.emplace_back(); @@ -137,9 +137,9 @@ template class HomeAssistantServiceCallAction : public Actionservice_.value(x...); resp.set_service(StringRef(service_value)); resp.is_event = this->flags_.is_event; - populate_service_map_(resp.data, this->data_, x...); - populate_service_map_(resp.data_template, this->data_template_, x...); - populate_service_map_(resp.variables, this->variables_, x...); + populate_service_map(resp.data, this->data_, x...); + populate_service_map(resp.data_template, this->data_template_, x...); + populate_service_map(resp.variables, this->variables_, x...); #ifdef USE_API_HOMEASSISTANT_ACTION_RESPONSES if (this->flags_.wants_status) { From 05efb6e9255c98e37542434e62bd6d019bab5113 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Mon, 13 Oct 2025 21:14:32 -1000 Subject: [PATCH 79/90] refactor to avoid move --- esphome/components/mdns/mdns_component.cpp | 5 ----- esphome/components/mdns/mdns_esp32.cpp | 5 +++++ esphome/components/mdns/mdns_esp8266.cpp | 5 +++++ esphome/components/mdns/mdns_libretiny.cpp | 5 +++++ esphome/components/mdns/mdns_rp2040.cpp | 5 +++++ 5 files changed, 20 insertions(+), 5 deletions(-) diff --git a/esphome/components/mdns/mdns_component.cpp b/esphome/components/mdns/mdns_component.cpp index 7b36fce229..d476136554 100644 --- a/esphome/components/mdns/mdns_component.cpp +++ b/esphome/components/mdns/mdns_component.cpp @@ -173,11 +173,6 @@ void MDNSComponent::compile_records_(StaticVectorservices_ = std::move(services); -#endif } void MDNSComponent::dump_config() { diff --git a/esphome/components/mdns/mdns_esp32.cpp b/esphome/components/mdns/mdns_esp32.cpp index da47be7dbc..f2cb2d3ef5 100644 --- a/esphome/components/mdns/mdns_esp32.cpp +++ b/esphome/components/mdns/mdns_esp32.cpp @@ -12,8 +12,13 @@ namespace mdns { static const char *const TAG = "mdns"; void MDNSComponent::setup() { +#ifdef USE_MDNS_STORE_SERVICES + this->compile_records_(this->services_); + const auto &services = this->services_; +#else StaticVector services; this->compile_records_(services); +#endif esp_err_t err = mdns_init(); if (err != ESP_OK) { diff --git a/esphome/components/mdns/mdns_esp8266.cpp b/esphome/components/mdns/mdns_esp8266.cpp index 06503742db..25a3defa7b 100644 --- a/esphome/components/mdns/mdns_esp8266.cpp +++ b/esphome/components/mdns/mdns_esp8266.cpp @@ -12,8 +12,13 @@ namespace esphome { namespace mdns { void MDNSComponent::setup() { +#ifdef USE_MDNS_STORE_SERVICES + this->compile_records_(this->services_); + const auto &services = this->services_; +#else StaticVector services; this->compile_records_(services); +#endif MDNS.begin(this->hostname_.c_str()); diff --git a/esphome/components/mdns/mdns_libretiny.cpp b/esphome/components/mdns/mdns_libretiny.cpp index a959482ff6..a3e317a2bf 100644 --- a/esphome/components/mdns/mdns_libretiny.cpp +++ b/esphome/components/mdns/mdns_libretiny.cpp @@ -12,8 +12,13 @@ namespace esphome { namespace mdns { void MDNSComponent::setup() { +#ifdef USE_MDNS_STORE_SERVICES + this->compile_records_(this->services_); + const auto &services = this->services_; +#else StaticVector services; this->compile_records_(services); +#endif MDNS.begin(this->hostname_.c_str()); diff --git a/esphome/components/mdns/mdns_rp2040.cpp b/esphome/components/mdns/mdns_rp2040.cpp index 9dfb05bda9..791fa3934d 100644 --- a/esphome/components/mdns/mdns_rp2040.cpp +++ b/esphome/components/mdns/mdns_rp2040.cpp @@ -12,8 +12,13 @@ namespace esphome { namespace mdns { void MDNSComponent::setup() { +#ifdef USE_MDNS_STORE_SERVICES + this->compile_records_(this->services_); + const auto &services = this->services_; +#else StaticVector services; this->compile_records_(services); +#endif MDNS.begin(this->hostname_.c_str()); From 92a6aade174f6433ca3a1c09fd794ae1e2546b51 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Mon, 13 Oct 2025 21:35:26 -1000 Subject: [PATCH 80/90] fixes --- .ai/instructions.md | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/.ai/instructions.md b/.ai/instructions.md index c608c9fd7e..5f314a0dc9 100644 --- a/.ai/instructions.md +++ b/.ai/instructions.md @@ -252,16 +252,18 @@ This document provides essential context for AI models interacting with this pro std::array buffer; ``` - 2. **Compile-time-known sizes with dynamic storage:** Use `StaticVector` from `esphome/core/helpers.h` when the maximum size is known at compile time but you need heap allocation. + 2. **Compile-time-known fixed sizes with vector-like API:** Use `StaticVector` from `esphome/core/helpers.h` for fixed-size stack allocation with `push_back()` interface. ```cpp // Bad - generates STL realloc code (_M_realloc_insert) std::vector services; services.reserve(5); // Still includes reallocation machinery - // Good - compile-time max size, heap allocated, no reallocation machinery - StaticVector services; // Max size known at compile time + // Good - compile-time fixed size, stack allocated, no reallocation machinery + StaticVector services; // Allocates all MAX_SERVICES on stack + services.push_back(record1); // Tracks count but all slots allocated ``` - Use `cg.add_define("MAX_SERVICES", count)` to set the maximum from Python configuration. + Use `cg.add_define("MAX_SERVICES", count)` to set the size from Python configuration. + Like `std::array` but with vector-like API (`push_back()`, `size()`) and no STL reallocation code. 3. **Runtime-known sizes:** Use `FixedVector` from `esphome/core/helpers.h` when the size is only known at runtime initialization. ```cpp From e241e430647ba589bbd31917b9148424d5f6d327 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Mon, 13 Oct 2025 21:41:49 -1000 Subject: [PATCH 81/90] preen --- .../components/api/homeassistant_service.h | 26 +++++++++---------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/esphome/components/api/homeassistant_service.h b/esphome/components/api/homeassistant_service.h index b24a6470fe..bd668d3cf8 100644 --- a/esphome/components/api/homeassistant_service.h +++ b/esphome/components/api/homeassistant_service.h @@ -122,24 +122,14 @@ template class HomeAssistantServiceCallAction : public Action *get_error_trigger() const { return this->error_trigger_; } #endif // USE_API_HOMEASSISTANT_ACTION_RESPONSES - template - static void populate_service_map(VectorType &dest, SourceType &source, Ts... x) { - dest.init(source.size()); - for (auto &it : source) { - auto &kv = dest.emplace_back(); - kv.set_key(StringRef(it.key)); - kv.value = it.value.value(x...); - } - } - void play(Ts... x) override { HomeassistantActionRequest resp; std::string service_value = this->service_.value(x...); resp.set_service(StringRef(service_value)); resp.is_event = this->flags_.is_event; - populate_service_map(resp.data, this->data_, x...); - populate_service_map(resp.data_template, this->data_template_, x...); - populate_service_map(resp.variables, this->variables_, x...); + this->populate_service_map_(resp.data, this->data_, x...); + this->populate_service_map_(resp.data_template, this->data_template_, x...); + this->populate_service_map_(resp.variables, this->variables_, x...); #ifdef USE_API_HOMEASSISTANT_ACTION_RESPONSES if (this->flags_.wants_status) { @@ -184,6 +174,16 @@ template class HomeAssistantServiceCallAction : public Action + static void populate_service_map(VectorType &dest, SourceType &source, Ts... x) { + dest.init(source.size()); + for (auto &it : source) { + auto &kv = dest.emplace_back(); + kv.set_key(StringRef(it.key)); + kv.value = it.value.value(x...); + } + } + APIServer *parent_; TemplatableStringValue service_{}; std::vector> data_; From 0fca842afe9212189b8503908475fb26ab9dd756 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Mon, 13 Oct 2025 21:41:57 -1000 Subject: [PATCH 82/90] preen --- esphome/components/api/homeassistant_service.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/esphome/components/api/homeassistant_service.h b/esphome/components/api/homeassistant_service.h index bd668d3cf8..9c2844fbed 100644 --- a/esphome/components/api/homeassistant_service.h +++ b/esphome/components/api/homeassistant_service.h @@ -175,7 +175,7 @@ template class HomeAssistantServiceCallAction : public Action - static void populate_service_map(VectorType &dest, SourceType &source, Ts... x) { + static void populate_service_map_(VectorType &dest, SourceType &source, Ts... x) { dest.init(source.size()); for (auto &it : source) { auto &kv = dest.emplace_back(); From 5ebb68b719088f34447a8bf29f030d87cefe8193 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Mon, 13 Oct 2025 21:45:52 -1000 Subject: [PATCH 83/90] fixed --- esphome/components/api/homeassistant_service.h | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/esphome/components/api/homeassistant_service.h b/esphome/components/api/homeassistant_service.h index 9c2844fbed..46e89cb39f 100644 --- a/esphome/components/api/homeassistant_service.h +++ b/esphome/components/api/homeassistant_service.h @@ -127,9 +127,9 @@ template class HomeAssistantServiceCallAction : public Actionservice_.value(x...); resp.set_service(StringRef(service_value)); resp.is_event = this->flags_.is_event; - this->populate_service_map_(resp.data, this->data_, x...); - this->populate_service_map_(resp.data_template, this->data_template_, x...); - this->populate_service_map_(resp.variables, this->variables_, x...); + this->populate_service_map(resp.data, this->data_, x...); + this->populate_service_map(resp.data_template, this->data_template_, x...); + this->populate_service_map(resp.variables, this->variables_, x...); #ifdef USE_API_HOMEASSISTANT_ACTION_RESPONSES if (this->flags_.wants_status) { @@ -175,7 +175,7 @@ template class HomeAssistantServiceCallAction : public Action - static void populate_service_map_(VectorType &dest, SourceType &source, Ts... x) { + static void populate_service_map(VectorType &dest, SourceType &source, Ts... x) { dest.init(source.size()); for (auto &it : source) { auto &kv = dest.emplace_back(); From ce3bd55a389917f908cf56abd4b028f7e16722f9 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Mon, 13 Oct 2025 21:52:24 -1000 Subject: [PATCH 84/90] [api] Use FixedVector for ListEntitiesServicesResponse args --- esphome/components/api/api.proto | 2 +- esphome/components/api/api_pb2.h | 2 +- esphome/components/api/user_services.h | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/esphome/components/api/api.proto b/esphome/components/api/api.proto index 9b714d00f1..f7b65c34e3 100644 --- a/esphome/components/api/api.proto +++ b/esphome/components/api/api.proto @@ -866,7 +866,7 @@ message ListEntitiesServicesResponse { string name = 1; fixed32 key = 2; - repeated ListEntitiesServicesArgument args = 3; + repeated ListEntitiesServicesArgument args = 3 [(fixed_vector) = true]; } message ExecuteServiceArgument { option (ifdef) = "USE_API_SERVICES"; diff --git a/esphome/components/api/api_pb2.h b/esphome/components/api/api_pb2.h index 1798458393..64419eaaa8 100644 --- a/esphome/components/api/api_pb2.h +++ b/esphome/components/api/api_pb2.h @@ -1263,7 +1263,7 @@ class ListEntitiesServicesResponse final : public ProtoMessage { StringRef name_ref_{}; void set_name(const StringRef &ref) { this->name_ref_ = ref; } uint32_t key{0}; - std::vector args{}; + FixedVector args{}; void encode(ProtoWriteBuffer buffer) const override; void calculate_size(ProtoSize &size) const override; #ifdef HAS_PROTO_MESSAGE_DUMP diff --git a/esphome/components/api/user_services.h b/esphome/components/api/user_services.h index 3996c921a9..29843a2f78 100644 --- a/esphome/components/api/user_services.h +++ b/esphome/components/api/user_services.h @@ -35,9 +35,9 @@ template class UserServiceBase : public UserServiceDescriptor { msg.set_name(StringRef(this->name_)); msg.key = this->key_; std::array arg_types = {to_service_arg_type()...}; + msg.args.init(sizeof...(Ts)); for (size_t i = 0; i < sizeof...(Ts); i++) { - msg.args.emplace_back(); - auto &arg = msg.args.back(); + auto &arg = msg.args.emplace_back(); arg.type = arg_types[i]; arg.set_name(StringRef(this->arg_names_[i])); } From 1acd7d4672267a14297eada2aa55192eb3817f28 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Mon, 13 Oct 2025 21:56:11 -1000 Subject: [PATCH 85/90] Update esphome/core/helpers.h Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- esphome/core/helpers.h | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/esphome/core/helpers.h b/esphome/core/helpers.h index 75ca600292..326718e974 100644 --- a/esphome/core/helpers.h +++ b/esphome/core/helpers.h @@ -201,9 +201,12 @@ template class FixedVector { /// This enables brace initialization: FixedVector v = {1, 2, 3}; FixedVector(std::initializer_list init_list) { init(init_list.size()); + size_t idx = 0; for (const auto &item : init_list) { - push_back(item); + new (data_ + idx) T(item); + ++idx; } + size_ = init_list.size(); } ~FixedVector() { cleanup_(); } From 87ae07e7be5d592298b38abcacdfb8bac20df208 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Mon, 13 Oct 2025 22:27:41 -1000 Subject: [PATCH 86/90] [light] Use FixedVector for LightState effects list --- esphome/components/light/light_state.cpp | 5 +++-- esphome/components/light/light_state.h | 6 +++--- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/esphome/components/light/light_state.cpp b/esphome/components/light/light_state.cpp index f18d5ba1de..1d139e49e7 100644 --- a/esphome/components/light/light_state.cpp +++ b/esphome/components/light/light_state.cpp @@ -177,9 +177,10 @@ void LightState::set_gamma_correct(float gamma_correct) { this->gamma_correct_ = void LightState::set_restore_mode(LightRestoreMode restore_mode) { this->restore_mode_ = restore_mode; } void LightState::set_initial_state(const LightStateRTCState &initial_state) { this->initial_state_ = initial_state; } bool LightState::supports_effects() { return !this->effects_.empty(); } -const std::vector &LightState::get_effects() const { return this->effects_; } +const FixedVector &LightState::get_effects() const { return this->effects_; } void LightState::add_effects(const std::vector &effects) { - this->effects_.reserve(this->effects_.size() + effects.size()); + // Called once from Python codegen during setup with all effects from YAML config + this->effects_.init(effects.size()); for (auto *effect : effects) { this->effects_.push_back(effect); } diff --git a/esphome/components/light/light_state.h b/esphome/components/light/light_state.h index 1427c02c35..87a509cba6 100644 --- a/esphome/components/light/light_state.h +++ b/esphome/components/light/light_state.h @@ -11,7 +11,7 @@ #include "light_traits.h" #include "light_transformer.h" -#include +#include "esphome/core/helpers.h" #include namespace esphome { @@ -159,7 +159,7 @@ class LightState : public EntityBase, public Component { bool supports_effects(); /// Get all effects for this light state. - const std::vector &get_effects() const; + const FixedVector &get_effects() const; /// Add effects for this light state. void add_effects(const std::vector &effects); @@ -260,7 +260,7 @@ class LightState : public EntityBase, public Component { /// The currently active transformer for this light (transition/flash). std::unique_ptr transformer_{nullptr}; /// List of effects for this light. - std::vector effects_; + FixedVector effects_; /// Object used to store the persisted values of the light. ESPPreferenceObject rtc_; /// Value for storing the index of the currently active effect. 0 if no effect is active From 3cf24a259cbb85fe314d793896ebecd713639fd0 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Mon, 13 Oct 2025 22:46:45 -1000 Subject: [PATCH 87/90] [web_server_idf] Use std::vector instead of std::set for SSE sessions --- .../components/web_server_idf/web_server_idf.cpp | 13 +++++++------ esphome/components/web_server_idf/web_server_idf.h | 6 ++++-- 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/esphome/components/web_server_idf/web_server_idf.cpp b/esphome/components/web_server_idf/web_server_idf.cpp index d90efd18bc..c3ba7ddc2b 100644 --- a/esphome/components/web_server_idf/web_server_idf.cpp +++ b/esphome/components/web_server_idf/web_server_idf.cpp @@ -380,24 +380,25 @@ void AsyncEventSource::handleRequest(AsyncWebServerRequest *request) { if (this->on_connect_) { this->on_connect_(rsp); } - this->sessions_.insert(rsp); + this->sessions_.push_back(rsp); } void AsyncEventSource::loop() { // Clean up dead sessions safely // This follows the ESP-IDF pattern where free_ctx marks resources as dead // and the main loop handles the actual cleanup to avoid race conditions - auto it = this->sessions_.begin(); - while (it != this->sessions_.end()) { - auto *ses = *it; + for (size_t i = 0; i < this->sessions_.size();) { + auto *ses = this->sessions_[i]; // If the session has a dead socket (marked by destroy callback) if (ses->fd_.load() == 0) { ESP_LOGD(TAG, "Removing dead event source session"); - it = this->sessions_.erase(it); delete ses; // NOLINT(cppcoreguidelines-owning-memory) + // Remove by swapping with last element (O(1) removal, order doesn't matter for sessions) + this->sessions_[i] = this->sessions_.back(); + this->sessions_.pop_back(); } else { ses->loop(); - ++it; + ++i; } } } diff --git a/esphome/components/web_server_idf/web_server_idf.h b/esphome/components/web_server_idf/web_server_idf.h index bf93dcbd34..5ec6fec009 100644 --- a/esphome/components/web_server_idf/web_server_idf.h +++ b/esphome/components/web_server_idf/web_server_idf.h @@ -8,7 +8,6 @@ #include #include #include -#include #include #include #include @@ -315,7 +314,10 @@ class AsyncEventSource : public AsyncWebHandler { protected: std::string url_; - std::set sessions_; + // Use vector instead of set: SSE sessions are typically 1-5 connections (browsers, dashboards). + // Linear search is faster than red-black tree overhead for this small dataset. + // Only operations needed: add session, remove session, iterate sessions - no need for sorted order. + std::vector sessions_; connect_handler_t on_connect_{}; esphome::web_server::WebServer *web_server_; }; From 793e75a09398082ace5c07675532fa65b303ee71 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Tue, 14 Oct 2025 09:28:56 -1000 Subject: [PATCH 88/90] [core] Use FixedVector for automation condition vectors to save 384 bytes flash --- esphome/core/base_automation.h | 13 +++++++------ esphome/core/helpers.h | 12 ++++++++++++ 2 files changed, 19 insertions(+), 6 deletions(-) diff --git a/esphome/core/base_automation.h b/esphome/core/base_automation.h index ba942e5e43..f1248e0035 100644 --- a/esphome/core/base_automation.h +++ b/esphome/core/base_automation.h @@ -7,6 +7,7 @@ #include "esphome/core/preferences.h" #include "esphome/core/scheduler.h" #include "esphome/core/application.h" +#include "esphome/core/helpers.h" #include @@ -14,7 +15,7 @@ namespace esphome { template class AndCondition : public Condition { public: - explicit AndCondition(const std::vector *> &conditions) : conditions_(conditions) {} + explicit AndCondition(std::initializer_list *> conditions) : conditions_(conditions) {} bool check(Ts... x) override { for (auto *condition : this->conditions_) { if (!condition->check(x...)) @@ -25,12 +26,12 @@ template class AndCondition : public Condition { } protected: - std::vector *> conditions_; + FixedVector *> conditions_; }; template class OrCondition : public Condition { public: - explicit OrCondition(const std::vector *> &conditions) : conditions_(conditions) {} + explicit OrCondition(std::initializer_list *> conditions) : conditions_(conditions) {} bool check(Ts... x) override { for (auto *condition : this->conditions_) { if (condition->check(x...)) @@ -41,7 +42,7 @@ template class OrCondition : public Condition { } protected: - std::vector *> conditions_; + FixedVector *> conditions_; }; template class NotCondition : public Condition { @@ -55,7 +56,7 @@ template class NotCondition : public Condition { template class XorCondition : public Condition { public: - explicit XorCondition(const std::vector *> &conditions) : conditions_(conditions) {} + explicit XorCondition(std::initializer_list *> conditions) : conditions_(conditions) {} bool check(Ts... x) override { size_t result = 0; for (auto *condition : this->conditions_) { @@ -66,7 +67,7 @@ template class XorCondition : public Condition { } protected: - std::vector *> conditions_; + FixedVector *> conditions_; }; template class LambdaCondition : public Condition { diff --git a/esphome/core/helpers.h b/esphome/core/helpers.h index e352c9c415..326718e974 100644 --- a/esphome/core/helpers.h +++ b/esphome/core/helpers.h @@ -197,6 +197,18 @@ template class FixedVector { public: FixedVector() = default; + /// Constructor from initializer list - allocates exact size needed + /// This enables brace initialization: FixedVector v = {1, 2, 3}; + FixedVector(std::initializer_list init_list) { + init(init_list.size()); + size_t idx = 0; + for (const auto &item : init_list) { + new (data_ + idx) T(item); + ++idx; + } + size_ = init_list.size(); + } + ~FixedVector() { cleanup_(); } // Disable copy operations (avoid accidental expensive copies) From 837a0bf6df011224be5649d991d1fb8277c2b035 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Tue, 14 Oct 2025 12:18:12 -1000 Subject: [PATCH 89/90] [pzemac, pzemdc, sdm_meter] Fix pin conflicts in ESP32-IDF tests --- tests/components/pzemac/test.esp32-idf.yaml | 1 + tests/components/pzemdc/test.esp32-idf.yaml | 1 + tests/components/sdm_meter/test.esp32-idf.yaml | 1 + 3 files changed, 3 insertions(+) diff --git a/tests/components/pzemac/test.esp32-idf.yaml b/tests/components/pzemac/test.esp32-idf.yaml index 37d98696cc..b631e16677 100644 --- a/tests/components/pzemac/test.esp32-idf.yaml +++ b/tests/components/pzemac/test.esp32-idf.yaml @@ -1,6 +1,7 @@ substitutions: tx_pin: GPIO4 rx_pin: GPIO5 + flow_control_pin: GPIO13 packages: modbus: !include ../../test_build_components/common/modbus/esp32-idf.yaml diff --git a/tests/components/pzemdc/test.esp32-idf.yaml b/tests/components/pzemdc/test.esp32-idf.yaml index 37d98696cc..b631e16677 100644 --- a/tests/components/pzemdc/test.esp32-idf.yaml +++ b/tests/components/pzemdc/test.esp32-idf.yaml @@ -1,6 +1,7 @@ substitutions: tx_pin: GPIO4 rx_pin: GPIO5 + flow_control_pin: GPIO13 packages: modbus: !include ../../test_build_components/common/modbus/esp32-idf.yaml diff --git a/tests/components/sdm_meter/test.esp32-idf.yaml b/tests/components/sdm_meter/test.esp32-idf.yaml index 37d98696cc..b631e16677 100644 --- a/tests/components/sdm_meter/test.esp32-idf.yaml +++ b/tests/components/sdm_meter/test.esp32-idf.yaml @@ -1,6 +1,7 @@ substitutions: tx_pin: GPIO4 rx_pin: GPIO5 + flow_control_pin: GPIO13 packages: modbus: !include ../../test_build_components/common/modbus/esp32-idf.yaml From b927cea0d61795ad112c7819a9fa7d35fe704b89 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Tue, 14 Oct 2025 16:23:06 -1000 Subject: [PATCH 90/90] [git] Automatically recover from broken git repositories in external_components --- esphome/git.py | 59 +++++++++---- tests/unit_tests/test_git.py | 160 +++++++++++++++++++++++++++++++++++ 2 files changed, 204 insertions(+), 15 deletions(-) diff --git a/esphome/git.py b/esphome/git.py index 62fe37a3fe..7f023e7834 100644 --- a/esphome/git.py +++ b/esphome/git.py @@ -5,6 +5,7 @@ import hashlib import logging from pathlib import Path import re +import shutil import subprocess import urllib.parse @@ -55,6 +56,7 @@ def clone_or_update( username: str = None, password: str = None, submodules: list[str] | None = None, + _recover_broken: bool = True, ) -> tuple[Path, Callable[[], None] | None]: key = f"{url}@{ref}" @@ -80,7 +82,7 @@ def clone_or_update( if submodules is not None: _LOGGER.info( - "Initialising submodules (%s) for %s", ", ".join(submodules), key + "Initializing submodules (%s) for %s", ", ".join(submodules), key ) run_git_command( ["git", "submodule", "update", "--init"] + submodules, str(repo_dir) @@ -99,20 +101,47 @@ def clone_or_update( file_timestamp = Path(repo_dir / ".git" / "HEAD") age = datetime.now() - datetime.fromtimestamp(file_timestamp.stat().st_mtime) if refresh is None or age.total_seconds() > refresh.total_seconds: - old_sha = run_git_command(["git", "rev-parse", "HEAD"], str(repo_dir)) - _LOGGER.info("Updating %s", key) - _LOGGER.debug("Location: %s", repo_dir) - # Stash local changes (if any) - run_git_command( - ["git", "stash", "push", "--include-untracked"], str(repo_dir) - ) - # Fetch remote ref - cmd = ["git", "fetch", "--", "origin"] - if ref is not None: - cmd.append(ref) - run_git_command(cmd, str(repo_dir)) - # Hard reset to FETCH_HEAD (short-lived git ref corresponding to most recent fetch) - run_git_command(["git", "reset", "--hard", "FETCH_HEAD"], str(repo_dir)) + # Try to update the repository, recovering from broken state if needed + old_sha: str | None = None + try: + old_sha = run_git_command(["git", "rev-parse", "HEAD"], str(repo_dir)) + _LOGGER.info("Updating %s", key) + _LOGGER.debug("Location: %s", repo_dir) + # Stash local changes (if any) + run_git_command( + ["git", "stash", "push", "--include-untracked"], str(repo_dir) + ) + # Fetch remote ref + cmd = ["git", "fetch", "--", "origin"] + if ref is not None: + cmd.append(ref) + run_git_command(cmd, str(repo_dir)) + # Hard reset to FETCH_HEAD (short-lived git ref corresponding to most recent fetch) + run_git_command(["git", "reset", "--hard", "FETCH_HEAD"], str(repo_dir)) + except cv.Invalid as err: + # Repository is in a broken state or update failed + # Only attempt recovery once to prevent infinite recursion + if not _recover_broken: + raise + + _LOGGER.warning( + "Repository %s has issues (%s), removing and re-cloning", + key, + err, + ) + shutil.rmtree(repo_dir) + # Recursively call clone_or_update to re-clone + # Set _recover_broken=False to prevent infinite recursion + return clone_or_update( + url=url, + ref=ref, + refresh=refresh, + domain=domain, + username=username, + password=password, + submodules=submodules, + _recover_broken=False, + ) if submodules is not None: _LOGGER.info( diff --git a/tests/unit_tests/test_git.py b/tests/unit_tests/test_git.py index 6a51206ec2..748d384018 100644 --- a/tests/unit_tests/test_git.py +++ b/tests/unit_tests/test_git.py @@ -6,7 +6,10 @@ import os from pathlib import Path from unittest.mock import Mock +import pytest + from esphome import git +import esphome.config_validation as cv from esphome.core import CORE, TimePeriodSeconds @@ -244,3 +247,160 @@ def test_clone_or_update_with_none_refresh_always_updates( if len(call[0]) > 0 and "fetch" in call[0][0] ] assert len(fetch_calls) > 0 + + +@pytest.mark.parametrize( + ("fail_command", "error_message"), + [ + ( + "rev-parse", + "ambiguous argument 'HEAD': unknown revision or path not in the working tree.", + ), + ("stash", "fatal: unable to write new index file"), + ( + "fetch", + "fatal: unable to access 'https://github.com/test/repo/': Could not resolve host", + ), + ("reset", "fatal: Could not reset index file to revision 'FETCH_HEAD'"), + ], +) +def test_clone_or_update_recovers_from_git_failures( + tmp_path: Path, mock_run_git_command: Mock, fail_command: str, error_message: str +) -> None: + """Test that repos are re-cloned when various git commands fail.""" + # Set up CORE.config_path so data_dir uses tmp_path + CORE.config_path = tmp_path / "test.yaml" + + url = "https://github.com/test/repo" + ref = "main" + key = f"{url}@{ref}" + domain = "test" + + h = hashlib.new("sha256") + h.update(key.encode()) + repo_dir = tmp_path / ".esphome" / domain / h.hexdigest()[:8] + + # Create repo directory + repo_dir.mkdir(parents=True) + git_dir = repo_dir / ".git" + git_dir.mkdir() + + fetch_head = git_dir / "FETCH_HEAD" + fetch_head.write_text("test") + old_time = datetime.now() - timedelta(days=2) + fetch_head.touch() + os.utime(fetch_head, (old_time.timestamp(), old_time.timestamp())) + + # Track command call counts to make first call fail, subsequent calls succeed + call_counts: dict[str, int] = {} + + def git_command_side_effect(cmd: list[str], cwd: str | None = None) -> str: + # Determine which command this is + cmd_type = None + if "rev-parse" in cmd: + cmd_type = "rev-parse" + elif "stash" in cmd: + cmd_type = "stash" + elif "fetch" in cmd: + cmd_type = "fetch" + elif "reset" in cmd: + cmd_type = "reset" + elif "clone" in cmd: + cmd_type = "clone" + + # Track call count for this command type + if cmd_type: + call_counts[cmd_type] = call_counts.get(cmd_type, 0) + 1 + + # Fail on first call to the specified command, succeed on subsequent calls + if cmd_type == fail_command and call_counts[cmd_type] == 1: + raise cv.Invalid(error_message) + + # Default successful responses + if cmd_type == "rev-parse": + return "abc123" + return "" + + mock_run_git_command.side_effect = git_command_side_effect + + refresh = TimePeriodSeconds(days=1) + result_dir, revert = git.clone_or_update( + url=url, + ref=ref, + refresh=refresh, + domain=domain, + ) + + # Verify recovery happened + call_list = mock_run_git_command.call_args_list + + # Should have attempted the failing command + assert any(fail_command in str(c) for c in call_list) + + # Should have called clone for recovery + assert any("clone" in str(c) for c in call_list) + + # Verify the repo directory path is returned + assert result_dir == repo_dir + + +def test_clone_or_update_fails_when_recovery_also_fails( + tmp_path: Path, mock_run_git_command: Mock +) -> None: + """Test that we don't infinitely recurse when recovery also fails.""" + # Set up CORE.config_path so data_dir uses tmp_path + CORE.config_path = tmp_path / "test.yaml" + + url = "https://github.com/test/repo" + ref = "main" + key = f"{url}@{ref}" + domain = "test" + + h = hashlib.new("sha256") + h.update(key.encode()) + repo_dir = tmp_path / ".esphome" / domain / h.hexdigest()[:8] + + # Create repo directory + repo_dir.mkdir(parents=True) + git_dir = repo_dir / ".git" + git_dir.mkdir() + + fetch_head = git_dir / "FETCH_HEAD" + fetch_head.write_text("test") + old_time = datetime.now() - timedelta(days=2) + fetch_head.touch() + os.utime(fetch_head, (old_time.timestamp(), old_time.timestamp())) + + # Mock git command to fail on clone (simulating network failure during recovery) + def git_command_side_effect(cmd: list[str], cwd: str | None = None) -> str: + if "rev-parse" in cmd: + # First time fails (broken repo) + raise cv.Invalid( + "ambiguous argument 'HEAD': unknown revision or path not in the working tree." + ) + if "clone" in cmd: + # Clone also fails (recovery fails) + raise cv.Invalid("fatal: unable to access repository") + return "" + + mock_run_git_command.side_effect = git_command_side_effect + + refresh = TimePeriodSeconds(days=1) + + # Should raise after one recovery attempt fails + with pytest.raises(cv.Invalid, match="fatal: unable to access repository"): + git.clone_or_update( + url=url, + ref=ref, + refresh=refresh, + domain=domain, + ) + + # Verify we only tried to clone once (no infinite recursion) + call_list = mock_run_git_command.call_args_list + clone_calls = [c for c in call_list if "clone" in c[0][0]] + # Should have exactly one clone call (the recovery attempt that failed) + assert len(clone_calls) == 1 + # Should have tried rev-parse once (which failed and triggered recovery) + rev_parse_calls = [c for c in call_list if "rev-parse" in c[0][0]] + assert len(rev_parse_calls) == 1