From 7c02f2f10adab07a9c63c574c573cc278999bf4d Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Mon, 13 Oct 2025 15:00:49 -1000 Subject: [PATCH 01/10] [socket] Split LWIP socket classes to reduce memory overhead on ESP8266/RP2040 (#11172) --- .../components/socket/lwip_raw_tcp_impl.cpp | 190 +++++++++++------- 1 file changed, 113 insertions(+), 77 deletions(-) diff --git a/esphome/components/socket/lwip_raw_tcp_impl.cpp b/esphome/components/socket/lwip_raw_tcp_impl.cpp index 3377682474..4dedeffb6a 100644 --- a/esphome/components/socket/lwip_raw_tcp_impl.cpp +++ b/esphome/components/socket/lwip_raw_tcp_impl.cpp @@ -40,33 +40,14 @@ class LWIPRawImpl : public Socket { void init() { LWIP_LOG("init(%p)", pcb_); tcp_arg(pcb_, this); - tcp_accept(pcb_, LWIPRawImpl::s_accept_fn); tcp_recv(pcb_, LWIPRawImpl::s_recv_fn); tcp_err(pcb_, LWIPRawImpl::s_err_fn); } std::unique_ptr accept(struct sockaddr *addr, socklen_t *addrlen) override { - if (pcb_ == nullptr) { - errno = EBADF; - return nullptr; - } - if (this->accepted_socket_count_ == 0) { - errno = EWOULDBLOCK; - return nullptr; - } - // Take from front for FIFO ordering - std::unique_ptr sock = std::move(this->accepted_sockets_[0]); - // Shift remaining sockets forward - for (uint8_t i = 1; i < this->accepted_socket_count_; i++) { - this->accepted_sockets_[i - 1] = std::move(this->accepted_sockets_[i]); - } - this->accepted_socket_count_--; - LWIP_LOG("Connection accepted by application, queue size: %d", this->accepted_socket_count_); - if (addr != nullptr) { - sock->getpeername(addr, addrlen); - } - LWIP_LOG("accept(%p)", sock.get()); - return std::unique_ptr(std::move(sock)); + // Non-listening sockets return error + errno = EINVAL; + return nullptr; } int bind(const struct sockaddr *name, socklen_t addrlen) override { if (pcb_ == nullptr) { @@ -292,25 +273,10 @@ class LWIPRawImpl : public Socket { return -1; } int listen(int backlog) override { - if (pcb_ == nullptr) { - errno = EBADF; - return -1; - } - LWIP_LOG("tcp_listen_with_backlog(%p backlog=%d)", pcb_, backlog); - struct tcp_pcb *listen_pcb = tcp_listen_with_backlog(pcb_, backlog); - if (listen_pcb == nullptr) { - tcp_abort(pcb_); - pcb_ = nullptr; - errno = EOPNOTSUPP; - return -1; - } - // tcp_listen reallocates the pcb, replace ours - pcb_ = listen_pcb; - // set callbacks on new pcb - LWIP_LOG("tcp_arg(%p)", pcb_); - tcp_arg(pcb_, this); - tcp_accept(pcb_, LWIPRawImpl::s_accept_fn); - return 0; + // Regular sockets can't be converted to listening - this shouldn't happen + // as listen() should only be called on sockets created for listening + errno = EOPNOTSUPP; + return -1; } ssize_t read(void *buf, size_t len) override { if (pcb_ == nullptr) { @@ -491,29 +457,6 @@ class LWIPRawImpl : public Socket { return 0; } - err_t accept_fn(struct tcp_pcb *newpcb, err_t err) { - LWIP_LOG("accept(newpcb=%p err=%d)", newpcb, err); - if (err != ERR_OK || newpcb == nullptr) { - // "An error code if there has been an error accepting. Only return ERR_ABRT if you have - // called tcp_abort from within the callback function!" - // https://www.nongnu.org/lwip/2_1_x/tcp_8h.html#a00517abce6856d6c82f0efebdafb734d - // nothing to do here, we just don't push it to the queue - return ERR_OK; - } - // Check if we've reached the maximum accept queue size - if (this->accepted_socket_count_ >= MAX_ACCEPTED_SOCKETS) { - LWIP_LOG("Rejecting connection, queue full (%d)", this->accepted_socket_count_); - // Abort the connection when queue is full - tcp_abort(newpcb); - // Must return ERR_ABRT since we called tcp_abort() - return ERR_ABRT; - } - auto sock = make_unique(family_, newpcb); - sock->init(); - this->accepted_sockets_[this->accepted_socket_count_++] = std::move(sock); - LWIP_LOG("Accepted connection, queue size: %d", this->accepted_socket_count_); - return ERR_OK; - } void err_fn(err_t err) { LWIP_LOG("err(err=%d)", err); // "If a connection is aborted because of an error, the application is alerted of this event by @@ -545,11 +488,6 @@ class LWIPRawImpl : public Socket { return ERR_OK; } - static err_t s_accept_fn(void *arg, struct tcp_pcb *newpcb, err_t err) { - LWIPRawImpl *arg_this = reinterpret_cast(arg); - return arg_this->accept_fn(newpcb, err); - } - static void s_err_fn(void *arg, err_t err) { LWIPRawImpl *arg_this = reinterpret_cast(arg); arg_this->err_fn(err); @@ -601,7 +539,107 @@ class LWIPRawImpl : public Socket { return -1; } + // Member ordering optimized to minimize padding on 32-bit systems + // Largest members first (4 bytes), then smaller members (1 byte each) struct tcp_pcb *pcb_; + pbuf *rx_buf_ = nullptr; + size_t rx_buf_offset_ = 0; + bool rx_closed_ = false; + // don't use lwip nodelay flag, it sometimes causes reconnect + // instead use it for determining whether to call lwip_output + bool nodelay_ = false; + sa_family_t family_ = 0; +}; + +// Listening socket class - only allocates accept queue when needed (for bind+listen sockets) +// This saves 16 bytes (12 bytes array + 1 byte count + 3 bytes padding) for regular connected sockets on ESP8266/RP2040 +class LWIPRawListenImpl : public LWIPRawImpl { + public: + LWIPRawListenImpl(sa_family_t family, struct tcp_pcb *pcb) : LWIPRawImpl(family, pcb) {} + + void init() { + LWIP_LOG("init(%p)", pcb_); + tcp_arg(pcb_, this); + tcp_accept(pcb_, LWIPRawListenImpl::s_accept_fn); + tcp_err(pcb_, LWIPRawImpl::s_err_fn); // Use base class error handler + } + + std::unique_ptr accept(struct sockaddr *addr, socklen_t *addrlen) override { + if (pcb_ == nullptr) { + errno = EBADF; + return nullptr; + } + if (accepted_socket_count_ == 0) { + errno = EWOULDBLOCK; + return nullptr; + } + // Take from front for FIFO ordering + std::unique_ptr sock = std::move(accepted_sockets_[0]); + // Shift remaining sockets forward + for (uint8_t i = 1; i < accepted_socket_count_; i++) { + accepted_sockets_[i - 1] = std::move(accepted_sockets_[i]); + } + accepted_socket_count_--; + LWIP_LOG("Connection accepted by application, queue size: %d", accepted_socket_count_); + if (addr != nullptr) { + sock->getpeername(addr, addrlen); + } + LWIP_LOG("accept(%p)", sock.get()); + return std::unique_ptr(std::move(sock)); + } + + int listen(int backlog) override { + if (pcb_ == nullptr) { + errno = EBADF; + return -1; + } + LWIP_LOG("tcp_listen_with_backlog(%p backlog=%d)", pcb_, backlog); + struct tcp_pcb *listen_pcb = tcp_listen_with_backlog(pcb_, backlog); + if (listen_pcb == nullptr) { + tcp_abort(pcb_); + pcb_ = nullptr; + errno = EOPNOTSUPP; + return -1; + } + // tcp_listen reallocates the pcb, replace ours + pcb_ = listen_pcb; + // set callbacks on new pcb + LWIP_LOG("tcp_arg(%p)", pcb_); + tcp_arg(pcb_, this); + tcp_accept(pcb_, LWIPRawListenImpl::s_accept_fn); + return 0; + } + + private: + err_t accept_fn(struct tcp_pcb *newpcb, err_t err) { + LWIP_LOG("accept(newpcb=%p err=%d)", newpcb, err); + if (err != ERR_OK || newpcb == nullptr) { + // "An error code if there has been an error accepting. Only return ERR_ABRT if you have + // called tcp_abort from within the callback function!" + // https://www.nongnu.org/lwip/2_1_x/tcp_8h.html#a00517abce6856d6c82f0efebdafb734d + // nothing to do here, we just don't push it to the queue + return ERR_OK; + } + // Check if we've reached the maximum accept queue size + if (accepted_socket_count_ >= MAX_ACCEPTED_SOCKETS) { + LWIP_LOG("Rejecting connection, queue full (%d)", accepted_socket_count_); + // Abort the connection when queue is full + tcp_abort(newpcb); + // Must return ERR_ABRT since we called tcp_abort() + return ERR_ABRT; + } + auto sock = make_unique(family_, newpcb); + sock->init(); + accepted_sockets_[accepted_socket_count_++] = std::move(sock); + LWIP_LOG("Accepted connection, queue size: %d", accepted_socket_count_); + return ERR_OK; + } + + static err_t s_accept_fn(void *arg, struct tcp_pcb *newpcb, err_t err) { + LWIPRawListenImpl *arg_this = reinterpret_cast(arg); + return arg_this->accept_fn(newpcb, err); + } + // Accept queue - holds incoming connections briefly until the event loop calls accept() // This is NOT a connection pool - just a temporary queue between LWIP callbacks and the main loop // 3 slots is plenty since connections are pulled out quickly by the event loop @@ -613,23 +651,21 @@ class LWIPRawImpl : public Socket { // - std::array<3>: 12 bytes fixed (3 pointers × 4 bytes) // Saves ~44+ bytes RAM per listening socket + avoids ALL heap allocations // Used on ESP8266 and RP2040 (platforms using LWIP_TCP implementation) + // + // By using a separate listening socket class, regular connected sockets save + // 16 bytes (12 bytes array + 1 byte count + 3 bytes padding) of memory overhead on 32-bit systems static constexpr size_t MAX_ACCEPTED_SOCKETS = 3; std::array, MAX_ACCEPTED_SOCKETS> accepted_sockets_; uint8_t accepted_socket_count_ = 0; // Number of sockets currently in queue - bool rx_closed_ = false; - pbuf *rx_buf_ = nullptr; - size_t rx_buf_offset_ = 0; - // don't use lwip nodelay flag, it sometimes causes reconnect - // instead use it for determining whether to call lwip_output - bool nodelay_ = false; - sa_family_t family_ = 0; }; std::unique_ptr socket(int domain, int type, int protocol) { auto *pcb = tcp_new(); if (pcb == nullptr) return nullptr; - auto *sock = new LWIPRawImpl((sa_family_t) domain, pcb); // NOLINT(cppcoreguidelines-owning-memory) + // Create listening socket implementation since user sockets typically bind+listen + // Accepted connections are created directly as LWIPRawImpl in the accept callback + auto *sock = new LWIPRawListenImpl((sa_family_t) domain, pcb); // NOLINT(cppcoreguidelines-owning-memory) sock->init(); return std::unique_ptr{sock}; } From 5bb69a968cd45dc52da458faef2ea0ca41124c1e Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Mon, 13 Oct 2025 16:33:33 -1000 Subject: [PATCH 02/10] [esp32_ble] Replace handler vectors with StaticVector for 560B-2KB memory savings (#11200) Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- esphome/components/esp32_ble/__init__.py | 88 ++++++++++++++++++- esphome/components/esp32_ble/ble.cpp | 56 +++++++----- esphome/components/esp32_ble/ble.h | 32 ++++--- .../components/esp32_ble_beacon/__init__.py | 2 +- .../components/esp32_ble_server/__init__.py | 4 +- .../components/esp32_ble_tracker/__init__.py | 8 +- esphome/core/defines.h | 5 ++ 7 files changed, 153 insertions(+), 42 deletions(-) diff --git a/esphome/components/esp32_ble/__init__.py b/esphome/components/esp32_ble/__init__.py index 1c142ca7bd..fc8c65a2f4 100644 --- a/esphome/components/esp32_ble/__init__.py +++ b/esphome/components/esp32_ble/__init__.py @@ -1,4 +1,5 @@ from collections.abc import Callable, MutableMapping +from dataclasses import dataclass from enum import Enum import logging import re @@ -16,7 +17,7 @@ from esphome.const import ( CONF_NAME, CONF_NAME_ADD_MAC_SUFFIX, ) -from esphome.core import CORE, TimePeriod +from esphome.core import CORE, CoroPriority, TimePeriod, coroutine_with_priority import esphome.final_validate as fv DEPENDENCIES = ["esp32"] @@ -111,6 +112,58 @@ class BTLoggers(Enum): _required_loggers: set[BTLoggers] = set() +# Dataclass for handler registration counts +@dataclass +class HandlerCounts: + gap_event: int = 0 + gap_scan_event: int = 0 + gattc_event: int = 0 + gatts_event: int = 0 + ble_status_event: int = 0 + + +# Track handler registration counts for StaticVector sizing +_handler_counts = HandlerCounts() + + +def register_gap_event_handler(parent_var: cg.MockObj, handler_var: cg.MockObj) -> None: + """Register a GAP event handler and track the count.""" + _handler_counts.gap_event += 1 + cg.add(parent_var.register_gap_event_handler(handler_var)) + + +def register_gap_scan_event_handler( + parent_var: cg.MockObj, handler_var: cg.MockObj +) -> None: + """Register a GAP scan event handler and track the count.""" + _handler_counts.gap_scan_event += 1 + cg.add(parent_var.register_gap_scan_event_handler(handler_var)) + + +def register_gattc_event_handler( + parent_var: cg.MockObj, handler_var: cg.MockObj +) -> None: + """Register a GATTc event handler and track the count.""" + _handler_counts.gattc_event += 1 + cg.add(parent_var.register_gattc_event_handler(handler_var)) + + +def register_gatts_event_handler( + parent_var: cg.MockObj, handler_var: cg.MockObj +) -> None: + """Register a GATTs event handler and track the count.""" + _handler_counts.gatts_event += 1 + cg.add(parent_var.register_gatts_event_handler(handler_var)) + + +def register_ble_status_event_handler( + parent_var: cg.MockObj, handler_var: cg.MockObj +) -> None: + """Register a BLE status event handler and track the count.""" + _handler_counts.ble_status_event += 1 + cg.add(parent_var.register_ble_status_event_handler(handler_var)) + + def register_bt_logger(*loggers: BTLoggers) -> None: """Register Bluetooth logger categories that a component needs. @@ -374,6 +427,36 @@ def final_validation(config): FINAL_VALIDATE_SCHEMA = final_validation +# This needs to be run as a job with CoroPriority.FINAL priority so that all components have +# a chance to register their handlers before the counts are added to defines. +@coroutine_with_priority(CoroPriority.FINAL) +async def _add_ble_handler_defines(): + # Add defines for StaticVector sizing based on handler registration counts + # Only define if count > 0 to avoid allocating unnecessary memory + if _handler_counts.gap_event > 0: + cg.add_define( + "ESPHOME_ESP32_BLE_GAP_EVENT_HANDLER_COUNT", _handler_counts.gap_event + ) + if _handler_counts.gap_scan_event > 0: + cg.add_define( + "ESPHOME_ESP32_BLE_GAP_SCAN_EVENT_HANDLER_COUNT", + _handler_counts.gap_scan_event, + ) + if _handler_counts.gattc_event > 0: + cg.add_define( + "ESPHOME_ESP32_BLE_GATTC_EVENT_HANDLER_COUNT", _handler_counts.gattc_event + ) + if _handler_counts.gatts_event > 0: + cg.add_define( + "ESPHOME_ESP32_BLE_GATTS_EVENT_HANDLER_COUNT", _handler_counts.gatts_event + ) + if _handler_counts.ble_status_event > 0: + cg.add_define( + "ESPHOME_ESP32_BLE_BLE_STATUS_EVENT_HANDLER_COUNT", + _handler_counts.ble_status_event, + ) + + async def to_code(config): var = cg.new_Pvariable(config[CONF_ID]) cg.add(var.set_enable_on_boot(config[CONF_ENABLE_ON_BOOT])) @@ -428,6 +511,9 @@ async def to_code(config): cg.add_define("USE_ESP32_BLE_ADVERTISING") cg.add_define("USE_ESP32_BLE_UUID") + # Schedule the handler defines to be added after all components register + CORE.add_job(_add_ble_handler_defines) + @automation.register_condition("ble.enabled", BLEEnabledCondition, cv.Schema({})) async def ble_enabled_to_code(config, condition_id, template_arg, args): diff --git a/esphome/components/esp32_ble/ble.cpp b/esphome/components/esp32_ble/ble.cpp index 1f6bc6d0a0..9f8ca1d149 100644 --- a/esphome/components/esp32_ble/ble.cpp +++ b/esphome/components/esp32_ble/ble.cpp @@ -181,31 +181,27 @@ bool ESP32BLE::ble_setup_() { return false; } - if (!this->gap_event_handlers_.empty()) { - err = esp_ble_gap_register_callback(ESP32BLE::gap_event_handler); - if (err != ESP_OK) { - ESP_LOGE(TAG, "esp_ble_gap_register_callback failed: %d", err); - return false; - } - } - -#ifdef USE_ESP32_BLE_SERVER - if (!this->gatts_event_handlers_.empty()) { - err = esp_ble_gatts_register_callback(ESP32BLE::gatts_event_handler); - if (err != ESP_OK) { - ESP_LOGE(TAG, "esp_ble_gatts_register_callback failed: %d", err); - return false; - } +#ifdef ESPHOME_ESP32_BLE_GAP_EVENT_HANDLER_COUNT + err = esp_ble_gap_register_callback(ESP32BLE::gap_event_handler); + if (err != ESP_OK) { + ESP_LOGE(TAG, "esp_ble_gap_register_callback failed: %d", err); + return false; } #endif -#ifdef USE_ESP32_BLE_CLIENT - if (!this->gattc_event_handlers_.empty()) { - err = esp_ble_gattc_register_callback(ESP32BLE::gattc_event_handler); - if (err != ESP_OK) { - ESP_LOGE(TAG, "esp_ble_gattc_register_callback failed: %d", err); - return false; - } +#if defined(USE_ESP32_BLE_SERVER) && defined(ESPHOME_ESP32_BLE_GATTS_EVENT_HANDLER_COUNT) + err = esp_ble_gatts_register_callback(ESP32BLE::gatts_event_handler); + if (err != ESP_OK) { + ESP_LOGE(TAG, "esp_ble_gatts_register_callback failed: %d", err); + return false; + } +#endif + +#if defined(USE_ESP32_BLE_CLIENT) && defined(ESPHOME_ESP32_BLE_GATTC_EVENT_HANDLER_COUNT) + err = esp_ble_gattc_register_callback(ESP32BLE::gattc_event_handler); + if (err != ESP_OK) { + ESP_LOGE(TAG, "esp_ble_gattc_register_callback failed: %d", err); + return false; } #endif @@ -302,9 +298,11 @@ void ESP32BLE::loop() { case BLE_COMPONENT_STATE_DISABLE: { ESP_LOGD(TAG, "Disabling"); +#ifdef ESPHOME_ESP32_BLE_BLE_STATUS_EVENT_HANDLER_COUNT for (auto *ble_event_handler : this->ble_status_event_handlers_) { ble_event_handler->ble_before_disabled_event_handler(); } +#endif if (!ble_dismantle_()) { ESP_LOGE(TAG, "Could not be dismantled"); @@ -334,7 +332,7 @@ void ESP32BLE::loop() { BLEEvent *ble_event = this->ble_events_.pop(); while (ble_event != nullptr) { switch (ble_event->type_) { -#ifdef USE_ESP32_BLE_SERVER +#if defined(USE_ESP32_BLE_SERVER) && defined(ESPHOME_ESP32_BLE_GATTS_EVENT_HANDLER_COUNT) case BLEEvent::GATTS: { esp_gatts_cb_event_t event = ble_event->event_.gatts.gatts_event; esp_gatt_if_t gatts_if = ble_event->event_.gatts.gatts_if; @@ -346,7 +344,7 @@ void ESP32BLE::loop() { break; } #endif -#ifdef USE_ESP32_BLE_CLIENT +#if defined(USE_ESP32_BLE_CLIENT) && defined(ESPHOME_ESP32_BLE_GATTC_EVENT_HANDLER_COUNT) case BLEEvent::GATTC: { esp_gattc_cb_event_t event = ble_event->event_.gattc.gattc_event; esp_gatt_if_t gattc_if = ble_event->event_.gattc.gattc_if; @@ -362,10 +360,12 @@ void ESP32BLE::loop() { esp_gap_ble_cb_event_t gap_event = ble_event->event_.gap.gap_event; switch (gap_event) { case ESP_GAP_BLE_SCAN_RESULT_EVT: +#ifdef ESPHOME_ESP32_BLE_GAP_SCAN_EVENT_HANDLER_COUNT // Use the new scan event handler - no memcpy! for (auto *scan_handler : this->gap_scan_event_handlers_) { scan_handler->gap_scan_event_handler(ble_event->scan_result()); } +#endif break; // Scan complete events @@ -377,10 +377,12 @@ void ESP32BLE::loop() { // This is verified at compile-time by static_assert checks in ble_event.h // The struct already contains our copy of the status (copied in BLEEvent constructor) ESP_LOGV(TAG, "gap_event_handler - %d", gap_event); +#ifdef ESPHOME_ESP32_BLE_GAP_EVENT_HANDLER_COUNT for (auto *gap_handler : this->gap_event_handlers_) { gap_handler->gap_event_handler( gap_event, reinterpret_cast(&ble_event->event_.gap.scan_complete)); } +#endif break; // Advertising complete events @@ -391,19 +393,23 @@ void ESP32BLE::loop() { case ESP_GAP_BLE_ADV_STOP_COMPLETE_EVT: // All advertising complete events have the same structure with just status ESP_LOGV(TAG, "gap_event_handler - %d", gap_event); +#ifdef ESPHOME_ESP32_BLE_GAP_EVENT_HANDLER_COUNT for (auto *gap_handler : this->gap_event_handlers_) { gap_handler->gap_event_handler( gap_event, reinterpret_cast(&ble_event->event_.gap.adv_complete)); } +#endif break; // RSSI complete event case ESP_GAP_BLE_READ_RSSI_COMPLETE_EVT: ESP_LOGV(TAG, "gap_event_handler - %d", gap_event); +#ifdef ESPHOME_ESP32_BLE_GAP_EVENT_HANDLER_COUNT for (auto *gap_handler : this->gap_event_handlers_) { gap_handler->gap_event_handler( gap_event, reinterpret_cast(&ble_event->event_.gap.read_rssi_complete)); } +#endif break; // Security events @@ -413,10 +419,12 @@ void ESP32BLE::loop() { case ESP_GAP_BLE_PASSKEY_REQ_EVT: case ESP_GAP_BLE_NC_REQ_EVT: ESP_LOGV(TAG, "gap_event_handler - %d", gap_event); +#ifdef ESPHOME_ESP32_BLE_GAP_EVENT_HANDLER_COUNT for (auto *gap_handler : this->gap_event_handlers_) { gap_handler->gap_event_handler( gap_event, reinterpret_cast(&ble_event->event_.gap.security)); } +#endif break; default: diff --git a/esphome/components/esp32_ble/ble.h b/esphome/components/esp32_ble/ble.h index 1aa3bc86ef..dc973f0e82 100644 --- a/esphome/components/esp32_ble/ble.h +++ b/esphome/components/esp32_ble/ble.h @@ -125,19 +125,25 @@ class ESP32BLE : public Component { void advertising_register_raw_advertisement_callback(std::function &&callback); #endif +#ifdef ESPHOME_ESP32_BLE_GAP_EVENT_HANDLER_COUNT void register_gap_event_handler(GAPEventHandler *handler) { this->gap_event_handlers_.push_back(handler); } +#endif +#ifdef ESPHOME_ESP32_BLE_GAP_SCAN_EVENT_HANDLER_COUNT void register_gap_scan_event_handler(GAPScanEventHandler *handler) { this->gap_scan_event_handlers_.push_back(handler); } -#ifdef USE_ESP32_BLE_CLIENT +#endif +#if defined(USE_ESP32_BLE_CLIENT) && defined(ESPHOME_ESP32_BLE_GATTC_EVENT_HANDLER_COUNT) void register_gattc_event_handler(GATTcEventHandler *handler) { this->gattc_event_handlers_.push_back(handler); } #endif -#ifdef USE_ESP32_BLE_SERVER +#if defined(USE_ESP32_BLE_SERVER) && defined(ESPHOME_ESP32_BLE_GATTS_EVENT_HANDLER_COUNT) void register_gatts_event_handler(GATTsEventHandler *handler) { this->gatts_event_handlers_.push_back(handler); } #endif +#ifdef ESPHOME_ESP32_BLE_BLE_STATUS_EVENT_HANDLER_COUNT void register_ble_status_event_handler(BLEStatusEventHandler *handler) { this->ble_status_event_handlers_.push_back(handler); } +#endif void set_enable_on_boot(bool enable_on_boot) { this->enable_on_boot_ = enable_on_boot; } protected: @@ -159,16 +165,22 @@ class ESP32BLE : public Component { private: template friend void enqueue_ble_event(Args... args); - // Vectors (12 bytes each on 32-bit, naturally aligned to 4 bytes) - std::vector gap_event_handlers_; - std::vector gap_scan_event_handlers_; -#ifdef USE_ESP32_BLE_CLIENT - std::vector gattc_event_handlers_; + // Handler vectors - use StaticVector when counts are known at compile time +#ifdef ESPHOME_ESP32_BLE_GAP_EVENT_HANDLER_COUNT + StaticVector gap_event_handlers_; #endif -#ifdef USE_ESP32_BLE_SERVER - std::vector gatts_event_handlers_; +#ifdef ESPHOME_ESP32_BLE_GAP_SCAN_EVENT_HANDLER_COUNT + StaticVector gap_scan_event_handlers_; +#endif +#if defined(USE_ESP32_BLE_CLIENT) && defined(ESPHOME_ESP32_BLE_GATTC_EVENT_HANDLER_COUNT) + StaticVector gattc_event_handlers_; +#endif +#if defined(USE_ESP32_BLE_SERVER) && defined(ESPHOME_ESP32_BLE_GATTS_EVENT_HANDLER_COUNT) + StaticVector gatts_event_handlers_; +#endif +#ifdef ESPHOME_ESP32_BLE_BLE_STATUS_EVENT_HANDLER_COUNT + StaticVector ble_status_event_handlers_; #endif - std::vector ble_status_event_handlers_; // Large objects (size depends on template parameters, but typically aligned to 4 bytes) esphome::LockFreeQueue ble_events_; diff --git a/esphome/components/esp32_ble_beacon/__init__.py b/esphome/components/esp32_ble_beacon/__init__.py index 794f5637a4..ba5ae4331c 100644 --- a/esphome/components/esp32_ble_beacon/__init__.py +++ b/esphome/components/esp32_ble_beacon/__init__.py @@ -74,7 +74,7 @@ async def to_code(config): var = cg.new_Pvariable(config[CONF_ID], uuid_arr) parent = await cg.get_variable(config[esp32_ble.CONF_BLE_ID]) - cg.add(parent.register_gap_event_handler(var)) + esp32_ble.register_gap_event_handler(parent, var) await cg.register_component(var, config) cg.add(var.set_major(config[CONF_MAJOR])) diff --git a/esphome/components/esp32_ble_server/__init__.py b/esphome/components/esp32_ble_server/__init__.py index 10fa09fcc3..55310f3275 100644 --- a/esphome/components/esp32_ble_server/__init__.py +++ b/esphome/components/esp32_ble_server/__init__.py @@ -546,8 +546,8 @@ async def to_code(config): await cg.register_component(var, config) parent = await cg.get_variable(config[esp32_ble.CONF_BLE_ID]) - cg.add(parent.register_gatts_event_handler(var)) - cg.add(parent.register_ble_status_event_handler(var)) + esp32_ble.register_gatts_event_handler(parent, var) + esp32_ble.register_ble_status_event_handler(parent, var) cg.add(var.set_parent(parent)) cg.add(parent.advertising_set_appearance(config[CONF_APPEARANCE])) if CONF_MANUFACTURER_DATA in config: diff --git a/esphome/components/esp32_ble_tracker/__init__.py b/esphome/components/esp32_ble_tracker/__init__.py index 8c7f3e3930..5910be67af 100644 --- a/esphome/components/esp32_ble_tracker/__init__.py +++ b/esphome/components/esp32_ble_tracker/__init__.py @@ -246,10 +246,10 @@ async def to_code(config): await cg.register_component(var, config) parent = await cg.get_variable(config[esp32_ble.CONF_BLE_ID]) - cg.add(parent.register_gap_event_handler(var)) - cg.add(parent.register_gap_scan_event_handler(var)) - cg.add(parent.register_gattc_event_handler(var)) - cg.add(parent.register_ble_status_event_handler(var)) + esp32_ble.register_gap_event_handler(parent, var) + esp32_ble.register_gap_scan_event_handler(parent, var) + esp32_ble.register_gattc_event_handler(parent, var) + esp32_ble.register_ble_status_event_handler(parent, var) cg.add(var.set_parent(parent)) params = config[CONF_SCAN_PARAMETERS] diff --git a/esphome/core/defines.h b/esphome/core/defines.h index ed30efd0b5..f770edaa00 100644 --- a/esphome/core/defines.h +++ b/esphome/core/defines.h @@ -178,6 +178,11 @@ #define USE_ESP32_BLE_SERVER_ON_DISCONNECT #define ESPHOME_ESP32_BLE_TRACKER_LISTENER_COUNT 1 #define ESPHOME_ESP32_BLE_TRACKER_CLIENT_COUNT 1 +#define ESPHOME_ESP32_BLE_GAP_EVENT_HANDLER_COUNT 2 +#define ESPHOME_ESP32_BLE_GAP_SCAN_EVENT_HANDLER_COUNT 1 +#define ESPHOME_ESP32_BLE_GATTC_EVENT_HANDLER_COUNT 1 +#define ESPHOME_ESP32_BLE_GATTS_EVENT_HANDLER_COUNT 1 +#define ESPHOME_ESP32_BLE_BLE_STATUS_EVENT_HANDLER_COUNT 2 #define USE_ESP32_CAMERA_JPEG_ENCODER #define USE_I2C #define USE_IMPROV From 69df07ddcf90bb7ccec9e80cd457c3b4e41a757b Mon Sep 17 00:00:00 2001 From: Jesse Hills <3060199+jesserockz@users.noreply.github.com> Date: Mon, 13 Oct 2025 12:58:30 +1300 Subject: [PATCH 03/10] [media_player.speaker] Dynamic auto load (#11084) Co-authored-by: J. Nick Koston --- esphome/components/psram/__init__.py | 2 ++ .../speaker/media_player/__init__.py | 19 ++++++++++++++++--- 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/esphome/components/psram/__init__.py b/esphome/components/psram/__init__.py index 6b85e7f720..8e4f9d7eac 100644 --- a/esphome/components/psram/__init__.py +++ b/esphome/components/psram/__init__.py @@ -63,6 +63,8 @@ SPIRAM_SPEEDS = { def supported() -> bool: + if not CORE.is_esp32: + return False variant = get_esp32_variant() return variant in SPIRAM_MODES diff --git a/esphome/components/speaker/media_player/__init__.py b/esphome/components/speaker/media_player/__init__.py index 69ea0a53c6..7537a61e4e 100644 --- a/esphome/components/speaker/media_player/__init__.py +++ b/esphome/components/speaker/media_player/__init__.py @@ -6,7 +6,7 @@ from pathlib import Path from esphome import automation, external_files import esphome.codegen as cg -from esphome.components import audio, esp32, media_player, speaker +from esphome.components import audio, esp32, media_player, psram, speaker import esphome.config_validation as cv from esphome.const import ( CONF_BUFFER_SIZE, @@ -26,10 +26,21 @@ from esphome.const import ( from esphome.core import CORE, HexInt from esphome.core.entity_helpers import inherit_property_from from esphome.external_files import download_content +from esphome.types import ConfigType _LOGGER = logging.getLogger(__name__) -AUTO_LOAD = ["audio", "psram"] + +def AUTO_LOAD(config: ConfigType) -> list[str]: + load = ["audio"] + if ( + not config + or config.get(CONF_TASK_STACK_IN_PSRAM) + or config.get(CONF_CODEC_SUPPORT_ENABLED) + ): + return load + ["psram"] + return load + CODEOWNERS = ["@kahrendt", "@synesthesiam"] DOMAIN = "media_player" @@ -279,7 +290,9 @@ CONFIG_SCHEMA = cv.All( cv.Optional(CONF_BUFFER_SIZE, default=1000000): cv.int_range( min=4000, max=4000000 ), - cv.Optional(CONF_CODEC_SUPPORT_ENABLED, default=True): cv.boolean, + cv.Optional( + CONF_CODEC_SUPPORT_ENABLED, default=psram.supported() + ): cv.boolean, cv.Optional(CONF_FILES): cv.ensure_list(MEDIA_FILE_TYPE_SCHEMA), cv.Optional(CONF_TASK_STACK_IN_PSRAM, default=False): cv.boolean, cv.Optional(CONF_VOLUME_INCREMENT, default=0.05): cv.percentage, From 8627b56e36a3b618a31d5e0e42e075df36316b83 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 13 Oct 2025 09:34:12 -1000 Subject: [PATCH 04/10] Bump esphome-dashboard from 20251009.0 to 20251013.0 (#11212) Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 64946263ea..9937709c1c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -11,7 +11,7 @@ pyserial==3.5 platformio==6.1.18 # When updating platformio, also update /docker/Dockerfile esptool==5.1.0 click==8.1.7 -esphome-dashboard==20251009.0 +esphome-dashboard==20251013.0 aioesphomeapi==41.14.0 zeroconf==0.148.0 puremagic==1.30 From b666b8e2615886e6ae0cef8699cd335dc32d5170 Mon Sep 17 00:00:00 2001 From: Jonathan Swoboda <154711427+swoboda1337@users.noreply.github.com> Date: Mon, 13 Oct 2025 17:12:45 -0400 Subject: [PATCH 05/10] [core] Properly clean the build dir in the HA addon (#11208) --- esphome/writer.py | 25 +++++++++++------- tests/unit_tests/test_writer.py | 46 +++++++++++++++++++++++++++++++++ 2 files changed, 62 insertions(+), 9 deletions(-) diff --git a/esphome/writer.py b/esphome/writer.py index b5cfd9b667..8eee445cf1 100644 --- a/esphome/writer.py +++ b/esphome/writer.py @@ -15,6 +15,8 @@ from esphome.const import ( from esphome.core import CORE, EsphomeError from esphome.helpers import ( copy_file_if_changed, + get_str_env, + is_ha_addon, read_file, walk_files, write_file_if_changed, @@ -338,16 +340,21 @@ def clean_build(): def clean_all(configuration: list[str]): import shutil - # Clean entire build dir - for dir in configuration: - build_dir = Path(dir) / ".esphome" - if build_dir.is_dir(): - _LOGGER.info("Cleaning %s", build_dir) - # Don't remove storage as it will cause the dashboard to regenerate all configs - for item in build_dir.iterdir(): - if item.is_file(): + data_dirs = [Path(dir) / ".esphome" for dir in configuration] + if is_ha_addon(): + data_dirs.append(Path("/data")) + if "ESPHOME_DATA_DIR" in os.environ: + data_dirs.append(Path(get_str_env("ESPHOME_DATA_DIR", None))) + + # Clean build dir + for dir in data_dirs: + if dir.is_dir(): + _LOGGER.info("Cleaning %s", dir) + # Don't remove storage or .json files which are needed by the dashboard + for item in dir.iterdir(): + if item.is_file() and not item.name.endswith(".json"): item.unlink() - elif item.name != "storage" and item.is_dir(): + elif item.is_dir() and item.name != "storage": shutil.rmtree(item) # Clean PlatformIO project files diff --git a/tests/unit_tests/test_writer.py b/tests/unit_tests/test_writer.py index bffd2b3881..a4490fbbc0 100644 --- a/tests/unit_tests/test_writer.py +++ b/tests/unit_tests/test_writer.py @@ -985,3 +985,49 @@ def test_clean_all_removes_non_storage_directories( # Verify logging mentions cleaning assert "Cleaning" in caplog.text assert str(build_dir) in caplog.text + + +@patch("esphome.writer.CORE") +def test_clean_all_preserves_json_files( + mock_core: MagicMock, + tmp_path: Path, + caplog: pytest.LogCaptureFixture, +) -> None: + """Test clean_all preserves .json files.""" + # Create build directory with various files + config_dir = tmp_path / "config" + config_dir.mkdir() + + build_dir = config_dir / ".esphome" + build_dir.mkdir() + + # Create .json files (should be preserved) + (build_dir / "config.json").write_text('{"config": "data"}') + (build_dir / "metadata.json").write_text('{"metadata": "info"}') + + # Create non-.json files (should be removed) + (build_dir / "dummy.txt").write_text("x") + (build_dir / "other.log").write_text("log content") + + # Call clean_all + from esphome.writer import clean_all + + with caplog.at_level("INFO"): + clean_all([str(config_dir)]) + + # Verify .esphome directory still exists + assert build_dir.exists() + + # Verify .json files are preserved + assert (build_dir / "config.json").exists() + assert (build_dir / "config.json").read_text() == '{"config": "data"}' + assert (build_dir / "metadata.json").exists() + assert (build_dir / "metadata.json").read_text() == '{"metadata": "info"}' + + # Verify non-.json files were removed + assert not (build_dir / "dummy.txt").exists() + assert not (build_dir / "other.log").exists() + + # Verify logging mentions cleaning + assert "Cleaning" in caplog.text + assert str(build_dir) in caplog.text From 07504c82081366744ecb920753dd6da841da2170 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Mon, 13 Oct 2025 11:23:44 -1000 Subject: [PATCH 06/10] Fix log retrieval with FQDN when mDNS is disabled (#11202) --- esphome/__main__.py | 15 +++++++----- tests/unit_tests/test_main.py | 45 ++++++++++++++++++++++++++++------- 2 files changed, 46 insertions(+), 14 deletions(-) diff --git a/esphome/__main__.py b/esphome/__main__.py index be4551f6b5..8e0c475525 100644 --- a/esphome/__main__.py +++ b/esphome/__main__.py @@ -268,8 +268,10 @@ def has_ip_address() -> bool: def has_resolvable_address() -> bool: - """Check if CORE.address is resolvable (via mDNS or is an IP address).""" - return has_mdns() or has_ip_address() + """Check if CORE.address is resolvable (via mDNS, DNS, or is an IP address).""" + # Any address (IP, mDNS hostname, or regular DNS hostname) is resolvable + # The resolve_ip_address() function in helpers.py handles all types via AsyncResolver + return CORE.address is not None def mqtt_get_ip(config: ConfigType, username: str, password: str, client_id: str): @@ -578,11 +580,12 @@ def show_logs(config: ConfigType, args: ArgsProtocol, devices: list[str]) -> int if has_api(): addresses_to_use: list[str] | None = None - if port_type == "NETWORK" and (has_mdns() or is_ip_address(port)): + if port_type == "NETWORK": + # Network addresses (IPs, mDNS names, or regular DNS hostnames) can be used + # The resolve_ip_address() function in helpers.py handles all types addresses_to_use = devices - elif port_type in ("NETWORK", "MQTT", "MQTTIP") and has_mqtt_ip_lookup(): - # Only use MQTT IP lookup if the first condition didn't match - # (for MQTT/MQTTIP types, or for NETWORK when mdns/ip check fails) + elif port_type in ("MQTT", "MQTTIP") and has_mqtt_ip_lookup(): + # Use MQTT IP lookup for MQTT/MQTTIP types addresses_to_use = mqtt_get_ip( config, args.username, args.password, args.client_id ) diff --git a/tests/unit_tests/test_main.py b/tests/unit_tests/test_main.py index e35378145a..becf911fa3 100644 --- a/tests/unit_tests/test_main.py +++ b/tests/unit_tests/test_main.py @@ -1203,6 +1203,31 @@ def test_show_logs_api( ) +@patch("esphome.components.api.client.run_logs") +def test_show_logs_api_with_fqdn_mdns_disabled( + mock_run_logs: Mock, +) -> None: + """Test show_logs with API using FQDN when mDNS is disabled.""" + setup_core( + config={ + "logger": {}, + CONF_API: {}, + CONF_MDNS: {CONF_DISABLED: True}, + }, + platform=PLATFORM_ESP32, + ) + mock_run_logs.return_value = 0 + + args = MockArgs() + devices = ["device.example.com"] + + result = show_logs(CORE.config, args, devices) + + assert result == 0 + # Should use the FQDN directly, not try MQTT lookup + mock_run_logs.assert_called_once_with(CORE.config, ["device.example.com"]) + + @patch("esphome.components.api.client.run_logs") def test_show_logs_api_with_mqtt_fallback( mock_run_logs: Mock, @@ -1222,7 +1247,7 @@ def test_show_logs_api_with_mqtt_fallback( mock_mqtt_get_ip.return_value = ["192.168.1.200"] args = MockArgs(username="user", password="pass", client_id="client") - devices = ["device.local"] + devices = ["MQTTIP"] result = show_logs(CORE.config, args, devices) @@ -1487,27 +1512,31 @@ def test_mqtt_get_ip() -> None: def test_has_resolvable_address() -> None: """Test has_resolvable_address function.""" - # Test with mDNS enabled and hostname address + # Test with mDNS enabled and .local hostname address setup_core(config={}, address="esphome-device.local") assert has_resolvable_address() is True - # Test with mDNS disabled and hostname address + # Test with mDNS disabled and .local hostname address (still resolvable via DNS) setup_core( config={CONF_MDNS: {CONF_DISABLED: True}}, address="esphome-device.local" ) - assert has_resolvable_address() is False + assert has_resolvable_address() is True - # Test with IP address (mDNS doesn't matter) + # Test with mDNS disabled and regular DNS hostname (resolvable) + setup_core(config={CONF_MDNS: {CONF_DISABLED: True}}, address="device.example.com") + assert has_resolvable_address() is True + + # Test with IP address (always resolvable, mDNS doesn't matter) setup_core(config={}, address="192.168.1.100") assert has_resolvable_address() is True - # Test with IP address and mDNS disabled + # Test with IP address and mDNS disabled (still resolvable) setup_core(config={CONF_MDNS: {CONF_DISABLED: True}}, address="192.168.1.100") assert has_resolvable_address() is True - # Test with no address but mDNS enabled (can still resolve mDNS names) + # Test with no address setup_core(config={}, address=None) - assert has_resolvable_address() is True + assert has_resolvable_address() is False # Test with no address and mDNS disabled setup_core(config={CONF_MDNS: {CONF_DISABLED: True}}, address=None) From d02ed41eb4adf2001df33b745431a04226022e9c Mon Sep 17 00:00:00 2001 From: Jesse Hills <3060199+jesserockz@users.noreply.github.com> Date: Tue, 14 Oct 2025 15:38:15 +1300 Subject: [PATCH 07/10] Bump version to 2025.10.0b3 --- Doxyfile | 2 +- esphome/const.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Doxyfile b/Doxyfile index 37ec90c851..d664ca5d63 100644 --- a/Doxyfile +++ b/Doxyfile @@ -48,7 +48,7 @@ PROJECT_NAME = ESPHome # could be handy for archiving the generated documentation or if some version # control system is used. -PROJECT_NUMBER = 2025.10.0b2 +PROJECT_NUMBER = 2025.10.0b3 # Using the PROJECT_BRIEF tag one can provide an optional one line description # for a project that appears at the top of each page and should give viewer a diff --git a/esphome/const.py b/esphome/const.py index a086f90385..76943ff59d 100644 --- a/esphome/const.py +++ b/esphome/const.py @@ -4,7 +4,7 @@ from enum import Enum from esphome.enum import StrEnum -__version__ = "2025.10.0b2" +__version__ = "2025.10.0b3" ALLOWED_NAME_CHARS = "abcdefghijklmnopqrstuvwxyz0123456789-_" VALID_SUBSTITUTIONS_CHARACTERS = ( From 8a15c18066d9d65515c81e8de3d5521a90d908f5 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Mon, 13 Oct 2025 17:05:13 -1000 Subject: [PATCH 08/10] [bluetooth_proxy] Use FixedVector for GATT characteristics and descriptors (#11214) Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- esphome/components/api/api.proto | 4 +- esphome/components/api/api_options.proto | 6 + esphome/components/api/api_pb2.h | 4 +- esphome/components/api/proto.h | 26 +++- .../bluetooth_proxy/bluetooth_connection.cpp | 13 +- esphome/core/helpers.h | 121 ++++++++++++++++-- script/api_protobuf/api_protobuf.py | 4 + 7 files changed, 146 insertions(+), 32 deletions(-) diff --git a/esphome/components/api/api.proto b/esphome/components/api/api.proto index 87f477799d..9b714d00f1 100644 --- a/esphome/components/api/api.proto +++ b/esphome/components/api/api.proto @@ -1519,7 +1519,7 @@ message BluetoothGATTCharacteristic { repeated uint64 uuid = 1 [(fixed_array_size) = 2, (fixed_array_skip_zero) = true]; uint32 handle = 2; uint32 properties = 3; - repeated BluetoothGATTDescriptor descriptors = 4; + repeated BluetoothGATTDescriptor descriptors = 4 [(fixed_vector) = true]; // New field for efficient UUID (v1.12+) // Only one of uuid or short_uuid will be set. @@ -1531,7 +1531,7 @@ message BluetoothGATTCharacteristic { message BluetoothGATTService { repeated uint64 uuid = 1 [(fixed_array_size) = 2, (fixed_array_skip_zero) = true]; uint32 handle = 2; - repeated BluetoothGATTCharacteristic characteristics = 3; + repeated BluetoothGATTCharacteristic characteristics = 3 [(fixed_vector) = true]; // New field for efficient UUID (v1.12+) // Only one of uuid or short_uuid will be set. diff --git a/esphome/components/api/api_options.proto b/esphome/components/api/api_options.proto index 633f39b552..ead8ac0bbc 100644 --- a/esphome/components/api/api_options.proto +++ b/esphome/components/api/api_options.proto @@ -64,4 +64,10 @@ extend google.protobuf.FieldOptions { // This is typically done through methods returning const T& or special accessor // methods like get_options() or supported_modes_for_api_(). optional string container_pointer = 50001; + + // fixed_vector: Use FixedVector instead of std::vector for repeated fields + // When set, the repeated field will use FixedVector which requires calling + // init(size) before adding elements. This eliminates std::vector template overhead + // and is ideal when the exact size is known before populating the array. + optional bool fixed_vector = 50013 [default=false]; } diff --git a/esphome/components/api/api_pb2.h b/esphome/components/api/api_pb2.h index d9e68ece9b..1798458393 100644 --- a/esphome/components/api/api_pb2.h +++ b/esphome/components/api/api_pb2.h @@ -1923,7 +1923,7 @@ class BluetoothGATTCharacteristic final : public ProtoMessage { std::array uuid{}; uint32_t handle{0}; uint32_t properties{0}; - std::vector descriptors{}; + FixedVector descriptors{}; uint32_t short_uuid{0}; void encode(ProtoWriteBuffer buffer) const override; void calculate_size(ProtoSize &size) const override; @@ -1937,7 +1937,7 @@ class BluetoothGATTService final : public ProtoMessage { public: std::array uuid{}; uint32_t handle{0}; - std::vector characteristics{}; + FixedVector characteristics{}; uint32_t short_uuid{0}; void encode(ProtoWriteBuffer buffer) const override; void calculate_size(ProtoSize &size) const override; diff --git a/esphome/components/api/proto.h b/esphome/components/api/proto.h index 9d780692ec..a6a09bf7c5 100644 --- a/esphome/components/api/proto.h +++ b/esphome/components/api/proto.h @@ -749,13 +749,29 @@ class ProtoSize { template inline void add_repeated_message(uint32_t field_id_size, const std::vector &messages) { // Skip if the vector is empty - if (messages.empty()) { - return; + if (!messages.empty()) { + // Use the force version for all messages in the repeated field + for (const auto &message : messages) { + add_message_object_force(field_id_size, message); + } } + } - // Use the force version for all messages in the repeated field - for (const auto &message : messages) { - add_message_object_force(field_id_size, message); + /** + * @brief Calculates and adds the sizes of all messages in a repeated field to the total message size (FixedVector + * version) + * + * @tparam MessageType The type of the nested messages in the FixedVector + * @param messages FixedVector of message objects + */ + template + inline void add_repeated_message(uint32_t field_id_size, const FixedVector &messages) { + // Skip if the fixed vector is empty + if (!messages.empty()) { + // Use the force version for all messages in the repeated field + for (const auto &message : messages) { + add_message_object_force(field_id_size, message); + } } } }; diff --git a/esphome/components/bluetooth_proxy/bluetooth_connection.cpp b/esphome/components/bluetooth_proxy/bluetooth_connection.cpp index cde82fbfb0..fcc344dda9 100644 --- a/esphome/components/bluetooth_proxy/bluetooth_connection.cpp +++ b/esphome/components/bluetooth_proxy/bluetooth_connection.cpp @@ -230,8 +230,8 @@ void BluetoothConnection::send_service_for_discovery_() { service_resp.handle = service_result.start_handle; if (total_char_count > 0) { - // Reserve space and process characteristics - service_resp.characteristics.reserve(total_char_count); + // Initialize FixedVector with exact count and process characteristics + service_resp.characteristics.init(total_char_count); uint16_t char_offset = 0; esp_gattc_char_elem_t char_result; while (true) { // characteristics @@ -253,9 +253,7 @@ void BluetoothConnection::send_service_for_discovery_() { service_resp.characteristics.emplace_back(); auto &characteristic_resp = service_resp.characteristics.back(); - fill_gatt_uuid(characteristic_resp.uuid, characteristic_resp.short_uuid, char_result.uuid, use_efficient_uuids); - characteristic_resp.handle = char_result.char_handle; characteristic_resp.properties = char_result.properties; char_offset++; @@ -271,12 +269,11 @@ void BluetoothConnection::send_service_for_discovery_() { return; } if (total_desc_count == 0) { - // No descriptors, continue to next characteristic continue; } - // Reserve space and process descriptors - characteristic_resp.descriptors.reserve(total_desc_count); + // Initialize FixedVector with exact count and process descriptors + characteristic_resp.descriptors.init(total_desc_count); uint16_t desc_offset = 0; esp_gattc_descr_elem_t desc_result; while (true) { // descriptors @@ -297,9 +294,7 @@ void BluetoothConnection::send_service_for_discovery_() { characteristic_resp.descriptors.emplace_back(); auto &descriptor_resp = characteristic_resp.descriptors.back(); - fill_gatt_uuid(descriptor_resp.uuid, descriptor_resp.short_uuid, desc_result.uuid, use_efficient_uuids); - descriptor_resp.handle = desc_result.handle; desc_offset++; } diff --git a/esphome/core/helpers.h b/esphome/core/helpers.h index ed1d5ac108..e352c9c415 100644 --- a/esphome/core/helpers.h +++ b/esphome/core/helpers.h @@ -168,26 +168,83 @@ template class FixedVector { size_t size_{0}; size_t capacity_{0}; - public: - FixedVector() = default; - - ~FixedVector() { - if (data_ != nullptr) { - delete[] data_; + // Helper to destroy all elements without freeing memory + void destroy_elements_() { + // Only call destructors for non-trivially destructible types + if constexpr (!std::is_trivially_destructible::value) { + for (size_t i = 0; i < size_; i++) { + data_[i].~T(); + } } } - // Disable copy to avoid accidental copies + // Helper to destroy elements and free memory + void cleanup_() { + if (data_ != nullptr) { + destroy_elements_(); + // Free raw memory + ::operator delete(data_); + } + } + + // Helper to reset pointers after cleanup + void reset_() { + data_ = nullptr; + capacity_ = 0; + size_ = 0; + } + + public: + FixedVector() = default; + + ~FixedVector() { cleanup_(); } + + // Disable copy operations (avoid accidental expensive copies) FixedVector(const FixedVector &) = delete; FixedVector &operator=(const FixedVector &) = delete; - // Allocate capacity - can only be called once on empty vector - void init(size_t n) { - if (data_ == nullptr && n > 0) { - data_ = new T[n]; - capacity_ = n; - size_ = 0; + // Enable move semantics (allows use in move-only containers like std::vector) + FixedVector(FixedVector &&other) noexcept : data_(other.data_), size_(other.size_), capacity_(other.capacity_) { + other.reset_(); + } + + FixedVector &operator=(FixedVector &&other) noexcept { + if (this != &other) { + // Delete our current data + cleanup_(); + // Take ownership of other's data + data_ = other.data_; + size_ = other.size_; + capacity_ = other.capacity_; + // Leave other in valid empty state + other.reset_(); } + return *this; + } + + // Allocate capacity - can be called multiple times to reinit + void init(size_t n) { + cleanup_(); + reset_(); + if (n > 0) { + // Allocate raw memory without calling constructors + // sizeof(T) is correct here for any type T (value types, pointers, etc.) + // NOLINTNEXTLINE(bugprone-sizeof-expression) + data_ = static_cast(::operator new(n * sizeof(T))); + capacity_ = n; + } + } + + // Clear the vector (destroy all elements, reset size to 0, keep capacity) + void clear() { + destroy_elements_(); + size_ = 0; + } + + // Shrink capacity to fit current size (frees all memory) + void shrink_to_fit() { + cleanup_(); + reset_(); } /// Add element without bounds checking @@ -195,16 +252,52 @@ template class FixedVector { /// Silently ignores pushes beyond capacity (no exception or assertion) void push_back(const T &value) { if (size_ < capacity_) { - data_[size_++] = value; + // Use placement new to construct the object in pre-allocated memory + new (&data_[size_]) T(value); + size_++; } } + /// Add element by move without bounds checking + /// Caller must ensure sufficient capacity was allocated via init() + /// Silently ignores pushes beyond capacity (no exception or assertion) + void push_back(T &&value) { + if (size_ < capacity_) { + // Use placement new to move-construct the object in pre-allocated memory + new (&data_[size_]) T(std::move(value)); + size_++; + } + } + + /// Emplace element without bounds checking - constructs in-place + /// Caller must ensure sufficient capacity was allocated via init() + /// Returns reference to the newly constructed element + /// NOTE: Caller MUST ensure size_ < capacity_ before calling + T &emplace_back() { + // Use placement new to default-construct the object in pre-allocated memory + new (&data_[size_]) T(); + size_++; + return data_[size_ - 1]; + } + + /// Access last element (no bounds checking - matches std::vector behavior) + /// Caller must ensure vector is not empty (size() > 0) + T &back() { return data_[size_ - 1]; } + const T &back() const { return data_[size_ - 1]; } + size_t size() const { return size_; } + bool empty() const { return size_ == 0; } /// Access element without bounds checking (matches std::vector behavior) /// Caller must ensure index is valid (i < size()) T &operator[](size_t i) { return data_[i]; } const T &operator[](size_t i) const { return data_[i]; } + + // Iterator support for range-based for loops + T *begin() { return data_; } + T *end() { return data_ + size_; } + const T *begin() const { return data_; } + const T *end() const { return data_ + size_; } }; ///@} diff --git a/script/api_protobuf/api_protobuf.py b/script/api_protobuf/api_protobuf.py index 487c187372..9a55f1d136 100755 --- a/script/api_protobuf/api_protobuf.py +++ b/script/api_protobuf/api_protobuf.py @@ -1415,6 +1415,8 @@ class RepeatedTypeInfo(TypeInfo): # Check if this is a pointer field by looking for container_pointer option self._container_type = get_field_opt(field, pb.container_pointer, "") self._use_pointer = bool(self._container_type) + # Check if this should use FixedVector instead of std::vector + self._use_fixed_vector = get_field_opt(field, pb.fixed_vector, False) # For repeated fields, we need to get the base type info # but we can't call create_field_type_info as it would cause recursion @@ -1438,6 +1440,8 @@ class RepeatedTypeInfo(TypeInfo): if "<" in self._container_type and ">" in self._container_type: return f"const {self._container_type}*" return f"const {self._container_type}<{self._ti.cpp_type}>*" + if self._use_fixed_vector: + return f"FixedVector<{self._ti.cpp_type}>" return f"std::vector<{self._ti.cpp_type}>" @property From 4c688a4b008cb89fff6f60e70a68da7dd6483c54 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Mon, 13 Oct 2025 17:54:33 -1000 Subject: [PATCH 09/10] [network] Optimize get_use_address() to return const reference instead of a copy (#11218) Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- .../ethernet/ethernet_component.cpp | 11 +++-------- .../components/ethernet/ethernet_component.h | 2 +- esphome/components/network/util.cpp | 19 +++++++++++-------- esphome/components/network/util.h | 2 +- esphome/components/wifi/wifi_component.cpp | 11 +++-------- esphome/components/wifi/wifi_component.h | 2 +- 6 files changed, 20 insertions(+), 27 deletions(-) diff --git a/esphome/components/ethernet/ethernet_component.cpp b/esphome/components/ethernet/ethernet_component.cpp index 13adab8815..24b6e8154b 100644 --- a/esphome/components/ethernet/ethernet_component.cpp +++ b/esphome/components/ethernet/ethernet_component.cpp @@ -689,14 +689,9 @@ void EthernetComponent::add_phy_register(PHYRegister register_value) { this->phy void EthernetComponent::set_type(EthernetType type) { this->type_ = type; } void EthernetComponent::set_manual_ip(const ManualIP &manual_ip) { this->manual_ip_ = manual_ip; } -std::string EthernetComponent::get_use_address() const { - if (this->use_address_.empty()) { - // ".local" suffix length for mDNS hostnames - constexpr size_t mdns_local_suffix_len = 5; - return make_name_with_suffix(App.get_name(), '.', "local", mdns_local_suffix_len); - } - return this->use_address_; -} +// set_use_address() is guaranteed to be called during component setup by Python code generation, +// so use_address_ will always be valid when get_use_address() is called - no fallback needed. +const std::string &EthernetComponent::get_use_address() const { return this->use_address_; } void EthernetComponent::set_use_address(const std::string &use_address) { this->use_address_ = use_address; } diff --git a/esphome/components/ethernet/ethernet_component.h b/esphome/components/ethernet/ethernet_component.h index 6b4e342df5..d5dda3e3ae 100644 --- a/esphome/components/ethernet/ethernet_component.h +++ b/esphome/components/ethernet/ethernet_component.h @@ -88,7 +88,7 @@ class EthernetComponent : public Component { network::IPAddresses get_ip_addresses(); network::IPAddress get_dns_address(uint8_t num); - std::string get_use_address() const; + const std::string &get_use_address() const; void set_use_address(const std::string &use_address); void get_eth_mac_address_raw(uint8_t *mac); std::string get_eth_mac_address_pretty(); diff --git a/esphome/components/network/util.cpp b/esphome/components/network/util.cpp index bf76aefc30..27ad9448a4 100644 --- a/esphome/components/network/util.cpp +++ b/esphome/components/network/util.cpp @@ -85,22 +85,25 @@ network::IPAddresses get_ip_addresses() { return {}; } -std::string get_use_address() { +const std::string &get_use_address() { + // Global component pointers are guaranteed to be set by component constructors when USE_* is defined #ifdef USE_ETHERNET - if (ethernet::global_eth_component != nullptr) - return ethernet::global_eth_component->get_use_address(); + return ethernet::global_eth_component->get_use_address(); #endif #ifdef USE_MODEM - if (modem::global_modem_component != nullptr) - return modem::global_modem_component->get_use_address(); + return modem::global_modem_component->get_use_address(); #endif #ifdef USE_WIFI - if (wifi::global_wifi_component != nullptr) - return wifi::global_wifi_component->get_use_address(); + return wifi::global_wifi_component->get_use_address(); +#endif + +#if !defined(USE_ETHERNET) && !defined(USE_MODEM) && !defined(USE_WIFI) + // Fallback when no network component is defined (e.g., host platform) + static const std::string empty; + return empty; #endif - return ""; } } // namespace network diff --git a/esphome/components/network/util.h b/esphome/components/network/util.h index b518696e68..b4a92f8bee 100644 --- a/esphome/components/network/util.h +++ b/esphome/components/network/util.h @@ -12,7 +12,7 @@ bool is_connected(); /// Return whether the network is disabled (only wifi for now) bool is_disabled(); /// Get the active network hostname -std::string get_use_address(); +const std::string &get_use_address(); IPAddresses get_ip_addresses(); } // namespace network diff --git a/esphome/components/wifi/wifi_component.cpp b/esphome/components/wifi/wifi_component.cpp index 0f9f879181..aa197b36e5 100644 --- a/esphome/components/wifi/wifi_component.cpp +++ b/esphome/components/wifi/wifi_component.cpp @@ -265,14 +265,9 @@ network::IPAddress WiFiComponent::get_dns_address(int num) { return this->wifi_dns_ip_(num); return {}; } -std::string WiFiComponent::get_use_address() const { - if (this->use_address_.empty()) { - // ".local" suffix length for mDNS hostnames - constexpr size_t mdns_local_suffix_len = 5; - return make_name_with_suffix(App.get_name(), '.', "local", mdns_local_suffix_len); - } - return this->use_address_; -} +// set_use_address() is guaranteed to be called during component setup by Python code generation, +// so use_address_ will always be valid when get_use_address() is called - no fallback needed. +const std::string &WiFiComponent::get_use_address() const { return this->use_address_; } void WiFiComponent::set_use_address(const std::string &use_address) { this->use_address_ = use_address; } #ifdef USE_WIFI_AP diff --git a/esphome/components/wifi/wifi_component.h b/esphome/components/wifi/wifi_component.h index ee62ec1a69..0e0f89d2c1 100644 --- a/esphome/components/wifi/wifi_component.h +++ b/esphome/components/wifi/wifi_component.h @@ -275,7 +275,7 @@ class WiFiComponent : public Component { network::IPAddress get_dns_address(int num); network::IPAddresses get_ip_addresses(); - std::string get_use_address() const; + const std::string &get_use_address() const; void set_use_address(const std::string &use_address); const std::vector &get_scan_result() const { return scan_result_; } From 8e9a68a107c12a8e4ff5256adcecc40badce3db9 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 13 Oct 2025 18:12:52 -1000 Subject: [PATCH 10/10] Bump aioesphomeapi from 41.16.0 to 41.16.1 (#11221) Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 1142c587b6..4425a4644e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -12,7 +12,7 @@ platformio==6.1.18 # When updating platformio, also update /docker/Dockerfile esptool==5.1.0 click==8.1.7 esphome-dashboard==20251013.0 -aioesphomeapi==41.16.0 +aioesphomeapi==41.16.1 zeroconf==0.148.0 puremagic==1.30 ruamel.yaml==0.18.15 # dashboard_import