From 7ab9083d778d47b7a88db8ec1d0c56c13027677b Mon Sep 17 00:00:00 2001 From: Edward Firmo <94725493+edwardtfn@users.noreply.github.com> Date: Sun, 22 Jun 2025 22:56:50 +0200 Subject: [PATCH 01/17] [nextion] Revert to `millis()` on `recv_ret_string_` (#9168) --- esphome/components/nextion/nextion.cpp | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/esphome/components/nextion/nextion.cpp b/esphome/components/nextion/nextion.cpp index 4f08fcb393..042a595ff8 100644 --- a/esphome/components/nextion/nextion.cpp +++ b/esphome/components/nextion/nextion.cpp @@ -963,13 +963,12 @@ uint16_t Nextion::recv_ret_string_(std::string &response, uint32_t timeout, bool uint16_t ret = 0; uint8_t c = 0; uint8_t nr_of_ff_bytes = 0; - uint64_t start; bool exit_flag = false; bool ff_flag = false; - start = App.get_loop_component_start_time(); + const uint32_t start = millis(); - while ((timeout == 0 && this->available()) || App.get_loop_component_start_time() - start <= timeout) { + while ((timeout == 0 && this->available()) || millis() - start <= timeout) { if (!this->available()) { App.feed_wdt(); delay(1); @@ -1038,7 +1037,7 @@ void Nextion::add_no_result_to_queue_(const std::string &variable_name) { nextion_queue->component = new nextion::NextionComponentBase; nextion_queue->component->set_variable_name(variable_name); - nextion_queue->queue_time = App.get_loop_component_start_time(); + nextion_queue->queue_time = millis(); this->nextion_queue_.push_back(nextion_queue); From dc5cbd4df8e658c48218f93e43dd1ecfc4d14d61 Mon Sep 17 00:00:00 2001 From: Jesse Hills <3060199+jesserockz@users.noreply.github.com> Date: Mon, 23 Jun 2025 09:54:49 +1200 Subject: [PATCH 02/17] [const] Move ``CONF_DEVICES`` to ``const.py`` (#9179) --- esphome/components/usb_host/__init__.py | 3 +-- esphome/const.py | 1 + 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/esphome/components/usb_host/__init__.py b/esphome/components/usb_host/__init__.py index 3204562dc8..0fe3310127 100644 --- a/esphome/components/usb_host/__init__.py +++ b/esphome/components/usb_host/__init__.py @@ -6,7 +6,7 @@ from esphome.components.esp32 import ( only_on_variant, ) import esphome.config_validation as cv -from esphome.const import CONF_ID +from esphome.const import CONF_DEVICES, CONF_ID from esphome.cpp_types import Component AUTO_LOAD = ["bytebuffer"] @@ -16,7 +16,6 @@ usb_host_ns = cg.esphome_ns.namespace("usb_host") USBHost = usb_host_ns.class_("USBHost", Component) USBClient = usb_host_ns.class_("USBClient", Component) -CONF_DEVICES = "devices" CONF_VID = "vid" CONF_PID = "pid" CONF_ENABLE_HUBS = "enable_hubs" diff --git a/esphome/const.py b/esphome/const.py index 69d75c81ce..e61af6c5b5 100644 --- a/esphome/const.py +++ b/esphome/const.py @@ -217,6 +217,7 @@ CONF_DEST = "dest" CONF_DEVICE = "device" CONF_DEVICE_CLASS = "device_class" CONF_DEVICE_FACTOR = "device_factor" +CONF_DEVICES = "devices" CONF_DIELECTRIC_CONSTANT = "dielectric_constant" CONF_DIMENSIONS = "dimensions" CONF_DIO_PIN = "dio_pin" From 59889a6286eed421c33880a1f931c0bc72733a95 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Mon, 23 Jun 2025 04:06:02 +0200 Subject: [PATCH 03/17] Reduce Logger memory usage by optimizing variable sizes (#9161) --- esphome/components/logger/__init__.py | 4 +- esphome/components/logger/logger.cpp | 20 ++--- esphome/components/logger/logger.h | 111 ++++++++++++++------------ esphome/core/log.cpp | 4 +- 4 files changed, 76 insertions(+), 63 deletions(-) diff --git a/esphome/components/logger/__init__.py b/esphome/components/logger/__init__.py index 26516e1506..af62d8a73f 100644 --- a/esphome/components/logger/__init__.py +++ b/esphome/components/logger/__init__.py @@ -184,7 +184,9 @@ CONFIG_SCHEMA = cv.All( { cv.GenerateID(): cv.declare_id(Logger), cv.Optional(CONF_BAUD_RATE, default=115200): cv.positive_int, - cv.Optional(CONF_TX_BUFFER_SIZE, default=512): cv.validate_bytes, + cv.Optional(CONF_TX_BUFFER_SIZE, default=512): cv.All( + cv.validate_bytes, cv.int_range(min=160, max=65535) + ), cv.Optional(CONF_DEASSERT_RTS_DTR, default=False): cv.boolean, cv.SplitDefault( CONF_TASK_LOG_BUFFER_SIZE, diff --git a/esphome/components/logger/logger.cpp b/esphome/components/logger/logger.cpp index 28a66b23b7..b42496af66 100644 --- a/esphome/components/logger/logger.cpp +++ b/esphome/components/logger/logger.cpp @@ -24,7 +24,7 @@ static const char *const TAG = "logger"; // - Messages are serialized through main loop for proper console output // - Fallback to emergency console logging only if ring buffer is full // - WITHOUT task log buffer: Only emergency console output, no callbacks -void HOT Logger::log_vprintf_(int level, const char *tag, int line, const char *format, va_list args) { // NOLINT +void HOT Logger::log_vprintf_(uint8_t level, const char *tag, int line, const char *format, va_list args) { // NOLINT if (level > this->level_for(tag)) return; @@ -46,8 +46,8 @@ void HOT Logger::log_vprintf_(int level, const char *tag, int line, const char * bool message_sent = false; #ifdef USE_ESPHOME_TASK_LOG_BUFFER // For non-main tasks, queue the message for callbacks - but only if we have any callbacks registered - message_sent = this->log_buffer_->send_message_thread_safe(static_cast(level), tag, - static_cast(line), current_task, format, args); + message_sent = + this->log_buffer_->send_message_thread_safe(level, tag, static_cast(line), current_task, format, args); #endif // USE_ESPHOME_TASK_LOG_BUFFER // Emergency console logging for non-main tasks when ring buffer is full or disabled @@ -58,7 +58,7 @@ void HOT Logger::log_vprintf_(int level, const char *tag, int line, const char * // Maximum size for console log messages (includes null terminator) static const size_t MAX_CONSOLE_LOG_MSG_SIZE = 144; char console_buffer[MAX_CONSOLE_LOG_MSG_SIZE]; // MUST be stack allocated for thread safety - int buffer_at = 0; // Initialize buffer position + uint16_t buffer_at = 0; // Initialize buffer position this->format_log_to_buffer_with_terminator_(level, tag, line, format, args, console_buffer, &buffer_at, MAX_CONSOLE_LOG_MSG_SIZE); this->write_msg_(console_buffer); @@ -69,7 +69,7 @@ void HOT Logger::log_vprintf_(int level, const char *tag, int line, const char * } #else // Implementation for all other platforms -void HOT Logger::log_vprintf_(int level, const char *tag, int line, const char *format, va_list args) { // NOLINT +void HOT Logger::log_vprintf_(uint8_t level, const char *tag, int line, const char *format, va_list args) { // NOLINT if (level > this->level_for(tag) || global_recursion_guard_) return; @@ -85,7 +85,7 @@ void HOT Logger::log_vprintf_(int level, const char *tag, int line, const char * #ifdef USE_STORE_LOG_STR_IN_FLASH // Implementation for ESP8266 with flash string support. // Note: USE_STORE_LOG_STR_IN_FLASH is only defined for ESP8266. -void Logger::log_vprintf_(int level, const char *tag, int line, const __FlashStringHelper *format, +void Logger::log_vprintf_(uint8_t level, const char *tag, int line, const __FlashStringHelper *format, va_list args) { // NOLINT if (level > this->level_for(tag) || global_recursion_guard_) return; @@ -122,7 +122,7 @@ void Logger::log_vprintf_(int level, const char *tag, int line, const __FlashStr } #endif // USE_STORE_LOG_STR_IN_FLASH -inline int Logger::level_for(const char *tag) { +inline uint8_t Logger::level_for(const char *tag) { auto it = this->log_levels_.find(tag); if (it != this->log_levels_.end()) return it->second; @@ -195,13 +195,13 @@ void Logger::loop() { #endif void Logger::set_baud_rate(uint32_t baud_rate) { this->baud_rate_ = baud_rate; } -void Logger::set_log_level(const std::string &tag, int log_level) { this->log_levels_[tag] = log_level; } +void Logger::set_log_level(const std::string &tag, uint8_t log_level) { this->log_levels_[tag] = log_level; } #if defined(USE_ESP32) || defined(USE_ESP8266) || defined(USE_RP2040) || defined(USE_LIBRETINY) UARTSelection Logger::get_uart() const { return this->uart_; } #endif -void Logger::add_on_log_callback(std::function &&callback) { +void Logger::add_on_log_callback(std::function &&callback) { this->log_callback_.add(std::move(callback)); } float Logger::get_setup_priority() const { return setup_priority::BUS + 500.0f; } @@ -230,7 +230,7 @@ void Logger::dump_config() { } } -void Logger::set_log_level(int level) { +void Logger::set_log_level(uint8_t level) { if (level > ESPHOME_LOG_LEVEL) { level = ESPHOME_LOG_LEVEL; ESP_LOGW(TAG, "Cannot set log level higher than pre-compiled %s", LOG_LEVELS[ESPHOME_LOG_LEVEL]); diff --git a/esphome/components/logger/logger.h b/esphome/components/logger/logger.h index 9f09208b66..ea82764393 100644 --- a/esphome/components/logger/logger.h +++ b/esphome/components/logger/logger.h @@ -61,7 +61,7 @@ static const char *const LOG_LEVEL_LETTERS[] = { * * Advanced configuration (pin selection, etc) is not supported. */ -enum UARTSelection { +enum UARTSelection : uint8_t { #ifdef USE_LIBRETINY UART_SELECTION_DEFAULT = 0, UART_SELECTION_UART0, @@ -129,10 +129,10 @@ class Logger : public Component { #endif /// Set the default log level for this logger. - void set_log_level(int level); + void set_log_level(uint8_t level); /// Set the log level of the specified tag. - void set_log_level(const std::string &tag, int log_level); - int get_log_level() { return this->current_level_; } + void set_log_level(const std::string &tag, uint8_t log_level); + uint8_t get_log_level() { return this->current_level_; } // ========== INTERNAL METHODS ========== // (In most use cases you won't need these) @@ -140,19 +140,20 @@ class Logger : public Component { void pre_setup(); void dump_config() override; - inline int level_for(const char *tag); + inline uint8_t level_for(const char *tag); /// Register a callback that will be called for every log message sent - void add_on_log_callback(std::function &&callback); + void add_on_log_callback(std::function &&callback); // add a listener for log level changes - void add_listener(std::function &&callback) { this->level_callback_.add(std::move(callback)); } + void add_listener(std::function &&callback) { this->level_callback_.add(std::move(callback)); } float get_setup_priority() const override; - void log_vprintf_(int level, const char *tag, int line, const char *format, va_list args); // NOLINT + void log_vprintf_(uint8_t level, const char *tag, int line, const char *format, va_list args); // NOLINT #ifdef USE_STORE_LOG_STR_IN_FLASH - void log_vprintf_(int level, const char *tag, int line, const __FlashStringHelper *format, va_list args); // NOLINT + void log_vprintf_(uint8_t level, const char *tag, int line, const __FlashStringHelper *format, + va_list args); // NOLINT #endif protected: @@ -160,8 +161,9 @@ class Logger : public Component { // Format a log message with printf-style arguments and write it to a buffer with header, footer, and null terminator // It's the caller's responsibility to initialize buffer_at (typically to 0) - inline void HOT format_log_to_buffer_with_terminator_(int level, const char *tag, int line, const char *format, - va_list args, char *buffer, int *buffer_at, int buffer_size) { + inline void HOT format_log_to_buffer_with_terminator_(uint8_t level, const char *tag, int line, const char *format, + va_list args, char *buffer, uint16_t *buffer_at, + uint16_t buffer_size) { #if defined(USE_ESP32) || defined(USE_LIBRETINY) this->write_header_to_buffer_(level, tag, line, this->get_thread_name_(), buffer, buffer_at, buffer_size); #else @@ -180,7 +182,7 @@ class Logger : public Component { } // Helper to format and send a log message to both console and callbacks - inline void HOT log_message_to_buffer_and_send_(int level, const char *tag, int line, const char *format, + inline void HOT log_message_to_buffer_and_send_(uint8_t level, const char *tag, int line, const char *format, va_list args) { // Format to tx_buffer and prepare for output this->tx_buffer_at_ = 0; // Initialize buffer position @@ -194,11 +196,12 @@ class Logger : public Component { } // Write the body of the log message to the buffer - inline void write_body_to_buffer_(const char *value, size_t length, char *buffer, int *buffer_at, int buffer_size) { + inline void write_body_to_buffer_(const char *value, size_t length, char *buffer, uint16_t *buffer_at, + uint16_t buffer_size) { // Calculate available space - const int available = buffer_size - *buffer_at; - if (available <= 0) + if (*buffer_at >= buffer_size) return; + const uint16_t available = buffer_size - *buffer_at; // Determine copy length (minimum of remaining capacity and string length) const size_t copy_len = (length < static_cast(available)) ? length : available; @@ -211,7 +214,7 @@ class Logger : public Component { } // Format string to explicit buffer with varargs - inline void printf_to_buffer_(char *buffer, int *buffer_at, int buffer_size, const char *format, ...) { + inline void printf_to_buffer_(char *buffer, uint16_t *buffer_at, uint16_t buffer_size, const char *format, ...) { va_list arg; va_start(arg, format); this->format_body_to_buffer_(buffer, buffer_at, buffer_size, format, arg); @@ -222,41 +225,50 @@ class Logger : public Component { const char *get_uart_selection_(); #endif + // Group 4-byte aligned members first uint32_t baud_rate_; char *tx_buffer_{nullptr}; - int tx_buffer_at_{0}; - int tx_buffer_size_{0}; +#ifdef USE_ARDUINO + Stream *hw_serial_{nullptr}; +#endif +#if defined(USE_ESP32) || defined(USE_LIBRETINY) + void *main_task_ = nullptr; // Only used for thread name identification +#endif +#ifdef USE_ESP32 + // Task-specific recursion guards: + // - Main task uses a dedicated member variable for efficiency + // - Other tasks use pthread TLS with a dynamically created key via pthread_key_create + pthread_key_t log_recursion_key_; // 4 bytes +#endif +#ifdef USE_ESP_IDF + uart_port_t uart_num_; // 4 bytes (enum defaults to int size) +#endif + + // Large objects (internally aligned) + std::map log_levels_{}; + CallbackManager log_callback_{}; + CallbackManager level_callback_{}; +#ifdef USE_ESPHOME_TASK_LOG_BUFFER + std::unique_ptr log_buffer_; // Will be initialized with init_log_buffer +#endif + + // Group smaller types together at the end + uint16_t tx_buffer_at_{0}; + uint16_t tx_buffer_size_{0}; + uint8_t current_level_{ESPHOME_LOG_LEVEL_VERY_VERBOSE}; #if defined(USE_ESP32) || defined(USE_ESP8266) || defined(USE_RP2040) UARTSelection uart_{UART_SELECTION_UART0}; #endif #ifdef USE_LIBRETINY UARTSelection uart_{UART_SELECTION_DEFAULT}; #endif -#ifdef USE_ARDUINO - Stream *hw_serial_{nullptr}; -#endif -#ifdef USE_ESP_IDF - uart_port_t uart_num_; -#endif - std::map log_levels_{}; - CallbackManager log_callback_{}; - int current_level_{ESPHOME_LOG_LEVEL_VERY_VERBOSE}; -#ifdef USE_ESPHOME_TASK_LOG_BUFFER - std::unique_ptr log_buffer_; // Will be initialized with init_log_buffer -#endif #ifdef USE_ESP32 - // Task-specific recursion guards: - // - Main task uses a dedicated member variable for efficiency - // - Other tasks use pthread TLS with a dynamically created key via pthread_key_create bool main_task_recursion_guard_{false}; - pthread_key_t log_recursion_key_; #else bool global_recursion_guard_{false}; // Simple global recursion guard for single-task platforms #endif - CallbackManager level_callback_{}; #if defined(USE_ESP32) || defined(USE_LIBRETINY) - void *main_task_ = nullptr; // Only used for thread name identification const char *HOT get_thread_name_() { TaskHandle_t current_task = xTaskGetCurrentTaskHandle(); if (current_task == main_task_) { @@ -297,11 +309,10 @@ class Logger : public Component { } #endif - inline void HOT write_header_to_buffer_(int level, const char *tag, int line, const char *thread_name, char *buffer, - int *buffer_at, int buffer_size) { + inline void HOT write_header_to_buffer_(uint8_t level, const char *tag, int line, const char *thread_name, + char *buffer, uint16_t *buffer_at, uint16_t buffer_size) { // Format header - if (level < 0) - level = 0; + // uint8_t level is already bounded 0-255, just ensure it's <= 7 if (level > 7) level = 7; @@ -320,12 +331,12 @@ class Logger : public Component { this->printf_to_buffer_(buffer, buffer_at, buffer_size, "%s[%s][%s:%03u]: ", color, letter, tag, line); } - inline void HOT format_body_to_buffer_(char *buffer, int *buffer_at, int buffer_size, const char *format, + inline void HOT format_body_to_buffer_(char *buffer, uint16_t *buffer_at, uint16_t buffer_size, const char *format, va_list args) { // Get remaining capacity in the buffer - const int remaining = buffer_size - *buffer_at; - if (remaining <= 0) + if (*buffer_at >= buffer_size) return; + const uint16_t remaining = buffer_size - *buffer_at; const int ret = vsnprintf(buffer + *buffer_at, remaining, format, args); @@ -334,7 +345,7 @@ class Logger : public Component { } // Update buffer_at with the formatted length (handle truncation) - int formatted_len = (ret >= remaining) ? remaining : ret; + uint16_t formatted_len = (ret >= remaining) ? remaining : ret; *buffer_at += formatted_len; // Remove all trailing newlines right after formatting @@ -343,18 +354,18 @@ class Logger : public Component { } } - inline void HOT write_footer_to_buffer_(char *buffer, int *buffer_at, int buffer_size) { - static const int RESET_COLOR_LEN = strlen(ESPHOME_LOG_RESET_COLOR); + inline void HOT write_footer_to_buffer_(char *buffer, uint16_t *buffer_at, uint16_t buffer_size) { + static const uint16_t RESET_COLOR_LEN = strlen(ESPHOME_LOG_RESET_COLOR); this->write_body_to_buffer_(ESPHOME_LOG_RESET_COLOR, RESET_COLOR_LEN, buffer, buffer_at, buffer_size); } }; extern Logger *global_logger; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables) -class LoggerMessageTrigger : public Trigger { +class LoggerMessageTrigger : public Trigger { public: - explicit LoggerMessageTrigger(Logger *parent, int level) { + explicit LoggerMessageTrigger(Logger *parent, uint8_t level) { this->level_ = level; - parent->add_on_log_callback([this](int level, const char *tag, const char *message) { + parent->add_on_log_callback([this](uint8_t level, const char *tag, const char *message) { if (level <= this->level_) { this->trigger(level, tag, message); } @@ -362,7 +373,7 @@ class LoggerMessageTrigger : public Trigger { } protected: - int level_; + uint8_t level_; }; } // namespace logger diff --git a/esphome/core/log.cpp b/esphome/core/log.cpp index 424154d253..909319dd28 100644 --- a/esphome/core/log.cpp +++ b/esphome/core/log.cpp @@ -29,7 +29,7 @@ void HOT esp_log_vprintf_(int level, const char *tag, int line, const char *form if (log == nullptr) return; - log->log_vprintf_(level, tag, line, format, args); + log->log_vprintf_(static_cast(level), tag, line, format, args); #endif } @@ -41,7 +41,7 @@ void HOT esp_log_vprintf_(int level, const char *tag, int line, const __FlashStr if (log == nullptr) return; - log->log_vprintf_(level, tag, line, format, args); + log->log_vprintf_(static_cast(level), tag, line, format, args); #endif } #endif From 04f592ba6d5e6d16b0978de04814abd3882f3666 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Mon, 23 Jun 2025 04:07:53 +0200 Subject: [PATCH 04/17] Fix slow noise handshake by reading multiple messages per loop (#9130) --- esphome/components/api/api_connection.cpp | 60 ++++++++++++--------- esphome/components/api/api_frame_helper.cpp | 17 ++++-- 2 files changed, 48 insertions(+), 29 deletions(-) diff --git a/esphome/components/api/api_connection.cpp b/esphome/components/api/api_connection.cpp index ca5689bdf6..ef791d462c 100644 --- a/esphome/components/api/api_connection.cpp +++ b/esphome/components/api/api_connection.cpp @@ -28,6 +28,12 @@ namespace esphome { namespace api { +// Read a maximum of 5 messages per loop iteration to prevent starving other components. +// This is a balance between API responsiveness and allowing other components to run. +// Since each message could contain multiple protobuf messages when using packet batching, +// this limits the number of messages processed, not the number of TCP packets. +static constexpr uint8_t MAX_MESSAGES_PER_LOOP = 5; + static const char *const TAG = "api.connection"; static const int ESP32_CAMERA_STOP_STREAM = 5000; @@ -109,33 +115,38 @@ void APIConnection::loop() { return; } + const uint32_t now = App.get_loop_component_start_time(); // Check if socket has data ready before attempting to read if (this->helper_->is_socket_ready()) { - ReadPacketBuffer buffer; - err = this->helper_->read_packet(&buffer); - if (err == APIError::WOULD_BLOCK) { - // pass - } else if (err != APIError::OK) { - on_fatal_error(); - if (err == APIError::SOCKET_READ_FAILED && errno == ECONNRESET) { - ESP_LOGW(TAG, "%s: Connection reset", this->get_client_combined_info().c_str()); - } else if (err == APIError::CONNECTION_CLOSED) { - ESP_LOGW(TAG, "%s: Connection closed", this->get_client_combined_info().c_str()); - } else { - ESP_LOGW(TAG, "%s: Reading failed: %s errno=%d", this->get_client_combined_info().c_str(), - api_error_to_str(err), errno); - } - return; - } else { - this->last_traffic_ = App.get_loop_component_start_time(); - // read a packet - if (buffer.data_len > 0) { - this->read_message(buffer.data_len, buffer.type, &buffer.container[buffer.data_offset]); - } else { - this->read_message(0, buffer.type, nullptr); - } - if (this->remove_) + // Read up to MAX_MESSAGES_PER_LOOP messages per loop to improve throughput + for (uint8_t message_count = 0; message_count < MAX_MESSAGES_PER_LOOP; message_count++) { + ReadPacketBuffer buffer; + err = this->helper_->read_packet(&buffer); + if (err == APIError::WOULD_BLOCK) { + // No more data available + break; + } else if (err != APIError::OK) { + on_fatal_error(); + if (err == APIError::SOCKET_READ_FAILED && errno == ECONNRESET) { + ESP_LOGW(TAG, "%s: Connection reset", this->get_client_combined_info().c_str()); + } else if (err == APIError::CONNECTION_CLOSED) { + ESP_LOGW(TAG, "%s: Connection closed", this->get_client_combined_info().c_str()); + } else { + ESP_LOGW(TAG, "%s: Reading failed: %s errno=%d", this->get_client_combined_info().c_str(), + api_error_to_str(err), errno); + } return; + } else { + this->last_traffic_ = now; + // read a packet + if (buffer.data_len > 0) { + this->read_message(buffer.data_len, buffer.type, &buffer.container[buffer.data_offset]); + } else { + this->read_message(0, buffer.type, nullptr); + } + if (this->remove_) + return; + } } } @@ -152,7 +163,6 @@ void APIConnection::loop() { static uint8_t max_ping_retries = 60; static uint16_t ping_retry_interval = 1000; - const uint32_t now = App.get_loop_component_start_time(); if (this->sent_ping_) { // Disconnect if not responded within 2.5*keepalive if (now - this->last_traffic_ > (KEEPALIVE_TIMEOUT_MS * 5) / 2) { diff --git a/esphome/components/api/api_frame_helper.cpp b/esphome/components/api/api_frame_helper.cpp index e0eb94836d..ff660f439e 100644 --- a/esphome/components/api/api_frame_helper.cpp +++ b/esphome/components/api/api_frame_helper.cpp @@ -274,12 +274,21 @@ APIError APINoiseFrameHelper::init() { } /// Run through handshake messages (if in that phase) APIError APINoiseFrameHelper::loop() { - APIError err = state_action_(); - if (err != APIError::OK && err != APIError::WOULD_BLOCK) { - return err; + // During handshake phase, process as many actions as possible until we can't progress + // socket_->ready() stays true until next main loop, but state_action() will return + // WOULD_BLOCK when no more data is available to read + while (state_ != State::DATA && this->socket_->ready()) { + APIError err = state_action_(); + if (err != APIError::OK && err != APIError::WOULD_BLOCK) { + return err; + } + if (err == APIError::WOULD_BLOCK) { + break; + } } + if (!this->tx_buf_.empty()) { - err = try_send_tx_buf_(); + APIError err = try_send_tx_buf_(); if (err != APIError::OK && err != APIError::WOULD_BLOCK) { return err; } From 7fc5bfd787794431cc386c6f1be09b42fef51d0e Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Mon, 23 Jun 2025 04:09:34 +0200 Subject: [PATCH 05/17] Reduce RAM usage for scheduled tasks (#9180) --- esphome/core/scheduler.cpp | 4 ++++ esphome/core/scheduler.h | 8 ++++++-- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/esphome/core/scheduler.cpp b/esphome/core/scheduler.cpp index eed222c974..8144435163 100644 --- a/esphome/core/scheduler.cpp +++ b/esphome/core/scheduler.cpp @@ -319,13 +319,17 @@ bool HOT Scheduler::cancel_item_(Component *component, const std::string &name, return ret; } uint64_t Scheduler::millis_() { + // Get the current 32-bit millis value const uint32_t now = millis(); + // Check for rollover by comparing with last value if (now < this->last_millis_) { + // Detected rollover (happens every ~49.7 days) this->millis_major_++; ESP_LOGD(TAG, "Incrementing scheduler major at %" PRIu64 "ms", now + (static_cast(this->millis_major_) << 32)); } this->last_millis_ = now; + // Combine major (high 32 bits) and now (low 32 bits) into 64-bit time return now + (static_cast(this->millis_major_) << 32); } diff --git a/esphome/core/scheduler.h b/esphome/core/scheduler.h index 872a8bd6f6..1284bcd4a7 100644 --- a/esphome/core/scheduler.h +++ b/esphome/core/scheduler.h @@ -29,12 +29,16 @@ class Scheduler { protected: struct SchedulerItem { + // Ordered by size to minimize padding Component *component; - std::string name; - enum Type { TIMEOUT, INTERVAL } type; uint32_t interval; + // 64-bit time to handle millis() rollover. The scheduler combines the 32-bit millis() + // with a 16-bit rollover counter to create a 64-bit time that won't roll over for + // billions of years. This ensures correct scheduling even when devices run for months. uint64_t next_execution_; + std::string name; std::function callback; + enum Type : uint8_t { TIMEOUT, INTERVAL } type; bool remove; static bool cmp(const std::unique_ptr &a, const std::unique_ptr &b); From 2a45467bf62fcde5933d55213a582d19627946b6 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Mon, 23 Jun 2025 04:10:09 +0200 Subject: [PATCH 06/17] Pre-reserve looping components vector to reduce memory allocations (#9177) --- esphome/core/application.cpp | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/esphome/core/application.cpp b/esphome/core/application.cpp index 49c1e5fd61..f64070fa3d 100644 --- a/esphome/core/application.cpp +++ b/esphome/core/application.cpp @@ -257,6 +257,17 @@ void Application::teardown_components(uint32_t timeout_ms) { } void Application::calculate_looping_components_() { + // Count total components that need looping + size_t total_looping = 0; + for (auto *obj : this->components_) { + if (obj->has_overridden_loop()) { + total_looping++; + } + } + + // Pre-reserve vector to avoid reallocations + this->looping_components_.reserve(total_looping); + // First add all active components for (auto *obj : this->components_) { if (obj->has_overridden_loop() && From 7aea82a273867b97e588a012bffbe4fb63527d06 Mon Sep 17 00:00:00 2001 From: Jesse Hills <3060199+jesserockz@users.noreply.github.com> Date: Mon, 23 Jun 2025 14:15:10 +1200 Subject: [PATCH 07/17] Move define --- esphome/core/defines.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/esphome/core/defines.h b/esphome/core/defines.h index 043ab13f7a..62aac4382c 100644 --- a/esphome/core/defines.h +++ b/esphome/core/defines.h @@ -17,7 +17,6 @@ // logger #define ESPHOME_LOG_LEVEL ESPHOME_LOG_LEVEL_VERY_VERBOSE -#define USE_ESPHOME_TASK_LOG_BUFFER // Feature flags #define USE_ALARM_CONTROL_PANEL @@ -131,6 +130,8 @@ // ESP32-specific feature flags #ifdef USE_ESP32 +#define USE_ESPHOME_TASK_LOG_BUFFER + #define USE_BLUETOOTH_PROXY #define USE_CAPTIVE_PORTAL #define USE_ESP32_BLE From 78ec9856fbcf9a01e760f57ec5c2233e1cc8ef4d Mon Sep 17 00:00:00 2001 From: Clyde Stubbs <2366188+clydebarrow@users.noreply.github.com> Date: Mon, 23 Jun 2025 12:23:41 +1000 Subject: [PATCH 08/17] [lvgl] Add start_value to bar; make values templatable and updateable (#9056) --- esphome/components/lvgl/schemas.py | 8 ++- esphome/components/lvgl/widgets/lv_bar.py | 59 +++++++++++++++-------- tests/components/lvgl/lvgl-package.yaml | 3 ++ 3 files changed, 49 insertions(+), 21 deletions(-) diff --git a/esphome/components/lvgl/schemas.py b/esphome/components/lvgl/schemas.py index fdc8750d1d..a0be65c928 100644 --- a/esphome/components/lvgl/schemas.py +++ b/esphome/components/lvgl/schemas.py @@ -454,9 +454,13 @@ def container_validator(schema, widget_type: WidgetType): """ def validator(value): - result = schema if w_sch := widget_type.schema: - result = result.extend(w_sch) + if isinstance(w_sch, dict): + w_sch = cv.Schema(w_sch) + # order is important here to preserve extras + result = w_sch.extend(schema) + else: + result = schema ltype = df.TYPE_NONE if value and (layout := value.get(df.CONF_LAYOUT)): if not isinstance(layout, dict): diff --git a/esphome/components/lvgl/widgets/lv_bar.py b/esphome/components/lvgl/widgets/lv_bar.py index 57209370c0..f0fdd6d278 100644 --- a/esphome/components/lvgl/widgets/lv_bar.py +++ b/esphome/components/lvgl/widgets/lv_bar.py @@ -1,8 +1,15 @@ import esphome.config_validation as cv from esphome.const import CONF_MAX_VALUE, CONF_MIN_VALUE, CONF_MODE, CONF_VALUE -from ..defines import BAR_MODES, CONF_ANIMATED, CONF_INDICATOR, CONF_MAIN, literal -from ..lv_validation import animated, get_start_value, lv_float +from ..defines import ( + BAR_MODES, + CONF_ANIMATED, + CONF_INDICATOR, + CONF_MAIN, + CONF_START_VALUE, + literal, +) +from ..lv_validation import animated, lv_int from ..lvcode import lv from ..types import LvNumber, NumberType from . import Widget @@ -10,22 +17,30 @@ from . import Widget # Note this file cannot be called "bar.py" because that name is disallowed. CONF_BAR = "bar" -BAR_MODIFY_SCHEMA = cv.Schema( - { - cv.Optional(CONF_VALUE): lv_float, - cv.Optional(CONF_ANIMATED, default=True): animated, - } -) + + +def validate_bar(config): + if config.get(CONF_MODE) != "LV_BAR_MODE_RANGE" and CONF_START_VALUE in config: + raise cv.Invalid( + f"{CONF_START_VALUE} is only allowed when {CONF_MODE} is set to 'RANGE'" + ) + if (CONF_MIN_VALUE in config) != (CONF_MAX_VALUE in config): + raise cv.Invalid( + f"If either {CONF_MIN_VALUE} or {CONF_MAX_VALUE} is set, both must be set" + ) + return config + BAR_SCHEMA = cv.Schema( { - cv.Optional(CONF_VALUE): lv_float, - cv.Optional(CONF_MIN_VALUE, default=0): cv.int_, - cv.Optional(CONF_MAX_VALUE, default=100): cv.int_, - cv.Optional(CONF_MODE, default="NORMAL"): BAR_MODES.one_of, + cv.Optional(CONF_VALUE): lv_int, + cv.Optional(CONF_START_VALUE): lv_int, + cv.Optional(CONF_MIN_VALUE): lv_int, + cv.Optional(CONF_MAX_VALUE): lv_int, + cv.Optional(CONF_MODE): BAR_MODES.one_of, cv.Optional(CONF_ANIMATED, default=True): animated, } -) +).add_extra(validate_bar) class BarType(NumberType): @@ -35,17 +50,23 @@ class BarType(NumberType): LvNumber("lv_bar_t"), parts=(CONF_MAIN, CONF_INDICATOR), schema=BAR_SCHEMA, - modify_schema=BAR_MODIFY_SCHEMA, ) async def to_code(self, w: Widget, config): var = w.obj + if mode := config.get(CONF_MODE): + lv.bar_set_mode(var, literal(mode)) + is_animated = literal(config[CONF_ANIMATED]) if CONF_MIN_VALUE in config: - lv.bar_set_range(var, config[CONF_MIN_VALUE], config[CONF_MAX_VALUE]) - lv.bar_set_mode(var, literal(config[CONF_MODE])) - value = await get_start_value(config) - if value is not None: - lv.bar_set_value(var, value, literal(config[CONF_ANIMATED])) + lv.bar_set_range( + var, + await lv_int.process(config[CONF_MIN_VALUE]), + await lv_int.process(config[CONF_MAX_VALUE]), + ) + if value := await lv_int.process(config.get(CONF_VALUE)): + lv.bar_set_value(var, value, is_animated) + if start_value := await lv_int.process(config.get(CONF_START_VALUE)): + lv.bar_set_start_value(var, start_value, is_animated) @property def animated(self): diff --git a/tests/components/lvgl/lvgl-package.yaml b/tests/components/lvgl/lvgl-package.yaml index d8452bdd2a..f32b09d0e6 100644 --- a/tests/components/lvgl/lvgl-package.yaml +++ b/tests/components/lvgl/lvgl-package.yaml @@ -728,12 +728,15 @@ lvgl: value: 30 max_value: 100 min_value: 10 + start_value: 20 mode: range on_click: then: - lvgl.bar.update: id: bar_id value: !lambda return (int)((float)rand() / RAND_MAX * 100); + start_value: !lambda return (int)((float)rand() / RAND_MAX * 100); + mode: symmetrical - logger.log: format: "bar value %f" args: [x] From 41c78521280e0376f5f2e0967cba784f44428504 Mon Sep 17 00:00:00 2001 From: Clyde Stubbs <2366188+clydebarrow@users.noreply.github.com> Date: Mon, 23 Jun 2025 12:25:26 +1000 Subject: [PATCH 09/17] [lvgl] Use styles instead of object properties for themes (#9116) --- esphome/components/lvgl/styles.py | 40 ++++++++++++--------- esphome/components/lvgl/widgets/__init__.py | 14 ++++++-- 2 files changed, 36 insertions(+), 18 deletions(-) diff --git a/esphome/components/lvgl/styles.py b/esphome/components/lvgl/styles.py index b59ff513e2..426dd3f229 100644 --- a/esphome/components/lvgl/styles.py +++ b/esphome/components/lvgl/styles.py @@ -3,7 +3,6 @@ import esphome.codegen as cg import esphome.config_validation as cv from esphome.const import CONF_ID from esphome.core import ID -from esphome.cpp_generator import MockObj from .defines import ( CONF_STYLE_DEFINITIONS, @@ -13,12 +12,13 @@ from .defines import ( literal, ) from .helpers import add_lv_use -from .lvcode import LambdaContext, LocalVariable, lv, lv_assign, lv_variable +from .lvcode import LambdaContext, LocalVariable, lv from .schemas import ALL_STYLES, FULL_STYLE_SCHEMA, STYLE_REMAP -from .types import ObjUpdateAction, lv_lambda_t, lv_obj_t, lv_obj_t_ptr, lv_style_t +from .types import ObjUpdateAction, lv_obj_t, lv_style_t from .widgets import ( Widget, add_widgets, + collect_parts, set_obj_properties, theme_widget_map, wait_for_widgets, @@ -37,12 +37,18 @@ async def style_set(svar, style): lv.call(f"style_set_{remapped_prop}", svar, literal(value)) +async def create_style(style, id_name): + style_id = ID(id_name, True, lv_style_t) + svar = cg.new_Pvariable(style_id) + lv.style_init(svar) + await style_set(svar, style) + return svar + + async def styles_to_code(config): """Convert styles to C__ code.""" for style in config.get(CONF_STYLE_DEFINITIONS, ()): - svar = cg.new_Pvariable(style[CONF_ID]) - lv.style_init(svar) - await style_set(svar, style) + await create_style(style, style[CONF_ID].id) @automation.register_action( @@ -68,16 +74,18 @@ async def theme_to_code(config): if theme := config.get(CONF_THEME): add_lv_use(CONF_THEME) for w_name, style in theme.items(): - if not isinstance(style, dict): - continue - - lname = "lv_theme_apply_" + w_name - apply = lv_variable(lv_lambda_t, lname) - theme_widget_map[w_name] = apply - ow = Widget.create("obj", MockObj(ID("obj")), obj_spec) - async with LambdaContext([(lv_obj_t_ptr, "obj")], where=w_name) as context: - await set_obj_properties(ow, style) - lv_assign(apply, await context.get_lambda()) + # Work around Python 3.10 bug with nested async comprehensions + # With Python 3.11 this could be simplified + styles = {} + for part, states in collect_parts(style).items(): + styles[part] = { + state: await create_style( + props, + "_lv_theme_style_" + w_name + "_" + part + "_" + state, + ) + for state, props in states.items() + } + theme_widget_map[w_name] = styles async def add_top_layer(lv_component, config): diff --git a/esphome/components/lvgl/widgets/__init__.py b/esphome/components/lvgl/widgets/__init__.py index 9d53c0df26..a8cb8dce33 100644 --- a/esphome/components/lvgl/widgets/__init__.py +++ b/esphome/components/lvgl/widgets/__init__.py @@ -6,7 +6,7 @@ from esphome.config_validation import Invalid from esphome.const import CONF_DEFAULT, CONF_GROUP, CONF_ID, CONF_STATE, CONF_TYPE from esphome.core import ID, TimePeriod from esphome.coroutine import FakeAwaitable -from esphome.cpp_generator import CallExpression, MockObj +from esphome.cpp_generator import MockObj from ..defines import ( CONF_FLEX_ALIGN_CROSS, @@ -453,7 +453,17 @@ async def widget_to_code(w_cnfig, w_type: WidgetType, parent): w = Widget.create(wid, var, spec, w_cnfig) if theme := theme_widget_map.get(w_type): - lv_add(CallExpression(theme, w.obj)) + for part, states in theme.items(): + part = "LV_PART_" + part.upper() + for state, style in states.items(): + state = "LV_STATE_" + state.upper() + if state == "LV_STATE_DEFAULT": + lv_state = literal(part) + elif part == "LV_PART_MAIN": + lv_state = literal(state) + else: + lv_state = join_enums((state, part)) + lv.obj_add_style(w.obj, style, lv_state) await set_obj_properties(w, w_cnfig) await add_widgets(w, w_cnfig) await spec.to_code(w, w_cnfig) From 87df3596a2236ad3c503fc4b1c75bbbe843364d4 Mon Sep 17 00:00:00 2001 From: Jesse Hills <3060199+jesserockz@users.noreply.github.com> Date: Mon, 23 Jun 2025 15:41:06 +1200 Subject: [PATCH 10/17] [config validation] Add more ip address / network validators (#9181) --- esphome/config_validation.py | 45 +++++++++++++++++++++++++++++++++++- esphome/yaml_util.py | 3 ++- 2 files changed, 46 insertions(+), 2 deletions(-) diff --git a/esphome/config_validation.py b/esphome/config_validation.py index 964f533215..bf69b81bb5 100644 --- a/esphome/config_validation.py +++ b/esphome/config_validation.py @@ -3,7 +3,15 @@ from contextlib import contextmanager from dataclasses import dataclass from datetime import datetime -from ipaddress import AddressValueError, IPv4Address, ip_address +from ipaddress import ( + AddressValueError, + IPv4Address, + IPv4Network, + IPv6Address, + IPv6Network, + ip_address, + ip_network, +) import logging import os import re @@ -1176,6 +1184,14 @@ def ipv4address(value): return address +def ipv6address(value): + try: + address = IPv6Address(value) + except AddressValueError as exc: + raise Invalid(f"{value} is not a valid IPv6 address") from exc + return address + + def ipv4address_multi_broadcast(value): address = ipv4address(value) if not (address.is_multicast or (address == IPv4Address("255.255.255.255"))): @@ -1193,6 +1209,33 @@ def ipaddress(value): return address +def ipv4network(value): + """Validate that the value is a valid IPv4 network.""" + try: + network = IPv4Network(value, strict=False) + except ValueError as exc: + raise Invalid(f"{value} is not a valid IPv4 network") from exc + return network + + +def ipv6network(value): + """Validate that the value is a valid IPv6 network.""" + try: + network = IPv6Network(value, strict=False) + except ValueError as exc: + raise Invalid(f"{value} is not a valid IPv6 network") from exc + return network + + +def ipnetwork(value): + """Validate that the value is a valid IP network.""" + try: + network = ip_network(value, strict=False) + except ValueError as exc: + raise Invalid(f"{value} is not a valid IP network") from exc + return network + + def _valid_topic(value): """Validate that this is a valid topic name/filter.""" if value is None: # Used to disable publishing and subscribing diff --git a/esphome/yaml_util.py b/esphome/yaml_util.py index 78deec8e65..bd1806affc 100644 --- a/esphome/yaml_util.py +++ b/esphome/yaml_util.py @@ -5,7 +5,7 @@ import fnmatch import functools import inspect from io import BytesIO, TextIOBase, TextIOWrapper -from ipaddress import _BaseAddress +from ipaddress import _BaseAddress, _BaseNetwork import logging import math import os @@ -621,6 +621,7 @@ ESPHomeDumper.add_multi_representer(str, ESPHomeDumper.represent_stringify) ESPHomeDumper.add_multi_representer(int, ESPHomeDumper.represent_int) ESPHomeDumper.add_multi_representer(float, ESPHomeDumper.represent_float) ESPHomeDumper.add_multi_representer(_BaseAddress, ESPHomeDumper.represent_stringify) +ESPHomeDumper.add_multi_representer(_BaseNetwork, ESPHomeDumper.represent_stringify) ESPHomeDumper.add_multi_representer(MACAddress, ESPHomeDumper.represent_stringify) ESPHomeDumper.add_multi_representer(TimePeriod, ESPHomeDumper.represent_stringify) ESPHomeDumper.add_multi_representer(Lambda, ESPHomeDumper.represent_lambda) From aecaffa2f5a9e35d1aa0773b17f07faa10dc8c98 Mon Sep 17 00:00:00 2001 From: rwrozelle Date: Sun, 22 Jun 2025 23:41:29 -0400 Subject: [PATCH 11/17] Fixes for setup of OpenThread either using TLV or entering Credentials directly (#9157) Co-authored-by: Jesse Hills <3060199+jesserockz@users.noreply.github.com> --- esphome/components/openthread/__init__.py | 10 +++++----- esphome/components/openthread/openthread.cpp | 6 +++--- esphome/components/openthread/tlv.py | 7 +++++++ tests/components/openthread/test.esp32-c6-idf.yaml | 2 ++ 4 files changed, 17 insertions(+), 8 deletions(-) diff --git a/esphome/components/openthread/__init__.py b/esphome/components/openthread/__init__.py index 5b1ea491e3..393c47e720 100644 --- a/esphome/components/openthread/__init__.py +++ b/esphome/components/openthread/__init__.py @@ -46,7 +46,7 @@ def set_sdkconfig_options(config): add_idf_sdkconfig_option("CONFIG_OPENTHREAD_NETWORK_PANID", config[CONF_PAN_ID]) add_idf_sdkconfig_option("CONFIG_OPENTHREAD_NETWORK_CHANNEL", config[CONF_CHANNEL]) add_idf_sdkconfig_option( - "CONFIG_OPENTHREAD_NETWORK_MASTERKEY", f"{config[CONF_NETWORK_KEY]:X}" + "CONFIG_OPENTHREAD_NETWORK_MASTERKEY", f"{config[CONF_NETWORK_KEY]:X}".lower() ) if network_name := config.get(CONF_NETWORK_NAME): @@ -54,14 +54,14 @@ def set_sdkconfig_options(config): if (ext_pan_id := config.get(CONF_EXT_PAN_ID)) is not None: add_idf_sdkconfig_option( - "CONFIG_OPENTHREAD_NETWORK_EXTPANID", f"{ext_pan_id:X}" + "CONFIG_OPENTHREAD_NETWORK_EXTPANID", f"{ext_pan_id:X}".lower() ) if (mesh_local_prefix := config.get(CONF_MESH_LOCAL_PREFIX)) is not None: add_idf_sdkconfig_option( - "CONFIG_OPENTHREAD_MESH_LOCAL_PREFIX", f"{mesh_local_prefix:X}" + "CONFIG_OPENTHREAD_MESH_LOCAL_PREFIX", f"{mesh_local_prefix}".lower() ) if (pskc := config.get(CONF_PSKC)) is not None: - add_idf_sdkconfig_option("CONFIG_OPENTHREAD_NETWORK_PSKC", f"{pskc:X}") + add_idf_sdkconfig_option("CONFIG_OPENTHREAD_NETWORK_PSKC", f"{pskc:X}".lower()) if CONF_FORCE_DATASET in config: if config[CONF_FORCE_DATASET]: @@ -98,7 +98,7 @@ _CONNECTION_SCHEMA = cv.Schema( cv.Optional(CONF_EXT_PAN_ID): cv.hex_int, cv.Optional(CONF_NETWORK_NAME): cv.string_strict, cv.Optional(CONF_PSKC): cv.hex_int, - cv.Optional(CONF_MESH_LOCAL_PREFIX): cv.hex_int, + cv.Optional(CONF_MESH_LOCAL_PREFIX): cv.ipv6network, } ) diff --git a/esphome/components/openthread/openthread.cpp b/esphome/components/openthread/openthread.cpp index f40a56952a..24b3c23960 100644 --- a/esphome/components/openthread/openthread.cpp +++ b/esphome/components/openthread/openthread.cpp @@ -137,7 +137,7 @@ void OpenThreadSrpComponent::setup() { // Copy the mdns services to our local instance so that the c_str pointers remain valid for the lifetime of this // component this->mdns_services_ = this->mdns_->get_services(); - ESP_LOGW(TAG, "Setting up SRP services. count = %d\n", this->mdns_services_.size()); + ESP_LOGD(TAG, "Setting up SRP services. count = %d\n", this->mdns_services_.size()); for (const auto &service : this->mdns_services_) { otSrpClientBuffersServiceEntry *entry = otSrpClientBuffersAllocateService(instance); if (!entry) { @@ -185,11 +185,11 @@ void OpenThreadSrpComponent::setup() { if (error != OT_ERROR_NONE) { ESP_LOGW(TAG, "Failed to add service: %s", otThreadErrorToString(error)); } - ESP_LOGW(TAG, "Added service: %s", full_service.c_str()); + ESP_LOGD(TAG, "Added service: %s", full_service.c_str()); } otSrpClientEnableAutoStartMode(instance, srp_start_callback, nullptr); - ESP_LOGW(TAG, "Finished SRP setup"); + ESP_LOGD(TAG, "Finished SRP setup"); } void *OpenThreadSrpComponent::pool_alloc_(size_t size) { diff --git a/esphome/components/openthread/tlv.py b/esphome/components/openthread/tlv.py index 45c8c47227..4a7d21c47d 100644 --- a/esphome/components/openthread/tlv.py +++ b/esphome/components/openthread/tlv.py @@ -1,5 +1,6 @@ # Sourced from https://gist.github.com/agners/0338576e0003318b63ec1ea75adc90f9 import binascii +import ipaddress from esphome.const import CONF_CHANNEL @@ -37,6 +38,12 @@ def parse_tlv(tlv) -> dict: if tag in TLV_TYPES: if tag == 3: output[TLV_TYPES[tag]] = val.decode("utf-8") + elif tag == 7: + mesh_local_prefix = binascii.hexlify(val).decode("utf-8") + mesh_local_prefix_str = f"{mesh_local_prefix}0000000000000000" + ipv6_bytes = bytes.fromhex(mesh_local_prefix_str) + ipv6_address = ipaddress.IPv6Address(ipv6_bytes) + output[TLV_TYPES[tag]] = f"{ipv6_address}/64" else: output[TLV_TYPES[tag]] = int.from_bytes(val) return output diff --git a/tests/components/openthread/test.esp32-c6-idf.yaml b/tests/components/openthread/test.esp32-c6-idf.yaml index 482fd1a453..f53b323bec 100644 --- a/tests/components/openthread/test.esp32-c6-idf.yaml +++ b/tests/components/openthread/test.esp32-c6-idf.yaml @@ -8,4 +8,6 @@ openthread: pan_id: 0x8f28 ext_pan_id: 0xd63e8e3e495ebbc3 pskc: 0xc23a76e98f1a6483639b1ac1271e2e27 + mesh_local_prefix: fd53:145f:ed22:ad81::/64 force_dataset: true + From cd22723623c78933764ff246fabd92eadeac87af Mon Sep 17 00:00:00 2001 From: myhomeiot <70070601+myhomeiot@users.noreply.github.com> Date: Mon, 23 Jun 2025 06:42:20 +0300 Subject: [PATCH 12/17] Restore access to BLEScanResult as get_scan_result (#9148) --- esphome/components/esp32_ble_tracker/esp32_ble_tracker.cpp | 1 + esphome/components/esp32_ble_tracker/esp32_ble_tracker.h | 4 ++++ 2 files changed, 5 insertions(+) diff --git a/esphome/components/esp32_ble_tracker/esp32_ble_tracker.cpp b/esphome/components/esp32_ble_tracker/esp32_ble_tracker.cpp index 4785c29230..d950ccb5f1 100644 --- a/esphome/components/esp32_ble_tracker/esp32_ble_tracker.cpp +++ b/esphome/components/esp32_ble_tracker/esp32_ble_tracker.cpp @@ -522,6 +522,7 @@ optional ESPBLEiBeacon::from_manufacturer_data(const ServiceData } void ESPBTDevice::parse_scan_rst(const BLEScanResult &scan_result) { + this->scan_result_ = &scan_result; for (uint8_t i = 0; i < ESP_BD_ADDR_LEN; i++) this->address_[i] = scan_result.bda[i]; this->address_type_ = static_cast(scan_result.ble_addr_type); diff --git a/esphome/components/esp32_ble_tracker/esp32_ble_tracker.h b/esphome/components/esp32_ble_tracker/esp32_ble_tracker.h index 414c9f4b48..f5ed75a93e 100644 --- a/esphome/components/esp32_ble_tracker/esp32_ble_tracker.h +++ b/esphome/components/esp32_ble_tracker/esp32_ble_tracker.h @@ -85,6 +85,9 @@ class ESPBTDevice { const std::vector &get_service_datas() const { return service_datas_; } + // Exposed through a function for use in lambdas + const BLEScanResult &get_scan_result() const { return *scan_result_; } + bool resolve_irk(const uint8_t *irk) const; optional get_ibeacon() const { @@ -111,6 +114,7 @@ class ESPBTDevice { std::vector service_uuids_{}; std::vector manufacturer_datas_{}; std::vector service_datas_{}; + const BLEScanResult *scan_result_{nullptr}; }; class ESP32BLETracker; From 1a47164876cd59dc7e4f7aeafda5829916e07f6d Mon Sep 17 00:00:00 2001 From: JonasB2497 <45214989+JonasB2497@users.noreply.github.com> Date: Mon, 23 Jun 2025 06:47:47 +0200 Subject: [PATCH 13/17] Feature fontmetrics (#8978) --- esphome/components/font/__init__.py | 174 +++++++++++++++------------- esphome/components/font/font.cpp | 11 +- esphome/components/font/font.h | 21 +++- 3 files changed, 119 insertions(+), 87 deletions(-) diff --git a/esphome/components/font/__init__.py b/esphome/components/font/__init__.py index be88fdb957..7d9a35647e 100644 --- a/esphome/components/font/__init__.py +++ b/esphome/components/font/__init__.py @@ -1,6 +1,7 @@ from collections.abc import MutableMapping import functools import hashlib +from itertools import accumulate import logging import os from pathlib import Path @@ -468,8 +469,9 @@ class EFont: class GlyphInfo: - def __init__(self, data_len, advance, offset_x, offset_y, width, height): - self.data_len = data_len + def __init__(self, glyph, data, advance, offset_x, offset_y, width, height): + self.glyph = glyph + self.bitmap_data = data self.advance = advance self.offset_x = offset_x self.offset_y = offset_y @@ -477,6 +479,62 @@ class GlyphInfo: self.height = height +def glyph_to_glyphinfo(glyph, font, size, bpp): + scale = 256 // (1 << bpp) + if not font.is_scalable: + sizes = [pt_to_px(x.size) for x in font.available_sizes] + if size in sizes: + font.select_size(sizes.index(size)) + else: + font.set_pixel_sizes(size, 0) + flags = FT_LOAD_RENDER + if bpp != 1: + flags |= FT_LOAD_NO_BITMAP + else: + flags |= FT_LOAD_TARGET_MONO + font.load_char(glyph, flags) + width = font.glyph.bitmap.width + height = font.glyph.bitmap.rows + buffer = font.glyph.bitmap.buffer + pitch = font.glyph.bitmap.pitch + glyph_data = [0] * ((height * width * bpp + 7) // 8) + src_mode = font.glyph.bitmap.pixel_mode + pos = 0 + for y in range(height): + for x in range(width): + if src_mode == ft_pixel_mode_mono: + pixel = ( + (1 << bpp) - 1 + if buffer[y * pitch + x // 8] & (1 << (7 - x % 8)) + else 0 + ) + else: + pixel = buffer[y * pitch + x] // scale + for bit_num in range(bpp): + if pixel & (1 << (bpp - bit_num - 1)): + glyph_data[pos // 8] |= 0x80 >> (pos % 8) + pos += 1 + ascender = pt_to_px(font.size.ascender) + if ascender == 0: + if not font.is_scalable: + ascender = size + else: + _LOGGER.error( + "Unable to determine ascender of font %s %s", + font.family_name, + font.style_name, + ) + return GlyphInfo( + glyph, + glyph_data, + pt_to_px(font.glyph.metrics.horiAdvance), + font.glyph.bitmap_left, + ascender - font.glyph.bitmap_top, + width, + height, + ) + + async def to_code(config): """ Collect all glyph codepoints, construct a map from a codepoint to a font file. @@ -506,98 +564,47 @@ async def to_code(config): codepoints = list(point_set) codepoints.sort(key=functools.cmp_to_key(glyph_comparator)) - glyph_args = {} - data = [] bpp = config[CONF_BPP] - scale = 256 // (1 << bpp) size = config[CONF_SIZE] # create the data array for all glyphs - for codepoint in codepoints: - font = point_font_map[codepoint] - if not font.is_scalable: - sizes = [pt_to_px(x.size) for x in font.available_sizes] - if size in sizes: - font.select_size(sizes.index(size)) - else: - font.set_pixel_sizes(size, 0) - flags = FT_LOAD_RENDER - if bpp != 1: - flags |= FT_LOAD_NO_BITMAP - else: - flags |= FT_LOAD_TARGET_MONO - font.load_char(codepoint, flags) - width = font.glyph.bitmap.width - height = font.glyph.bitmap.rows - buffer = font.glyph.bitmap.buffer - pitch = font.glyph.bitmap.pitch - glyph_data = [0] * ((height * width * bpp + 7) // 8) - src_mode = font.glyph.bitmap.pixel_mode - pos = 0 - for y in range(height): - for x in range(width): - if src_mode == ft_pixel_mode_mono: - pixel = ( - (1 << bpp) - 1 - if buffer[y * pitch + x // 8] & (1 << (7 - x % 8)) - else 0 - ) - else: - pixel = buffer[y * pitch + x] // scale - for bit_num in range(bpp): - if pixel & (1 << (bpp - bit_num - 1)): - glyph_data[pos // 8] |= 0x80 >> (pos % 8) - pos += 1 - ascender = pt_to_px(font.size.ascender) - if ascender == 0: - if not font.is_scalable: - ascender = size - else: - _LOGGER.error( - "Unable to determine ascender of font %s", config[CONF_FILE] - ) - glyph_args[codepoint] = GlyphInfo( - len(data), - pt_to_px(font.glyph.metrics.horiAdvance), - font.glyph.bitmap_left, - ascender - font.glyph.bitmap_top, - width, - height, - ) - data += glyph_data - - rhs = [HexInt(x) for x in data] + glyph_args = [ + glyph_to_glyphinfo(x, point_font_map[x], size, bpp) for x in codepoints + ] + rhs = [HexInt(x) for x in flatten([x.bitmap_data for x in glyph_args])] prog_arr = cg.progmem_array(config[CONF_RAW_DATA_ID], rhs) # Create the glyph table that points to data in the above array. - glyph_initializer = [] - for codepoint in codepoints: - glyph_initializer.append( - cg.StructInitializer( - GlyphData, - ( - "a_char", - cg.RawExpression( - f"(const uint8_t *){cpp_string_escape(codepoint)}" - ), - ), - ( - "data", - cg.RawExpression( - f"{str(prog_arr)} + {str(glyph_args[codepoint].data_len)}" - ), - ), - ("advance", glyph_args[codepoint].advance), - ("offset_x", glyph_args[codepoint].offset_x), - ("offset_y", glyph_args[codepoint].offset_y), - ("width", glyph_args[codepoint].width), - ("height", glyph_args[codepoint].height), - ) + glyph_initializer = [ + cg.StructInitializer( + GlyphData, + ( + "a_char", + cg.RawExpression(f"(const uint8_t *){cpp_string_escape(x.glyph)}"), + ), + ( + "data", + cg.RawExpression(f"{str(prog_arr)} + {str(y - len(x.bitmap_data))}"), + ), + ("advance", x.advance), + ("offset_x", x.offset_x), + ("offset_y", x.offset_y), + ("width", x.width), + ("height", x.height), ) + for (x, y) in zip( + glyph_args, list(accumulate([len(x.bitmap_data) for x in glyph_args])) + ) + ] glyphs = cg.static_const_array(config[CONF_RAW_GLYPH_ID], glyph_initializer) font_height = pt_to_px(base_font.size.height) ascender = pt_to_px(base_font.size.ascender) + descender = abs(pt_to_px(base_font.size.descender)) + g = glyph_to_glyphinfo("x", base_font, size, bpp) + xheight = g.height if len(g.bitmap_data) > 1 else 0 + g = glyph_to_glyphinfo("X", base_font, size, bpp) + capheight = g.height if len(g.bitmap_data) > 1 else 0 if font_height == 0: if not base_font.is_scalable: font_height = size @@ -610,5 +617,8 @@ async def to_code(config): len(glyph_initializer), ascender, font_height, + descender, + xheight, + capheight, bpp, ) diff --git a/esphome/components/font/font.cpp b/esphome/components/font/font.cpp index 32464d87ee..8b2420ac07 100644 --- a/esphome/components/font/font.cpp +++ b/esphome/components/font/font.cpp @@ -45,8 +45,15 @@ void Glyph::scan_area(int *x1, int *y1, int *width, int *height) const { *height = this->glyph_data_->height; } -Font::Font(const GlyphData *data, int data_nr, int baseline, int height, uint8_t bpp) - : baseline_(baseline), height_(height), bpp_(bpp) { +Font::Font(const GlyphData *data, int data_nr, int baseline, int height, int descender, int xheight, int capheight, + uint8_t bpp) + : baseline_(baseline), + height_(height), + descender_(descender), + linegap_(height - baseline - descender), + xheight_(xheight), + capheight_(capheight), + bpp_(bpp) { glyphs_.reserve(data_nr); for (int i = 0; i < data_nr; ++i) glyphs_.emplace_back(&data[i]); diff --git a/esphome/components/font/font.h b/esphome/components/font/font.h index 992c77cb9f..28832d647d 100644 --- a/esphome/components/font/font.h +++ b/esphome/components/font/font.h @@ -50,11 +50,17 @@ class Font public: /** Construct the font with the given glyphs. * - * @param glyphs A vector of glyphs, must be sorted lexicographically. + * @param data A vector of glyphs, must be sorted lexicographically. + * @param data_nr The number of glyphs in data. * @param baseline The y-offset from the top of the text to the baseline. - * @param bottom The y-offset from the top of the text to the bottom (i.e. height). + * @param height The y-offset from the top of the text to the bottom. + * @param descender The y-offset from the baseline to the lowest stroke in the font (e.g. from letters like g or p). + * @param xheight The height of lowercase letters, usually measured at the "x" glyph. + * @param capheight The height of capital letters, usually measured at the "X" glyph. + * @param bpp The bits per pixel used for this font. Used to read data out of the glyph bitmaps. */ - Font(const GlyphData *data, int data_nr, int baseline, int height, uint8_t bpp = 1); + Font(const GlyphData *data, int data_nr, int baseline, int height, int descender, int xheight, int capheight, + uint8_t bpp = 1); int match_next_glyph(const uint8_t *str, int *match_length); @@ -65,6 +71,11 @@ class Font #endif inline int get_baseline() { return this->baseline_; } inline int get_height() { return this->height_; } + inline int get_ascender() { return this->baseline_; } + inline int get_descender() { return this->descender_; } + inline int get_linegap() { return this->linegap_; } + inline int get_xheight() { return this->xheight_; } + inline int get_capheight() { return this->capheight_; } inline int get_bpp() { return this->bpp_; } const std::vector> &get_glyphs() const { return glyphs_; } @@ -73,6 +84,10 @@ class Font std::vector> glyphs_; int baseline_; int height_; + int descender_; + int linegap_; + int xheight_; + int capheight_; uint8_t bpp_; // bits per pixel }; From 6afa8141c08968ae077e0730565e3686e1d41469 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Mon, 23 Jun 2025 09:00:46 +0200 Subject: [PATCH 14/17] Update esphome/components/logger/logger.cpp --- esphome/components/logger/logger.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/esphome/components/logger/logger.cpp b/esphome/components/logger/logger.cpp index cfc059c29f..6316eb6991 100644 --- a/esphome/components/logger/logger.cpp +++ b/esphome/components/logger/logger.cpp @@ -151,7 +151,7 @@ void Logger::init_log_buffer(size_t total_buffer_size) { } #endif -#if defined(USE_LOGGER_USB_CDC) || defined(USE_ESPHOME_TASK_LOG_BUFFER) +#if defined(USE_LOGGER_USB_CDC) || defined(USE_ESP32) void Logger::loop() { #if defined(USE_LOGGER_USB_CDC) && defined(USE_ARDUINO) if (this->uart_ == UART_SELECTION_USB_CDC) { From f0369893615f66c405bb34640faa3354df3895d3 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Mon, 23 Jun 2025 09:01:01 +0200 Subject: [PATCH 15/17] Update esphome/components/logger/logger.h --- esphome/components/logger/logger.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/esphome/components/logger/logger.h b/esphome/components/logger/logger.h index fda9983098..fe0e4cd636 100644 --- a/esphome/components/logger/logger.h +++ b/esphome/components/logger/logger.h @@ -107,7 +107,7 @@ class Logger : public Component { #ifdef USE_ESPHOME_TASK_LOG_BUFFER void init_log_buffer(size_t total_buffer_size); #endif -#if defined(USE_LOGGER_USB_CDC) || defined(USE_ESPHOME_TASK_LOG_BUFFER) +#if defined(USE_LOGGER_USB_CDC) || defined(USE_ESP32) void loop() override; #endif /// Manually set the baud rate for serial, set to 0 to disable. From 9f489c9f273d8e954a471a914e2103b9493f6ce3 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Mon, 23 Jun 2025 09:01:21 +0200 Subject: [PATCH 16/17] Update esphome/components/logger/logger.h --- esphome/components/logger/logger.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/esphome/components/logger/logger.h b/esphome/components/logger/logger.h index fe0e4cd636..38faf73d84 100644 --- a/esphome/components/logger/logger.h +++ b/esphome/components/logger/logger.h @@ -359,7 +359,7 @@ class Logger : public Component { this->write_body_to_buffer_(ESPHOME_LOG_RESET_COLOR, RESET_COLOR_LEN, buffer, buffer_at, buffer_size); } -#ifdef USE_ESPHOME_TASK_LOG_BUFFER +#ifdef USE_ESP32 // Disable loop when task buffer is empty (with USB CDC check) inline void disable_loop_when_buffer_empty_() { // Thread safety note: This is safe even if another task calls enable_loop_soon_any_context() From ed57e7c6b000a298eb1c561b162c05a62cd71acb Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Mon, 23 Jun 2025 09:02:22 +0200 Subject: [PATCH 17/17] Update esphome/components/logger/logger.cpp --- esphome/components/logger/logger.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/esphome/components/logger/logger.cpp b/esphome/components/logger/logger.cpp index 6316eb6991..a2c2aa0320 100644 --- a/esphome/components/logger/logger.cpp +++ b/esphome/components/logger/logger.cpp @@ -46,8 +46,8 @@ void HOT Logger::log_vprintf_(uint8_t level, const char *tag, int line, const ch bool message_sent = false; #ifdef USE_ESPHOME_TASK_LOG_BUFFER // For non-main tasks, queue the message for callbacks - but only if we have any callbacks registered - message_sent = this->log_buffer_->send_message_thread_safe(level, tag, - static_cast(line), current_task, format, args); + message_sent = + this->log_buffer_->send_message_thread_safe(level, tag, static_cast(line), current_task, format, args); if (message_sent) { // Enable logger loop to process the buffered message // This is safe to call from any context including ISRs