Reduce API memory footprint through bitfield consolidation and type sizing (#9252)

esphome/components/api/__init__.py

@@ -110,9 +110,10 @@ CONFIG_SCHEMA = cv.All(
             ): ACTIONS_SCHEMA,
             cv.Exclusive(CONF_ACTIONS, group_of_exclusion=CONF_ACTIONS): ACTIONS_SCHEMA,
             cv.Optional(CONF_ENCRYPTION): _encryption_schema,
-            cv.Optional(
-                CONF_BATCH_DELAY, default="100ms"
-            ): cv.positive_time_period_milliseconds,
+            cv.Optional(CONF_BATCH_DELAY, default="100ms"): cv.All(
+                cv.positive_time_period_milliseconds,
+                cv.Range(max=cv.TimePeriod(milliseconds=65535)),
+            ),
             cv.Optional(CONF_ON_CLIENT_CONNECTED): automation.validate_automation(
                 single=True
             ),

esphome/components/api/api_connection.cpp

@@ -93,21 +93,21 @@ APIConnection::~APIConnection() {
 #ifdef HAS_PROTO_MESSAGE_DUMP
 void APIConnection::log_batch_item_(const DeferredBatch::BatchItem &item) {
   // Set log-only mode
-  this->log_only_mode_ = true;
+  this->flags_.log_only_mode = true;
 
   // Call the creator - it will create the message and log it via encode_message_to_buffer
   item.creator(item.entity, this, std::numeric_limits<uint16_t>::max(), true, item.message_type);
 
   // Clear log-only mode
-  this->log_only_mode_ = false;
+  this->flags_.log_only_mode = false;
 }
 #endif
 
 void APIConnection::loop() {
-  if (this->next_close_) {
+  if (this->flags_.next_close) {
     // requested a disconnect
     this->helper_->close();
-    this->remove_ = true;
+    this->flags_.remove = true;
     return;
   }
 
@@ -148,15 +148,14 @@ void APIConnection::loop() {
         } else {
           this->read_message(0, buffer.type, nullptr);
         }
-        if (this->remove_)
+        if (this->flags_.remove)
           return;
       }
     }
   }
 
   // Process deferred batch if scheduled
-  if (this->deferred_batch_.batch_scheduled &&
-      now - this->deferred_batch_.batch_start_time >= this->get_batch_delay_ms_()) {
+  if (this->flags_.batch_scheduled && now - this->deferred_batch_.batch_start_time >= this->get_batch_delay_ms_()) {
     this->process_batch_();
   }
 
@@ -166,7 +165,7 @@ void APIConnection::loop() {
     this->initial_state_iterator_.advance();
   }
 
-  if (this->sent_ping_) {
+  if (this->flags_.sent_ping) {
     // Disconnect if not responded within 2.5*keepalive
     if (now - this->last_traffic_ > KEEPALIVE_DISCONNECT_TIMEOUT) {
       on_fatal_error();
@@ -174,13 +173,13 @@ void APIConnection::loop() {
     }
   } else if (now - this->last_traffic_ > KEEPALIVE_TIMEOUT_MS) {
     ESP_LOGVV(TAG, "Sending keepalive PING");
-    this->sent_ping_ = this->send_message(PingRequest());
-    if (!this->sent_ping_) {
+    this->flags_.sent_ping = this->send_message(PingRequest());
+    if (!this->flags_.sent_ping) {
       // If we can't send the ping request directly (tx_buffer full),
       // schedule it at the front of the batch so it will be sent with priority
       ESP_LOGW(TAG, "Buffer full, ping queued");
       this->schedule_message_front_(nullptr, &APIConnection::try_send_ping_request, PingRequest::MESSAGE_TYPE);
-      this->sent_ping_ = true;  // Mark as sent to avoid scheduling multiple pings
+      this->flags_.sent_ping = true;  // Mark as sent to avoid scheduling multiple pings
     }
   }
 
@@ -240,13 +239,13 @@ DisconnectResponse APIConnection::disconnect(const DisconnectRequest &msg) {
   // don't close yet, we still need to send the disconnect response
   // close will happen on next loop
   ESP_LOGD(TAG, "%s disconnected", this->get_client_combined_info().c_str());
-  this->next_close_ = true;
+  this->flags_.next_close = true;
   DisconnectResponse resp;
   return resp;
 }
 void APIConnection::on_disconnect_response(const DisconnectResponse &value) {
   this->helper_->close();
-  this->remove_ = true;
+  this->flags_.remove = true;
 }
 
 // Encodes a message to the buffer and returns the total number of bytes used,
@@ -255,7 +254,7 @@ uint16_t APIConnection::encode_message_to_buffer(ProtoMessage &msg, uint16_t mes
                                                  uint32_t remaining_size, bool is_single) {
 #ifdef HAS_PROTO_MESSAGE_DUMP
   // If in log-only mode, just log and return
-  if (conn->log_only_mode_) {
+  if (conn->flags_.log_only_mode) {
     conn->log_send_message_(msg.message_name(), msg.dump());
     return 1;  // Return non-zero to indicate "success" for logging
   }
@@ -1118,7 +1117,7 @@ void APIConnection::media_player_command(const MediaPlayerCommandRequest &msg) {
 
 #ifdef USE_ESP32_CAMERA
 void APIConnection::set_camera_state(std::shared_ptr<esp32_camera::CameraImage> image) {
-  if (!this->state_subscription_)
+  if (!this->flags_.state_subscription)
     return;
   if (this->image_reader_.available())
     return;
@@ -1459,7 +1458,7 @@ void APIConnection::update_command(const UpdateCommandRequest &msg) {
 #endif
 
 bool APIConnection::try_send_log_message(int level, const char *tag, const char *line) {
-  if (this->log_subscription_ < level)
+  if (this->flags_.log_subscription < level)
     return false;
 
   // Pre-calculate message size to avoid reallocations
@@ -1500,7 +1499,7 @@ HelloResponse APIConnection::hello(const HelloRequest &msg) {
   resp.server_info = App.get_name() + " (esphome v" ESPHOME_VERSION ")";
   resp.name = App.get_name();
 
-  this->connection_state_ = ConnectionState::CONNECTED;
+  this->flags_.connection_state = static_cast<uint8_t>(ConnectionState::CONNECTED);
   return resp;
 }
 ConnectResponse APIConnection::connect(const ConnectRequest &msg) {
@@ -1511,7 +1510,7 @@ ConnectResponse APIConnection::connect(const ConnectRequest &msg) {
   resp.invalid_password = !correct;
   if (correct) {
     ESP_LOGD(TAG, "%s connected", this->get_client_combined_info().c_str());
-    this->connection_state_ = ConnectionState::AUTHENTICATED;
+    this->flags_.connection_state = static_cast<uint8_t>(ConnectionState::AUTHENTICATED);
     this->parent_->get_client_connected_trigger()->trigger(this->client_info_, this->client_peername_);
 #ifdef USE_HOMEASSISTANT_TIME
     if (homeassistant::global_homeassistant_time != nullptr) {
@@ -1625,7 +1624,7 @@ void APIConnection::subscribe_home_assistant_states(const SubscribeHomeAssistant
   state_subs_at_ = 0;
 }
 bool APIConnection::try_to_clear_buffer(bool log_out_of_space) {
-  if (this->remove_)
+  if (this->flags_.remove)
     return false;
   if (this->helper_->can_write_without_blocking())
     return true;
@@ -1675,7 +1674,7 @@ void APIConnection::on_no_setup_connection() {
 }
 void APIConnection::on_fatal_error() {
   this->helper_->close();
-  this->remove_ = true;
+  this->flags_.remove = true;
 }
 
 void APIConnection::DeferredBatch::add_item(EntityBase *entity, MessageCreator creator, uint16_t message_type) {
@@ -1700,8 +1699,8 @@ void APIConnection::DeferredBatch::add_item_front(EntityBase *entity, MessageCre
 }
 
 bool APIConnection::schedule_batch_() {
-  if (!this->deferred_batch_.batch_scheduled) {
-    this->deferred_batch_.batch_scheduled = true;
+  if (!this->flags_.batch_scheduled) {
+    this->flags_.batch_scheduled = true;
     this->deferred_batch_.batch_start_time = App.get_loop_component_start_time();
   }
   return true;
@@ -1710,14 +1709,14 @@ bool APIConnection::schedule_batch_() {
 ProtoWriteBuffer APIConnection::allocate_single_message_buffer(uint16_t size) { return this->create_buffer(size); }
 
 ProtoWriteBuffer APIConnection::allocate_batch_message_buffer(uint16_t size) {
-  ProtoWriteBuffer result = this->prepare_message_buffer(size, this->batch_first_message_);
-  this->batch_first_message_ = false;
+  ProtoWriteBuffer result = this->prepare_message_buffer(size, this->flags_.batch_first_message);
+  this->flags_.batch_first_message = false;
   return result;
 }
 
 void APIConnection::process_batch_() {
   if (this->deferred_batch_.empty()) {
-    this->deferred_batch_.batch_scheduled = false;
+    this->flags_.batch_scheduled = false;
     return;
   }
 
@@ -1770,7 +1769,7 @@ void APIConnection::process_batch_() {
 
   // Reserve based on estimated size (much more accurate than 24-byte worst-case)
   this->parent_->get_shared_buffer_ref().reserve(total_estimated_size + total_overhead);
-  this->batch_first_message_ = true;
+  this->flags_.batch_first_message = true;
 
   size_t items_processed = 0;
   uint16_t remaining_size = std::numeric_limits<uint16_t>::max();

esphome/components/api/api_connection.h

@@ -107,7 +107,7 @@ class APIConnection : public APIServerConnection {
 #endif
   bool try_send_log_message(int level, const char *tag, const char *line);
   void send_homeassistant_service_call(const HomeassistantServiceResponse &call) {
-    if (!this->service_call_subscription_)
+    if (!this->flags_.service_call_subscription)
       return;
     this->send_message(call);
   }
@@ -164,7 +164,7 @@ class APIConnection : public APIServerConnection {
   void on_disconnect_response(const DisconnectResponse &value) override;
   void on_ping_response(const PingResponse &value) override {
     // we initiated ping
-    this->sent_ping_ = false;
+    this->flags_.sent_ping = false;
   }
   void on_home_assistant_state_response(const HomeAssistantStateResponse &msg) override;
 #ifdef USE_HOMEASSISTANT_TIME
@@ -177,16 +177,16 @@ class APIConnection : public APIServerConnection {
   DeviceInfoResponse device_info(const DeviceInfoRequest &msg) override;
   void list_entities(const ListEntitiesRequest &msg) override { this->list_entities_iterator_.begin(); }
   void subscribe_states(const SubscribeStatesRequest &msg) override {
-    this->state_subscription_ = true;
+    this->flags_.state_subscription = true;
     this->initial_state_iterator_.begin();
   }
   void subscribe_logs(const SubscribeLogsRequest &msg) override {
-    this->log_subscription_ = msg.level;
+    this->flags_.log_subscription = msg.level;
     if (msg.dump_config)
       App.schedule_dump_config();
   }
   void subscribe_homeassistant_services(const SubscribeHomeassistantServicesRequest &msg) override {
-    this->service_call_subscription_ = true;
+    this->flags_.service_call_subscription = true;
   }
   void subscribe_home_assistant_states(const SubscribeHomeAssistantStatesRequest &msg) override;
   GetTimeResponse get_time(const GetTimeRequest &msg) override {
@@ -198,9 +198,12 @@ class APIConnection : public APIServerConnection {
   NoiseEncryptionSetKeyResponse noise_encryption_set_key(const NoiseEncryptionSetKeyRequest &msg) override;
 #endif
 
-  bool is_authenticated() override { return this->connection_state_ == ConnectionState::AUTHENTICATED; }
+  bool is_authenticated() override {
+    return static_cast<ConnectionState>(this->flags_.connection_state) == ConnectionState::AUTHENTICATED;
+  }
   bool is_connection_setup() override {
-    return this->connection_state_ == ConnectionState ::CONNECTED || this->is_authenticated();
+    return static_cast<ConnectionState>(this->flags_.connection_state) == ConnectionState::CONNECTED ||
+           this->is_authenticated();
   }
   void on_fatal_error() override;
   void on_unauthenticated_access() override;
@@ -423,49 +426,28 @@ class APIConnection : public APIServerConnection {
   static uint16_t try_send_ping_request(EntityBase *entity, APIConnection *conn, uint32_t remaining_size,
                                         bool is_single);
 
-  // Pointers first (4 bytes each, naturally aligned)
+  // === Optimal member ordering for 32-bit systems ===
+
+  // Group 1: Pointers (4 bytes each on 32-bit)
   std::unique_ptr<APIFrameHelper> helper_;
   APIServer *parent_;
 
-  // 4-byte aligned types
-  uint32_t last_traffic_;
-  int state_subs_at_ = -1;
-
-  // Strings (12 bytes each on 32-bit)
-  std::string client_info_;
-  std::string client_peername_;
-
-  // 2-byte aligned types
-  uint16_t client_api_version_major_{0};
-  uint16_t client_api_version_minor_{0};
-
-  // Group all 1-byte types together to minimize padding
-  enum class ConnectionState : uint8_t {
-    WAITING_FOR_HELLO,
-    CONNECTED,
-    AUTHENTICATED,
-  } connection_state_{ConnectionState::WAITING_FOR_HELLO};
-  uint8_t log_subscription_{ESPHOME_LOG_LEVEL_NONE};
-  bool remove_{false};
-  bool state_subscription_{false};
-  bool sent_ping_{false};
-  bool service_call_subscription_{false};
-  bool next_close_ = false;
-  // 7 bytes used, 1 byte padding
-#ifdef HAS_PROTO_MESSAGE_DUMP
-  // When true, encode_message_to_buffer will only log, not encode
-  bool log_only_mode_{false};
-#endif
-  uint8_t ping_retries_{0};
-  // 8 bytes used, no padding needed
-
-  // Larger objects at the end
+  // Group 2: Larger objects (must be 4-byte aligned)
+  // These contain vectors/pointers internally, so putting them early ensures good alignment
   InitialStateIterator initial_state_iterator_;
   ListEntitiesIterator list_entities_iterator_;
 #ifdef USE_ESP32_CAMERA
   esp32_camera::CameraImageReader image_reader_;
 #endif
 
+  // Group 3: Strings (12 bytes each on 32-bit, 4-byte aligned)
+  std::string client_info_;
+  std::string client_peername_;
+
+  // Group 4: 4-byte types
+  uint32_t last_traffic_;
+  int state_subs_at_ = -1;
+
   // Function pointer type for message encoding
   using MessageCreatorPtr = uint16_t (*)(EntityBase *, APIConnection *, uint32_t remaining_size, bool is_single);
 
@@ -575,7 +557,6 @@ class APIConnection : public APIServerConnection {
 
     std::vector<BatchItem> items;
     uint32_t batch_start_time{0};
-    bool batch_scheduled{false};
 
     DeferredBatch() {
       // Pre-allocate capacity for typical batch sizes to avoid reallocation
@@ -588,13 +569,47 @@ class APIConnection : public APIServerConnection {
     void add_item_front(EntityBase *entity, MessageCreator creator, uint16_t message_type);
     void clear() {
       items.clear();
-      batch_scheduled = false;
       batch_start_time = 0;
     }
     bool empty() const { return items.empty(); }
   };
 
+  // DeferredBatch here (16 bytes, 4-byte aligned)
   DeferredBatch deferred_batch_;
+
+  // ConnectionState enum for type safety
+  enum class ConnectionState : uint8_t {
+    WAITING_FOR_HELLO = 0,
+    CONNECTED = 1,
+    AUTHENTICATED = 2,
+  };
+
+  // Group 5: Pack all small members together to minimize padding
+  // This group starts at a 4-byte boundary after DeferredBatch
+  struct APIFlags {
+    // Connection state only needs 2 bits (3 states)
+    uint8_t connection_state : 2;
+    // Log subscription needs 3 bits (log levels 0-7)
+    uint8_t log_subscription : 3;
+    // Boolean flags (1 bit each)
+    uint8_t remove : 1;
+    uint8_t state_subscription : 1;
+    uint8_t sent_ping : 1;
+
+    uint8_t service_call_subscription : 1;
+    uint8_t next_close : 1;
+    uint8_t batch_scheduled : 1;
+    uint8_t batch_first_message : 1;  // For batch buffer allocation
+#ifdef HAS_PROTO_MESSAGE_DUMP
+    uint8_t log_only_mode : 1;
+#endif
+  } flags_{};  // 2 bytes total
+
+  // 2-byte types immediately after flags_ (no padding between them)
+  uint16_t client_api_version_major_{0};
+  uint16_t client_api_version_minor_{0};
+  // Total: 2 (flags) + 2 + 2 = 6 bytes, then 2 bytes padding to next 4-byte boundary
+
   uint32_t get_batch_delay_ms_() const;
   // Message will use 8 more bytes than the minimum size, and typical
   // MTU is 1500. Sometimes users will see as low as 1460 MTU.
@@ -612,9 +627,6 @@ class APIConnection : public APIServerConnection {
   bool schedule_batch_();
   void process_batch_();
 
-  // State for batch buffer allocation
-  bool batch_first_message_{false};
-
 #ifdef HAS_PROTO_MESSAGE_DUMP
   void log_batch_item_(const DeferredBatch::BatchItem &item);
 #endif

esphome/components/api/api_server.cpp

@@ -104,7 +104,7 @@ void APIServer::setup() {
         return;
       }
       for (auto &c : this->clients_) {
-        if (!c->remove_)
+        if (!c->flags_.remove)
           c->try_send_log_message(level, tag, message);
       }
     });
@@ -116,7 +116,7 @@ void APIServer::setup() {
     esp32_camera::global_esp32_camera->add_image_callback(
         [this](const std::shared_ptr<esp32_camera::CameraImage> &image) {
           for (auto &c : this->clients_) {
-            if (!c->remove_)
+            if (!c->flags_.remove)
               c->set_camera_state(image);
           }
         });
@@ -176,7 +176,7 @@ void APIServer::loop() {
   while (client_index < this->clients_.size()) {
     auto &client = this->clients_[client_index];
 
-    if (!client->remove_) {
+    if (!client->flags_.remove) {
       // Common case: process active client
       client->loop();
       client_index++;
@@ -431,7 +431,7 @@ void APIServer::set_port(uint16_t port) { this->port_ = port; }
 
 void APIServer::set_password(const std::string &password) { this->password_ = password; }
 
-void APIServer::set_batch_delay(uint32_t batch_delay) { this->batch_delay_ = batch_delay; }
+void APIServer::set_batch_delay(uint16_t batch_delay) { this->batch_delay_ = batch_delay; }
 
 void APIServer::send_homeassistant_service_call(const HomeassistantServiceResponse &call) {
   for (auto &client : this->clients_) {
@@ -502,7 +502,7 @@ bool APIServer::save_noise_psk(psk_t psk, bool make_active) {
 #ifdef USE_HOMEASSISTANT_TIME
 void APIServer::request_time() {
   for (auto &client : this->clients_) {
-    if (!client->remove_ && client->is_authenticated())
+    if (!client->flags_.remove && client->is_authenticated())
       client->send_time_request();
   }
 }

esphome/components/api/api_server.h

@@ -40,8 +40,8 @@ class APIServer : public Component, public Controller {
   void set_port(uint16_t port);
   void set_password(const std::string &password);
   void set_reboot_timeout(uint32_t reboot_timeout);
-  void set_batch_delay(uint32_t batch_delay);
-  uint32_t get_batch_delay() const { return batch_delay_; }
+  void set_batch_delay(uint16_t batch_delay);
+  uint16_t get_batch_delay() const { return batch_delay_; }
 
   // Get reference to shared buffer for API connections
   std::vector<uint8_t> &get_shared_buffer_ref() { return shared_write_buffer_; }
@@ -150,7 +150,6 @@ class APIServer : public Component, public Controller {
 
   // 4-byte aligned types
   uint32_t reboot_timeout_{300000};
-  uint32_t batch_delay_{100};
 
   // Vectors and strings (12 bytes each on 32-bit)
   std::vector<std::unique_ptr<APIConnection>> clients_;
@@ -161,8 +160,9 @@ class APIServer : public Component, public Controller {
 
   // Group smaller types together
   uint16_t port_{6053};
+  uint16_t batch_delay_{100};
   bool shutting_down_ = false;
-  // 3 bytes used, 1 byte padding
+  // 5 bytes used, 3 bytes padding
 
 #ifdef USE_API_NOISE
   std::shared_ptr<APINoiseContext> noise_ctx_ = std::make_shared<APINoiseContext>();