[bluetooth_proxy] Replace dynamic vector with fixed array for BLE advertisements (#10174)

esphome/components/api/api.proto

@@ -1438,7 +1438,7 @@ message BluetoothLERawAdvertisementsResponse {
   option (ifdef) = "USE_BLUETOOTH_PROXY";
   option (no_delay) = true;
 
-  repeated BluetoothLERawAdvertisement advertisements = 1;
+  repeated BluetoothLERawAdvertisement advertisements = 1 [(fixed_array_with_length_define) = "BLUETOOTH_PROXY_ADVERTISEMENT_BATCH_SIZE"];
 }
 
 enum BluetoothDeviceRequestType {

esphome/components/api/api_options.proto

@@ -30,6 +30,7 @@ extend google.protobuf.FieldOptions {
     optional bool no_zero_copy = 50008 [default=false];
     optional bool fixed_array_skip_zero = 50009 [default=false];
     optional string fixed_array_size_define = 50010;
+    optional string fixed_array_with_length_define = 50011;
 
     // container_pointer: Zero-copy optimization for repeated fields.
     //

esphome/components/api/api_pb2.cpp

@@ -1843,12 +1843,14 @@ void BluetoothLERawAdvertisement::calculate_size(ProtoSize &size) const {
   size.add_length(1, this->data_len);
 }
 void BluetoothLERawAdvertisementsResponse::encode(ProtoWriteBuffer buffer) const {
-  for (auto &it : this->advertisements) {
+  for (uint16_t i = 0; i < this->advertisements_len; i++) {
-    buffer.encode_message(1, it, true);
+    buffer.encode_message(1, this->advertisements[i], true);
   }
 }
 void BluetoothLERawAdvertisementsResponse::calculate_size(ProtoSize &size) const {
-  size.add_repeated_message(1, this->advertisements);
+  for (uint16_t i = 0; i < this->advertisements_len; i++) {
+    size.add_message_object_force(1, this->advertisements[i]);
+  }
 }
 bool BluetoothDeviceRequest::decode_varint(uint32_t field_id, ProtoVarInt value) {
   switch (field_id) {

esphome/components/api/api_pb2.h

@@ -1788,11 +1788,12 @@ class BluetoothLERawAdvertisement : public ProtoMessage {
 class BluetoothLERawAdvertisementsResponse : public ProtoMessage {
  public:
   static constexpr uint8_t MESSAGE_TYPE = 93;
-  static constexpr uint8_t ESTIMATED_SIZE = 34;
+  static constexpr uint8_t ESTIMATED_SIZE = 136;
 #ifdef HAS_PROTO_MESSAGE_DUMP
   const char *message_name() const override { return "bluetooth_le_raw_advertisements_response"; }
 #endif
-  std::vector<BluetoothLERawAdvertisement> advertisements{};
+  std::array<BluetoothLERawAdvertisement, BLUETOOTH_PROXY_ADVERTISEMENT_BATCH_SIZE> advertisements{};
+  uint16_t advertisements_len{0};
   void encode(ProtoWriteBuffer buffer) const override;
   void calculate_size(ProtoSize &size) const override;
 #ifdef HAS_PROTO_MESSAGE_DUMP

esphome/components/api/api_pb2_dump.cpp

@@ -1534,9 +1534,9 @@ void BluetoothLERawAdvertisement::dump_to(std::string &out) const {
 }
 void BluetoothLERawAdvertisementsResponse::dump_to(std::string &out) const {
   MessageDumpHelper helper(out, "BluetoothLERawAdvertisementsResponse");
-  for (const auto &it : this->advertisements) {
+  for (uint16_t i = 0; i < this->advertisements_len; i++) {
     out.append("  advertisements: ");
-    it.dump_to(out);
+    this->advertisements[i].dump_to(out);
     out.append("\n");
   }
 }

esphome/components/bluetooth_proxy/__init__.py

@@ -118,6 +118,12 @@ async def to_code(config):
     connection_count = len(config.get(CONF_CONNECTIONS, []))
     cg.add_define("BLUETOOTH_PROXY_MAX_CONNECTIONS", connection_count)
 
+    # Define batch size for BLE advertisements
+    # Each advertisement is up to 80 bytes when packaged (including protocol overhead)
+    # 16 advertisements × 80 bytes (worst case) = 1280 bytes out of ~1320 bytes usable payload
+    # This achieves ~97% WiFi MTU utilization while staying under the limit
+    cg.add_define("BLUETOOTH_PROXY_ADVERTISEMENT_BATCH_SIZE", 16)
+
     for connection_conf in config.get(CONF_CONNECTIONS, []):
         connection_var = cg.new_Pvariable(connection_conf[CONF_ID])
         await cg.register_component(connection_var, connection_conf)

esphome/components/bluetooth_proxy/bluetooth_proxy.cpp

@@ -11,12 +11,8 @@ namespace esphome::bluetooth_proxy {
 
 static const char *const TAG = "bluetooth_proxy";
 
-// Batch size for BLE advertisements to maximize WiFi efficiency
+// BLUETOOTH_PROXY_ADVERTISEMENT_BATCH_SIZE is defined during code generation
-// Each advertisement is up to 80 bytes when packaged (including protocol overhead)
+// It sets the batch size for BLE advertisements to maximize WiFi efficiency
-// Most advertisements are 20-30 bytes, allowing even more to fit per packet
-// 16 advertisements × 80 bytes (worst case) = 1280 bytes out of ~1320 bytes usable payload
-// This achieves ~97% WiFi MTU utilization while staying under the limit
-static constexpr size_t FLUSH_BATCH_SIZE = 16;
 
 // Verify BLE advertisement data array size matches the BLE specification (31 bytes adv + 31 bytes scan response)
 static_assert(sizeof(((api::BluetoothLERawAdvertisement *) nullptr)->data) == 62,
@@ -25,13 +21,6 @@ static_assert(sizeof(((api::BluetoothLERawAdvertisement *) nullptr)->data) == 62
 BluetoothProxy::BluetoothProxy() { global_bluetooth_proxy = this; }
 
 void BluetoothProxy::setup() {
-  // Reserve capacity but start with size 0
-  // Reserve 50% since we'll grow naturally and flush at FLUSH_BATCH_SIZE
-  this->response_.advertisements.reserve(FLUSH_BATCH_SIZE / 2);
-
-  // Don't pre-allocate pool - let it grow only if needed in busy environments
-  // Many devices in quiet areas will never need the overflow pool
-
   this->connections_free_response_.limit = BLUETOOTH_PROXY_MAX_CONNECTIONS;
   this->connections_free_response_.free = BLUETOOTH_PROXY_MAX_CONNECTIONS;
 
@@ -88,33 +77,21 @@ bool BluetoothProxy::parse_devices(const esp32_ble::BLEScanResult *scan_results,
     auto &result = scan_results[i];
     uint8_t length = result.adv_data_len + result.scan_rsp_len;
 
-    // Check if we need to expand the vector
-    if (this->advertisement_count_ >= advertisements.size()) {
-      if (this->advertisement_pool_.empty()) {
-        // No room in pool, need to allocate
-        advertisements.emplace_back();
-      } else {
-        // Pull from pool
-        advertisements.push_back(std::move(this->advertisement_pool_.back()));
-        this->advertisement_pool_.pop_back();
-      }
-    }
-
     // Fill in the data directly at current position
-    auto &adv = advertisements[this->advertisement_count_];
+    auto &adv = advertisements[this->response_.advertisements_len];
     adv.address = esp32_ble::ble_addr_to_uint64(result.bda);
     adv.rssi = result.rssi;
     adv.address_type = result.ble_addr_type;
     adv.data_len = length;
     std::memcpy(adv.data, result.ble_adv, length);
 
-    this->advertisement_count_++;
+    this->response_.advertisements_len++;
 
     ESP_LOGV(TAG, "Queuing raw packet from %02X:%02X:%02X:%02X:%02X:%02X, length %d. RSSI: %d dB", result.bda[0],
              result.bda[1], result.bda[2], result.bda[3], result.bda[4], result.bda[5], length, result.rssi);
 
-    // Flush if we have reached FLUSH_BATCH_SIZE
+    // Flush if we have reached BLUETOOTH_PROXY_ADVERTISEMENT_BATCH_SIZE
-    if (this->advertisement_count_ >= FLUSH_BATCH_SIZE) {
+    if (this->response_.advertisements_len >= BLUETOOTH_PROXY_ADVERTISEMENT_BATCH_SIZE) {
       this->flush_pending_advertisements();
     }
   }
@@ -123,27 +100,17 @@ bool BluetoothProxy::parse_devices(const esp32_ble::BLEScanResult *scan_results,
 }
 
 void BluetoothProxy::flush_pending_advertisements() {
-  if (this->advertisement_count_ == 0 || !api::global_api_server->is_connected() || this->api_connection_ == nullptr)
+  if (this->response_.advertisements_len == 0 || !api::global_api_server->is_connected() ||
+      this->api_connection_ == nullptr)
     return;
 
-  auto &advertisements = this->response_.advertisements;
-
-  // Return any items beyond advertisement_count_ to the pool
-  if (advertisements.size() > this->advertisement_count_) {
-    // Move unused items back to pool
-    this->advertisement_pool_.insert(this->advertisement_pool_.end(),
-                                     std::make_move_iterator(advertisements.begin() + this->advertisement_count_),
-                                     std::make_move_iterator(advertisements.end()));
-
-    // Resize to actual count
-    advertisements.resize(this->advertisement_count_);
-  }
-
   // Send the message
   this->api_connection_->send_message(this->response_, api::BluetoothLERawAdvertisementsResponse::MESSAGE_TYPE);
 
-  // Reset count - existing items will be overwritten in next batch
+  ESP_LOGV(TAG, "Sent batch of %u BLE advertisements", this->response_.advertisements_len);
-  this->advertisement_count_ = 0;
+
+  // Reset the length for the next batch
+  this->response_.advertisements_len = 0;
 }
 
 void BluetoothProxy::dump_config() {

esphome/components/bluetooth_proxy/bluetooth_proxy.h

@@ -150,7 +150,6 @@ class BluetoothProxy : public esp32_ble_tracker::ESPBTDeviceListener, public Com
   std::array<BluetoothConnection *, BLUETOOTH_PROXY_MAX_CONNECTIONS> connections_{};
 
   // BLE advertisement batching
-  std::vector<api::BluetoothLERawAdvertisement> advertisement_pool_;
   api::BluetoothLERawAdvertisementsResponse response_;
 
   // Group 3: 4-byte types
@@ -161,9 +160,8 @@ class BluetoothProxy : public esp32_ble_tracker::ESPBTDeviceListener, public Com
 
   // Group 4: 1-byte types grouped together
   bool active_;
-  uint8_t advertisement_count_{0};
   uint8_t connection_count_{0};
-  // 3 bytes used, 1 byte padding
+  // 2 bytes used, 2 bytes padding
 };
 
 extern BluetoothProxy *global_bluetooth_proxy;  // NOLINT(cppcoreguidelines-avoid-non-const-global-variables)

esphome/core/defines.h

@@ -148,6 +148,7 @@
 
 #define USE_BLUETOOTH_PROXY
 #define BLUETOOTH_PROXY_MAX_CONNECTIONS 3
+#define BLUETOOTH_PROXY_ADVERTISEMENT_BATCH_SIZE 16
 #define USE_CAPTIVE_PORTAL
 #define USE_ESP32_BLE
 #define USE_ESP32_BLE_CLIENT

script/api_protobuf/api_protobuf.py

@@ -339,6 +339,11 @@ def create_field_type_info(
 ) -> TypeInfo:
     """Create the appropriate TypeInfo instance for a field, handling repeated fields and custom options."""
     if field.label == 3:  # repeated
+        # Check if this repeated field has fixed_array_with_length_define option
+        if (
+            fixed_size := get_field_opt(field, pb.fixed_array_with_length_define)
+        ) is not None:
+            return FixedArrayWithLengthRepeatedType(field, fixed_size)
         # Check if this repeated field has fixed_array_size option
         if (fixed_size := get_field_opt(field, pb.fixed_array_size)) is not None:
             return FixedArrayRepeatedType(field, fixed_size)
@@ -1052,7 +1057,7 @@ def _generate_array_dump_content(
     """
     o = f"for (const auto {'' if is_bool else '&'}it : {field_name}) {{\n"
     # Check if underlying type can use dump_field
-    if type(ti).can_use_dump_field():
+    if ti.can_use_dump_field():
         # For types that have dump_field overloads, use them with extra indent
         o += f'  dump_field(out, "{name}", {ti.dump_field_value("it")}, 4);\n'
     else:
@@ -1084,6 +1089,12 @@ class FixedArrayRepeatedType(TypeInfo):
         validate_field_type(field.type, field.name)
         self._ti: TypeInfo = TYPE_INFO[field.type](field)
 
+    def _encode_element(self, element: str) -> str:
+        """Helper to generate encode statement for a single element."""
+        if isinstance(self._ti, EnumType):
+            return f"buffer.{self._ti.encode_func}({self.number}, static_cast<uint32_t>({element}), true);"
+        return f"buffer.{self._ti.encode_func}({self.number}, {element}, true);"
+
     @property
     def cpp_type(self) -> str:
         return f"std::array<{self._ti.cpp_type}, {self.array_size}>"
@@ -1111,19 +1122,13 @@ class FixedArrayRepeatedType(TypeInfo):
 
     @property
     def encode_content(self) -> str:
-        # Helper to generate encode statement for a single element
-        def encode_element(element: str) -> str:
-            if isinstance(self._ti, EnumType):
-                return f"buffer.{self._ti.encode_func}({self.number}, static_cast<uint32_t>({element}), true);"
-            return f"buffer.{self._ti.encode_func}({self.number}, {element}, true);"
-
         # If skip_zero is enabled, wrap encoding in a zero check
         if self.skip_zero:
             if self.is_define:
                 # When using a define, we need to use a loop-based approach
                 o = f"for (const auto &it : this->{self.field_name}) {{\n"
                 o += "  if (it != 0) {\n"
-                o += f"    {encode_element('it')}\n"
+                o += f"    {self._encode_element('it')}\n"
                 o += "  }\n"
                 o += "}"
                 return o
@@ -1132,7 +1137,7 @@ class FixedArrayRepeatedType(TypeInfo):
                 [f"this->{self.field_name}[{i}] != 0" for i in range(self.array_size)]
             )
             encode_lines = [
-                f"  {encode_element(f'this->{self.field_name}[{i}]')}"
+                f"  {self._encode_element(f'this->{self.field_name}[{i}]')}"
                 for i in range(self.array_size)
             ]
             return f"if ({non_zero_checks}) {{\n" + "\n".join(encode_lines) + "\n}"
@@ -1140,23 +1145,23 @@ class FixedArrayRepeatedType(TypeInfo):
         # When using a define, always use loop-based approach
         if self.is_define:
             o = f"for (const auto &it : this->{self.field_name}) {{\n"
-            o += f"  {encode_element('it')}\n"
+            o += f"  {self._encode_element('it')}\n"
             o += "}"
             return o
 
         # Unroll small arrays for efficiency
         if self.array_size == 1:
-            return encode_element(f"this->{self.field_name}[0]")
+            return self._encode_element(f"this->{self.field_name}[0]")
         if self.array_size == 2:
             return (
-                encode_element(f"this->{self.field_name}[0]")
+                self._encode_element(f"this->{self.field_name}[0]")
                 + "\n  "
-                + encode_element(f"this->{self.field_name}[1]")
+                + self._encode_element(f"this->{self.field_name}[1]")
             )
 
         # Use loops for larger arrays
         o = f"for (const auto &it : this->{self.field_name}) {{\n"
-        o += f"  {encode_element('it')}\n"
+        o += f"  {self._encode_element('it')}\n"
         o += "}"
         return o
 
@@ -1230,6 +1235,66 @@ class FixedArrayRepeatedType(TypeInfo):
         return underlying_size * self.array_size
 
 
+class FixedArrayWithLengthRepeatedType(FixedArrayRepeatedType):
+    """Special type for fixed-size repeated fields with variable length tracking.
+
+    Similar to FixedArrayRepeatedType but generates an additional length field
+    to track how many elements are actually in use. Only encodes/sends elements
+    up to the current length.
+
+    Fixed arrays with length are only supported for encoding (SOURCE_SERVER) since
+    we cannot control how many items we receive when decoding.
+    """
+
+    @property
+    def public_content(self) -> list[str]:
+        # Return both the array and the length field
+        return [
+            f"{self.cpp_type} {self.field_name}{{}};",
+            f"uint16_t {self.field_name}_len{{0}};",
+        ]
+
+    @property
+    def encode_content(self) -> str:
+        # Always use a loop up to the current length
+        o = f"for (uint16_t i = 0; i < this->{self.field_name}_len; i++) {{\n"
+        o += f"  {self._encode_element(f'this->{self.field_name}[i]')}\n"
+        o += "}"
+        return o
+
+    @property
+    def dump_content(self) -> str:
+        # Dump only the active elements
+        o = f"for (uint16_t i = 0; i < this->{self.field_name}_len; i++) {{\n"
+        # Check if underlying type can use dump_field
+        if self._ti.can_use_dump_field():
+            o += f'  dump_field(out, "{self.name}", {self._ti.dump_field_value(f"this->{self.field_name}[i]")}, 4);\n'
+        else:
+            o += f'  out.append("  {self.name}: ");\n'
+            o += indent(self._ti.dump(f"this->{self.field_name}[i]")) + "\n"
+            o += '  out.append("\\n");\n'
+        o += "}"
+        return o
+
+    def get_size_calculation(self, name: str, force: bool = False) -> str:
+        # Calculate size only for active elements
+        o = f"for (uint16_t i = 0; i < {name}_len; i++) {{\n"
+        o += f"  {self._ti.get_size_calculation(f'{name}[i]', True)}\n"
+        o += "}"
+        return o
+
+    def get_estimated_size(self) -> int:
+        # For fixed arrays with length, estimate based on typical usage
+        # Assume on average half the array is used
+        underlying_size = self._ti.get_estimated_size()
+        if self.is_define:
+            # When using a define, estimate 8 elements as typical
+            return underlying_size * 8
+        return underlying_size * (
+            self.array_size // 2 if self.array_size > 2 else self.array_size
+        )
+
+
 class RepeatedTypeInfo(TypeInfo):
     def __init__(self, field: descriptor.FieldDescriptorProto) -> None:
         super().__init__(field)
@@ -1711,6 +1776,19 @@ def build_message_type(
                 f"since we cannot trust or control the number of items received from clients."
             )
 
+        # Validate that fixed_array_with_length_define is only used in encode-only messages
+        if (
+            needs_decode
+            and field.label == 3
+            and get_field_opt(field, pb.fixed_array_with_length_define) is not None
+        ):
+            raise ValueError(
+                f"Message '{desc.name}' uses fixed_array_with_length_define on field '{field.name}' "
+                f"but has source={SOURCE_NAMES[source]}. "
+                f"Fixed arrays with length are only supported for SOURCE_SERVER (encode-only) messages "
+                f"since we cannot trust or control the number of items received from clients."
+            )
+
         ti = create_field_type_info(field, needs_decode, needs_encode)
 
         # Skip field declarations for fields that are in the base class