diff --git a/esphome/components/api/api.proto b/esphome/components/api/api.proto index 9d77ecdfa8..6b19f2026a 100644 --- a/esphome/components/api/api.proto +++ b/esphome/components/api/api.proto @@ -1438,7 +1438,7 @@ message BluetoothLERawAdvertisementsResponse { option (ifdef) = "USE_BLUETOOTH_PROXY"; option (no_delay) = true; - repeated BluetoothLERawAdvertisement advertisements = 1; + repeated BluetoothLERawAdvertisement advertisements = 1 [(fixed_array_with_length_define) = "BLUETOOTH_PROXY_ADVERTISEMENT_BATCH_SIZE"]; } enum BluetoothDeviceRequestType { diff --git a/esphome/components/api/api_options.proto b/esphome/components/api/api_options.proto index ed0e0d7455..50c43b96fd 100644 --- a/esphome/components/api/api_options.proto +++ b/esphome/components/api/api_options.proto @@ -30,6 +30,7 @@ extend google.protobuf.FieldOptions { optional bool no_zero_copy = 50008 [default=false]; optional bool fixed_array_skip_zero = 50009 [default=false]; optional string fixed_array_size_define = 50010; + optional string fixed_array_with_length_define = 50011; // container_pointer: Zero-copy optimization for repeated fields. // diff --git a/esphome/components/api/api_pb2.cpp b/esphome/components/api/api_pb2.cpp index 5dddc79b49..476e3c88d0 100644 --- a/esphome/components/api/api_pb2.cpp +++ b/esphome/components/api/api_pb2.cpp @@ -1843,12 +1843,14 @@ void BluetoothLERawAdvertisement::calculate_size(ProtoSize &size) const { size.add_length(1, this->data_len); } void BluetoothLERawAdvertisementsResponse::encode(ProtoWriteBuffer buffer) const { - for (auto &it : this->advertisements) { - buffer.encode_message(1, it, true); + for (uint16_t i = 0; i < this->advertisements_len; i++) { + buffer.encode_message(1, this->advertisements[i], true); } } void BluetoothLERawAdvertisementsResponse::calculate_size(ProtoSize &size) const { - size.add_repeated_message(1, this->advertisements); + for (uint16_t i = 0; i < this->advertisements_len; i++) { + size.add_message_object_force(1, this->advertisements[i]); + } } bool BluetoothDeviceRequest::decode_varint(uint32_t field_id, ProtoVarInt value) { switch (field_id) { diff --git a/esphome/components/api/api_pb2.h b/esphome/components/api/api_pb2.h index d43d3c61b7..edf839be55 100644 --- a/esphome/components/api/api_pb2.h +++ b/esphome/components/api/api_pb2.h @@ -1788,11 +1788,12 @@ class BluetoothLERawAdvertisement : public ProtoMessage { class BluetoothLERawAdvertisementsResponse : public ProtoMessage { public: static constexpr uint8_t MESSAGE_TYPE = 93; - static constexpr uint8_t ESTIMATED_SIZE = 34; + static constexpr uint8_t ESTIMATED_SIZE = 136; #ifdef HAS_PROTO_MESSAGE_DUMP const char *message_name() const override { return "bluetooth_le_raw_advertisements_response"; } #endif - std::vector advertisements{}; + std::array advertisements{}; + uint16_t advertisements_len{0}; void encode(ProtoWriteBuffer buffer) const override; void calculate_size(ProtoSize &size) const override; #ifdef HAS_PROTO_MESSAGE_DUMP diff --git a/esphome/components/api/api_pb2_dump.cpp b/esphome/components/api/api_pb2_dump.cpp index b212353ad8..7af322f96d 100644 --- a/esphome/components/api/api_pb2_dump.cpp +++ b/esphome/components/api/api_pb2_dump.cpp @@ -1534,9 +1534,9 @@ void BluetoothLERawAdvertisement::dump_to(std::string &out) const { } void BluetoothLERawAdvertisementsResponse::dump_to(std::string &out) const { MessageDumpHelper helper(out, "BluetoothLERawAdvertisementsResponse"); - for (const auto &it : this->advertisements) { + for (uint16_t i = 0; i < this->advertisements_len; i++) { out.append(" advertisements: "); - it.dump_to(out); + this->advertisements[i].dump_to(out); out.append("\n"); } } diff --git a/esphome/components/bluetooth_proxy/__init__.py b/esphome/components/bluetooth_proxy/__init__.py index fb7f7a37c0..112faa27e5 100644 --- a/esphome/components/bluetooth_proxy/__init__.py +++ b/esphome/components/bluetooth_proxy/__init__.py @@ -118,6 +118,12 @@ async def to_code(config): connection_count = len(config.get(CONF_CONNECTIONS, [])) cg.add_define("BLUETOOTH_PROXY_MAX_CONNECTIONS", connection_count) + # Define batch size for BLE advertisements + # Each advertisement is up to 80 bytes when packaged (including protocol overhead) + # 16 advertisements × 80 bytes (worst case) = 1280 bytes out of ~1320 bytes usable payload + # This achieves ~97% WiFi MTU utilization while staying under the limit + cg.add_define("BLUETOOTH_PROXY_ADVERTISEMENT_BATCH_SIZE", 16) + for connection_conf in config.get(CONF_CONNECTIONS, []): connection_var = cg.new_Pvariable(connection_conf[CONF_ID]) await cg.register_component(connection_var, connection_conf) diff --git a/esphome/components/bluetooth_proxy/bluetooth_proxy.cpp b/esphome/components/bluetooth_proxy/bluetooth_proxy.cpp index 04b85fc3f0..723466a5ff 100644 --- a/esphome/components/bluetooth_proxy/bluetooth_proxy.cpp +++ b/esphome/components/bluetooth_proxy/bluetooth_proxy.cpp @@ -11,12 +11,8 @@ namespace esphome::bluetooth_proxy { static const char *const TAG = "bluetooth_proxy"; -// Batch size for BLE advertisements to maximize WiFi efficiency -// Each advertisement is up to 80 bytes when packaged (including protocol overhead) -// Most advertisements are 20-30 bytes, allowing even more to fit per packet -// 16 advertisements × 80 bytes (worst case) = 1280 bytes out of ~1320 bytes usable payload -// This achieves ~97% WiFi MTU utilization while staying under the limit -static constexpr size_t FLUSH_BATCH_SIZE = 16; +// BLUETOOTH_PROXY_ADVERTISEMENT_BATCH_SIZE is defined during code generation +// It sets the batch size for BLE advertisements to maximize WiFi efficiency // Verify BLE advertisement data array size matches the BLE specification (31 bytes adv + 31 bytes scan response) static_assert(sizeof(((api::BluetoothLERawAdvertisement *) nullptr)->data) == 62, @@ -25,13 +21,6 @@ static_assert(sizeof(((api::BluetoothLERawAdvertisement *) nullptr)->data) == 62 BluetoothProxy::BluetoothProxy() { global_bluetooth_proxy = this; } void BluetoothProxy::setup() { - // Reserve capacity but start with size 0 - // Reserve 50% since we'll grow naturally and flush at FLUSH_BATCH_SIZE - this->response_.advertisements.reserve(FLUSH_BATCH_SIZE / 2); - - // Don't pre-allocate pool - let it grow only if needed in busy environments - // Many devices in quiet areas will never need the overflow pool - this->connections_free_response_.limit = BLUETOOTH_PROXY_MAX_CONNECTIONS; this->connections_free_response_.free = BLUETOOTH_PROXY_MAX_CONNECTIONS; @@ -88,33 +77,21 @@ bool BluetoothProxy::parse_devices(const esp32_ble::BLEScanResult *scan_results, auto &result = scan_results[i]; uint8_t length = result.adv_data_len + result.scan_rsp_len; - // Check if we need to expand the vector - if (this->advertisement_count_ >= advertisements.size()) { - if (this->advertisement_pool_.empty()) { - // No room in pool, need to allocate - advertisements.emplace_back(); - } else { - // Pull from pool - advertisements.push_back(std::move(this->advertisement_pool_.back())); - this->advertisement_pool_.pop_back(); - } - } - // Fill in the data directly at current position - auto &adv = advertisements[this->advertisement_count_]; + auto &adv = advertisements[this->response_.advertisements_len]; adv.address = esp32_ble::ble_addr_to_uint64(result.bda); adv.rssi = result.rssi; adv.address_type = result.ble_addr_type; adv.data_len = length; std::memcpy(adv.data, result.ble_adv, length); - this->advertisement_count_++; + this->response_.advertisements_len++; ESP_LOGV(TAG, "Queuing raw packet from %02X:%02X:%02X:%02X:%02X:%02X, length %d. RSSI: %d dB", result.bda[0], result.bda[1], result.bda[2], result.bda[3], result.bda[4], result.bda[5], length, result.rssi); - // Flush if we have reached FLUSH_BATCH_SIZE - if (this->advertisement_count_ >= FLUSH_BATCH_SIZE) { + // Flush if we have reached BLUETOOTH_PROXY_ADVERTISEMENT_BATCH_SIZE + if (this->response_.advertisements_len >= BLUETOOTH_PROXY_ADVERTISEMENT_BATCH_SIZE) { this->flush_pending_advertisements(); } } @@ -123,27 +100,17 @@ bool BluetoothProxy::parse_devices(const esp32_ble::BLEScanResult *scan_results, } void BluetoothProxy::flush_pending_advertisements() { - if (this->advertisement_count_ == 0 || !api::global_api_server->is_connected() || this->api_connection_ == nullptr) + if (this->response_.advertisements_len == 0 || !api::global_api_server->is_connected() || + this->api_connection_ == nullptr) return; - auto &advertisements = this->response_.advertisements; - - // Return any items beyond advertisement_count_ to the pool - if (advertisements.size() > this->advertisement_count_) { - // Move unused items back to pool - this->advertisement_pool_.insert(this->advertisement_pool_.end(), - std::make_move_iterator(advertisements.begin() + this->advertisement_count_), - std::make_move_iterator(advertisements.end())); - - // Resize to actual count - advertisements.resize(this->advertisement_count_); - } - // Send the message this->api_connection_->send_message(this->response_, api::BluetoothLERawAdvertisementsResponse::MESSAGE_TYPE); - // Reset count - existing items will be overwritten in next batch - this->advertisement_count_ = 0; + ESP_LOGV(TAG, "Sent batch of %u BLE advertisements", this->response_.advertisements_len); + + // Reset the length for the next batch + this->response_.advertisements_len = 0; } void BluetoothProxy::dump_config() { diff --git a/esphome/components/bluetooth_proxy/bluetooth_proxy.h b/esphome/components/bluetooth_proxy/bluetooth_proxy.h index 21695d9819..bc8d3ed762 100644 --- a/esphome/components/bluetooth_proxy/bluetooth_proxy.h +++ b/esphome/components/bluetooth_proxy/bluetooth_proxy.h @@ -150,7 +150,6 @@ class BluetoothProxy : public esp32_ble_tracker::ESPBTDeviceListener, public Com std::array connections_{}; // BLE advertisement batching - std::vector advertisement_pool_; api::BluetoothLERawAdvertisementsResponse response_; // Group 3: 4-byte types @@ -161,9 +160,8 @@ class BluetoothProxy : public esp32_ble_tracker::ESPBTDeviceListener, public Com // Group 4: 1-byte types grouped together bool active_; - uint8_t advertisement_count_{0}; uint8_t connection_count_{0}; - // 3 bytes used, 1 byte padding + // 2 bytes used, 2 bytes padding }; extern BluetoothProxy *global_bluetooth_proxy; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables) diff --git a/esphome/core/defines.h b/esphome/core/defines.h index 7631ff54f3..01f6811e05 100644 --- a/esphome/core/defines.h +++ b/esphome/core/defines.h @@ -148,6 +148,7 @@ #define USE_BLUETOOTH_PROXY #define BLUETOOTH_PROXY_MAX_CONNECTIONS 3 +#define BLUETOOTH_PROXY_ADVERTISEMENT_BATCH_SIZE 16 #define USE_CAPTIVE_PORTAL #define USE_ESP32_BLE #define USE_ESP32_BLE_CLIENT diff --git a/script/api_protobuf/api_protobuf.py b/script/api_protobuf/api_protobuf.py index fa2f87d98d..3396e5ad05 100755 --- a/script/api_protobuf/api_protobuf.py +++ b/script/api_protobuf/api_protobuf.py @@ -339,6 +339,11 @@ def create_field_type_info( ) -> TypeInfo: """Create the appropriate TypeInfo instance for a field, handling repeated fields and custom options.""" if field.label == 3: # repeated + # Check if this repeated field has fixed_array_with_length_define option + if ( + fixed_size := get_field_opt(field, pb.fixed_array_with_length_define) + ) is not None: + return FixedArrayWithLengthRepeatedType(field, fixed_size) # Check if this repeated field has fixed_array_size option if (fixed_size := get_field_opt(field, pb.fixed_array_size)) is not None: return FixedArrayRepeatedType(field, fixed_size) @@ -1052,7 +1057,7 @@ def _generate_array_dump_content( """ o = f"for (const auto {'' if is_bool else '&'}it : {field_name}) {{\n" # Check if underlying type can use dump_field - if type(ti).can_use_dump_field(): + if ti.can_use_dump_field(): # For types that have dump_field overloads, use them with extra indent o += f' dump_field(out, "{name}", {ti.dump_field_value("it")}, 4);\n' else: @@ -1084,6 +1089,12 @@ class FixedArrayRepeatedType(TypeInfo): validate_field_type(field.type, field.name) self._ti: TypeInfo = TYPE_INFO[field.type](field) + def _encode_element(self, element: str) -> str: + """Helper to generate encode statement for a single element.""" + if isinstance(self._ti, EnumType): + return f"buffer.{self._ti.encode_func}({self.number}, static_cast({element}), true);" + return f"buffer.{self._ti.encode_func}({self.number}, {element}, true);" + @property def cpp_type(self) -> str: return f"std::array<{self._ti.cpp_type}, {self.array_size}>" @@ -1111,19 +1122,13 @@ class FixedArrayRepeatedType(TypeInfo): @property def encode_content(self) -> str: - # Helper to generate encode statement for a single element - def encode_element(element: str) -> str: - if isinstance(self._ti, EnumType): - return f"buffer.{self._ti.encode_func}({self.number}, static_cast({element}), true);" - return f"buffer.{self._ti.encode_func}({self.number}, {element}, true);" - # If skip_zero is enabled, wrap encoding in a zero check if self.skip_zero: if self.is_define: # When using a define, we need to use a loop-based approach o = f"for (const auto &it : this->{self.field_name}) {{\n" o += " if (it != 0) {\n" - o += f" {encode_element('it')}\n" + o += f" {self._encode_element('it')}\n" o += " }\n" o += "}" return o @@ -1132,7 +1137,7 @@ class FixedArrayRepeatedType(TypeInfo): [f"this->{self.field_name}[{i}] != 0" for i in range(self.array_size)] ) encode_lines = [ - f" {encode_element(f'this->{self.field_name}[{i}]')}" + f" {self._encode_element(f'this->{self.field_name}[{i}]')}" for i in range(self.array_size) ] return f"if ({non_zero_checks}) {{\n" + "\n".join(encode_lines) + "\n}" @@ -1140,23 +1145,23 @@ class FixedArrayRepeatedType(TypeInfo): # When using a define, always use loop-based approach if self.is_define: o = f"for (const auto &it : this->{self.field_name}) {{\n" - o += f" {encode_element('it')}\n" + o += f" {self._encode_element('it')}\n" o += "}" return o # Unroll small arrays for efficiency if self.array_size == 1: - return encode_element(f"this->{self.field_name}[0]") + return self._encode_element(f"this->{self.field_name}[0]") if self.array_size == 2: return ( - encode_element(f"this->{self.field_name}[0]") + self._encode_element(f"this->{self.field_name}[0]") + "\n " - + encode_element(f"this->{self.field_name}[1]") + + self._encode_element(f"this->{self.field_name}[1]") ) # Use loops for larger arrays o = f"for (const auto &it : this->{self.field_name}) {{\n" - o += f" {encode_element('it')}\n" + o += f" {self._encode_element('it')}\n" o += "}" return o @@ -1230,6 +1235,66 @@ class FixedArrayRepeatedType(TypeInfo): return underlying_size * self.array_size +class FixedArrayWithLengthRepeatedType(FixedArrayRepeatedType): + """Special type for fixed-size repeated fields with variable length tracking. + + Similar to FixedArrayRepeatedType but generates an additional length field + to track how many elements are actually in use. Only encodes/sends elements + up to the current length. + + Fixed arrays with length are only supported for encoding (SOURCE_SERVER) since + we cannot control how many items we receive when decoding. + """ + + @property + def public_content(self) -> list[str]: + # Return both the array and the length field + return [ + f"{self.cpp_type} {self.field_name}{{}};", + f"uint16_t {self.field_name}_len{{0}};", + ] + + @property + def encode_content(self) -> str: + # Always use a loop up to the current length + o = f"for (uint16_t i = 0; i < this->{self.field_name}_len; i++) {{\n" + o += f" {self._encode_element(f'this->{self.field_name}[i]')}\n" + o += "}" + return o + + @property + def dump_content(self) -> str: + # Dump only the active elements + o = f"for (uint16_t i = 0; i < this->{self.field_name}_len; i++) {{\n" + # Check if underlying type can use dump_field + if self._ti.can_use_dump_field(): + o += f' dump_field(out, "{self.name}", {self._ti.dump_field_value(f"this->{self.field_name}[i]")}, 4);\n' + else: + o += f' out.append(" {self.name}: ");\n' + o += indent(self._ti.dump(f"this->{self.field_name}[i]")) + "\n" + o += ' out.append("\\n");\n' + o += "}" + return o + + def get_size_calculation(self, name: str, force: bool = False) -> str: + # Calculate size only for active elements + o = f"for (uint16_t i = 0; i < {name}_len; i++) {{\n" + o += f" {self._ti.get_size_calculation(f'{name}[i]', True)}\n" + o += "}" + return o + + def get_estimated_size(self) -> int: + # For fixed arrays with length, estimate based on typical usage + # Assume on average half the array is used + underlying_size = self._ti.get_estimated_size() + if self.is_define: + # When using a define, estimate 8 elements as typical + return underlying_size * 8 + return underlying_size * ( + self.array_size // 2 if self.array_size > 2 else self.array_size + ) + + class RepeatedTypeInfo(TypeInfo): def __init__(self, field: descriptor.FieldDescriptorProto) -> None: super().__init__(field) @@ -1711,6 +1776,19 @@ def build_message_type( f"since we cannot trust or control the number of items received from clients." ) + # Validate that fixed_array_with_length_define is only used in encode-only messages + if ( + needs_decode + and field.label == 3 + and get_field_opt(field, pb.fixed_array_with_length_define) is not None + ): + raise ValueError( + f"Message '{desc.name}' uses fixed_array_with_length_define on field '{field.name}' " + f"but has source={SOURCE_NAMES[source]}. " + f"Fixed arrays with length are only supported for SOURCE_SERVER (encode-only) messages " + f"since we cannot trust or control the number of items received from clients." + ) + ti = create_field_type_info(field, needs_decode, needs_encode) # Skip field declarations for fields that are in the base class