sascha/esphome
1
0
Fork 0
mirror of https://github.com/esphome/esphome.git synced 2025-09-01 10:52:19 +01:00
Code Issues Releases Wiki Activity

[bluetooth_proxy] Replace dynamic vector with fixed array for BLE advertisements (#10174)

Browse Source
This commit is contained in:
J. Nick Koston
2025-08-11 15:47:46 -05:00
committed by GitHub
parent 9aa21956c8
commit 42aee53dde
10 changed files with 124 additions and 70 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
esphome/components/api/api.proto
View File

@@ -1438,7 +1438,7 @@ message BluetoothLERawAdvertisementsResponse {
option (ifdef) = "USE_BLUETOOTH_PROXY";
option (no_delay) = true;
repeated BluetoothLERawAdvertisement advertisements = 1;
repeated BluetoothLERawAdvertisement advertisements = 1 [(fixed_array_with_length_define) = "BLUETOOTH_PROXY_ADVERTISEMENT_BATCH_SIZE"];
}
enum BluetoothDeviceRequestType {

1
esphome/components/api/api_options.proto
View File

@@ -30,6 +30,7 @@ extend google.protobuf.FieldOptions {
optional bool no_zero_copy = 50008 [default=false];
optional bool fixed_array_skip_zero = 50009 [default=false];
optional string fixed_array_size_define = 50010;
optional string fixed_array_with_length_define = 50011;
// container_pointer: Zero-copy optimization for repeated fields.
//

8
esphome/components/api/api_pb2.cpp
View File

@@ -1843,12 +1843,14 @@ void BluetoothLERawAdvertisement::calculate_size(ProtoSize &size) const {
size.add_length(1, this->data_len);
}
void BluetoothLERawAdvertisementsResponse::encode(ProtoWriteBuffer buffer) const {
for (auto &it : this->advertisements) {
buffer.encode_message(1, it, true);
for (uint16_t i = 0; i < this->advertisements_len; i++) {
buffer.encode_message(1, this->advertisements[i], true);
}
}
void BluetoothLERawAdvertisementsResponse::calculate_size(ProtoSize &size) const {
size.add_repeated_message(1, this->advertisements);
for (uint16_t i = 0; i < this->advertisements_len; i++) {
size.add_message_object_force(1, this->advertisements[i]);
}
}
bool BluetoothDeviceRequest::decode_varint(uint32_t field_id, ProtoVarInt value) {
switch (field_id) {

5
esphome/components/api/api_pb2.h
View File

@@ -1788,11 +1788,12 @@ class BluetoothLERawAdvertisement : public ProtoMessage {
class BluetoothLERawAdvertisementsResponse : public ProtoMessage {
public:
static constexpr uint8_t MESSAGE_TYPE = 93;
static constexpr uint8_t ESTIMATED_SIZE = 34;
static constexpr uint8_t ESTIMATED_SIZE = 136;
#ifdef HAS_PROTO_MESSAGE_DUMP
const char *message_name() const override { return "bluetooth_le_raw_advertisements_response"; }
#endif
std::vector<BluetoothLERawAdvertisement> advertisements{};
std::array<BluetoothLERawAdvertisement, BLUETOOTH_PROXY_ADVERTISEMENT_BATCH_SIZE> advertisements{};
uint16_t advertisements_len{0};
void encode(ProtoWriteBuffer buffer) const override;
void calculate_size(ProtoSize &size) const override;
#ifdef HAS_PROTO_MESSAGE_DUMP

4
esphome/components/api/api_pb2_dump.cpp
View File

@@ -1534,9 +1534,9 @@ void BluetoothLERawAdvertisement::dump_to(std::string &out) const {
}
void BluetoothLERawAdvertisementsResponse::dump_to(std::string &out) const {
MessageDumpHelper helper(out, "BluetoothLERawAdvertisementsResponse");
for (const auto &it : this->advertisements) {
for (uint16_t i = 0; i < this->advertisements_len; i++) {
out.append(" advertisements: ");
it.dump_to(out);
this->advertisements[i].dump_to(out);
out.append("\n");
}
}

6
esphome/components/bluetooth_proxy/__init__.py
View File

@@ -118,6 +118,12 @@ async def to_code(config):
connection_count = len(config.get(CONF_CONNECTIONS, []))
cg.add_define("BLUETOOTH_PROXY_MAX_CONNECTIONS", connection_count)
# Define batch size for BLE advertisements
# Each advertisement is up to 80 bytes when packaged (including protocol overhead)
# 16 advertisements × 80 bytes (worst case) = 1280 bytes out of ~1320 bytes usable payload
# This achieves ~97% WiFi MTU utilization while staying under the limit
cg.add_define("BLUETOOTH_PROXY_ADVERTISEMENT_BATCH_SIZE", 16)
for connection_conf in config.get(CONF_CONNECTIONS, []):
connection_var = cg.new_Pvariable(connection_conf[CONF_ID])
await cg.register_component(connection_var, connection_conf)

57
esphome/components/bluetooth_proxy/bluetooth_proxy.cpp
View File

@@ -11,12 +11,8 @@ namespace esphome::bluetooth_proxy {
static const char *const TAG = "bluetooth_proxy";
// Batch size for BLE advertisements to maximize WiFi efficiency
// Each advertisement is up to 80 bytes when packaged (including protocol overhead)
// Most advertisements are 20-30 bytes, allowing even more to fit per packet
// 16 advertisements × 80 bytes (worst case) = 1280 bytes out of ~1320 bytes usable payload
// This achieves ~97% WiFi MTU utilization while staying under the limit
static constexpr size_t FLUSH_BATCH_SIZE = 16;
// BLUETOOTH_PROXY_ADVERTISEMENT_BATCH_SIZE is defined during code generation
// It sets the batch size for BLE advertisements to maximize WiFi efficiency
// Verify BLE advertisement data array size matches the BLE specification (31 bytes adv + 31 bytes scan response)
static_assert(sizeof(((api::BluetoothLERawAdvertisement *) nullptr)->data) == 62,
@@ -25,13 +21,6 @@ static_assert(sizeof(((api::BluetoothLERawAdvertisement *) nullptr)->data) == 62
BluetoothProxy::BluetoothProxy() { global_bluetooth_proxy = this; }
void BluetoothProxy::setup() {
// Reserve capacity but start with size 0
// Reserve 50% since we'll grow naturally and flush at FLUSH_BATCH_SIZE
this->response_.advertisements.reserve(FLUSH_BATCH_SIZE / 2);
// Don't pre-allocate pool - let it grow only if needed in busy environments
// Many devices in quiet areas will never need the overflow pool
this->connections_free_response_.limit = BLUETOOTH_PROXY_MAX_CONNECTIONS;
this->connections_free_response_.free = BLUETOOTH_PROXY_MAX_CONNECTIONS;
@@ -88,33 +77,21 @@ bool BluetoothProxy::parse_devices(const esp32_ble::BLEScanResult *scan_results,
auto &result = scan_results[i];
uint8_t length = result.adv_data_len + result.scan_rsp_len;
// Check if we need to expand the vector
if (this->advertisement_count_ >= advertisements.size()) {
if (this->advertisement_pool_.empty()) {
// No room in pool, need to allocate
advertisements.emplace_back();
} else {
// Pull from pool
advertisements.push_back(std::move(this->advertisement_pool_.back()));
this->advertisement_pool_.pop_back();
}
}
// Fill in the data directly at current position
auto &adv = advertisements[this->advertisement_count_];
auto &adv = advertisements[this->response_.advertisements_len];
adv.address = esp32_ble::ble_addr_to_uint64(result.bda);
adv.rssi = result.rssi;
adv.address_type = result.ble_addr_type;
adv.data_len = length;
std::memcpy(adv.data, result.ble_adv, length);
this->advertisement_count_++;
this->response_.advertisements_len++;
ESP_LOGV(TAG, "Queuing raw packet from %02X:%02X:%02X:%02X:%02X:%02X, length %d. RSSI: %d dB", result.bda[0],
result.bda[1], result.bda[2], result.bda[3], result.bda[4], result.bda[5], length, result.rssi);
// Flush if we have reached FLUSH_BATCH_SIZE
if (this->advertisement_count_ >= FLUSH_BATCH_SIZE) {
// Flush if we have reached BLUETOOTH_PROXY_ADVERTISEMENT_BATCH_SIZE
if (this->response_.advertisements_len >= BLUETOOTH_PROXY_ADVERTISEMENT_BATCH_SIZE) {
this->flush_pending_advertisements();
}
}
@@ -123,27 +100,17 @@ bool BluetoothProxy::parse_devices(const esp32_ble::BLEScanResult *scan_results,
}
void BluetoothProxy::flush_pending_advertisements() {
if (this->advertisement_count_ == 0 || !api::global_api_server->is_connected() || this->api_connection_ == nullptr)
if (this->response_.advertisements_len == 0 || !api::global_api_server->is_connected() ||
this->api_connection_ == nullptr)
return;
auto &advertisements = this->response_.advertisements;
// Return any items beyond advertisement_count_ to the pool
if (advertisements.size() > this->advertisement_count_) {
// Move unused items back to pool
this->advertisement_pool_.insert(this->advertisement_pool_.end(),
std::make_move_iterator(advertisements.begin() + this->advertisement_count_),
std::make_move_iterator(advertisements.end()));
// Resize to actual count
advertisements.resize(this->advertisement_count_);
}
// Send the message
this->api_connection_->send_message(this->response_, api::BluetoothLERawAdvertisementsResponse::MESSAGE_TYPE);
// Reset count - existing items will be overwritten in next batch
this->advertisement_count_ = 0;
ESP_LOGV(TAG, "Sent batch of %u BLE advertisements", this->response_.advertisements_len);
// Reset the length for the next batch
this->response_.advertisements_len = 0;
}
void BluetoothProxy::dump_config() {

4
esphome/components/bluetooth_proxy/bluetooth_proxy.h
View File

@@ -150,7 +150,6 @@ class BluetoothProxy : public esp32_ble_tracker::ESPBTDeviceListener, public Com
std::array<BluetoothConnection *, BLUETOOTH_PROXY_MAX_CONNECTIONS> connections_{};
// BLE advertisement batching
std::vector<api::BluetoothLERawAdvertisement> advertisement_pool_;
api::BluetoothLERawAdvertisementsResponse response_;
// Group 3: 4-byte types
@@ -161,9 +160,8 @@ class BluetoothProxy : public esp32_ble_tracker::ESPBTDeviceListener, public Com
// Group 4: 1-byte types grouped together
bool active_;
uint8_t advertisement_count_{0};
uint8_t connection_count_{0};
// 3 bytes used, 1 byte padding
// 2 bytes used, 2 bytes padding
};
extern BluetoothProxy *global_bluetooth_proxy; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables)

1
esphome/core/defines.h
View File

@@ -148,6 +148,7 @@
#define USE_BLUETOOTH_PROXY
#define BLUETOOTH_PROXY_MAX_CONNECTIONS 3
#define BLUETOOTH_PROXY_ADVERTISEMENT_BATCH_SIZE 16
#define USE_CAPTIVE_PORTAL
#define USE_ESP32_BLE
#define USE_ESP32_BLE_CLIENT

106
script/api_protobuf/api_protobuf.py
View File

@@ -339,6 +339,11 @@ def create_field_type_info(
) -> TypeInfo:
"""Create the appropriate TypeInfo instance for a field, handling repeated fields and custom options."""
if field.label == 3: # repeated
# Check if this repeated field has fixed_array_with_length_define option
if (
fixed_size := get_field_opt(field, pb.fixed_array_with_length_define)
) is not None:
return FixedArrayWithLengthRepeatedType(field, fixed_size)
# Check if this repeated field has fixed_array_size option
if (fixed_size := get_field_opt(field, pb.fixed_array_size)) is not None:
return FixedArrayRepeatedType(field, fixed_size)
@@ -1052,7 +1057,7 @@ def _generate_array_dump_content(
"""
o = f"for (const auto {'' if is_bool else '&'}it : {field_name}) {{\n"
# Check if underlying type can use dump_field
if type(ti).can_use_dump_field():
if ti.can_use_dump_field():
# For types that have dump_field overloads, use them with extra indent
o += f' dump_field(out, "{name}", {ti.dump_field_value("it")}, 4);\n'
else:
@@ -1084,6 +1089,12 @@ class FixedArrayRepeatedType(TypeInfo):
validate_field_type(field.type, field.name)
self._ti: TypeInfo = TYPE_INFO[field.type](field)
def _encode_element(self, element: str) -> str:
"""Helper to generate encode statement for a single element."""
if isinstance(self._ti, EnumType):
return f"buffer.{self._ti.encode_func}({self.number}, static_cast<uint32_t>({element}), true);"
return f"buffer.{self._ti.encode_func}({self.number}, {element}, true);"
@property
def cpp_type(self) -> str:
return f"std::array<{self._ti.cpp_type}, {self.array_size}>"
@@ -1111,19 +1122,13 @@ class FixedArrayRepeatedType(TypeInfo):
@property
def encode_content(self) -> str:
# Helper to generate encode statement for a single element
def encode_element(element: str) -> str:
if isinstance(self._ti, EnumType):
return f"buffer.{self._ti.encode_func}({self.number}, static_cast<uint32_t>({element}), true);"
return f"buffer.{self._ti.encode_func}({self.number}, {element}, true);"
# If skip_zero is enabled, wrap encoding in a zero check
if self.skip_zero:
if self.is_define:
# When using a define, we need to use a loop-based approach
o = f"for (const auto &it : this->{self.field_name}) {{\n"
o += " if (it != 0) {\n"
o += f" {encode_element('it')}\n"
o += f" {self._encode_element('it')}\n"
o += " }\n"
o += "}"
return o
@@ -1132,7 +1137,7 @@ class FixedArrayRepeatedType(TypeInfo):
[f"this->{self.field_name}[{i}] != 0" for i in range(self.array_size)]
)
encode_lines = [
f" {encode_element(f'this->{self.field_name}[{i}]')}"
f" {self._encode_element(f'this->{self.field_name}[{i}]')}"
for i in range(self.array_size)
]
return f"if ({non_zero_checks}) {{\n" + "\n".join(encode_lines) + "\n}"
@@ -1140,23 +1145,23 @@ class FixedArrayRepeatedType(TypeInfo):
# When using a define, always use loop-based approach
if self.is_define:
o = f"for (const auto &it : this->{self.field_name}) {{\n"
o += f" {encode_element('it')}\n"
o += f" {self._encode_element('it')}\n"
o += "}"
return o
# Unroll small arrays for efficiency
if self.array_size == 1:
return encode_element(f"this->{self.field_name}[0]")
return self._encode_element(f"this->{self.field_name}[0]")
if self.array_size == 2:
return (
encode_element(f"this->{self.field_name}[0]")
self._encode_element(f"this->{self.field_name}[0]")
+ "\n "
+ encode_element(f"this->{self.field_name}[1]")
+ self._encode_element(f"this->{self.field_name}[1]")
)
# Use loops for larger arrays
o = f"for (const auto &it : this->{self.field_name}) {{\n"
o += f" {encode_element('it')}\n"
o += f" {self._encode_element('it')}\n"
o += "}"
return o
@@ -1230,6 +1235,66 @@ class FixedArrayRepeatedType(TypeInfo):
return underlying_size * self.array_size
class FixedArrayWithLengthRepeatedType(FixedArrayRepeatedType):
"""Special type for fixed-size repeated fields with variable length tracking.
Similar to FixedArrayRepeatedType but generates an additional length field
to track how many elements are actually in use. Only encodes/sends elements
up to the current length.
Fixed arrays with length are only supported for encoding (SOURCE_SERVER) since
we cannot control how many items we receive when decoding.
"""
@property
def public_content(self) -> list[str]:
# Return both the array and the length field
return [
f"{self.cpp_type} {self.field_name}{{}};",
f"uint16_t {self.field_name}_len{{0}};",
]
@property
def encode_content(self) -> str:
# Always use a loop up to the current length
o = f"for (uint16_t i = 0; i < this->{self.field_name}_len; i++) {{\n"
o += f" {self._encode_element(f'this->{self.field_name}[i]')}\n"
o += "}"
return o
@property
def dump_content(self) -> str:
# Dump only the active elements
o = f"for (uint16_t i = 0; i < this->{self.field_name}_len; i++) {{\n"
# Check if underlying type can use dump_field
if self._ti.can_use_dump_field():
o += f' dump_field(out, "{self.name}", {self._ti.dump_field_value(f"this->{self.field_name}[i]")}, 4);\n'
else:
o += f' out.append(" {self.name}: ");\n'
o += indent(self._ti.dump(f"this->{self.field_name}[i]")) + "\n"
o += ' out.append("\\n");\n'
o += "}"
return o
def get_size_calculation(self, name: str, force: bool = False) -> str:
# Calculate size only for active elements
o = f"for (uint16_t i = 0; i < {name}_len; i++) {{\n"
o += f" {self._ti.get_size_calculation(f'{name}[i]', True)}\n"
o += "}"
return o
def get_estimated_size(self) -> int:
# For fixed arrays with length, estimate based on typical usage
# Assume on average half the array is used
underlying_size = self._ti.get_estimated_size()
if self.is_define:
# When using a define, estimate 8 elements as typical
return underlying_size * 8
return underlying_size * (
self.array_size // 2 if self.array_size > 2 else self.array_size
)
class RepeatedTypeInfo(TypeInfo):
def __init__(self, field: descriptor.FieldDescriptorProto) -> None:
super().__init__(field)
@@ -1711,6 +1776,19 @@ def build_message_type(
f"since we cannot trust or control the number of items received from clients."
)
# Validate that fixed_array_with_length_define is only used in encode-only messages
if (
needs_decode
and field.label == 3
and get_field_opt(field, pb.fixed_array_with_length_define) is not None
):
raise ValueError(
f"Message '{desc.name}' uses fixed_array_with_length_define on field '{field.name}' "
f"but has source={SOURCE_NAMES[source]}. "
f"Fixed arrays with length are only supported for SOURCE_SERVER (encode-only) messages "
f"since we cannot trust or control the number of items received from clients."
)
ti = create_field_type_info(field, needs_decode, needs_encode)
# Skip field declarations for fields that are in the base class