From 3a80aac6e8356889b1c394f9d349e3571953c915 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Thu, 31 Jul 2025 06:35:19 -1000 Subject: [PATCH 01/11] ble dynamic batch --- .../bluetooth_proxy/bluetooth_connection.cpp | 82 +++++++++++++++++-- .../bluetooth_proxy/bluetooth_proxy.h | 1 - 2 files changed, 73 insertions(+), 10 deletions(-) diff --git a/esphome/components/bluetooth_proxy/bluetooth_connection.cpp b/esphome/components/bluetooth_proxy/bluetooth_connection.cpp index 4f312fce30..931bae7247 100644 --- a/esphome/components/bluetooth_proxy/bluetooth_connection.cpp +++ b/esphome/components/bluetooth_proxy/bluetooth_connection.cpp @@ -37,6 +37,25 @@ static void fill_gatt_uuid(std::array &uuid_128, uint32_t &short_uu } } +// Constants for size estimation +static constexpr uint8_t SERVICE_OVERHEAD_LEGACY = 25; // UUID(20) + handle(4) + overhead(1) +static constexpr uint8_t SERVICE_OVERHEAD_EFFICIENT = 10; // UUID(6) + handle(4) +static constexpr uint8_t CHAR_SIZE_128BIT = 35; // UUID(20) + handle(4) + props(4) + overhead(7) +static constexpr uint8_t DESC_SIZE_128BIT = 25; // UUID(20) + handle(4) + overhead(1) +static constexpr uint8_t DESC_SIZE_16BIT = 10; // UUID(6) + handle(4) +static constexpr uint8_t DESC_PER_CHAR = 2; // Assume 2 descriptors per characteristic + +// Helper to estimate service size before fetching all data +static size_t estimate_service_size(uint16_t char_count, bool use_efficient_uuids) { + size_t service_overhead = use_efficient_uuids ? SERVICE_OVERHEAD_EFFICIENT : SERVICE_OVERHEAD_LEGACY; + // Always assume 128-bit UUIDs for characteristics to be safe + size_t char_size = CHAR_SIZE_128BIT; + // Assume mix of descriptor types: one 128-bit + one 16-bit per characteristic + size_t desc_size = (DESC_SIZE_128BIT + DESC_SIZE_16BIT) * DESC_PER_CHAR; + + return service_overhead + (char_size + desc_size) * char_count; +} + bool BluetoothConnection::supports_efficient_uuids_() const { auto *api_conn = this->proxy_->get_api_connection(); return api_conn && api_conn->client_supports_api_version(1, 12); @@ -95,16 +114,21 @@ void BluetoothConnection::send_service_for_discovery_() { // Check if client supports efficient UUIDs bool use_efficient_uuids = this->supports_efficient_uuids_(); - // Prepare response for up to 3 services + // Prepare response api::BluetoothGATTGetServicesResponse resp; resp.address = this->address_; - // Process up to 3 services in this iteration - uint8_t services_to_process = - std::min(MAX_SERVICES_PER_BATCH, static_cast(this->service_count_ - this->send_service_)); - resp.services.reserve(services_to_process); + // Dynamic batching based on actual size + static constexpr size_t MAX_PACKET_SIZE = 1390; // MTU limit for API messages + static constexpr size_t NEXT_SERVICE_BUFFER = 400; // Reserve space for next service - for (int service_idx = 0; service_idx < services_to_process; service_idx++) { + // Keep running total of actual message size + size_t current_size = 0; + api::ProtoSize size; + resp.calculate_size(size); + current_size = size.get_size(); + + while (this->send_service_ < this->service_count_) { esp_gattc_service_elem_t service_result; uint16_t service_count = 1; esp_gatt_status_t service_status = esp_ble_gattc_get_service(this->gattc_if_, this->conn_id_, nullptr, @@ -118,7 +142,6 @@ void BluetoothConnection::send_service_for_discovery_() { return; } - this->send_service_++; resp.services.emplace_back(); auto &service_resp = resp.services.back(); @@ -139,8 +162,16 @@ void BluetoothConnection::send_service_for_discovery_() { return; } + // If this service likely won't fit, send current batch (unless it's the first) + if (!resp.services.empty() && + (current_size + estimate_service_size(total_char_count, use_efficient_uuids) > MAX_PACKET_SIZE)) { + // This service likely won't fit, send current batch + break; + } + if (total_char_count == 0) { - // No characteristics, continue to next service + // No characteristics, increment and continue to next service + this->send_service_++; continue; } @@ -221,9 +252,42 @@ void BluetoothConnection::send_service_for_discovery_() { desc_offset++; } } + + // Calculate the actual size of just this service + api::ProtoSize service_size; + service_resp.calculate_size(service_size); + + // Update running total + current_size += service_size.get_size() + 1; // +1 for field tag + + // Check if we've exceeded the limit (worst case scenario) + // Our estimation above should have caught this, but if we're here it means + // this service is extraordinarily large (many characteristics/descriptors) + if (current_size > MAX_PACKET_SIZE) { + // We've gone over - pop the last service if we have more than one + if (resp.services.size() > 1) { + resp.services.pop_back(); + // Don't increment send_service_ - we'll retry this service in next batch + } else { + // This single service is too large, but we have to send it anyway + // Increment so we don't get stuck + this->send_service_++; + } + // Send what we have + break; + } + + // Successfully added this service, increment counter + this->send_service_++; + + // Check if we have room for another service + if (current_size > MAX_PACKET_SIZE - NEXT_SERVICE_BUFFER) { + // Getting close to limit, send this batch + break; + } } - // Send the message with 1-3 services + // Send the message with dynamically batched services api_conn->send_message(resp, api::BluetoothGATTGetServicesResponse::MESSAGE_TYPE); } diff --git a/esphome/components/bluetooth_proxy/bluetooth_proxy.h b/esphome/components/bluetooth_proxy/bluetooth_proxy.h index b33460339b..d249515fdf 100644 --- a/esphome/components/bluetooth_proxy/bluetooth_proxy.h +++ b/esphome/components/bluetooth_proxy/bluetooth_proxy.h @@ -22,7 +22,6 @@ namespace esphome::bluetooth_proxy { static const esp_err_t ESP_GATT_NOT_CONNECTED = -1; static const int DONE_SENDING_SERVICES = -2; -static const uint8_t MAX_SERVICES_PER_BATCH = 3; using namespace esp32_ble_client; From 551bff33c2c9727f86f4ac75fb27d1964d491e51 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Thu, 31 Jul 2025 06:47:31 -1000 Subject: [PATCH 02/11] preen --- .../bluetooth_proxy/bluetooth_connection.cpp | 155 +++++++++--------- 1 file changed, 77 insertions(+), 78 deletions(-) diff --git a/esphome/components/bluetooth_proxy/bluetooth_connection.cpp b/esphome/components/bluetooth_proxy/bluetooth_connection.cpp index 931bae7247..974ecfb3e2 100644 --- a/esphome/components/bluetooth_proxy/bluetooth_connection.cpp +++ b/esphome/components/bluetooth_proxy/bluetooth_connection.cpp @@ -56,6 +56,13 @@ static size_t estimate_service_size(uint16_t char_count, bool use_efficient_uuid return service_overhead + (char_size + desc_size) * char_count; } +// Helper to calculate actual service size +static size_t get_service_size(api::BluetoothGATTService &service) { + api::ProtoSize service_size; + service.calculate_size(service_size); + return service_size.get_size(); +} + bool BluetoothConnection::supports_efficient_uuids_() const { auto *api_conn = this->proxy_->get_api_connection(); return api_conn && api_conn->client_supports_api_version(1, 12); @@ -169,96 +176,88 @@ void BluetoothConnection::send_service_for_discovery_() { break; } - if (total_char_count == 0) { - // No characteristics, increment and continue to next service - this->send_service_++; - continue; - } - - // Reserve space and process characteristics - service_resp.characteristics.reserve(total_char_count); - uint16_t char_offset = 0; - esp_gattc_char_elem_t char_result; - while (true) { // characteristics - uint16_t char_count = 1; - esp_gatt_status_t char_status = - esp_ble_gattc_get_all_char(this->gattc_if_, this->conn_id_, service_result.start_handle, - service_result.end_handle, &char_result, &char_count, char_offset); - if (char_status == ESP_GATT_INVALID_OFFSET || char_status == ESP_GATT_NOT_FOUND) { - break; - } - if (char_status != ESP_GATT_OK) { - ESP_LOGE(TAG, "[%d] [%s] esp_ble_gattc_get_all_char error, status=%d", this->connection_index_, - this->address_str().c_str(), char_status); - this->send_service_ = DONE_SENDING_SERVICES; - return; - } - if (char_count == 0) { - break; - } - - service_resp.characteristics.emplace_back(); - auto &characteristic_resp = service_resp.characteristics.back(); - - fill_gatt_uuid(characteristic_resp.uuid, characteristic_resp.short_uuid, char_result.uuid, use_efficient_uuids); - - characteristic_resp.handle = char_result.char_handle; - characteristic_resp.properties = char_result.properties; - char_offset++; - - // Get the number of descriptors directly with one call - uint16_t total_desc_count = 0; - esp_gatt_status_t desc_count_status = esp_ble_gattc_get_attr_count( - this->gattc_if_, this->conn_id_, ESP_GATT_DB_DESCRIPTOR, 0, 0, char_result.char_handle, &total_desc_count); - - if (desc_count_status != ESP_GATT_OK) { - ESP_LOGE(TAG, "[%d] [%s] Error getting descriptor count for char handle %d, status=%d", this->connection_index_, - this->address_str().c_str(), char_result.char_handle, desc_count_status); - this->send_service_ = DONE_SENDING_SERVICES; - return; - } - if (total_desc_count == 0) { - // No descriptors, continue to next characteristic - continue; - } - - // Reserve space and process descriptors - characteristic_resp.descriptors.reserve(total_desc_count); - uint16_t desc_offset = 0; - esp_gattc_descr_elem_t desc_result; - while (true) { // descriptors - uint16_t desc_count = 1; - esp_gatt_status_t desc_status = esp_ble_gattc_get_all_descr( - this->gattc_if_, this->conn_id_, char_result.char_handle, &desc_result, &desc_count, desc_offset); - if (desc_status == ESP_GATT_INVALID_OFFSET || desc_status == ESP_GATT_NOT_FOUND) { + if (total_char_count > 0) { + // Reserve space and process characteristics + service_resp.characteristics.reserve(total_char_count); + uint16_t char_offset = 0; + esp_gattc_char_elem_t char_result; + while (true) { // characteristics + uint16_t char_count = 1; + esp_gatt_status_t char_status = + esp_ble_gattc_get_all_char(this->gattc_if_, this->conn_id_, service_result.start_handle, + service_result.end_handle, &char_result, &char_count, char_offset); + if (char_status == ESP_GATT_INVALID_OFFSET || char_status == ESP_GATT_NOT_FOUND) { break; } - if (desc_status != ESP_GATT_OK) { - ESP_LOGE(TAG, "[%d] [%s] esp_ble_gattc_get_all_descr error, status=%d", this->connection_index_, - this->address_str().c_str(), desc_status); + if (char_status != ESP_GATT_OK) { + ESP_LOGE(TAG, "[%d] [%s] esp_ble_gattc_get_all_char error, status=%d", this->connection_index_, + this->address_str().c_str(), char_status); this->send_service_ = DONE_SENDING_SERVICES; return; } - if (desc_count == 0) { - break; // No more descriptors + if (char_count == 0) { + break; } - characteristic_resp.descriptors.emplace_back(); - auto &descriptor_resp = characteristic_resp.descriptors.back(); + service_resp.characteristics.emplace_back(); + auto &characteristic_resp = service_resp.characteristics.back(); - fill_gatt_uuid(descriptor_resp.uuid, descriptor_resp.short_uuid, desc_result.uuid, use_efficient_uuids); + fill_gatt_uuid(characteristic_resp.uuid, characteristic_resp.short_uuid, char_result.uuid, use_efficient_uuids); - descriptor_resp.handle = desc_result.handle; - desc_offset++; + characteristic_resp.handle = char_result.char_handle; + characteristic_resp.properties = char_result.properties; + char_offset++; + + // Get the number of descriptors directly with one call + uint16_t total_desc_count = 0; + esp_gatt_status_t desc_count_status = esp_ble_gattc_get_attr_count( + this->gattc_if_, this->conn_id_, ESP_GATT_DB_DESCRIPTOR, 0, 0, char_result.char_handle, &total_desc_count); + + if (desc_count_status != ESP_GATT_OK) { + ESP_LOGE(TAG, "[%d] [%s] Error getting descriptor count for char handle %d, status=%d", + this->connection_index_, this->address_str().c_str(), char_result.char_handle, desc_count_status); + this->send_service_ = DONE_SENDING_SERVICES; + return; + } + if (total_desc_count == 0) { + // No descriptors, continue to next characteristic + continue; + } + + // Reserve space and process descriptors + characteristic_resp.descriptors.reserve(total_desc_count); + uint16_t desc_offset = 0; + esp_gattc_descr_elem_t desc_result; + while (true) { // descriptors + uint16_t desc_count = 1; + esp_gatt_status_t desc_status = esp_ble_gattc_get_all_descr( + this->gattc_if_, this->conn_id_, char_result.char_handle, &desc_result, &desc_count, desc_offset); + if (desc_status == ESP_GATT_INVALID_OFFSET || desc_status == ESP_GATT_NOT_FOUND) { + break; + } + if (desc_status != ESP_GATT_OK) { + ESP_LOGE(TAG, "[%d] [%s] esp_ble_gattc_get_all_descr error, status=%d", this->connection_index_, + this->address_str().c_str(), desc_status); + this->send_service_ = DONE_SENDING_SERVICES; + return; + } + if (desc_count == 0) { + break; // No more descriptors + } + + characteristic_resp.descriptors.emplace_back(); + auto &descriptor_resp = characteristic_resp.descriptors.back(); + + fill_gatt_uuid(descriptor_resp.uuid, descriptor_resp.short_uuid, desc_result.uuid, use_efficient_uuids); + + descriptor_resp.handle = desc_result.handle; + desc_offset++; + } } - } + } // end if (total_char_count > 0) // Calculate the actual size of just this service - api::ProtoSize service_size; - service_resp.calculate_size(service_size); - - // Update running total - current_size += service_size.get_size() + 1; // +1 for field tag + current_size += get_service_size(service_resp) + 1; // +1 for field tag // Check if we've exceeded the limit (worst case scenario) // Our estimation above should have caught this, but if we're here it means From 1225df594fa9e7929509b7dd3066baaac967ee52 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Thu, 31 Jul 2025 06:59:10 -1000 Subject: [PATCH 03/11] preen --- .../bluetooth_proxy/bluetooth_connection.cpp | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/esphome/components/bluetooth_proxy/bluetooth_connection.cpp b/esphome/components/bluetooth_proxy/bluetooth_connection.cpp index 974ecfb3e2..84d9939d27 100644 --- a/esphome/components/bluetooth_proxy/bluetooth_connection.cpp +++ b/esphome/components/bluetooth_proxy/bluetooth_connection.cpp @@ -126,14 +126,15 @@ void BluetoothConnection::send_service_for_discovery_() { resp.address = this->address_; // Dynamic batching based on actual size - static constexpr size_t MAX_PACKET_SIZE = 1390; // MTU limit for API messages - static constexpr size_t NEXT_SERVICE_BUFFER = 400; // Reserve space for next service + static constexpr size_t MAX_PACKET_SIZE = 1390; // MTU limit for API messages // Keep running total of actual message size size_t current_size = 0; api::ProtoSize size; resp.calculate_size(size); current_size = size.get_size(); + ESP_LOGD(TAG, "[%d] [%s] Starting batch with base size: %d", this->connection_index_, this->address_str().c_str(), + current_size); while (this->send_service_ < this->service_count_) { esp_gattc_service_elem_t service_result; @@ -257,7 +258,10 @@ void BluetoothConnection::send_service_for_discovery_() { } // end if (total_char_count > 0) // Calculate the actual size of just this service - current_size += get_service_size(service_resp) + 1; // +1 for field tag + size_t service_size = get_service_size(service_resp) + 1; // +1 for field tag + current_size += service_size; + ESP_LOGD(TAG, "[%d] [%s] Service %d size: %d, total size now: %d", this->connection_index_, + this->address_str().c_str(), this->send_service_ - 1, service_size, current_size); // Check if we've exceeded the limit (worst case scenario) // Our estimation above should have caught this, but if we're here it means @@ -278,15 +282,11 @@ void BluetoothConnection::send_service_for_discovery_() { // Successfully added this service, increment counter this->send_service_++; - - // Check if we have room for another service - if (current_size > MAX_PACKET_SIZE - NEXT_SERVICE_BUFFER) { - // Getting close to limit, send this batch - break; - } } // Send the message with dynamically batched services + ESP_LOGD(TAG, "[%d] [%s] Sending batch with %d services, total size %d", this->connection_index_, + this->address_str().c_str(), resp.services.size(), current_size); api_conn->send_message(resp, api::BluetoothGATTGetServicesResponse::MESSAGE_TYPE); } From b66141e5ba450b256ed56e1fa4091fbe46cf12b1 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Thu, 31 Jul 2025 07:12:53 -1000 Subject: [PATCH 04/11] fix --- .../bluetooth_proxy/bluetooth_connection.cpp | 32 ++++++++++++------- 1 file changed, 21 insertions(+), 11 deletions(-) diff --git a/esphome/components/bluetooth_proxy/bluetooth_connection.cpp b/esphome/components/bluetooth_proxy/bluetooth_connection.cpp index 84d9939d27..a49d06994b 100644 --- a/esphome/components/bluetooth_proxy/bluetooth_connection.cpp +++ b/esphome/components/bluetooth_proxy/bluetooth_connection.cpp @@ -60,7 +60,10 @@ static size_t estimate_service_size(uint16_t char_count, bool use_efficient_uuid static size_t get_service_size(api::BluetoothGATTService &service) { api::ProtoSize service_size; service.calculate_size(service_size); - return service_size.get_size(); + size_t size = service_size.get_size(); + ESP_LOGD(TAG, "Service size calculation: uuid[0]=%llx uuid[1]=%llx short_uuid=%u handle=%u -> size=%d", + service.uuid[0], service.uuid[1], service.short_uuid, service.handle, size); + return size; } bool BluetoothConnection::supports_efficient_uuids_() const { @@ -133,8 +136,8 @@ void BluetoothConnection::send_service_for_discovery_() { api::ProtoSize size; resp.calculate_size(size); current_size = size.get_size(); - ESP_LOGD(TAG, "[%d] [%s] Starting batch with base size: %d", this->connection_index_, this->address_str().c_str(), - current_size); + ESP_LOGD(TAG, "[%d] [%s] Starting batch with base size: %d, send_service_: %d", this->connection_index_, + this->address_str().c_str(), current_size, this->send_service_); while (this->send_service_ < this->service_count_) { esp_gattc_service_elem_t service_result; @@ -157,6 +160,10 @@ void BluetoothConnection::send_service_for_discovery_() { service_resp.handle = service_result.start_handle; + ESP_LOGD(TAG, "[%d] [%s] Service UUID: %llx,%llx short:%u handle:%u", this->connection_index_, + this->address_str().c_str(), service_resp.uuid[0], service_resp.uuid[1], service_resp.short_uuid, + service_resp.handle); + // Get the number of characteristics directly with one call uint16_t total_char_count = 0; esp_gatt_status_t char_count_status = @@ -259,20 +266,18 @@ void BluetoothConnection::send_service_for_discovery_() { // Calculate the actual size of just this service size_t service_size = get_service_size(service_resp) + 1; // +1 for field tag - current_size += service_size; - ESP_LOGD(TAG, "[%d] [%s] Service %d size: %d, total size now: %d", this->connection_index_, - this->address_str().c_str(), this->send_service_ - 1, service_size, current_size); - // Check if we've exceeded the limit (worst case scenario) - // Our estimation above should have caught this, but if we're here it means - // this service is extraordinarily large (many characteristics/descriptors) - if (current_size > MAX_PACKET_SIZE) { - // We've gone over - pop the last service if we have more than one + // Check if adding this service would exceed the limit + if (current_size + service_size > MAX_PACKET_SIZE) { + // We would go over - pop the last service if we have more than one if (resp.services.size() > 1) { resp.services.pop_back(); // Don't increment send_service_ - we'll retry this service in next batch } else { // This single service is too large, but we have to send it anyway + current_size += service_size; + ESP_LOGW(TAG, "[%d] [%s] Service %d is too large (%d bytes) but sending anyway", this->connection_index_, + this->address_str().c_str(), this->send_service_, service_size); // Increment so we don't get stuck this->send_service_++; } @@ -280,6 +285,11 @@ void BluetoothConnection::send_service_for_discovery_() { break; } + // Now we know we're keeping this service, add its size + current_size += service_size; + ESP_LOGD(TAG, "[%d] [%s] Service %d size: %d, total size now: %d", this->connection_index_, + this->address_str().c_str(), this->send_service_, service_size, current_size); + // Successfully added this service, increment counter this->send_service_++; } From fe2b2d5280055507fb3dc9cc44759b7b3f1307de Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Thu, 31 Jul 2025 07:19:03 -1000 Subject: [PATCH 05/11] fix --- .../bluetooth_proxy/bluetooth_connection.cpp | 25 ++++++++++--------- 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/esphome/components/bluetooth_proxy/bluetooth_connection.cpp b/esphome/components/bluetooth_proxy/bluetooth_connection.cpp index a49d06994b..360bd471b8 100644 --- a/esphome/components/bluetooth_proxy/bluetooth_connection.cpp +++ b/esphome/components/bluetooth_proxy/bluetooth_connection.cpp @@ -153,18 +153,7 @@ void BluetoothConnection::send_service_for_discovery_() { return; } - resp.services.emplace_back(); - auto &service_resp = resp.services.back(); - - fill_gatt_uuid(service_resp.uuid, service_resp.short_uuid, service_result.uuid, use_efficient_uuids); - - service_resp.handle = service_result.start_handle; - - ESP_LOGD(TAG, "[%d] [%s] Service UUID: %llx,%llx short:%u handle:%u", this->connection_index_, - this->address_str().c_str(), service_resp.uuid[0], service_resp.uuid[1], service_resp.short_uuid, - service_resp.handle); - - // Get the number of characteristics directly with one call + // Get the number of characteristics BEFORE adding to response uint16_t total_char_count = 0; esp_gatt_status_t char_count_status = esp_ble_gattc_get_attr_count(this->gattc_if_, this->conn_id_, ESP_GATT_DB_CHARACTERISTIC, @@ -184,6 +173,18 @@ void BluetoothConnection::send_service_for_discovery_() { break; } + // Now add the service since we know it will likely fit + resp.services.emplace_back(); + auto &service_resp = resp.services.back(); + + fill_gatt_uuid(service_resp.uuid, service_resp.short_uuid, service_result.uuid, use_efficient_uuids); + + service_resp.handle = service_result.start_handle; + + ESP_LOGD(TAG, "[%d] [%s] Service UUID: %llx,%llx short:%u handle:%u", this->connection_index_, + this->address_str().c_str(), service_resp.uuid[0], service_resp.uuid[1], service_resp.short_uuid, + service_resp.handle); + if (total_char_count > 0) { // Reserve space and process characteristics service_resp.characteristics.reserve(total_char_count); From 38e2b6c5f3c5a4b79cb917c566b72525cf13c748 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Thu, 31 Jul 2025 07:33:14 -1000 Subject: [PATCH 06/11] wip --- .../bluetooth_proxy/bluetooth_connection.cpp | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/esphome/components/bluetooth_proxy/bluetooth_connection.cpp b/esphome/components/bluetooth_proxy/bluetooth_connection.cpp index 360bd471b8..bcd8eafb8c 100644 --- a/esphome/components/bluetooth_proxy/bluetooth_connection.cpp +++ b/esphome/components/bluetooth_proxy/bluetooth_connection.cpp @@ -61,7 +61,7 @@ static size_t get_service_size(api::BluetoothGATTService &service) { api::ProtoSize service_size; service.calculate_size(service_size); size_t size = service_size.get_size(); - ESP_LOGD(TAG, "Service size calculation: uuid[0]=%llx uuid[1]=%llx short_uuid=%u handle=%u -> size=%d", + ESP_LOGV(TAG, "Service size calculation: uuid[0]=%llx uuid[1]=%llx short_uuid=%u handle=%u -> size=%d", service.uuid[0], service.uuid[1], service.short_uuid, service.handle, size); return size; } @@ -136,7 +136,7 @@ void BluetoothConnection::send_service_for_discovery_() { api::ProtoSize size; resp.calculate_size(size); current_size = size.get_size(); - ESP_LOGD(TAG, "[%d] [%s] Starting batch with base size: %d, send_service_: %d", this->connection_index_, + ESP_LOGV(TAG, "[%d] [%s] Starting batch with base size: %d, send_service_: %d", this->connection_index_, this->address_str().c_str(), current_size, this->send_service_); while (this->send_service_ < this->service_count_) { @@ -181,7 +181,7 @@ void BluetoothConnection::send_service_for_discovery_() { service_resp.handle = service_result.start_handle; - ESP_LOGD(TAG, "[%d] [%s] Service UUID: %llx,%llx short:%u handle:%u", this->connection_index_, + ESP_LOGV(TAG, "[%d] [%s] Service UUID: %llx,%llx short:%u handle:%u", this->connection_index_, this->address_str().c_str(), service_resp.uuid[0], service_resp.uuid[1], service_resp.short_uuid, service_resp.handle); @@ -277,7 +277,7 @@ void BluetoothConnection::send_service_for_discovery_() { } else { // This single service is too large, but we have to send it anyway current_size += service_size; - ESP_LOGW(TAG, "[%d] [%s] Service %d is too large (%d bytes) but sending anyway", this->connection_index_, + ESP_LOGV(TAG, "[%d] [%s] Service %d is too large (%d bytes) but sending anyway", this->connection_index_, this->address_str().c_str(), this->send_service_, service_size); // Increment so we don't get stuck this->send_service_++; @@ -288,7 +288,7 @@ void BluetoothConnection::send_service_for_discovery_() { // Now we know we're keeping this service, add its size current_size += service_size; - ESP_LOGD(TAG, "[%d] [%s] Service %d size: %d, total size now: %d", this->connection_index_, + ESP_LOGV(TAG, "[%d] [%s] Service %d size: %d, total size now: %d", this->connection_index_, this->address_str().c_str(), this->send_service_, service_size, current_size); // Successfully added this service, increment counter From 255cf4b6618cf44b82c1fc7e92fa577d89a9e739 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Thu, 31 Jul 2025 07:34:02 -1000 Subject: [PATCH 07/11] wip --- esphome/components/bluetooth_proxy/bluetooth_connection.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/esphome/components/bluetooth_proxy/bluetooth_connection.cpp b/esphome/components/bluetooth_proxy/bluetooth_connection.cpp index bcd8eafb8c..9ca1e7c6e8 100644 --- a/esphome/components/bluetooth_proxy/bluetooth_connection.cpp +++ b/esphome/components/bluetooth_proxy/bluetooth_connection.cpp @@ -273,6 +273,9 @@ void BluetoothConnection::send_service_for_discovery_() { // We would go over - pop the last service if we have more than one if (resp.services.size() > 1) { resp.services.pop_back(); + ESP_LOGD(TAG, "[%d] [%s] Service %d would exceed limit (current: %d + service: %d > %d), sending current batch", + this->connection_index_, this->address_str().c_str(), this->send_service_, current_size, service_size, + MAX_PACKET_SIZE); // Don't increment send_service_ - we'll retry this service in next batch } else { // This single service is too large, but we have to send it anyway From d6776804aeff69daf98497c3283402440284b4a4 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Thu, 31 Jul 2025 07:45:03 -1000 Subject: [PATCH 08/11] tweak --- .../bluetooth_proxy/bluetooth_connection.cpp | 21 ++++++++++++------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/esphome/components/bluetooth_proxy/bluetooth_connection.cpp b/esphome/components/bluetooth_proxy/bluetooth_connection.cpp index 9ca1e7c6e8..05ef181a7c 100644 --- a/esphome/components/bluetooth_proxy/bluetooth_connection.cpp +++ b/esphome/components/bluetooth_proxy/bluetooth_connection.cpp @@ -43,15 +43,15 @@ static constexpr uint8_t SERVICE_OVERHEAD_EFFICIENT = 10; // UUID(6) + handle(4 static constexpr uint8_t CHAR_SIZE_128BIT = 35; // UUID(20) + handle(4) + props(4) + overhead(7) static constexpr uint8_t DESC_SIZE_128BIT = 25; // UUID(20) + handle(4) + overhead(1) static constexpr uint8_t DESC_SIZE_16BIT = 10; // UUID(6) + handle(4) -static constexpr uint8_t DESC_PER_CHAR = 2; // Assume 2 descriptors per characteristic +static constexpr uint8_t DESC_PER_CHAR = 1; // Assume 1 descriptor per characteristic // Helper to estimate service size before fetching all data static size_t estimate_service_size(uint16_t char_count, bool use_efficient_uuids) { size_t service_overhead = use_efficient_uuids ? SERVICE_OVERHEAD_EFFICIENT : SERVICE_OVERHEAD_LEGACY; // Always assume 128-bit UUIDs for characteristics to be safe size_t char_size = CHAR_SIZE_128BIT; - // Assume mix of descriptor types: one 128-bit + one 16-bit per characteristic - size_t desc_size = (DESC_SIZE_128BIT + DESC_SIZE_16BIT) * DESC_PER_CHAR; + // Assume one 128-bit descriptor per characteristic + size_t desc_size = DESC_SIZE_128BIT * DESC_PER_CHAR; return service_overhead + (char_size + desc_size) * char_count; } @@ -129,7 +129,8 @@ void BluetoothConnection::send_service_for_discovery_() { resp.address = this->address_; // Dynamic batching based on actual size - static constexpr size_t MAX_PACKET_SIZE = 1390; // MTU limit for API messages + static constexpr size_t MAX_PACKET_SIZE = + 1360; // Conservative MTU limit for API messages (accounts for WPA3 overhead) // Keep running total of actual message size size_t current_size = 0; @@ -167,8 +168,8 @@ void BluetoothConnection::send_service_for_discovery_() { } // If this service likely won't fit, send current batch (unless it's the first) - if (!resp.services.empty() && - (current_size + estimate_service_size(total_char_count, use_efficient_uuids) > MAX_PACKET_SIZE)) { + size_t estimated_size = estimate_service_size(total_char_count, use_efficient_uuids); + if (!resp.services.empty() && (current_size + estimated_size > MAX_PACKET_SIZE)) { // This service likely won't fit, send current batch break; } @@ -291,8 +292,12 @@ void BluetoothConnection::send_service_for_discovery_() { // Now we know we're keeping this service, add its size current_size += service_size; - ESP_LOGV(TAG, "[%d] [%s] Service %d size: %d, total size now: %d", this->connection_index_, - this->address_str().c_str(), this->send_service_, service_size, current_size); + + // Log the difference between estimate and actual size + int size_diff = (int) service_size - (int) estimated_size; + ESP_LOGD(TAG, "[%d] [%s] Service %d actual: %d, estimated: %d, diff: %+d", this->connection_index_, + this->address_str().c_str(), this->send_service_, service_size, estimated_size, size_diff); + ESP_LOGV(TAG, "[%d] [%s] Total size now: %d", this->connection_index_, this->address_str().c_str(), current_size); // Successfully added this service, increment counter this->send_service_++; From 0356e24baed223f8fc5287ab77d54eb3ebaabf5f Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Thu, 31 Jul 2025 07:46:18 -1000 Subject: [PATCH 09/11] tweak --- esphome/components/bluetooth_proxy/bluetooth_connection.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/esphome/components/bluetooth_proxy/bluetooth_connection.cpp b/esphome/components/bluetooth_proxy/bluetooth_connection.cpp index 05ef181a7c..1f4bfd9806 100644 --- a/esphome/components/bluetooth_proxy/bluetooth_connection.cpp +++ b/esphome/components/bluetooth_proxy/bluetooth_connection.cpp @@ -295,7 +295,7 @@ void BluetoothConnection::send_service_for_discovery_() { // Log the difference between estimate and actual size int size_diff = (int) service_size - (int) estimated_size; - ESP_LOGD(TAG, "[%d] [%s] Service %d actual: %d, estimated: %d, diff: %+d", this->connection_index_, + ESP_LOGV(TAG, "[%d] [%s] Service %d actual: %d, estimated: %d, diff: %+d", this->connection_index_, this->address_str().c_str(), this->send_service_, service_size, estimated_size, size_diff); ESP_LOGV(TAG, "[%d] [%s] Total size now: %d", this->connection_index_, this->address_str().c_str(), current_size); From 0ae7dcdb62a500be32278c328b64f819c3429a51 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Thu, 31 Jul 2025 07:48:54 -1000 Subject: [PATCH 10/11] tweak --- .../bluetooth_proxy/bluetooth_connection.cpp | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/esphome/components/bluetooth_proxy/bluetooth_connection.cpp b/esphome/components/bluetooth_proxy/bluetooth_connection.cpp index 1f4bfd9806..fd67d5690f 100644 --- a/esphome/components/bluetooth_proxy/bluetooth_connection.cpp +++ b/esphome/components/bluetooth_proxy/bluetooth_connection.cpp @@ -61,7 +61,7 @@ static size_t get_service_size(api::BluetoothGATTService &service) { api::ProtoSize service_size; service.calculate_size(service_size); size_t size = service_size.get_size(); - ESP_LOGV(TAG, "Service size calculation: uuid[0]=%llx uuid[1]=%llx short_uuid=%u handle=%u -> size=%d", + ESP_LOGD(TAG, "Service size calculation: uuid[0]=%llx uuid[1]=%llx short_uuid=%u handle=%u -> size=%d", service.uuid[0], service.uuid[1], service.short_uuid, service.handle, size); return size; } @@ -137,7 +137,7 @@ void BluetoothConnection::send_service_for_discovery_() { api::ProtoSize size; resp.calculate_size(size); current_size = size.get_size(); - ESP_LOGV(TAG, "[%d] [%s] Starting batch with base size: %d, send_service_: %d", this->connection_index_, + ESP_LOGD(TAG, "[%d] [%s] Starting batch with base size: %d, send_service_: %d", this->connection_index_, this->address_str().c_str(), current_size, this->send_service_); while (this->send_service_ < this->service_count_) { @@ -182,7 +182,7 @@ void BluetoothConnection::send_service_for_discovery_() { service_resp.handle = service_result.start_handle; - ESP_LOGV(TAG, "[%d] [%s] Service UUID: %llx,%llx short:%u handle:%u", this->connection_index_, + ESP_LOGD(TAG, "[%d] [%s] Service UUID: %llx,%llx short:%u handle:%u", this->connection_index_, this->address_str().c_str(), service_resp.uuid[0], service_resp.uuid[1], service_resp.short_uuid, service_resp.handle); @@ -281,7 +281,7 @@ void BluetoothConnection::send_service_for_discovery_() { } else { // This single service is too large, but we have to send it anyway current_size += service_size; - ESP_LOGV(TAG, "[%d] [%s] Service %d is too large (%d bytes) but sending anyway", this->connection_index_, + ESP_LOGD(TAG, "[%d] [%s] Service %d is too large (%d bytes) but sending anyway", this->connection_index_, this->address_str().c_str(), this->send_service_, service_size); // Increment so we don't get stuck this->send_service_++; @@ -295,9 +295,9 @@ void BluetoothConnection::send_service_for_discovery_() { // Log the difference between estimate and actual size int size_diff = (int) service_size - (int) estimated_size; - ESP_LOGV(TAG, "[%d] [%s] Service %d actual: %d, estimated: %d, diff: %+d", this->connection_index_, + ESP_LOGD(TAG, "[%d] [%s] Service %d actual: %d, estimated: %d, diff: %+d", this->connection_index_, this->address_str().c_str(), this->send_service_, service_size, estimated_size, size_diff); - ESP_LOGV(TAG, "[%d] [%s] Total size now: %d", this->connection_index_, this->address_str().c_str(), current_size); + ESP_LOGD(TAG, "[%d] [%s] Total size now: %d", this->connection_index_, this->address_str().c_str(), current_size); // Successfully added this service, increment counter this->send_service_++; From f2b3f413fc26ce9ccab4a30939a1289005fee4a8 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Thu, 31 Jul 2025 07:56:57 -1000 Subject: [PATCH 11/11] back --- .../bluetooth_proxy/bluetooth_connection.cpp | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/esphome/components/bluetooth_proxy/bluetooth_connection.cpp b/esphome/components/bluetooth_proxy/bluetooth_connection.cpp index fd67d5690f..1f4bfd9806 100644 --- a/esphome/components/bluetooth_proxy/bluetooth_connection.cpp +++ b/esphome/components/bluetooth_proxy/bluetooth_connection.cpp @@ -61,7 +61,7 @@ static size_t get_service_size(api::BluetoothGATTService &service) { api::ProtoSize service_size; service.calculate_size(service_size); size_t size = service_size.get_size(); - ESP_LOGD(TAG, "Service size calculation: uuid[0]=%llx uuid[1]=%llx short_uuid=%u handle=%u -> size=%d", + ESP_LOGV(TAG, "Service size calculation: uuid[0]=%llx uuid[1]=%llx short_uuid=%u handle=%u -> size=%d", service.uuid[0], service.uuid[1], service.short_uuid, service.handle, size); return size; } @@ -137,7 +137,7 @@ void BluetoothConnection::send_service_for_discovery_() { api::ProtoSize size; resp.calculate_size(size); current_size = size.get_size(); - ESP_LOGD(TAG, "[%d] [%s] Starting batch with base size: %d, send_service_: %d", this->connection_index_, + ESP_LOGV(TAG, "[%d] [%s] Starting batch with base size: %d, send_service_: %d", this->connection_index_, this->address_str().c_str(), current_size, this->send_service_); while (this->send_service_ < this->service_count_) { @@ -182,7 +182,7 @@ void BluetoothConnection::send_service_for_discovery_() { service_resp.handle = service_result.start_handle; - ESP_LOGD(TAG, "[%d] [%s] Service UUID: %llx,%llx short:%u handle:%u", this->connection_index_, + ESP_LOGV(TAG, "[%d] [%s] Service UUID: %llx,%llx short:%u handle:%u", this->connection_index_, this->address_str().c_str(), service_resp.uuid[0], service_resp.uuid[1], service_resp.short_uuid, service_resp.handle); @@ -281,7 +281,7 @@ void BluetoothConnection::send_service_for_discovery_() { } else { // This single service is too large, but we have to send it anyway current_size += service_size; - ESP_LOGD(TAG, "[%d] [%s] Service %d is too large (%d bytes) but sending anyway", this->connection_index_, + ESP_LOGV(TAG, "[%d] [%s] Service %d is too large (%d bytes) but sending anyway", this->connection_index_, this->address_str().c_str(), this->send_service_, service_size); // Increment so we don't get stuck this->send_service_++; @@ -295,9 +295,9 @@ void BluetoothConnection::send_service_for_discovery_() { // Log the difference between estimate and actual size int size_diff = (int) service_size - (int) estimated_size; - ESP_LOGD(TAG, "[%d] [%s] Service %d actual: %d, estimated: %d, diff: %+d", this->connection_index_, + ESP_LOGV(TAG, "[%d] [%s] Service %d actual: %d, estimated: %d, diff: %+d", this->connection_index_, this->address_str().c_str(), this->send_service_, service_size, estimated_size, size_diff); - ESP_LOGD(TAG, "[%d] [%s] Total size now: %d", this->connection_index_, this->address_str().c_str(), current_size); + ESP_LOGV(TAG, "[%d] [%s] Total size now: %d", this->connection_index_, this->address_str().c_str(), current_size); // Successfully added this service, increment counter this->send_service_++;