diff --git a/esphome/components/api/api.proto b/esphome/components/api/api.proto index ad99de4b4a..c59ccc6e29 100644 --- a/esphome/components/api/api.proto +++ b/esphome/components/api/api.proto @@ -1465,7 +1465,7 @@ message BluetoothDeviceRequest { uint64 address = 1; BluetoothDeviceRequestType request_type = 2; - bool has_address_type = 3; + bool has_address_type = 3; // Deprecated, should be removed in 2027.8 - https://github.com/esphome/esphome/pull/10318 uint32 address_type = 4; } @@ -1571,7 +1571,7 @@ message BluetoothGATTWriteRequest { uint32 handle = 2; bool response = 3; - bytes data = 4; + bytes data = 4 [(pointer_to_buffer) = true]; } message BluetoothGATTReadDescriptorRequest { @@ -1591,7 +1591,7 @@ message BluetoothGATTWriteDescriptorRequest { uint64 address = 1; uint32 handle = 2; - bytes data = 3; + bytes data = 3 [(pointer_to_buffer) = true]; } message BluetoothGATTNotifyRequest { diff --git a/esphome/components/api/api_options.proto b/esphome/components/api/api_options.proto index 50c43b96fd..633f39b552 100644 --- a/esphome/components/api/api_options.proto +++ b/esphome/components/api/api_options.proto @@ -32,6 +32,13 @@ extend google.protobuf.FieldOptions { optional string fixed_array_size_define = 50010; optional string fixed_array_with_length_define = 50011; + // pointer_to_buffer: Use pointer instead of array for fixed-size byte fields + // When set, the field will be declared as a pointer (const uint8_t *data) + // instead of an array (uint8_t data[N]). This allows zero-copy on decode + // by pointing directly to the protobuf buffer. The buffer must remain valid + // until the message is processed (which is guaranteed for stack-allocated messages). + optional bool pointer_to_buffer = 50012 [default=false]; + // container_pointer: Zero-copy optimization for repeated fields. // // When container_pointer is set on a repeated field, the generated message will diff --git a/esphome/components/api/api_pb2.cpp b/esphome/components/api/api_pb2.cpp index 245933724b..08384c6869 100644 --- a/esphome/components/api/api_pb2.cpp +++ b/esphome/components/api/api_pb2.cpp @@ -2028,9 +2028,12 @@ bool BluetoothGATTWriteRequest::decode_varint(uint32_t field_id, ProtoVarInt val } bool BluetoothGATTWriteRequest::decode_length(uint32_t field_id, ProtoLengthDelimited value) { switch (field_id) { - case 4: - this->data = value.as_string(); + case 4: { + // Use raw data directly to avoid allocation + this->data = value.data(); + this->data_len = value.size(); break; + } default: return false; } @@ -2064,9 +2067,12 @@ bool BluetoothGATTWriteDescriptorRequest::decode_varint(uint32_t field_id, Proto } bool BluetoothGATTWriteDescriptorRequest::decode_length(uint32_t field_id, ProtoLengthDelimited value) { switch (field_id) { - case 3: - this->data = value.as_string(); + case 3: { + // Use raw data directly to avoid allocation + this->data = value.data(); + this->data_len = value.size(); break; + } default: return false; } diff --git a/esphome/components/api/api_pb2.h b/esphome/components/api/api_pb2.h index 248a4b1f82..adc06ad5b7 100644 --- a/esphome/components/api/api_pb2.h +++ b/esphome/components/api/api_pb2.h @@ -1985,14 +1985,15 @@ class BluetoothGATTReadResponse final : public ProtoMessage { class BluetoothGATTWriteRequest final : public ProtoDecodableMessage { public: static constexpr uint8_t MESSAGE_TYPE = 75; - static constexpr uint8_t ESTIMATED_SIZE = 19; + static constexpr uint8_t ESTIMATED_SIZE = 29; #ifdef HAS_PROTO_MESSAGE_DUMP const char *message_name() const override { return "bluetooth_gatt_write_request"; } #endif uint64_t address{0}; uint32_t handle{0}; bool response{false}; - std::string data{}; + const uint8_t *data{nullptr}; + uint16_t data_len{0}; #ifdef HAS_PROTO_MESSAGE_DUMP void dump_to(std::string &out) const override; #endif @@ -2020,13 +2021,14 @@ class BluetoothGATTReadDescriptorRequest final : public ProtoDecodableMessage { class BluetoothGATTWriteDescriptorRequest final : public ProtoDecodableMessage { public: static constexpr uint8_t MESSAGE_TYPE = 77; - static constexpr uint8_t ESTIMATED_SIZE = 17; + static constexpr uint8_t ESTIMATED_SIZE = 27; #ifdef HAS_PROTO_MESSAGE_DUMP const char *message_name() const override { return "bluetooth_gatt_write_descriptor_request"; } #endif uint64_t address{0}; uint32_t handle{0}; - std::string data{}; + const uint8_t *data{nullptr}; + uint16_t data_len{0}; #ifdef HAS_PROTO_MESSAGE_DUMP void dump_to(std::string &out) const override; #endif diff --git a/esphome/components/api/api_pb2_dump.cpp b/esphome/components/api/api_pb2_dump.cpp index ac43af6d54..e2a4772692 100644 --- a/esphome/components/api/api_pb2_dump.cpp +++ b/esphome/components/api/api_pb2_dump.cpp @@ -1649,7 +1649,7 @@ void BluetoothGATTWriteRequest::dump_to(std::string &out) const { dump_field(out, "handle", this->handle); dump_field(out, "response", this->response); out.append(" data: "); - out.append(format_hex_pretty(reinterpret_cast(this->data.data()), this->data.size())); + out.append(format_hex_pretty(this->data, this->data_len)); out.append("\n"); } void BluetoothGATTReadDescriptorRequest::dump_to(std::string &out) const { @@ -1662,7 +1662,7 @@ void BluetoothGATTWriteDescriptorRequest::dump_to(std::string &out) const { dump_field(out, "address", this->address); dump_field(out, "handle", this->handle); out.append(" data: "); - out.append(format_hex_pretty(reinterpret_cast(this->data.data()), this->data.size())); + out.append(format_hex_pretty(this->data, this->data_len)); out.append("\n"); } void BluetoothGATTNotifyRequest::dump_to(std::string &out) const { diff --git a/esphome/components/api/proto.h b/esphome/components/api/proto.h index 0e5ec61050..6be5f00e75 100644 --- a/esphome/components/api/proto.h +++ b/esphome/components/api/proto.h @@ -182,6 +182,10 @@ class ProtoLengthDelimited { explicit ProtoLengthDelimited(const uint8_t *value, size_t length) : value_(value), length_(length) {} std::string as_string() const { return std::string(reinterpret_cast(this->value_), this->length_); } + // Direct access to raw data without string allocation + const uint8_t *data() const { return this->value_; } + size_t size() const { return this->length_; } + /** * Decode the length-delimited data into an existing ProtoDecodableMessage instance. * diff --git a/esphome/components/bluetooth_proxy/bluetooth_connection.cpp b/esphome/components/bluetooth_proxy/bluetooth_connection.cpp index 540492f8c5..cde82fbfb0 100644 --- a/esphome/components/bluetooth_proxy/bluetooth_connection.cpp +++ b/esphome/components/bluetooth_proxy/bluetooth_connection.cpp @@ -514,7 +514,8 @@ esp_err_t BluetoothConnection::read_characteristic(uint16_t handle) { return this->check_and_log_error_("esp_ble_gattc_read_char", err); } -esp_err_t BluetoothConnection::write_characteristic(uint16_t handle, const std::string &data, bool response) { +esp_err_t BluetoothConnection::write_characteristic(uint16_t handle, const uint8_t *data, size_t length, + bool response) { if (!this->connected()) { this->log_gatt_not_connected_("write", "characteristic"); return ESP_GATT_NOT_CONNECTED; @@ -522,8 +523,11 @@ esp_err_t BluetoothConnection::write_characteristic(uint16_t handle, const std:: ESP_LOGV(TAG, "[%d] [%s] Writing GATT characteristic handle %d", this->connection_index_, this->address_str_.c_str(), handle); + // ESP-IDF's API requires a non-const uint8_t* but it doesn't modify the data + // The BTC layer immediately copies the data to its own buffer (see btc_gattc.c) + // const_cast is safe here and was previously hidden by a C-style cast esp_err_t err = - esp_ble_gattc_write_char(this->gattc_if_, this->conn_id_, handle, data.size(), (uint8_t *) data.data(), + esp_ble_gattc_write_char(this->gattc_if_, this->conn_id_, handle, length, const_cast(data), response ? ESP_GATT_WRITE_TYPE_RSP : ESP_GATT_WRITE_TYPE_NO_RSP, ESP_GATT_AUTH_REQ_NONE); return this->check_and_log_error_("esp_ble_gattc_write_char", err); } @@ -540,7 +544,7 @@ esp_err_t BluetoothConnection::read_descriptor(uint16_t handle) { return this->check_and_log_error_("esp_ble_gattc_read_char_descr", err); } -esp_err_t BluetoothConnection::write_descriptor(uint16_t handle, const std::string &data, bool response) { +esp_err_t BluetoothConnection::write_descriptor(uint16_t handle, const uint8_t *data, size_t length, bool response) { if (!this->connected()) { this->log_gatt_not_connected_("write", "descriptor"); return ESP_GATT_NOT_CONNECTED; @@ -548,8 +552,11 @@ esp_err_t BluetoothConnection::write_descriptor(uint16_t handle, const std::stri ESP_LOGV(TAG, "[%d] [%s] Writing GATT descriptor handle %d", this->connection_index_, this->address_str_.c_str(), handle); + // ESP-IDF's API requires a non-const uint8_t* but it doesn't modify the data + // The BTC layer immediately copies the data to its own buffer (see btc_gattc.c) + // const_cast is safe here and was previously hidden by a C-style cast esp_err_t err = esp_ble_gattc_write_char_descr( - this->gattc_if_, this->conn_id_, handle, data.size(), (uint8_t *) data.data(), + this->gattc_if_, this->conn_id_, handle, length, const_cast(data), response ? ESP_GATT_WRITE_TYPE_RSP : ESP_GATT_WRITE_TYPE_NO_RSP, ESP_GATT_AUTH_REQ_NONE); return this->check_and_log_error_("esp_ble_gattc_write_char_descr", err); } diff --git a/esphome/components/bluetooth_proxy/bluetooth_connection.h b/esphome/components/bluetooth_proxy/bluetooth_connection.h index e5d5ff2dd6..60bbc93e8b 100644 --- a/esphome/components/bluetooth_proxy/bluetooth_connection.h +++ b/esphome/components/bluetooth_proxy/bluetooth_connection.h @@ -18,9 +18,9 @@ class BluetoothConnection final : public esp32_ble_client::BLEClientBase { esp32_ble_tracker::AdvertisementParserType get_advertisement_parser_type() override; esp_err_t read_characteristic(uint16_t handle); - esp_err_t write_characteristic(uint16_t handle, const std::string &data, bool response); + esp_err_t write_characteristic(uint16_t handle, const uint8_t *data, size_t length, bool response); esp_err_t read_descriptor(uint16_t handle); - esp_err_t write_descriptor(uint16_t handle, const std::string &data, bool response); + esp_err_t write_descriptor(uint16_t handle, const uint8_t *data, size_t length, bool response); esp_err_t notify_characteristic(uint16_t handle, bool enable); diff --git a/esphome/components/bluetooth_proxy/bluetooth_proxy.cpp b/esphome/components/bluetooth_proxy/bluetooth_proxy.cpp index 532aff550e..cd7261d5e5 100644 --- a/esphome/components/bluetooth_proxy/bluetooth_proxy.cpp +++ b/esphome/components/bluetooth_proxy/bluetooth_proxy.cpp @@ -305,7 +305,7 @@ void BluetoothProxy::bluetooth_gatt_write(const api::BluetoothGATTWriteRequest & return; } - auto err = connection->write_characteristic(msg.handle, msg.data, msg.response); + auto err = connection->write_characteristic(msg.handle, msg.data, msg.data_len, msg.response); if (err != ESP_OK) { this->send_gatt_error(msg.address, msg.handle, err); } @@ -331,7 +331,7 @@ void BluetoothProxy::bluetooth_gatt_write_descriptor(const api::BluetoothGATTWri return; } - auto err = connection->write_descriptor(msg.handle, msg.data, true); + auto err = connection->write_descriptor(msg.handle, msg.data, msg.data_len, true); if (err != ESP_OK) { this->send_gatt_error(msg.address, msg.handle, err); } diff --git a/script/api_protobuf/api_protobuf.py b/script/api_protobuf/api_protobuf.py index fa04222c5d..22bebcbd29 100755 --- a/script/api_protobuf/api_protobuf.py +++ b/script/api_protobuf/api_protobuf.py @@ -353,12 +353,25 @@ def create_field_type_info( return FixedArrayRepeatedType(field, size_define) return RepeatedTypeInfo(field) - # Check for fixed_array_size option on bytes fields - if ( - field.type == 12 - and (fixed_size := get_field_opt(field, pb.fixed_array_size)) is not None - ): - return FixedArrayBytesType(field, fixed_size) + # Check for mutually exclusive options on bytes fields + if field.type == 12: + has_pointer_to_buffer = get_field_opt(field, pb.pointer_to_buffer, False) + fixed_size = get_field_opt(field, pb.fixed_array_size, None) + + if has_pointer_to_buffer and fixed_size is not None: + raise ValueError( + f"Field '{field.name}' has both pointer_to_buffer and fixed_array_size. " + "These options are mutually exclusive. Use pointer_to_buffer for zero-copy " + "or fixed_array_size for traditional array storage." + ) + + if has_pointer_to_buffer: + # Zero-copy pointer approach - no size needed, will use size_t for length + return PointerToBytesBufferType(field, None) + + if fixed_size is not None: + # Traditional fixed array approach with copy + return FixedArrayBytesType(field, fixed_size) # Special handling for bytes fields if field.type == 12: @@ -818,6 +831,91 @@ class BytesType(TypeInfo): return self.calculate_field_id_size() + 8 # field ID + 8 bytes typical bytes +class PointerToBytesBufferType(TypeInfo): + """Type for bytes fields that use pointer_to_buffer option for zero-copy.""" + + @classmethod + def can_use_dump_field(cls) -> bool: + return False + + def __init__( + self, field: descriptor.FieldDescriptorProto, size: int | None = None + ) -> None: + super().__init__(field) + # Size is not used for pointer_to_buffer - we always use size_t for length + self.array_size = 0 + + @property + def cpp_type(self) -> str: + return "const uint8_t*" + + @property + def default_value(self) -> str: + return "nullptr" + + @property + def reference_type(self) -> str: + return "const uint8_t*" + + @property + def const_reference_type(self) -> str: + return "const uint8_t*" + + @property + def public_content(self) -> list[str]: + # Use uint16_t for length - max packet size is well below 65535 + # Add pointer and length fields + return [ + f"const uint8_t* {self.field_name}{{nullptr}};", + f"uint16_t {self.field_name}_len{{0}};", + ] + + @property + def encode_content(self) -> str: + return f"buffer.encode_bytes({self.number}, this->{self.field_name}, this->{self.field_name}_len);" + + @property + def decode_length_content(self) -> str | None: + # Decode directly stores the pointer to avoid allocation + return f"""case {self.number}: {{ + // Use raw data directly to avoid allocation + this->{self.field_name} = value.data(); + this->{self.field_name}_len = value.size(); + break; + }}""" + + @property + def decode_length(self) -> str | None: + # This is handled in decode_length_content + return None + + @property + def wire_type(self) -> WireType: + """Get the wire type for this bytes field.""" + return WireType.LENGTH_DELIMITED # Uses wire type 2 + + def dump(self, name: str) -> str: + return ( + f"format_hex_pretty(this->{self.field_name}, this->{self.field_name}_len)" + ) + + @property + def dump_content(self) -> str: + # Custom dump that doesn't use dump_field template + return ( + f'out.append(" {self.name}: ");\n' + + f"out.append({self.dump(self.field_name)});\n" + + 'out.append("\\n");' + ) + + def get_size_calculation(self, name: str, force: bool = False) -> str: + return f"size.add_length({self.number}, this->{self.field_name}_len);" + + def get_estimated_size(self) -> int: + # field ID + length varint + typical data (assume small for pointer fields) + return self.calculate_field_id_size() + 2 + 16 + + class FixedArrayBytesType(TypeInfo): """Special type for fixed-size byte arrays."""