[api] Implement zero-copy API for bluetooth_proxy writes (#10840)

esphome/components/api/api.proto

@@ -1465,7 +1465,7 @@ message BluetoothDeviceRequest {
 
   uint64 address = 1;
   BluetoothDeviceRequestType request_type = 2;
-  bool has_address_type = 3;
+  bool has_address_type = 3;  // Deprecated, should be removed in 2027.8 - https://github.com/esphome/esphome/pull/10318
   uint32 address_type = 4;
 }
 
@@ -1571,7 +1571,7 @@ message BluetoothGATTWriteRequest {
   uint32 handle = 2;
   bool response = 3;
 
-  bytes data = 4;
+  bytes data = 4 [(pointer_to_buffer) = true];
 }
 
 message BluetoothGATTReadDescriptorRequest {
@@ -1591,7 +1591,7 @@ message BluetoothGATTWriteDescriptorRequest {
   uint64 address = 1;
   uint32 handle = 2;
 
-  bytes data = 3;
+  bytes data = 3 [(pointer_to_buffer) = true];
 }
 
 message BluetoothGATTNotifyRequest {

esphome/components/api/api_options.proto

@@ -32,6 +32,13 @@ extend google.protobuf.FieldOptions {
     optional string fixed_array_size_define = 50010;
     optional string fixed_array_with_length_define = 50011;
 
+    // pointer_to_buffer: Use pointer instead of array for fixed-size byte fields
+    // When set, the field will be declared as a pointer (const uint8_t *data)
+    // instead of an array (uint8_t data[N]). This allows zero-copy on decode
+    // by pointing directly to the protobuf buffer. The buffer must remain valid
+    // until the message is processed (which is guaranteed for stack-allocated messages).
+    optional bool pointer_to_buffer = 50012 [default=false];
+
     // container_pointer: Zero-copy optimization for repeated fields.
     //
     // When container_pointer is set on a repeated field, the generated message will

esphome/components/api/api_pb2.cpp

@@ -2028,9 +2028,12 @@ bool BluetoothGATTWriteRequest::decode_varint(uint32_t field_id, ProtoVarInt val
 }
 bool BluetoothGATTWriteRequest::decode_length(uint32_t field_id, ProtoLengthDelimited value) {
   switch (field_id) {
-    case 4:
-      this->data = value.as_string();
+    case 4: {
+      // Use raw data directly to avoid allocation
+      this->data = value.data();
+      this->data_len = value.size();
       break;
+    }
     default:
       return false;
   }
@@ -2064,9 +2067,12 @@ bool BluetoothGATTWriteDescriptorRequest::decode_varint(uint32_t field_id, Proto
 }
 bool BluetoothGATTWriteDescriptorRequest::decode_length(uint32_t field_id, ProtoLengthDelimited value) {
   switch (field_id) {
-    case 3:
-      this->data = value.as_string();
+    case 3: {
+      // Use raw data directly to avoid allocation
+      this->data = value.data();
+      this->data_len = value.size();
       break;
+    }
     default:
       return false;
   }

esphome/components/api/api_pb2.h

@@ -1985,14 +1985,15 @@ class BluetoothGATTReadResponse final : public ProtoMessage {
 class BluetoothGATTWriteRequest final : public ProtoDecodableMessage {
  public:
   static constexpr uint8_t MESSAGE_TYPE = 75;
-  static constexpr uint8_t ESTIMATED_SIZE = 19;
+  static constexpr uint8_t ESTIMATED_SIZE = 29;
 #ifdef HAS_PROTO_MESSAGE_DUMP
   const char *message_name() const override { return "bluetooth_gatt_write_request"; }
 #endif
   uint64_t address{0};
   uint32_t handle{0};
   bool response{false};
-  std::string data{};
+  const uint8_t *data{nullptr};
+  uint16_t data_len{0};
 #ifdef HAS_PROTO_MESSAGE_DUMP
   void dump_to(std::string &out) const override;
 #endif
@@ -2020,13 +2021,14 @@ class BluetoothGATTReadDescriptorRequest final : public ProtoDecodableMessage {
 class BluetoothGATTWriteDescriptorRequest final : public ProtoDecodableMessage {
  public:
   static constexpr uint8_t MESSAGE_TYPE = 77;
-  static constexpr uint8_t ESTIMATED_SIZE = 17;
+  static constexpr uint8_t ESTIMATED_SIZE = 27;
 #ifdef HAS_PROTO_MESSAGE_DUMP
   const char *message_name() const override { return "bluetooth_gatt_write_descriptor_request"; }
 #endif
   uint64_t address{0};
   uint32_t handle{0};
-  std::string data{};
+  const uint8_t *data{nullptr};
+  uint16_t data_len{0};
 #ifdef HAS_PROTO_MESSAGE_DUMP
   void dump_to(std::string &out) const override;
 #endif

esphome/components/api/api_pb2_dump.cpp

@@ -1649,7 +1649,7 @@ void BluetoothGATTWriteRequest::dump_to(std::string &out) const {
   dump_field(out, "handle", this->handle);
   dump_field(out, "response", this->response);
   out.append("  data: ");
-  out.append(format_hex_pretty(reinterpret_cast<const uint8_t *>(this->data.data()), this->data.size()));
+  out.append(format_hex_pretty(this->data, this->data_len));
   out.append("\n");
 }
 void BluetoothGATTReadDescriptorRequest::dump_to(std::string &out) const {
@@ -1662,7 +1662,7 @@ void BluetoothGATTWriteDescriptorRequest::dump_to(std::string &out) const {
   dump_field(out, "address", this->address);
   dump_field(out, "handle", this->handle);
   out.append("  data: ");
-  out.append(format_hex_pretty(reinterpret_cast<const uint8_t *>(this->data.data()), this->data.size()));
+  out.append(format_hex_pretty(this->data, this->data_len));
   out.append("\n");
 }
 void BluetoothGATTNotifyRequest::dump_to(std::string &out) const {

esphome/components/api/proto.h

@@ -182,6 +182,10 @@ class ProtoLengthDelimited {
   explicit ProtoLengthDelimited(const uint8_t *value, size_t length) : value_(value), length_(length) {}
   std::string as_string() const { return std::string(reinterpret_cast<const char *>(this->value_), this->length_); }
 
+  // Direct access to raw data without string allocation
+  const uint8_t *data() const { return this->value_; }
+  size_t size() const { return this->length_; }
+
   /**
    * Decode the length-delimited data into an existing ProtoDecodableMessage instance.
    *

esphome/components/bluetooth_proxy/bluetooth_connection.cpp

@@ -514,7 +514,8 @@ esp_err_t BluetoothConnection::read_characteristic(uint16_t handle) {
   return this->check_and_log_error_("esp_ble_gattc_read_char", err);
 }
 
-esp_err_t BluetoothConnection::write_characteristic(uint16_t handle, const std::string &data, bool response) {
+esp_err_t BluetoothConnection::write_characteristic(uint16_t handle, const uint8_t *data, size_t length,
+                                                    bool response) {
   if (!this->connected()) {
     this->log_gatt_not_connected_("write", "characteristic");
     return ESP_GATT_NOT_CONNECTED;
@@ -522,8 +523,11 @@ esp_err_t BluetoothConnection::write_characteristic(uint16_t handle, const std::
   ESP_LOGV(TAG, "[%d] [%s] Writing GATT characteristic handle %d", this->connection_index_, this->address_str_.c_str(),
            handle);
 
+  // ESP-IDF's API requires a non-const uint8_t* but it doesn't modify the data
+  // The BTC layer immediately copies the data to its own buffer (see btc_gattc.c)
+  // const_cast is safe here and was previously hidden by a C-style cast
   esp_err_t err =
-      esp_ble_gattc_write_char(this->gattc_if_, this->conn_id_, handle, data.size(), (uint8_t *) data.data(),
+      esp_ble_gattc_write_char(this->gattc_if_, this->conn_id_, handle, length, const_cast<uint8_t *>(data),
                                response ? ESP_GATT_WRITE_TYPE_RSP : ESP_GATT_WRITE_TYPE_NO_RSP, ESP_GATT_AUTH_REQ_NONE);
   return this->check_and_log_error_("esp_ble_gattc_write_char", err);
 }
@@ -540,7 +544,7 @@ esp_err_t BluetoothConnection::read_descriptor(uint16_t handle) {
   return this->check_and_log_error_("esp_ble_gattc_read_char_descr", err);
 }
 
-esp_err_t BluetoothConnection::write_descriptor(uint16_t handle, const std::string &data, bool response) {
+esp_err_t BluetoothConnection::write_descriptor(uint16_t handle, const uint8_t *data, size_t length, bool response) {
   if (!this->connected()) {
     this->log_gatt_not_connected_("write", "descriptor");
     return ESP_GATT_NOT_CONNECTED;
@@ -548,8 +552,11 @@ esp_err_t BluetoothConnection::write_descriptor(uint16_t handle, const std::stri
   ESP_LOGV(TAG, "[%d] [%s] Writing GATT descriptor handle %d", this->connection_index_, this->address_str_.c_str(),
            handle);
 
+  // ESP-IDF's API requires a non-const uint8_t* but it doesn't modify the data
+  // The BTC layer immediately copies the data to its own buffer (see btc_gattc.c)
+  // const_cast is safe here and was previously hidden by a C-style cast
   esp_err_t err = esp_ble_gattc_write_char_descr(
-      this->gattc_if_, this->conn_id_, handle, data.size(), (uint8_t *) data.data(),
+      this->gattc_if_, this->conn_id_, handle, length, const_cast<uint8_t *>(data),
       response ? ESP_GATT_WRITE_TYPE_RSP : ESP_GATT_WRITE_TYPE_NO_RSP, ESP_GATT_AUTH_REQ_NONE);
   return this->check_and_log_error_("esp_ble_gattc_write_char_descr", err);
 }

esphome/components/bluetooth_proxy/bluetooth_connection.h

@@ -18,9 +18,9 @@ class BluetoothConnection final : public esp32_ble_client::BLEClientBase {
   esp32_ble_tracker::AdvertisementParserType get_advertisement_parser_type() override;
 
   esp_err_t read_characteristic(uint16_t handle);
-  esp_err_t write_characteristic(uint16_t handle, const std::string &data, bool response);
+  esp_err_t write_characteristic(uint16_t handle, const uint8_t *data, size_t length, bool response);
   esp_err_t read_descriptor(uint16_t handle);
-  esp_err_t write_descriptor(uint16_t handle, const std::string &data, bool response);
+  esp_err_t write_descriptor(uint16_t handle, const uint8_t *data, size_t length, bool response);
 
   esp_err_t notify_characteristic(uint16_t handle, bool enable);
 

esphome/components/bluetooth_proxy/bluetooth_proxy.cpp

@@ -305,7 +305,7 @@ void BluetoothProxy::bluetooth_gatt_write(const api::BluetoothGATTWriteRequest &
     return;
   }
 
-  auto err = connection->write_characteristic(msg.handle, msg.data, msg.response);
+  auto err = connection->write_characteristic(msg.handle, msg.data, msg.data_len, msg.response);
   if (err != ESP_OK) {
     this->send_gatt_error(msg.address, msg.handle, err);
   }
@@ -331,7 +331,7 @@ void BluetoothProxy::bluetooth_gatt_write_descriptor(const api::BluetoothGATTWri
     return;
   }
 
-  auto err = connection->write_descriptor(msg.handle, msg.data, true);
+  auto err = connection->write_descriptor(msg.handle, msg.data, msg.data_len, true);
   if (err != ESP_OK) {
     this->send_gatt_error(msg.address, msg.handle, err);
   }

script/api_protobuf/api_protobuf.py

@@ -353,11 +353,24 @@ def create_field_type_info(
             return FixedArrayRepeatedType(field, size_define)
         return RepeatedTypeInfo(field)
 
-    # Check for fixed_array_size option on bytes fields
-    if (
-        field.type == 12
-        and (fixed_size := get_field_opt(field, pb.fixed_array_size)) is not None
-    ):
+    # Check for mutually exclusive options on bytes fields
+    if field.type == 12:
+        has_pointer_to_buffer = get_field_opt(field, pb.pointer_to_buffer, False)
+        fixed_size = get_field_opt(field, pb.fixed_array_size, None)
+
+        if has_pointer_to_buffer and fixed_size is not None:
+            raise ValueError(
+                f"Field '{field.name}' has both pointer_to_buffer and fixed_array_size. "
+                "These options are mutually exclusive. Use pointer_to_buffer for zero-copy "
+                "or fixed_array_size for traditional array storage."
+            )
+
+        if has_pointer_to_buffer:
+            # Zero-copy pointer approach - no size needed, will use size_t for length
+            return PointerToBytesBufferType(field, None)
+
+        if fixed_size is not None:
+            # Traditional fixed array approach with copy
             return FixedArrayBytesType(field, fixed_size)
 
     # Special handling for bytes fields
@@ -818,6 +831,91 @@ class BytesType(TypeInfo):
         return self.calculate_field_id_size() + 8  # field ID + 8 bytes typical bytes
 
 
+class PointerToBytesBufferType(TypeInfo):
+    """Type for bytes fields that use pointer_to_buffer option for zero-copy."""
+
+    @classmethod
+    def can_use_dump_field(cls) -> bool:
+        return False
+
+    def __init__(
+        self, field: descriptor.FieldDescriptorProto, size: int | None = None
+    ) -> None:
+        super().__init__(field)
+        # Size is not used for pointer_to_buffer - we always use size_t for length
+        self.array_size = 0
+
+    @property
+    def cpp_type(self) -> str:
+        return "const uint8_t*"
+
+    @property
+    def default_value(self) -> str:
+        return "nullptr"
+
+    @property
+    def reference_type(self) -> str:
+        return "const uint8_t*"
+
+    @property
+    def const_reference_type(self) -> str:
+        return "const uint8_t*"
+
+    @property
+    def public_content(self) -> list[str]:
+        # Use uint16_t for length - max packet size is well below 65535
+        # Add pointer and length fields
+        return [
+            f"const uint8_t* {self.field_name}{{nullptr}};",
+            f"uint16_t {self.field_name}_len{{0}};",
+        ]
+
+    @property
+    def encode_content(self) -> str:
+        return f"buffer.encode_bytes({self.number}, this->{self.field_name}, this->{self.field_name}_len);"
+
+    @property
+    def decode_length_content(self) -> str | None:
+        # Decode directly stores the pointer to avoid allocation
+        return f"""case {self.number}: {{
+      // Use raw data directly to avoid allocation
+      this->{self.field_name} = value.data();
+      this->{self.field_name}_len = value.size();
+      break;
+    }}"""
+
+    @property
+    def decode_length(self) -> str | None:
+        # This is handled in decode_length_content
+        return None
+
+    @property
+    def wire_type(self) -> WireType:
+        """Get the wire type for this bytes field."""
+        return WireType.LENGTH_DELIMITED  # Uses wire type 2
+
+    def dump(self, name: str) -> str:
+        return (
+            f"format_hex_pretty(this->{self.field_name}, this->{self.field_name}_len)"
+        )
+
+    @property
+    def dump_content(self) -> str:
+        # Custom dump that doesn't use dump_field template
+        return (
+            f'out.append("  {self.name}: ");\n'
+            + f"out.append({self.dump(self.field_name)});\n"
+            + 'out.append("\\n");'
+        )
+
+    def get_size_calculation(self, name: str, force: bool = False) -> str:
+        return f"size.add_length({self.number}, this->{self.field_name}_len);"
+
+    def get_estimated_size(self) -> int:
+        # field ID + length varint + typical data (assume small for pointer fields)
+        return self.calculate_field_id_size() + 2 + 16
+
+
 class FixedArrayBytesType(TypeInfo):
     """Special type for fixed-size byte arrays."""