From d7bff38ad956c202e62131ee2320590ea7f9b627 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Mon, 22 Sep 2025 22:30:30 -0500 Subject: [PATCH] Implement zero-copy API for zwave_proxy --- esphome/components/api/api.proto | 2 +- esphome/components/api/api_options.proto | 7 ++ esphome/components/api/api_pb2.cpp | 9 +- esphome/components/api/api_pb2.h | 4 +- esphome/components/api/api_pb2_dump.cpp | 7 +- esphome/components/api/proto.h | 4 + .../components/zwave_proxy/zwave_proxy.cpp | 8 +- esphome/components/zwave_proxy/zwave_proxy.h | 10 +- script/api_protobuf/api_protobuf.py | 99 +++++++++++++++++-- 9 files changed, 121 insertions(+), 29 deletions(-) diff --git a/esphome/components/api/api.proto b/esphome/components/api/api.proto index ad99de4b4a..eceee9d27d 100644 --- a/esphome/components/api/api.proto +++ b/esphome/components/api/api.proto @@ -2292,7 +2292,7 @@ message ZWaveProxyFrame { option (ifdef) = "USE_ZWAVE_PROXY"; option (no_delay) = true; - bytes data = 1 [(fixed_array_size) = 257]; + bytes data = 1 [(pointer_to_buffer) = true]; } enum ZWaveProxyRequestType { diff --git a/esphome/components/api/api_options.proto b/esphome/components/api/api_options.proto index 50c43b96fd..633f39b552 100644 --- a/esphome/components/api/api_options.proto +++ b/esphome/components/api/api_options.proto @@ -32,6 +32,13 @@ extend google.protobuf.FieldOptions { optional string fixed_array_size_define = 50010; optional string fixed_array_with_length_define = 50011; + // pointer_to_buffer: Use pointer instead of array for fixed-size byte fields + // When set, the field will be declared as a pointer (const uint8_t *data) + // instead of an array (uint8_t data[N]). This allows zero-copy on decode + // by pointing directly to the protobuf buffer. The buffer must remain valid + // until the message is processed (which is guaranteed for stack-allocated messages). + optional bool pointer_to_buffer = 50012 [default=false]; + // container_pointer: Zero-copy optimization for repeated fields. // // When container_pointer is set on a repeated field, the generated message will diff --git a/esphome/components/api/api_pb2.cpp b/esphome/components/api/api_pb2.cpp index 245933724b..e9c2cb2cff 100644 --- a/esphome/components/api/api_pb2.cpp +++ b/esphome/components/api/api_pb2.cpp @@ -3029,12 +3029,9 @@ bool UpdateCommandRequest::decode_32bit(uint32_t field_id, Proto32Bit value) { bool ZWaveProxyFrame::decode_length(uint32_t field_id, ProtoLengthDelimited value) { switch (field_id) { case 1: { - const std::string &data_str = value.as_string(); - this->data_len = data_str.size(); - if (this->data_len > 257) { - this->data_len = 257; - } - memcpy(this->data, data_str.data(), this->data_len); + // Use raw data directly to avoid allocation + this->data = value.data(); + this->data_len = value.size(); break; } default: diff --git a/esphome/components/api/api_pb2.h b/esphome/components/api/api_pb2.h index 248a4b1f82..5715f840c2 100644 --- a/esphome/components/api/api_pb2.h +++ b/esphome/components/api/api_pb2.h @@ -2929,11 +2929,11 @@ class UpdateCommandRequest final : public CommandProtoMessage { class ZWaveProxyFrame final : public ProtoDecodableMessage { public: static constexpr uint8_t MESSAGE_TYPE = 128; - static constexpr uint8_t ESTIMATED_SIZE = 33; + static constexpr uint8_t ESTIMATED_SIZE = 19; #ifdef HAS_PROTO_MESSAGE_DUMP const char *message_name() const override { return "z_wave_proxy_frame"; } #endif - uint8_t data[257]{}; + const uint8_t *data{nullptr}; uint16_t data_len{0}; void encode(ProtoWriteBuffer buffer) const override; void calculate_size(ProtoSize &size) const override; diff --git a/esphome/components/api/api_pb2_dump.cpp b/esphome/components/api/api_pb2_dump.cpp index ac43af6d54..b67f909bd0 100644 --- a/esphome/components/api/api_pb2_dump.cpp +++ b/esphome/components/api/api_pb2_dump.cpp @@ -2126,12 +2126,7 @@ void UpdateCommandRequest::dump_to(std::string &out) const { } #endif #ifdef USE_ZWAVE_PROXY -void ZWaveProxyFrame::dump_to(std::string &out) const { - MessageDumpHelper helper(out, "ZWaveProxyFrame"); - out.append(" data: "); - out.append(format_hex_pretty(this->data, this->data_len)); - out.append("\n"); -} +void ZWaveProxyFrame::dump_to(std::string &out) const { dump_field(out, "data", this->data); } void ZWaveProxyRequest::dump_to(std::string &out) const { MessageDumpHelper helper(out, "ZWaveProxyRequest"); dump_field(out, "type", static_cast(this->type)); diff --git a/esphome/components/api/proto.h b/esphome/components/api/proto.h index 0e5ec61050..6be5f00e75 100644 --- a/esphome/components/api/proto.h +++ b/esphome/components/api/proto.h @@ -182,6 +182,10 @@ class ProtoLengthDelimited { explicit ProtoLengthDelimited(const uint8_t *value, size_t length) : value_(value), length_(length) {} std::string as_string() const { return std::string(reinterpret_cast(this->value_), this->length_); } + // Direct access to raw data without string allocation + const uint8_t *data() const { return this->value_; } + size_t size() const { return this->length_; } + /** * Decode the length-delimited data into an existing ProtoDecodableMessage instance. * diff --git a/esphome/components/zwave_proxy/zwave_proxy.cpp b/esphome/components/zwave_proxy/zwave_proxy.cpp index 12c4ee0c0d..19e4182c1b 100644 --- a/esphome/components/zwave_proxy/zwave_proxy.cpp +++ b/esphome/components/zwave_proxy/zwave_proxy.cpp @@ -61,14 +61,14 @@ void ZWaveProxy::loop() { } ESP_LOGV(TAG, "Sending to client: %s", YESNO(this->api_connection_ != nullptr)); if (this->api_connection_ != nullptr) { - // minimize copying to reduce CPU overhead + // Zero-copy: point directly to our buffer + this->outgoing_proto_msg_.data = this->buffer_.data(); if (this->in_bootloader_) { this->outgoing_proto_msg_.data_len = this->buffer_index_; } else { // If this is a data frame, use frame length indicator + 2 (for SoF + checksum), else assume 1 for ACK/NAK/CAN this->outgoing_proto_msg_.data_len = this->buffer_[0] == ZWAVE_FRAME_TYPE_START ? this->buffer_[1] + 2 : 1; } - std::memcpy(this->outgoing_proto_msg_.data, this->buffer_.data(), this->outgoing_proto_msg_.data_len); this->api_connection_->send_message(this->outgoing_proto_msg_, api::ZWaveProxyFrame::MESSAGE_TYPE); } } @@ -228,7 +228,9 @@ void ZWaveProxy::parse_start_(uint8_t byte) { } // Forward response (ACK/NAK/CAN) back to client for processing if (this->api_connection_ != nullptr) { - this->outgoing_proto_msg_.data[0] = byte; + // Store single byte in buffer and point to it + this->buffer_[0] = byte; + this->outgoing_proto_msg_.data = this->buffer_.data(); this->outgoing_proto_msg_.data_len = 1; this->api_connection_->send_message(this->outgoing_proto_msg_, api::ZWaveProxyFrame::MESSAGE_TYPE); } diff --git a/esphome/components/zwave_proxy/zwave_proxy.h b/esphome/components/zwave_proxy/zwave_proxy.h index 5d908b328c..e5080e4f86 100644 --- a/esphome/components/zwave_proxy/zwave_proxy.h +++ b/esphome/components/zwave_proxy/zwave_proxy.h @@ -63,11 +63,11 @@ class ZWaveProxy : public uart::UARTDevice, public Component { api::APIConnection *api_connection_{nullptr}; // Current subscribed client - std::array home_id_{0, 0, 0, 0}; // Fixed buffer for home ID - std::array buffer_; // Fixed buffer for incoming data - uint8_t buffer_index_{0}; // Index for populating the data buffer - uint8_t end_frame_after_{0}; // Payload reception ends after this index - uint8_t last_response_{0}; // Last response type sent + std::array home_id_{0, 0, 0, 0}; // Fixed buffer for home ID + std::array buffer_; // Fixed buffer for incoming data + uint8_t buffer_index_{0}; // Index for populating the data buffer + uint8_t end_frame_after_{0}; // Payload reception ends after this index + uint8_t last_response_{0}; // Last response type sent ZWaveParsingState parsing_state_{ZWAVE_PARSING_STATE_WAIT_START}; bool in_bootloader_{false}; // True if the device is detected to be in bootloader mode diff --git a/script/api_protobuf/api_protobuf.py b/script/api_protobuf/api_protobuf.py index fa04222c5d..d212401bd2 100755 --- a/script/api_protobuf/api_protobuf.py +++ b/script/api_protobuf/api_protobuf.py @@ -353,12 +353,25 @@ def create_field_type_info( return FixedArrayRepeatedType(field, size_define) return RepeatedTypeInfo(field) - # Check for fixed_array_size option on bytes fields - if ( - field.type == 12 - and (fixed_size := get_field_opt(field, pb.fixed_array_size)) is not None - ): - return FixedArrayBytesType(field, fixed_size) + # Check for mutually exclusive options on bytes fields + if field.type == 12: + has_pointer_to_buffer = get_field_opt(field, pb.pointer_to_buffer, False) + fixed_size = get_field_opt(field, pb.fixed_array_size, None) + + if has_pointer_to_buffer and fixed_size is not None: + raise ValueError( + f"Field '{field.name}' has both pointer_to_buffer and fixed_array_size. " + "These options are mutually exclusive. Use pointer_to_buffer for zero-copy " + "or fixed_array_size for traditional array storage." + ) + + if has_pointer_to_buffer: + # Zero-copy pointer approach - no size needed, will use size_t for length + return PointerToBytesBufferType(field, None) + + if fixed_size is not None: + # Traditional fixed array approach with copy + return FixedArrayBytesType(field, fixed_size) # Special handling for bytes fields if field.type == 12: @@ -818,6 +831,80 @@ class BytesType(TypeInfo): return self.calculate_field_id_size() + 8 # field ID + 8 bytes typical bytes +class PointerToBytesBufferType(TypeInfo): + """Type for bytes fields that use pointer_to_buffer option for zero-copy.""" + + @classmethod + def can_use_dump_field(cls) -> bool: + return False + + def __init__( + self, field: descriptor.FieldDescriptorProto, size: int | None = None + ) -> None: + super().__init__(field) + # Size is not used for pointer_to_buffer - we always use size_t for length + self.array_size = 0 + + @property + def cpp_type(self) -> str: + return "const uint8_t*" + + @property + def default_value(self) -> str: + return "nullptr" + + @property + def reference_type(self) -> str: + return "const uint8_t*" + + @property + def const_reference_type(self) -> str: + return "const uint8_t*" + + @property + def public_content(self) -> list[str]: + # Use uint16_t for length - max packet size is well below 65535 + # Add pointer and length fields + return [ + f"const uint8_t* {self.field_name}{{nullptr}};", + f"uint16_t {self.field_name}_len{{0}};", + ] + + @property + def encode_content(self) -> str: + return f"buffer.encode_bytes({self.number}, this->{self.field_name}, this->{self.field_name}_len);" + + @property + def decode_length_content(self) -> str | None: + # Decode directly stores the pointer to avoid allocation + return f"""case {self.number}: {{ + // Use raw data directly to avoid allocation + this->{self.field_name} = value.data(); + this->{self.field_name}_len = value.size(); + break; + }}""" + + @property + def decode_length(self) -> str | None: + # This is handled in decode_length_content + return None + + @property + def wire_type(self) -> WireType: + """Get the wire type for this bytes field.""" + return WireType.LENGTH_DELIMITED # Uses wire type 2 + + def dump(self, name: str) -> str: + return f"format_hex_pretty(this->{name}, this->{name}_len)" + + def get_size_calculation(self, name: str, force: bool = False) -> str: + return f"size.add_length({self.number}, this->{self.field_name}_len);" + + def get_estimated_size(self) -> int: + # field ID + length varint + typical data (assume small for pointer fields) + return self.calculate_field_id_size() + 2 + 16 + + class FixedArrayBytesType(TypeInfo): """Special type for fixed-size byte arrays."""