Replace API deferred queue with efficient message batching system (#9012)

script/api_protobuf/api_protobuf.py

@@ -258,6 +258,14 @@ class TypeInfo(ABC):
             force: Whether to force encoding the field even if it has a default value
         """
 
+    @abstractmethod
+    def get_estimated_size(self) -> int:
+        """Get estimated size in bytes for this field with typical values.
+
+        Returns:
+            Estimated size in bytes including field ID and typical data
+        """
+
 
 TYPE_INFO: dict[int, TypeInfo] = {}
 
@@ -291,6 +299,9 @@ class DoubleType(TypeInfo):
         o = f"ProtoSize::add_fixed_field<8>(total_size, {field_id_size}, {name} != 0.0, {force_str(force)});"
         return o
 
+    def get_estimated_size(self) -> int:
+        return self.calculate_field_id_size() + 8  # field ID + 8 bytes for double
+
 
 @register_type(2)
 class FloatType(TypeInfo):
@@ -310,6 +321,9 @@ class FloatType(TypeInfo):
         o = f"ProtoSize::add_fixed_field<4>(total_size, {field_id_size}, {name} != 0.0f, {force_str(force)});"
         return o
 
+    def get_estimated_size(self) -> int:
+        return self.calculate_field_id_size() + 4  # field ID + 4 bytes for float
+
 
 @register_type(3)
 class Int64Type(TypeInfo):
@@ -329,6 +343,9 @@ class Int64Type(TypeInfo):
         o = f"ProtoSize::add_int64_field(total_size, {field_id_size}, {name}, {force_str(force)});"
         return o
 
+    def get_estimated_size(self) -> int:
+        return self.calculate_field_id_size() + 3  # field ID + 3 bytes typical varint
+
 
 @register_type(4)
 class UInt64Type(TypeInfo):
@@ -348,6 +365,9 @@ class UInt64Type(TypeInfo):
         o = f"ProtoSize::add_uint64_field(total_size, {field_id_size}, {name}, {force_str(force)});"
         return o
 
+    def get_estimated_size(self) -> int:
+        return self.calculate_field_id_size() + 3  # field ID + 3 bytes typical varint
+
 
 @register_type(5)
 class Int32Type(TypeInfo):
@@ -367,6 +387,9 @@ class Int32Type(TypeInfo):
         o = f"ProtoSize::add_int32_field(total_size, {field_id_size}, {name}, {force_str(force)});"
         return o
 
+    def get_estimated_size(self) -> int:
+        return self.calculate_field_id_size() + 3  # field ID + 3 bytes typical varint
+
 
 @register_type(6)
 class Fixed64Type(TypeInfo):
@@ -386,6 +409,9 @@ class Fixed64Type(TypeInfo):
         o = f"ProtoSize::add_fixed_field<8>(total_size, {field_id_size}, {name} != 0, {force_str(force)});"
         return o
 
+    def get_estimated_size(self) -> int:
+        return self.calculate_field_id_size() + 8  # field ID + 8 bytes fixed
+
 
 @register_type(7)
 class Fixed32Type(TypeInfo):
@@ -405,6 +431,9 @@ class Fixed32Type(TypeInfo):
         o = f"ProtoSize::add_fixed_field<4>(total_size, {field_id_size}, {name} != 0, {force_str(force)});"
         return o
 
+    def get_estimated_size(self) -> int:
+        return self.calculate_field_id_size() + 4  # field ID + 4 bytes fixed
+
 
 @register_type(8)
 class BoolType(TypeInfo):
@@ -423,6 +452,9 @@ class BoolType(TypeInfo):
         o = f"ProtoSize::add_bool_field(total_size, {field_id_size}, {name}, {force_str(force)});"
         return o
 
+    def get_estimated_size(self) -> int:
+        return self.calculate_field_id_size() + 1  # field ID + 1 byte
+
 
 @register_type(9)
 class StringType(TypeInfo):
@@ -443,6 +475,9 @@ class StringType(TypeInfo):
         o = f"ProtoSize::add_string_field(total_size, {field_id_size}, {name}, {force_str(force)});"
         return o
 
+    def get_estimated_size(self) -> int:
+        return self.calculate_field_id_size() + 8  # field ID + 8 bytes typical string
+
 
 @register_type(11)
 class MessageType(TypeInfo):
@@ -478,6 +513,11 @@ class MessageType(TypeInfo):
         o = f"ProtoSize::add_message_object(total_size, {field_id_size}, {name}, {force_str(force)});"
         return o
 
+    def get_estimated_size(self) -> int:
+        return (
+            self.calculate_field_id_size() + 16
+        )  # field ID + 16 bytes estimated submessage
+
 
 @register_type(12)
 class BytesType(TypeInfo):
@@ -498,6 +538,9 @@ class BytesType(TypeInfo):
         o = f"ProtoSize::add_string_field(total_size, {field_id_size}, {name}, {force_str(force)});"
         return o
 
+    def get_estimated_size(self) -> int:
+        return self.calculate_field_id_size() + 8  # field ID + 8 bytes typical bytes
+
 
 @register_type(13)
 class UInt32Type(TypeInfo):
@@ -517,6 +560,9 @@ class UInt32Type(TypeInfo):
         o = f"ProtoSize::add_uint32_field(total_size, {field_id_size}, {name}, {force_str(force)});"
         return o
 
+    def get_estimated_size(self) -> int:
+        return self.calculate_field_id_size() + 3  # field ID + 3 bytes typical varint
+
 
 @register_type(14)
 class EnumType(TypeInfo):
@@ -544,6 +590,9 @@ class EnumType(TypeInfo):
         o = f"ProtoSize::add_enum_field(total_size, {field_id_size}, static_cast<uint32_t>({name}), {force_str(force)});"
         return o
 
+    def get_estimated_size(self) -> int:
+        return self.calculate_field_id_size() + 1  # field ID + 1 byte typical enum
+
 
 @register_type(15)
 class SFixed32Type(TypeInfo):
@@ -563,6 +612,9 @@ class SFixed32Type(TypeInfo):
         o = f"ProtoSize::add_fixed_field<4>(total_size, {field_id_size}, {name} != 0, {force_str(force)});"
         return o
 
+    def get_estimated_size(self) -> int:
+        return self.calculate_field_id_size() + 4  # field ID + 4 bytes fixed
+
 
 @register_type(16)
 class SFixed64Type(TypeInfo):
@@ -582,6 +634,9 @@ class SFixed64Type(TypeInfo):
         o = f"ProtoSize::add_fixed_field<8>(total_size, {field_id_size}, {name} != 0, {force_str(force)});"
         return o
 
+    def get_estimated_size(self) -> int:
+        return self.calculate_field_id_size() + 8  # field ID + 8 bytes fixed
+
 
 @register_type(17)
 class SInt32Type(TypeInfo):
@@ -601,6 +656,9 @@ class SInt32Type(TypeInfo):
         o = f"ProtoSize::add_sint32_field(total_size, {field_id_size}, {name}, {force_str(force)});"
         return o
 
+    def get_estimated_size(self) -> int:
+        return self.calculate_field_id_size() + 3  # field ID + 3 bytes typical varint
+
 
 @register_type(18)
 class SInt64Type(TypeInfo):
@@ -620,6 +678,9 @@ class SInt64Type(TypeInfo):
         o = f"ProtoSize::add_sint64_field(total_size, {field_id_size}, {name}, {force_str(force)});"
         return o
 
+    def get_estimated_size(self) -> int:
+        return self.calculate_field_id_size() + 3  # field ID + 3 bytes typical varint
+
 
 class RepeatedTypeInfo(TypeInfo):
     def __init__(self, field: descriptor.FieldDescriptorProto) -> None:
@@ -738,6 +799,15 @@ class RepeatedTypeInfo(TypeInfo):
         o += "}"
         return o
 
+    def get_estimated_size(self) -> int:
+        # For repeated fields, estimate underlying type size * 2 (assume 2 items typically)
+        underlying_size = (
+            self._ti.get_estimated_size()
+            if hasattr(self._ti, "get_estimated_size")
+            else 8
+        )
+        return underlying_size * 2
+
 
 def build_enum_type(desc) -> tuple[str, str]:
     """Builds the enum type."""
@@ -762,6 +832,22 @@ def build_enum_type(desc) -> tuple[str, str]:
     return out, cpp
 
 
+def calculate_message_estimated_size(desc: descriptor.DescriptorProto) -> int:
+    """Calculate estimated size for a complete message based on typical values."""
+    total_size = 0
+
+    for field in desc.field:
+        if field.label == 3:  # repeated
+            ti = RepeatedTypeInfo(field)
+        else:
+            ti = TYPE_INFO[field.type](field)
+
+        # Add estimated size for this field
+        total_size += ti.get_estimated_size()
+
+    return total_size
+
+
 def build_message_type(desc: descriptor.DescriptorProto) -> tuple[str, str]:
     public_content: list[str] = []
     protected_content: list[str] = []
@@ -773,6 +859,28 @@ def build_message_type(desc: descriptor.DescriptorProto) -> tuple[str, str]:
     dump: list[str] = []
     size_calc: list[str] = []
 
+    # Get message ID if it's a service message
+    message_id: int | None = get_opt(desc, pb.id)
+
+    # Add MESSAGE_TYPE method if this is a service message
+    if message_id is not None:
+        # Add static constexpr for message type
+        public_content.append(f"static constexpr uint16_t MESSAGE_TYPE = {message_id};")
+
+        # Add estimated size constant
+        estimated_size = calculate_message_estimated_size(desc)
+        public_content.append(
+            f"static constexpr uint16_t ESTIMATED_SIZE = {estimated_size};"
+        )
+
+        # Add message_name method for debugging
+        public_content.append("#ifdef HAS_PROTO_MESSAGE_DUMP")
+        snake_name = camel_to_snake(desc.name)
+        public_content.append(
+            f'static constexpr const char *message_name() {{ return "{snake_name}"; }}'
+        )
+        public_content.append("#endif")
+
     for field in desc.field:
         if field.label == 3:
             ti = RepeatedTypeInfo(field)
@@ -941,24 +1049,18 @@ def build_service_message_type(
     hout = ""
     cout = ""
 
+    # Store ifdef for later use
     if ifdef is not None:
         ifdefs[str(mt.name)] = ifdef
-        hout += f"#ifdef {ifdef}\n"
-        cout += f"#ifdef {ifdef}\n"
 
     if source in (SOURCE_BOTH, SOURCE_SERVER):
-        # Generate send
-        func = f"send_{snake}"
-        hout += f"bool {func}(const {mt.name} &msg);\n"
-        cout += f"bool APIServerConnectionBase::{func}(const {mt.name} &msg) {{\n"
-        if log:
-            cout += "#ifdef HAS_PROTO_MESSAGE_DUMP\n"
-            cout += f'  ESP_LOGVV(TAG, "{func}: %s", msg.dump().c_str());\n'
-            cout += "#endif\n"
-        # cout += f'  this->set_nodelay({str(nodelay).lower()});\n'
-        cout += f"  return this->send_message_<{mt.name}>(msg, {id_});\n"
-        cout += "}\n"
+        # Don't generate individual send methods anymore
+        # The generic send_message method will be used instead
+        pass
     if source in (SOURCE_BOTH, SOURCE_CLIENT):
+        # Only add ifdef when we're actually generating content
+        if ifdef is not None:
+            hout += f"#ifdef {ifdef}\n"
         # Generate receive
         func = f"on_{snake}"
         hout += f"virtual void {func}(const {mt.name} &value){{}};\n"
@@ -977,9 +1079,9 @@ def build_service_message_type(
         case += "break;"
         RECEIVE_CASES[id_] = case
 
-    if ifdef is not None:
-        hout += "#endif\n"
-        cout += "#endif\n"
+        # Only close ifdef if we opened it
+        if ifdef is not None:
+            hout += "#endif\n"
 
     return hout, cout
 
@@ -1083,6 +1185,29 @@ def main() -> None:
     hpp += f"class {class_name} : public ProtoService {{\n"
     hpp += " public:\n"
 
+    # Add logging helper method declaration
+    hpp += "#ifdef HAS_PROTO_MESSAGE_DUMP\n"
+    hpp += " protected:\n"
+    hpp += "  void log_send_message_(const char *name, const std::string &dump);\n"
+    hpp += " public:\n"
+    hpp += "#endif\n\n"
+
+    # Add generic send_message method
+    hpp += "  template<typename T>\n"
+    hpp += "  bool send_message(const T &msg) {\n"
+    hpp += "#ifdef HAS_PROTO_MESSAGE_DUMP\n"
+    hpp += "    this->log_send_message_(T::message_name(), msg.dump());\n"
+    hpp += "#endif\n"
+    hpp += "    return this->send_message_(msg, T::MESSAGE_TYPE);\n"
+    hpp += "  }\n\n"
+
+    # Add logging helper method implementation to cpp
+    cpp += "#ifdef HAS_PROTO_MESSAGE_DUMP\n"
+    cpp += f"void {class_name}::log_send_message_(const char *name, const std::string &dump) {{\n"
+    cpp += '  ESP_LOGVV(TAG, "send_message %s: %s", name, dump.c_str());\n'
+    cpp += "}\n"
+    cpp += "#endif\n\n"
+
     for mt in file.message_type:
         obj = build_service_message_type(mt)
         if obj is None:
@@ -1155,8 +1280,7 @@ def main() -> None:
             body += f"this->{func}(msg);\n"
         else:
             body += f"{ret} ret = this->{func}(msg);\n"
-            ret_snake = camel_to_snake(ret)
-            body += f"if (!this->send_{ret_snake}(ret)) {{\n"
+            body += "if (!this->send_message(ret)) {\n"
             body += "  this->on_fatal_error();\n"
             body += "}\n"
         cpp += indent(body) + "\n" + "}\n"