[api] Add message size limits to prevent memory exhaustion

esphome/components/api/api_frame_helper.h

@@ -17,6 +17,16 @@ namespace esphome::api {
 // uncomment to log raw packets
 //#define HELPER_LOG_PACKETS
 
+// Maximum message size limits to prevent OOM on constrained devices
+// Voice Assistant is our largest user at 1024 bytes per audio chunk
+// Using 2048 + 256 bytes overhead = 2304 bytes total to support voice and future needs
+// ESP8266 has very limited RAM and cannot support voice assistant
+#ifdef USE_ESP8266
+static constexpr uint16_t MAX_MESSAGE_SIZE = 512;  // Keep small for memory constrained ESP8266
+#else
+static constexpr uint16_t MAX_MESSAGE_SIZE = 2304;  // Support voice (1024) + headroom for larger messages
+#endif
+
 // Forward declaration
 struct ClientInfo;
 

esphome/components/api/api_frame_helper_noise.cpp

@@ -184,6 +184,13 @@ APIError APINoiseFrameHelper::try_read_frame_(std::vector<uint8_t> *frame) {
     return APIError::BAD_HANDSHAKE_PACKET_LEN;
   }
 
+  // Check against maximum message size to prevent OOM
+  if (msg_size > MAX_MESSAGE_SIZE) {
+    state_ = State::FAILED;
+    HELPER_LOG("Bad packet: message size %u exceeds maximum %u", msg_size, MAX_MESSAGE_SIZE);
+    return APIError::BAD_DATA_PACKET;
+  }
+
   // reserve space for body
   if (rx_buf_.size() != msg_size) {
     rx_buf_.resize(msg_size);

esphome/components/api/api_frame_helper_plaintext.cpp

@@ -122,10 +122,10 @@ APIError APIPlaintextFrameHelper::try_read_frame_(std::vector<uint8_t> *frame) {
       continue;
     }
 
-    if (msg_size_varint->as_uint32() > std::numeric_limits<uint16_t>::max()) {
+    if (msg_size_varint->as_uint32() > MAX_MESSAGE_SIZE) {
       state_ = State::FAILED;
       HELPER_LOG("Bad packet: message size %" PRIu32 " exceeds maximum %u", msg_size_varint->as_uint32(),
-                 std::numeric_limits<uint16_t>::max());
+                 MAX_MESSAGE_SIZE);
       return APIError::BAD_DATA_PACKET;
     }
     rx_header_parsed_len_ = msg_size_varint->as_uint16();

tests/integration/test_oversized_payloads.py

@@ -15,7 +15,7 @@ async def test_oversized_payload_plaintext(
     run_compiled: RunCompiledFunction,
     api_client_connected_with_disconnect: APIClientConnectedWithDisconnectFactory,
 ) -> None:
-    """Test that oversized payloads (>100KiB) from client cause disconnection without crashing."""
+    """Test that oversized payloads (>2304 bytes) from client cause disconnection without crashing."""
     process_exited = False
     helper_log_found = False
 
@@ -39,8 +39,8 @@ async def test_oversized_payload_plaintext(
             assert device_info is not None
             assert device_info.name == "oversized-plaintext"
 
-            # Create an oversized payload (>100KiB)
+            # Create an oversized payload (>2304 bytes which is our new limit)
-            oversized_data = b"X" * (100 * 1024 + 1)  # 100KiB + 1 byte
+            oversized_data = b"X" * 3000  # ~3KiB, exceeds the 2304 byte limit
 
             # Access the internal connection to send raw data
             frame_helper = client._connection._frame_helper
@@ -132,22 +132,24 @@ async def test_oversized_payload_noise(
     run_compiled: RunCompiledFunction,
     api_client_connected_with_disconnect: APIClientConnectedWithDisconnectFactory,
 ) -> None:
-    """Test that oversized payloads (>100KiB) from client cause disconnection without crashing with noise encryption."""
+    """Test that oversized payloads from client cause disconnection without crashing with noise encryption."""
     noise_key = "N4Yle5YirwZhPiHHsdZLdOA73ndj/84veVaLhTvxCuU="
     process_exited = False
-    cipherstate_failed = False
+    helper_log_found = False
 
     def check_logs(line: str) -> None:
-        nonlocal process_exited, cipherstate_failed
+        nonlocal process_exited, helper_log_found
         # Check for signs that the process exited/crashed
         if "Segmentation fault" in line or "core dumped" in line:
             process_exited = True
-        # Check for the expected warning about decryption failure
+        # Check for HELPER_LOG message about message size exceeding maximum
+        # With our new protection, oversized messages are rejected at frame level
         if (
-            "[W][api.connection" in line
+            "[VV]" in line
-            and "Reading failed CIPHERSTATE_DECRYPT_FAILED" in line
+            and "Bad packet: message size" in line
+            and "exceeds maximum" in line
         ):
-            cipherstate_failed = True
+            helper_log_found = True
 
     async with run_compiled(yaml_config, line_callback=check_logs):
         async with api_client_connected_with_disconnect(noise_psk=noise_key) as (
@@ -159,8 +161,8 @@ async def test_oversized_payload_noise(
             assert device_info is not None
             assert device_info.name == "oversized-noise"
 
-            # Create an oversized payload (>100KiB)
+            # Create an oversized payload (>2304 bytes which is our new limit)
-            oversized_data = b"Y" * (100 * 1024 + 1)  # 100KiB + 1 byte
+            oversized_data = b"Y" * 3000  # ~3KiB, exceeds the 2304 byte limit
 
             # Access the internal connection to send raw data
             frame_helper = client._connection._frame_helper
@@ -175,9 +177,9 @@ async def test_oversized_payload_noise(
 
         # After disconnection, verify process didn't crash
         assert not process_exited, "ESPHome process should not crash"
-        # Verify we saw the expected warning message
+        # Verify we saw the expected HELPER_LOG message
-        assert cipherstate_failed, (
+        assert helper_log_found, (
-            "Expected to see warning about CIPHERSTATE_DECRYPT_FAILED"
+            "Expected to see HELPER_LOG about message size exceeding maximum"
         )
 
         # Try to reconnect to verify the process is still running