sascha/esphome
1
0
Fork 0
mirror of https://github.com/esphome/esphome.git synced 2025-10-29 22:24:26 +00:00
Code Issues Releases Wiki Activity

Replace API deferred queue with efficient message batching system (#9012)

Browse Source
This commit is contained in:
J. Nick Koston
2025-06-10 18:49:15 -05:00
committed by GitHub
parent 1467b704b8
commit 2ed5611a08
24 changed files with 2832 additions and 1669 deletions
Download Patch File Download Diff File Expand all files Collapse all files

55
tests/integration/fixtures/host_mode_batch_delay.yaml Normal file
View File

@@ -0,0 +1,55 @@
esphome:
name: host-batch-delay-test
host:
api:
batch_delay: 0ms
logger:
# Add multiple sensors to test batching
sensor:
- platform: template
name: "Test Sensor 1"
id: test_sensor1
lambda: |-
return 1.0;
update_interval: 0.1s
- platform: template
name: "Test Sensor 2"
id: test_sensor2
lambda: |-
return 2.0;
update_interval: 0.1s
- platform: template
name: "Test Sensor 3"
id: test_sensor3
lambda: |-
return 3.0;
update_interval: 0.1s
binary_sensor:
- platform: template
name: "Test Binary Sensor 1"
id: test_binary_sensor1
lambda: |-
return millis() % 1000 < 500;
- platform: template
name: "Test Binary Sensor 2"
id: test_binary_sensor2
lambda: |-
return millis() % 2000 < 1000;
switch:
- platform: template
name: "Test Switch 1"
id: test_switch1
turn_on_action:
- logger.log: "Switch 1 turned on"
turn_off_action:
- logger.log: "Switch 1 turned off"
- platform: template
name: "Test Switch 2"
id: test_switch2
turn_on_action:
- logger.log: "Switch 2 turned on"
turn_off_action:
- logger.log: "Switch 2 turned off"

322
tests/integration/fixtures/host_mode_many_entities.yaml Normal file
View File

@@ -0,0 +1,322 @@
esphome:
name: host-mode-many-entities
friendly_name: "Host Mode Many Entities Test"
logger:
host:
api:
sensor:
# 50 test sensors with predictable values for batching test
- platform: template
name: "Test Sensor 1"
lambda: return 1.0;
update_interval: 0.1s
- platform: template
name: "Test Sensor 2"
lambda: return 2.0;
update_interval: 0.1s
- platform: template
name: "Test Sensor 3"
lambda: return 3.0;
update_interval: 0.1s
- platform: template
name: "Test Sensor 4"
lambda: return 4.0;
update_interval: 0.1s
- platform: template
name: "Test Sensor 5"
lambda: return 5.0;
update_interval: 0.1s
- platform: template
name: "Test Sensor 6"
lambda: return 6.0;
update_interval: 0.1s
- platform: template
name: "Test Sensor 7"
lambda: return 7.0;
update_interval: 0.1s
- platform: template
name: "Test Sensor 8"
lambda: return 8.0;
update_interval: 0.1s
- platform: template
name: "Test Sensor 9"
lambda: return 9.0;
update_interval: 0.1s
- platform: template
name: "Test Sensor 10"
lambda: return 10.0;
update_interval: 0.1s
- platform: template
name: "Test Sensor 11"
lambda: return 11.0;
update_interval: 0.1s
- platform: template
name: "Test Sensor 12"
lambda: return 12.0;
update_interval: 0.1s
- platform: template
name: "Test Sensor 13"
lambda: return 13.0;
update_interval: 0.1s
- platform: template
name: "Test Sensor 14"
lambda: return 14.0;
update_interval: 0.1s
- platform: template
name: "Test Sensor 15"
lambda: return 15.0;
update_interval: 0.1s
- platform: template
name: "Test Sensor 16"
lambda: return 16.0;
update_interval: 0.1s
- platform: template
name: "Test Sensor 17"
lambda: return 17.0;
update_interval: 0.1s
- platform: template
name: "Test Sensor 18"
lambda: return 18.0;
update_interval: 0.1s
- platform: template
name: "Test Sensor 19"
lambda: return 19.0;
update_interval: 0.1s
- platform: template
name: "Test Sensor 20"
lambda: return 20.0;
update_interval: 0.1s
- platform: template
name: "Test Sensor 21"
lambda: return 21.0;
update_interval: 0.1s
- platform: template
name: "Test Sensor 22"
lambda: return 22.0;
update_interval: 0.1s
- platform: template
name: "Test Sensor 23"
lambda: return 23.0;
update_interval: 0.1s
- platform: template
name: "Test Sensor 24"
lambda: return 24.0;
update_interval: 0.1s
- platform: template
name: "Test Sensor 25"
lambda: return 25.0;
update_interval: 0.1s
- platform: template
name: "Test Sensor 26"
lambda: return 26.0;
update_interval: 0.1s
- platform: template
name: "Test Sensor 27"
lambda: return 27.0;
update_interval: 0.1s
- platform: template
name: "Test Sensor 28"
lambda: return 28.0;
update_interval: 0.1s
- platform: template
name: "Test Sensor 29"
lambda: return 29.0;
update_interval: 0.1s
- platform: template
name: "Test Sensor 30"
lambda: return 30.0;
update_interval: 0.1s
- platform: template
name: "Test Sensor 31"
lambda: return 31.0;
update_interval: 0.1s
- platform: template
name: "Test Sensor 32"
lambda: return 32.0;
update_interval: 0.1s
- platform: template
name: "Test Sensor 33"
lambda: return 33.0;
update_interval: 0.1s
- platform: template
name: "Test Sensor 34"
lambda: return 34.0;
update_interval: 0.1s
- platform: template
name: "Test Sensor 35"
lambda: return 35.0;
update_interval: 0.1s
- platform: template
name: "Test Sensor 36"
lambda: return 36.0;
update_interval: 0.1s
- platform: template
name: "Test Sensor 37"
lambda: return 37.0;
update_interval: 0.1s
- platform: template
name: "Test Sensor 38"
lambda: return 38.0;
update_interval: 0.1s
- platform: template
name: "Test Sensor 39"
lambda: return 39.0;
update_interval: 0.1s
- platform: template
name: "Test Sensor 40"
lambda: return 40.0;
update_interval: 0.1s
- platform: template
name: "Test Sensor 41"
lambda: return 41.0;
update_interval: 0.1s
- platform: template
name: "Test Sensor 42"
lambda: return 42.0;
update_interval: 0.1s
- platform: template
name: "Test Sensor 43"
lambda: return 43.0;
update_interval: 0.1s
- platform: template
name: "Test Sensor 44"
lambda: return 44.0;
update_interval: 0.1s
- platform: template
name: "Test Sensor 45"
lambda: return 45.0;
update_interval: 0.1s
- platform: template
name: "Test Sensor 46"
lambda: return 46.0;
update_interval: 0.1s
- platform: template
name: "Test Sensor 47"
lambda: return 47.0;
update_interval: 0.1s
- platform: template
name: "Test Sensor 48"
lambda: return 48.0;
update_interval: 0.1s
- platform: template
name: "Test Sensor 49"
lambda: return 49.0;
update_interval: 0.1s
- platform: template
name: "Test Sensor 50"
lambda: return 50.0;
update_interval: 0.1s
# Mixed entity types for comprehensive batching test
binary_sensor:
- platform: template
name: "Test Binary Sensor 1"
lambda: return millis() % 1000 < 500;
- platform: template
name: "Test Binary Sensor 2"
lambda: return millis() % 2000 < 1000;
switch:
- platform: template
name: "Test Switch 1"
lambda: return true;
turn_on_action:
- logger.log: "Switch 1 ON"
turn_off_action:
- logger.log: "Switch 1 OFF"
- platform: template
name: "Test Switch 2"
lambda: return false;
turn_on_action:
- logger.log: "Switch 2 ON"
turn_off_action:
- logger.log: "Switch 2 OFF"
text_sensor:
- platform: template
name: "Test Text Sensor 1"
lambda: return std::string("Test Value 1");
- platform: template
name: "Test Text Sensor 2"
lambda: return std::string("Test Value 2");
- platform: version
name: "ESPHome Version"
number:
- platform: template
name: "Test Number"
min_value: 0
max_value: 100
step: 1
lambda: return 50.0;
set_action:
- logger.log: "Number set"
select:
- platform: template
name: "Test Select"
options:
- "Option 1"
- "Option 2"
initial_option: "Option 1"
optimistic: true
set_action:
- logger.log: "Select changed"
text:
- platform: template
name: "Test Text"
mode: text
initial_value: "Hello"
set_action:
- logger.log: "Text changed"
valve:
- platform: template
name: "Test Valve"
open_action:
- logger.log: "Valve opening"
close_action:
- logger.log: "Valve closing"
stop_action:
- logger.log: "Valve stopping"
alarm_control_panel:
- platform: template
name: "Test Alarm"
codes:
- "1234"
arming_away_time: 0s
arming_home_time: 0s
pending_time: 0s
trigger_time: 300s
restore_mode: ALWAYS_DISARMED
on_disarmed:
- logger.log: "Alarm disarmed"
on_arming:
- logger.log: "Alarm arming"
on_armed_away:
- logger.log: "Alarm armed away"
on_armed_home:
- logger.log: "Alarm armed home"
on_pending:
- logger.log: "Alarm pending"
on_triggered:
- logger.log: "Alarm triggered"
event:
- platform: template
name: "Test Event"
event_types:
- first_event
- second_event
button:
- platform: template
name: "Test Button"
on_press:
- logger.log: "Button pressed"

136
tests/integration/fixtures/host_mode_many_entities_multiple_connections.yaml Normal file
View File

@@ -0,0 +1,136 @@
esphome:
name: host-mode-many-entities-multi
friendly_name: "Host Mode Many Entities Multiple Connections Test"
logger:
host:
api:
sensor:
# 20 test sensors for faster testing with multiple connections
- platform: template
name: "Test Sensor 1"
lambda: return 1.0;
update_interval: 0.1s
- platform: template
name: "Test Sensor 2"
lambda: return 2.0;
update_interval: 0.1s
- platform: template
name: "Test Sensor 3"
lambda: return 3.0;
update_interval: 0.1s
- platform: template
name: "Test Sensor 4"
lambda: return 4.0;
update_interval: 0.1s
- platform: template
name: "Test Sensor 5"
lambda: return 5.0;
update_interval: 0.1s
- platform: template
name: "Test Sensor 6"
lambda: return 6.0;
update_interval: 0.1s
- platform: template
name: "Test Sensor 7"
lambda: return 7.0;
update_interval: 0.1s
- platform: template
name: "Test Sensor 8"
lambda: return 8.0;
update_interval: 0.1s
- platform: template
name: "Test Sensor 9"
lambda: return 9.0;
update_interval: 0.1s
- platform: template
name: "Test Sensor 10"
lambda: return 10.0;
update_interval: 0.1s
- platform: template
name: "Test Sensor 11"
lambda: return 11.0;
update_interval: 0.1s
- platform: template
name: "Test Sensor 12"
lambda: return 12.0;
update_interval: 0.1s
- platform: template
name: "Test Sensor 13"
lambda: return 13.0;
update_interval: 0.1s
- platform: template
name: "Test Sensor 14"
lambda: return 14.0;
update_interval: 0.1s
- platform: template
name: "Test Sensor 15"
lambda: return 15.0;
update_interval: 0.1s
- platform: template
name: "Test Sensor 16"
lambda: return 16.0;
update_interval: 0.1s
- platform: template
name: "Test Sensor 17"
lambda: return 17.0;
update_interval: 0.1s
- platform: template
name: "Test Sensor 18"
lambda: return 18.0;
update_interval: 0.1s
- platform: template
name: "Test Sensor 19"
lambda: return 19.0;
update_interval: 0.1s
- platform: template
name: "Test Sensor 20"
lambda: return 20.0;
update_interval: 0.1s
# Mixed entity types for comprehensive batching test
binary_sensor:
- platform: template
name: "Test Binary Sensor 1"
lambda: return millis() % 1000 < 500;
- platform: template
name: "Test Binary Sensor 2"
lambda: return millis() % 2000 < 1000;
text_sensor:
- platform: template
name: "Test Text Sensor 1"
lambda: return std::string("Test Value 1");
- platform: template
name: "Test Text Sensor 2"
lambda: return std::string("Test Value 2");
- platform: version
name: "ESPHome Version"
switch:
- platform: template
name: "Test Switch 1"
lambda: return true;
turn_on_action:
- logger.log: "Switch 1 ON"
turn_off_action:
- logger.log: "Switch 1 OFF"
button:
- platform: template
name: "Test Button"
on_press:
- logger.log: "Button pressed"
number:
- platform: template
name: "Test Number"
min_value: 0
max_value: 100
step: 1
lambda: return 50.0;
set_action:
- logger.log: "Number set"

43
tests/integration/fixtures/host_mode_noise_encryption.yaml
View File

@@ -5,3 +5,46 @@ api:
encryption:
key: N4Yle5YirwZhPiHHsdZLdOA73ndj/84veVaLhTvxCuU=
logger:
# Test sensors to verify batching works with noise encryption
sensor:
- platform: template
name: "Noise Test Sensor 1"
lambda: return 1.0;
update_interval: 2s
- platform: template
name: "Noise Test Sensor 2"
lambda: return 2.0;
update_interval: 2s
- platform: template
name: "Noise Test Sensor 3"
lambda: return 3.0;
update_interval: 2s
- platform: template
name: "Noise Test Sensor 4"
lambda: return 4.0;
update_interval: 2s
- platform: template
name: "Noise Test Sensor 5"
lambda: return 5.0;
update_interval: 2s
- platform: template
name: "Noise Test Sensor 6"
lambda: return 6.0;
update_interval: 2s
- platform: template
name: "Noise Test Sensor 7"
lambda: return 7.0;
update_interval: 2s
- platform: template
name: "Noise Test Sensor 8"
lambda: return 8.0;
update_interval: 2s
- platform: template
name: "Noise Test Sensor 9"
lambda: return 9.0;
update_interval: 2s
- platform: template
name: "Noise Test Sensor 10"
lambda: return 10.0;
update_interval: 2s

80
tests/integration/test_host_mode_batch_delay.py Normal file
View File

@@ -0,0 +1,80 @@
"""Integration test for API batch_delay setting."""
from __future__ import annotations
import asyncio
import time
from aioesphomeapi import EntityState
import pytest
from .types import APIClientConnectedFactory, RunCompiledFunction
@pytest.mark.asyncio
async def test_host_mode_batch_delay(
yaml_config: str,
run_compiled: RunCompiledFunction,
api_client_connected: APIClientConnectedFactory,
) -> None:
"""Test API with batch_delay set to 0ms - messages should be sent immediately without batching."""
# Write, compile and run the ESPHome device, then connect to API
loop = asyncio.get_running_loop()
async with run_compiled(yaml_config), api_client_connected() as client:
# Verify we can get device info
device_info = await client.device_info()
assert device_info is not None
assert device_info.name == "host-batch-delay-test"
# Subscribe to state changes
states: dict[int, EntityState] = {}
state_timestamps: dict[int, float] = {}
entity_count_future: asyncio.Future[int] = loop.create_future()
def on_state(state: EntityState) -> None:
"""Track when states are received."""
states[state.key] = state
state_timestamps[state.key] = time.monotonic()
# When we have received all expected entities, resolve the future
if len(states) >= 7 and not entity_count_future.done():
entity_count_future.set_result(len(states))
client.subscribe_states(on_state)
# Wait for states from all entities with timeout
try:
entity_count = await asyncio.wait_for(entity_count_future, timeout=5.0)
except asyncio.TimeoutError:
pytest.fail(
f"Did not receive states from at least 7 entities within 5 seconds. "
f"Received {len(states)} states"
)
# Verify we received all states
assert entity_count >= 7, f"Expected at least 7 entities, got {entity_count}"
assert len(states) >= 7 # 3 sensors + 2 binary sensors + 2 switches
# When batch_delay is 0ms, states are sent immediately without batching
# This means each state arrives in its own packet, which may actually be slower
# than batching due to network overhead
if state_timestamps:
first_timestamp = min(state_timestamps.values())
last_timestamp = max(state_timestamps.values())
time_spread = last_timestamp - first_timestamp
# With batch_delay=0ms, states arrive individually which may take longer
# We just verify they all arrive within a reasonable time
assert time_spread < 1.0, f"States took {time_spread:.3f}s to arrive"
# Also test list_entities - with batch_delay=0ms each entity is sent separately
start_time = time.monotonic()
entity_info, services = await client.list_entities_services()
end_time = time.monotonic()
list_time = end_time - start_time
# Verify we got all expected entities
assert len(entity_info) >= 7 # 3 sensors + 2 binary sensors + 2 switches
# With batch_delay=0ms, listing sends each entity separately which may be slower
assert list_time < 1.0, f"list_entities took {list_time:.3f}s"

57
tests/integration/test_host_mode_many_entities.py Normal file
View File

@@ -0,0 +1,57 @@
"""Integration test for many entities to test API batching."""
from __future__ import annotations
import asyncio
from aioesphomeapi import EntityState
import pytest
from .types import APIClientConnectedFactory, RunCompiledFunction
@pytest.mark.asyncio
async def test_host_mode_many_entities(
yaml_config: str,
run_compiled: RunCompiledFunction,
api_client_connected: APIClientConnectedFactory,
) -> None:
"""Test API batching with many entities of different types."""
# Write, compile and run the ESPHome device, then connect to API
loop = asyncio.get_running_loop()
async with run_compiled(yaml_config), api_client_connected() as client:
# Subscribe to state changes
states: dict[int, EntityState] = {}
entity_count_future: asyncio.Future[int] = loop.create_future()
def on_state(state: EntityState) -> None:
states[state.key] = state
# When we have received states from a good number of entities, resolve the future
if len(states) >= 50 and not entity_count_future.done():
entity_count_future.set_result(len(states))
client.subscribe_states(on_state)
# Wait for states from at least 50 entities with timeout
try:
entity_count = await asyncio.wait_for(entity_count_future, timeout=10.0)
except asyncio.TimeoutError:
pytest.fail(
f"Did not receive states from at least 50 entities within 10 seconds. "
f"Received {len(states)} states: {list(states.keys())}"
)
# Verify we received a good number of entity states
assert entity_count >= 50, f"Expected at least 50 entities, got {entity_count}"
assert len(states) >= 50, f"Expected at least 50 states, got {len(states)}"
# Verify we have different entity types by checking some expected values
sensor_states = [
s
for s in states.values()
if hasattr(s, "state") and isinstance(s.state, float)
]
assert len(sensor_states) >= 50, (
f"Expected at least 50 sensor states, got {len(sensor_states)}"
)

71
tests/integration/test_host_mode_many_entities_multiple_connections.py Normal file
View File

@@ -0,0 +1,71 @@
"""Integration test for shared buffer optimization with multiple API connections."""
from __future__ import annotations
import asyncio
from aioesphomeapi import EntityState
import pytest
from .types import APIClientConnectedFactory, RunCompiledFunction
@pytest.mark.asyncio
async def test_host_mode_many_entities_multiple_connections(
yaml_config: str,
run_compiled: RunCompiledFunction,
api_client_connected: APIClientConnectedFactory,
) -> None:
"""Test shared buffer optimization with multiple API connections."""
# Write, compile and run the ESPHome device
loop = asyncio.get_running_loop()
async with (
run_compiled(yaml_config),
api_client_connected() as client1,
api_client_connected() as client2,
):
# Subscribe both clients to state changes
states1: dict[int, EntityState] = {}
states2: dict[int, EntityState] = {}
client1_ready = loop.create_future()
client2_ready = loop.create_future()
def on_state1(state: EntityState) -> None:
states1[state.key] = state
if len(states1) >= 20 and not client1_ready.done():
client1_ready.set_result(len(states1))
def on_state2(state: EntityState) -> None:
states2[state.key] = state
if len(states2) >= 20 and not client2_ready.done():
client2_ready.set_result(len(states2))
client1.subscribe_states(on_state1)
client2.subscribe_states(on_state2)
# Wait for both clients to receive states
try:
count1, count2 = await asyncio.gather(
asyncio.wait_for(client1_ready, timeout=10.0),
asyncio.wait_for(client2_ready, timeout=10.0),
)
except asyncio.TimeoutError:
pytest.fail(
f"One or both clients did not receive enough states within 10 seconds. "
f"Client1: {len(states1)}, Client2: {len(states2)}"
)
# Verify both clients received states successfully
assert count1 >= 20, (
f"Client 1 should have received at least 20 states, got {count1}"
)
assert count2 >= 20, (
f"Client 2 should have received at least 20 states, got {count2}"
)
# Verify both clients received the same entity keys (same device state)
common_keys = set(states1.keys()) & set(states2.keys())
assert len(common_keys) >= 20, (
f"Expected at least 20 common entity keys, got {len(common_keys)}"
)