Merge branch 'heap_scheduler_stress_component' into integration

tests/components/http_request/common.yaml

@@ -91,3 +91,5 @@ update:
     name: OTA Update
     id: ota_update
     source: http://my.ha.net:8123/local/esphome/manifest.json
+    on_update_available:
+      - logger.log: "A new update is available"

tests/components/update/common.yaml

@@ -26,3 +26,5 @@ update:
   - platform: http_request
     name: Firmware Update
     source: http://example.com/manifest.json
+    on_update_available:
+      - logger.log: "A new update is available"

tests/integration/fixtures/external_components/scheduler_bulk_cleanup_component/__init__.py

@@ -0,0 +1,21 @@
+import esphome.codegen as cg
+import esphome.config_validation as cv
+from esphome.const import CONF_ID
+
+scheduler_bulk_cleanup_component_ns = cg.esphome_ns.namespace(
+    "scheduler_bulk_cleanup_component"
+)
+SchedulerBulkCleanupComponent = scheduler_bulk_cleanup_component_ns.class_(
+    "SchedulerBulkCleanupComponent", cg.Component
+)
+
+CONFIG_SCHEMA = cv.Schema(
+    {
+        cv.GenerateID(): cv.declare_id(SchedulerBulkCleanupComponent),
+    }
+).extend(cv.COMPONENT_SCHEMA)
+
+
+async def to_code(config):
+    var = cg.new_Pvariable(config[CONF_ID])
+    await cg.register_component(var, config)

tests/integration/fixtures/external_components/scheduler_bulk_cleanup_component/scheduler_bulk_cleanup_component.cpp

@@ -0,0 +1,63 @@
+#include "scheduler_bulk_cleanup_component.h"
+#include "esphome/core/log.h"
+#include "esphome/core/helpers.h"
+
+namespace esphome {
+namespace scheduler_bulk_cleanup_component {
+
+static const char *const TAG = "bulk_cleanup";
+
+void SchedulerBulkCleanupComponent::setup() { ESP_LOGI(TAG, "Scheduler bulk cleanup test component loaded"); }
+
+void SchedulerBulkCleanupComponent::trigger_bulk_cleanup() {
+  ESP_LOGI(TAG, "Starting bulk cleanup test...");
+
+  // Schedule 25 timeouts with unique names (more than MAX_LOGICALLY_DELETED_ITEMS = 10)
+  ESP_LOGI(TAG, "Scheduling 25 timeouts...");
+  for (int i = 0; i < 25; i++) {
+    std::string name = "bulk_timeout_" + std::to_string(i);
+    App.scheduler.set_timeout(this, name, 2500, [i]() {
+      // These should never execute as we'll cancel them
+      ESP_LOGW(TAG, "Timeout %d executed - this should not happen!", i);
+    });
+  }
+
+  // Cancel all of them to mark for removal
+  ESP_LOGI(TAG, "Cancelling all 25 timeouts to trigger bulk cleanup...");
+  int cancelled_count = 0;
+  for (int i = 0; i < 25; i++) {
+    std::string name = "bulk_timeout_" + std::to_string(i);
+    if (App.scheduler.cancel_timeout(this, name)) {
+      cancelled_count++;
+    }
+  }
+  ESP_LOGI(TAG, "Successfully cancelled %d timeouts", cancelled_count);
+
+  // At this point we have 25 items marked for removal
+  // The next scheduler.call() should trigger the bulk cleanup path
+
+  // Schedule an interval that will execute multiple times to ensure cleanup happens
+  static int cleanup_check_count = 0;
+  App.scheduler.set_interval(this, "cleanup_checker", 25, [this]() {
+    cleanup_check_count++;
+    ESP_LOGI(TAG, "Cleanup check %d - scheduler still running", cleanup_check_count);
+
+    if (cleanup_check_count >= 5) {
+      // Cancel the interval and complete the test
+      App.scheduler.cancel_interval(this, "cleanup_checker");
+      ESP_LOGI(TAG, "Bulk cleanup triggered: removed %d items", 25);
+      ESP_LOGI(TAG, "Items before cleanup: 25+, after: <unknown>");
+      ESP_LOGI(TAG, "Bulk cleanup test complete");
+    }
+  });
+
+  // Also schedule some normal timeouts to ensure scheduler keeps working after cleanup
+  for (int i = 0; i < 5; i++) {
+    std::string name = "post_cleanup_" + std::to_string(i);
+    App.scheduler.set_timeout(this, name, 50 + i * 25,
+                              [i]() { ESP_LOGI(TAG, "Post-cleanup timeout %d executed correctly", i); });
+  }
+}
+
+}  // namespace scheduler_bulk_cleanup_component
+}  // namespace esphome

tests/integration/fixtures/external_components/scheduler_bulk_cleanup_component/scheduler_bulk_cleanup_component.h

@@ -0,0 +1,18 @@
+#pragma once
+
+#include "esphome/core/component.h"
+#include "esphome/core/application.h"
+
+namespace esphome {
+namespace scheduler_bulk_cleanup_component {
+
+class SchedulerBulkCleanupComponent : public Component {
+ public:
+  void setup() override;
+  float get_setup_priority() const override { return setup_priority::LATE; }
+
+  void trigger_bulk_cleanup();
+};
+
+}  // namespace scheduler_bulk_cleanup_component
+}  // namespace esphome

tests/integration/fixtures/external_components/scheduler_rapid_cancellation_component/rapid_cancellation_component.cpp

@@ -70,6 +70,9 @@ void SchedulerRapidCancellationComponent::run_rapid_cancellation_test() {
     ESP_LOGI(TAG, "  Implicit cancellations (replaced): %d", implicit_cancellations);
     ESP_LOGI(TAG, "  Total accounted: %d (executed + implicit cancellations)",
              this->total_executed_.load() + implicit_cancellations);
+
+    // Final message to signal test completion - ensures all stats are logged before test ends
+    ESP_LOGI(TAG, "Test finished - all statistics reported");
   });
 }
 

tests/integration/fixtures/scheduler_bulk_cleanup.yaml

@@ -0,0 +1,23 @@
+esphome:
+  name: scheduler-bulk-cleanup
+
+external_components:
+  - source:
+      type: local
+      path: EXTERNAL_COMPONENT_PATH
+
+host:
+
+logger:
+  level: DEBUG
+
+api:
+  services:
+    - service: trigger_bulk_cleanup
+      then:
+        - lambda: |-
+            auto component = id(bulk_cleanup_component);
+            component->trigger_bulk_cleanup();
+
+scheduler_bulk_cleanup_component:
+  id: bulk_cleanup_component

tests/integration/test_scheduler_bulk_cleanup.py

@@ -0,0 +1,123 @@
+"""Test that triggers the bulk cleanup path when to_remove_ > MAX_LOGICALLY_DELETED_ITEMS."""
+
+import asyncio
+from pathlib import Path
+import re
+
+from aioesphomeapi import UserService
+import pytest
+
+from .types import APIClientConnectedFactory, RunCompiledFunction
+
+
+@pytest.mark.asyncio
+async def test_scheduler_bulk_cleanup(
+    yaml_config: str,
+    run_compiled: RunCompiledFunction,
+    api_client_connected: APIClientConnectedFactory,
+) -> None:
+    """Test that bulk cleanup path is triggered when many items are cancelled."""
+
+    # Get the absolute path to the external components directory
+    external_components_path = str(
+        Path(__file__).parent / "fixtures" / "external_components"
+    )
+
+    # Replace the placeholder in the YAML config with the actual path
+    yaml_config = yaml_config.replace(
+        "EXTERNAL_COMPONENT_PATH", external_components_path
+    )
+
+    # Create a future to signal test completion
+    loop = asyncio.get_event_loop()
+    test_complete_future: asyncio.Future[None] = loop.create_future()
+    bulk_cleanup_triggered = False
+    cleanup_stats: dict[str, int] = {
+        "removed": 0,
+        "before": 0,
+        "after": 0,
+    }
+    post_cleanup_executed = 0
+
+    def on_log_line(line: str) -> None:
+        nonlocal bulk_cleanup_triggered, post_cleanup_executed
+
+        # Look for logs indicating bulk cleanup was triggered
+        # The actual cleanup happens silently, so we track the cancel operations
+        if "Successfully cancelled" in line and "timeouts" in line:
+            match = re.search(r"Successfully cancelled (\d+) timeouts", line)
+            if match and int(match.group(1)) > 10:
+                bulk_cleanup_triggered = True
+
+        # Track cleanup statistics
+        match = re.search(r"Bulk cleanup triggered: removed (\d+) items", line)
+        if match:
+            cleanup_stats["removed"] = int(match.group(1))
+
+        match = re.search(r"Items before cleanup: (\d+), after: (\d+)", line)
+        if match:
+            cleanup_stats["before"] = int(match.group(1))
+            cleanup_stats["after"] = int(match.group(2))
+
+        # Track post-cleanup timeout executions
+        if "Post-cleanup timeout" in line and "executed correctly" in line:
+            match = re.search(r"Post-cleanup timeout (\d+) executed correctly", line)
+            if match:
+                post_cleanup_executed += 1
+                # All 5 post-cleanup timeouts have executed
+                if post_cleanup_executed >= 5 and not test_complete_future.done():
+                    test_complete_future.set_result(None)
+
+        # Check for bulk cleanup completion (but don't end test yet)
+        if "Bulk cleanup test complete" in line:
+            # This just means the interval finished, not that all timeouts executed
+            pass
+
+    async with (
+        run_compiled(yaml_config, line_callback=on_log_line),
+        api_client_connected() as client,
+    ):
+        # Verify we can connect
+        device_info = await client.device_info()
+        assert device_info is not None
+        assert device_info.name == "scheduler-bulk-cleanup"
+
+        # List entities and services
+        _, services = await asyncio.wait_for(
+            client.list_entities_services(), timeout=5.0
+        )
+
+        # Find our test service
+        trigger_bulk_cleanup_service: UserService | None = None
+        for service in services:
+            if service.name == "trigger_bulk_cleanup":
+                trigger_bulk_cleanup_service = service
+                break
+
+        assert trigger_bulk_cleanup_service is not None, (
+            "trigger_bulk_cleanup service not found"
+        )
+
+        # Execute the test
+        client.execute_service(trigger_bulk_cleanup_service, {})
+
+        # Wait for test completion
+        try:
+            await asyncio.wait_for(test_complete_future, timeout=10.0)
+        except asyncio.TimeoutError:
+            pytest.fail("Bulk cleanup test timed out")
+
+        # Verify bulk cleanup was triggered
+        assert bulk_cleanup_triggered, (
+            "Bulk cleanup path was not triggered - MAX_LOGICALLY_DELETED_ITEMS threshold not reached"
+        )
+
+        # Verify cleanup statistics
+        assert cleanup_stats["removed"] > 10, (
+            f"Expected more than 10 items removed, got {cleanup_stats['removed']}"
+        )
+
+        # Verify scheduler still works after bulk cleanup
+        assert post_cleanup_executed == 5, (
+            f"Expected 5 post-cleanup timeouts to execute, but {post_cleanup_executed} executed"
+        )

tests/integration/test_scheduler_rapid_cancellation.py

@@ -74,9 +74,9 @@ async def test_scheduler_rapid_cancellation(
                 test_complete_future.set_exception(Exception(f"Crash detected: {line}"))
             return
 
-        # Check for completion
+        # Check for completion - wait for final message after all stats are logged
         if (
-            "Rapid cancellation test complete" in line
+            "Test finished - all statistics reported" in line
             and not test_complete_future.done()
         ):
             test_complete_future.set_result(None)