Merge branch 'determine_jobs_comp_split' into determine_jobs_comp_split_test

.github/workflows/ci.yml

@@ -180,6 +180,7 @@ jobs:
       memory_impact: ${{ steps.determine.outputs.memory-impact }}
       cpp-unit-tests-run-all: ${{ steps.determine.outputs.cpp-unit-tests-run-all }}
       cpp-unit-tests-components: ${{ steps.determine.outputs.cpp-unit-tests-components }}
+      component-test-batches: ${{ steps.determine.outputs.component-test-batches }}
     steps:
       - name: Check out code from GitHub
         uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
@@ -214,6 +215,7 @@ jobs:
           echo "memory-impact=$(echo "$output" | jq -c '.memory_impact')" >> $GITHUB_OUTPUT
           echo "cpp-unit-tests-run-all=$(echo "$output" | jq -r '.cpp_unit_tests_run_all')" >> $GITHUB_OUTPUT
           echo "cpp-unit-tests-components=$(echo "$output" | jq -c '.cpp_unit_tests_components')" >> $GITHUB_OUTPUT
+          echo "component-test-batches=$(echo "$output" | jq -c '.component_test_batches')" >> $GITHUB_OUTPUT
 
   integration-tests:
     name: Run integration tests
@@ -536,59 +538,18 @@ jobs:
         run: script/ci-suggest-changes
         if: always()
 
-  test-build-components-splitter:
-    name: Split components for intelligent grouping (40 weighted per batch)
-    runs-on: ubuntu-24.04
-    needs:
-      - common
-      - determine-jobs
-    if: github.event_name == 'pull_request' && fromJSON(needs.determine-jobs.outputs.component-test-count) > 0
-    outputs:
-      matrix: ${{ steps.split.outputs.components }}
-    steps:
-      - name: Check out code from GitHub
-        uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
-      - name: Restore Python
-        uses: ./.github/actions/restore-python
-        with:
-          python-version: ${{ env.DEFAULT_PYTHON }}
-          cache-key: ${{ needs.common.outputs.cache-key }}
-      - name: Split components intelligently based on bus configurations
-        id: split
-        run: |
-          . venv/bin/activate
-
-          # Use intelligent splitter that groups components with same bus configs
-          components='${{ needs.determine-jobs.outputs.changed-components-with-tests }}'
-
-          # Only isolate directly changed components when targeting dev branch
-          # For beta/release branches, group everything for faster CI
-          if [[ "${{ github.base_ref }}" == beta* ]] || [[ "${{ github.base_ref }}" == release* ]]; then
-            directly_changed='[]'
-            echo "Target branch: ${{ github.base_ref }} - grouping all components"
-          else
-            directly_changed='${{ needs.determine-jobs.outputs.directly-changed-components-with-tests }}'
-            echo "Target branch: ${{ github.base_ref }} - isolating directly changed components"
-          fi
-
-          echo "Splitting components intelligently..."
-          output=$(python3 script/split_components_for_ci.py --components "$components" --directly-changed "$directly_changed" --batch-size 40 --output github)
-
-          echo "$output" >> $GITHUB_OUTPUT
-
   test-build-components-split:
     name: Test components batch (${{ matrix.components }})
     runs-on: ubuntu-24.04
     needs:
       - common
       - determine-jobs
-      - test-build-components-splitter
     if: github.event_name == 'pull_request' && fromJSON(needs.determine-jobs.outputs.component-test-count) > 0
     strategy:
       fail-fast: false
       max-parallel: ${{ (startsWith(github.base_ref, 'beta') || startsWith(github.base_ref, 'release')) && 8 || 4 }}
       matrix:
-        components: ${{ fromJson(needs.test-build-components-splitter.outputs.matrix) }}
+        components: ${{ fromJson(needs.determine-jobs.outputs.component-test-batches) }}
     steps:
       - name: Show disk space
         run: |
@@ -980,7 +941,6 @@ jobs:
       - clang-tidy-nosplit
       - clang-tidy-split
       - determine-jobs
-      - test-build-components-splitter
       - test-build-components-split
       - pre-commit-ci-lite
       - memory-impact-target-branch

esphome/components/api/api_connection.cpp

@@ -1,3 +1,5 @@
+// random comment
+
 #include "api_connection.h"
 #ifdef USE_API
 #ifdef USE_API_NOISE

script/determine-jobs.py

@@ -43,12 +43,14 @@ from enum import StrEnum
 from functools import cache
 import json
 import os
+from pathlib import Path
 import subprocess
 import sys
 from typing import Any
 
 from helpers import (
     CPP_FILE_EXTENSIONS,
+    ESPHOME_TESTS_COMPONENTS_PATH,
     PYTHON_FILE_EXTENSIONS,
     changed_files,
     core_changed,
@@ -65,12 +67,17 @@ from helpers import (
     parse_test_filename,
     root_path,
 )
+from split_components_for_ci import create_intelligent_batches
 
 # Threshold for splitting clang-tidy jobs
 # For small PRs (< 65 files), use nosplit for faster CI
 # For large PRs (>= 65 files), use split for better parallelization
 CLANG_TIDY_SPLIT_THRESHOLD = 65
 
+# Component test batch size (weighted)
+# Isolated components count as 10x, groupable components count as 1x
+COMPONENT_TEST_BATCH_SIZE = 40
+
 
 class Platform(StrEnum):
     """Platform identifiers for memory impact analysis."""
@@ -686,6 +693,22 @@ def main() -> None:
     # Determine which C++ unit tests to run
     cpp_run_all, cpp_components = determine_cpp_unit_tests(args.branch)
 
+    # Split components into batches for CI testing
+    # This intelligently groups components with similar bus configurations
+    component_test_batches: list[str]
+    if changed_components_with_tests:
+        tests_dir = Path(root_path) / ESPHOME_TESTS_COMPONENTS_PATH
+        batches, _ = create_intelligent_batches(
+            components=changed_components_with_tests,
+            tests_dir=tests_dir,
+            batch_size=COMPONENT_TEST_BATCH_SIZE,
+            directly_changed=directly_changed_with_tests,
+        )
+        # Convert batches to space-separated strings for CI matrix
+        component_test_batches = [" ".join(batch) for batch in batches]
+    else:
+        component_test_batches = []
+
     output: dict[str, Any] = {
         "integration_tests": run_integration,
         "clang_tidy": run_clang_tidy,
@@ -703,6 +726,7 @@ def main() -> None:
         "memory_impact": memory_impact,
         "cpp_unit_tests_run_all": cpp_run_all,
         "cpp_unit_tests_components": cpp_components,
+        "component_test_batches": component_test_batches,
     }
 
     # Output as JSON

script/split_components_for_ci.py

@@ -62,6 +62,10 @@ def create_intelligent_batches(
 ) -> tuple[list[list[str]], dict[tuple[str, str], list[str]]]:
     """Create batches optimized for component grouping.
 
+    IMPORTANT: This function is called from both split_components_for_ci.py (standalone script)
+    and determine-jobs.py (integrated into job determination). Be careful when refactoring
+    to ensure changes work in both contexts.
+
     Args:
         components: List of component names to batch
         tests_dir: Path to tests/components directory

tests/script/test_determine_jobs.py

@@ -152,6 +152,14 @@ def test_main_all_tests_should_run(
     assert output["memory_impact"]["should_run"] == "false"
     assert output["cpp_unit_tests_run_all"] is False
     assert output["cpp_unit_tests_components"] == ["wifi", "api", "sensor"]
+    # component_test_batches should be present and be a list of space-separated strings
+    assert "component_test_batches" in output
+    assert isinstance(output["component_test_batches"], list)
+    # Each batch should be a space-separated string of component names
+    for batch in output["component_test_batches"]:
+        assert isinstance(batch, str)
+        # Should contain at least one component (no empty batches)
+        assert len(batch) > 0
 
 
 def test_main_no_tests_should_run(
@@ -209,6 +217,9 @@ def test_main_no_tests_should_run(
     assert output["memory_impact"]["should_run"] == "false"
     assert output["cpp_unit_tests_run_all"] is False
     assert output["cpp_unit_tests_components"] == []
+    # component_test_batches should be empty list
+    assert "component_test_batches" in output
+    assert output["component_test_batches"] == []
 
 
 def test_main_with_branch_argument(