[ci] Group all PR builds, isolate direct changes for full validation on dev (#11193)

.github/workflows/ci.yml

@@ -178,6 +178,7 @@ jobs:
       python-linters: ${{ steps.determine.outputs.python-linters }}
       changed-components: ${{ steps.determine.outputs.changed-components }}
       changed-components-with-tests: ${{ steps.determine.outputs.changed-components-with-tests }}
+      directly-changed-components-with-tests: ${{ steps.determine.outputs.directly-changed-components-with-tests }}
       component-test-count: ${{ steps.determine.outputs.component-test-count }}
     steps:
       - name: Check out code from GitHub
@@ -206,6 +207,7 @@ jobs:
           echo "python-linters=$(echo "$output" | jq -r '.python_linters')" >> $GITHUB_OUTPUT
           echo "changed-components=$(echo "$output" | jq -c '.changed_components')" >> $GITHUB_OUTPUT
           echo "changed-components-with-tests=$(echo "$output" | jq -c '.changed_components_with_tests')" >> $GITHUB_OUTPUT
+          echo "directly-changed-components-with-tests=$(echo "$output" | jq -c '.directly_changed_components_with_tests')" >> $GITHUB_OUTPUT
           echo "component-test-count=$(echo "$output" | jq -r '.component_test_count')" >> $GITHUB_OUTPUT
 
   integration-tests:
@@ -358,48 +360,13 @@ jobs:
         # yamllint disable-line rule:line-length
         if: always()
 
-  test-build-components:
-    name: Component test ${{ matrix.file }}
-    runs-on: ubuntu-24.04
-    needs:
-      - common
-      - determine-jobs
-    if: github.event_name == 'pull_request' && fromJSON(needs.determine-jobs.outputs.component-test-count) > 0 && fromJSON(needs.determine-jobs.outputs.component-test-count) < 100
-    strategy:
-      fail-fast: false
-      max-parallel: 2
-      matrix:
-        file: ${{ fromJson(needs.determine-jobs.outputs.changed-components-with-tests) }}
-    steps:
-      - name: Cache apt packages
-        uses: awalsh128/cache-apt-pkgs-action@acb598e5ddbc6f68a970c5da0688d2f3a9f04d05 # v1.5.3
-        with:
-          packages: libsdl2-dev
-          version: 1.0
-
-      - name: Check out code from GitHub
-        uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
-      - name: Restore Python
-        uses: ./.github/actions/restore-python
-        with:
-          python-version: ${{ env.DEFAULT_PYTHON }}
-          cache-key: ${{ needs.common.outputs.cache-key }}
-      - name: Validate config for ${{ matrix.file }}
-        run: |
-          . venv/bin/activate
-          python3 script/test_build_components.py -e config -c ${{ matrix.file }}
-      - name: Compile config for ${{ matrix.file }}
-        run: |
-          . venv/bin/activate
-          python3 script/test_build_components.py -e compile -c ${{ matrix.file }}
-
   test-build-components-splitter:
     name: Split components for intelligent grouping (40 weighted per batch)
     runs-on: ubuntu-24.04
     needs:
       - common
       - determine-jobs
-    if: github.event_name == 'pull_request' && fromJSON(needs.determine-jobs.outputs.component-test-count) >= 100
+    if: github.event_name == 'pull_request' && fromJSON(needs.determine-jobs.outputs.component-test-count) > 0
     outputs:
       matrix: ${{ steps.split.outputs.components }}
     steps:
@@ -417,9 +384,10 @@ jobs:
 
           # Use intelligent splitter that groups components with same bus configs
           components='${{ needs.determine-jobs.outputs.changed-components-with-tests }}'
+          directly_changed='${{ needs.determine-jobs.outputs.directly-changed-components-with-tests }}'
 
           echo "Splitting components intelligently..."
-          output=$(python3 script/split_components_for_ci.py --components "$components" --batch-size 40 --output github)
+          output=$(python3 script/split_components_for_ci.py --components "$components" --directly-changed "$directly_changed" --batch-size 40 --output github)
 
           echo "$output" >> $GITHUB_OUTPUT
 
@@ -430,7 +398,7 @@ jobs:
       - common
       - determine-jobs
       - test-build-components-splitter
-    if: github.event_name == 'pull_request' && fromJSON(needs.determine-jobs.outputs.component-test-count) >= 100
+    if: github.event_name == 'pull_request' && fromJSON(needs.determine-jobs.outputs.component-test-count) > 0
     strategy:
       fail-fast: false
       max-parallel: ${{ (github.base_ref == 'beta' || github.base_ref == 'release') && 8 || 4 }}
@@ -477,18 +445,34 @@ jobs:
           # Convert space-separated components to comma-separated for Python script
           components_csv=$(echo "${{ matrix.components }}" | tr ' ' ',')
 
-          echo "Testing components: $components_csv"
+          # Only isolate directly changed components when targeting dev branch
+          # For beta/release branches, group everything for faster CI
+          #
+          # WHY ISOLATE DIRECTLY CHANGED COMPONENTS?
+          # - Isolated tests run WITHOUT --testing-mode, enabling full validation
+          # - This catches pin conflicts and other issues in directly changed code
+          # - Grouped tests use --testing-mode to allow config merging (disables some checks)
+          # - Dependencies are safe to group since they weren't modified in this PR
+          if [ "${{ github.base_ref }}" = "beta" ] || [ "${{ github.base_ref }}" = "release" ]; then
+            directly_changed_csv=""
+            echo "Testing components: $components_csv"
+            echo "Target branch: ${{ github.base_ref }} - grouping all components"
+          else
+            directly_changed_csv=$(echo '${{ needs.determine-jobs.outputs.directly-changed-components-with-tests }}' | jq -r 'join(",")')
+            echo "Testing components: $components_csv"
+            echo "Target branch: ${{ github.base_ref }} - isolating directly changed components: $directly_changed_csv"
+          fi
           echo ""
 
-          # Run config validation with grouping
-          python3 script/test_build_components.py -e config -c "$components_csv" -f
+          # Run config validation with grouping and isolation
+          python3 script/test_build_components.py -e config -c "$components_csv" -f --isolate "$directly_changed_csv"
 
           echo ""
           echo "Config validation passed! Starting compilation..."
           echo ""
 
-          # Run compilation with grouping
-          python3 script/test_build_components.py -e compile -c "$components_csv" -f
+          # Run compilation with grouping and isolation
+          python3 script/test_build_components.py -e compile -c "$components_csv" -f --isolate "$directly_changed_csv"
 
   pre-commit-ci-lite:
     name: pre-commit.ci lite
@@ -521,7 +505,6 @@ jobs:
       - integration-tests
       - clang-tidy
       - determine-jobs
-      - test-build-components
       - test-build-components-splitter
       - test-build-components-split
       - pre-commit-ci-lite

script/determine-jobs.py

@@ -31,6 +31,7 @@ Options:
 from __future__ import annotations
 
 import argparse
+from functools import cache
 import json
 import os
 from pathlib import Path
@@ -45,7 +46,6 @@ from helpers import (
     changed_files,
     get_all_dependencies,
     get_components_from_integration_fixtures,
-    parse_list_components_output,
     root_path,
 )
 
@@ -212,6 +212,24 @@ def _any_changed_file_endswith(branch: str | None, extensions: tuple[str, ...])
     return any(file.endswith(extensions) for file in changed_files(branch))
 
 
+@cache
+def _component_has_tests(component: str) -> bool:
+    """Check if a component has test files.
+
+    Cached to avoid repeated filesystem operations for the same component.
+
+    Args:
+        component: Component name to check
+
+    Returns:
+        True if the component has test YAML files
+    """
+    tests_dir = Path(root_path) / "tests" / "components" / component
+    if not tests_dir.exists():
+        return False
+    return any(tests_dir.glob("test.*.yaml"))
+
+
 def main() -> None:
     """Main function that determines which CI jobs to run."""
     parser = argparse.ArgumentParser(
@@ -228,23 +246,37 @@ def main() -> None:
     run_clang_format = should_run_clang_format(args.branch)
     run_python_linters = should_run_python_linters(args.branch)
 
-    # Get changed components using list-components.py for exact compatibility
+    # Get both directly changed and all changed components (with dependencies) in one call
     script_path = Path(__file__).parent / "list-components.py"
-    cmd = [sys.executable, str(script_path), "--changed"]
+    cmd = [sys.executable, str(script_path), "--changed-with-deps"]
     if args.branch:
         cmd.extend(["-b", args.branch])
 
     result = subprocess.run(cmd, capture_output=True, text=True, check=True)
-    changed_components = parse_list_components_output(result.stdout)
+    component_data = json.loads(result.stdout)
+    directly_changed_components = component_data["directly_changed"]
+    changed_components = component_data["all_changed"]
 
     # Filter to only components that have test files
     # Components without tests shouldn't generate CI test jobs
-    tests_dir = Path(root_path) / "tests" / "components"
     changed_components_with_tests = [
+        component for component in changed_components if _component_has_tests(component)
+    ]
+
+    # Get directly changed components with tests (for isolated testing)
+    # These will be tested WITHOUT --testing-mode in CI to enable full validation
+    # (pin conflicts, etc.) since they contain the actual changes being reviewed
+    directly_changed_with_tests = [
         component
-        for component in changed_components
-        if (component_test_dir := tests_dir / component).exists()
-        and any(component_test_dir.glob("test.*.yaml"))
+        for component in directly_changed_components
+        if _component_has_tests(component)
+    ]
+
+    # Get dependency-only components (for grouped testing)
+    dependency_only_components = [
+        component
+        for component in changed_components_with_tests
+        if component not in directly_changed_components
     ]
 
     # Build output
@@ -255,7 +287,11 @@ def main() -> None:
         "python_linters": run_python_linters,
         "changed_components": changed_components,
         "changed_components_with_tests": changed_components_with_tests,
+        "directly_changed_components_with_tests": directly_changed_with_tests,
+        "dependency_only_components_with_tests": dependency_only_components,
         "component_test_count": len(changed_components_with_tests),
+        "directly_changed_count": len(directly_changed_with_tests),
+        "dependency_only_count": len(dependency_only_components),
     }
 
     # Output as JSON

script/list-components.py

@@ -185,18 +185,32 @@ def main():
         "-c",
         "--changed",
         action="store_true",
-        help="List all components required for testing based on changes",
+        help="List all components required for testing based on changes (includes dependencies)",
+    )
+    parser.add_argument(
+        "--changed-direct",
+        action="store_true",
+        help="List only directly changed components (without dependencies)",
+    )
+    parser.add_argument(
+        "--changed-with-deps",
+        action="store_true",
+        help="Output JSON with both directly changed and all changed components",
     )
     parser.add_argument(
         "-b", "--branch", help="Branch to compare changed files against"
     )
     args = parser.parse_args()
 
-    if args.branch and not args.changed:
-        parser.error("--branch requires --changed")
+    if args.branch and not (
+        args.changed or args.changed_direct or args.changed_with_deps
+    ):
+        parser.error(
+            "--branch requires --changed, --changed-direct, or --changed-with-deps"
+        )
 
-    if args.changed:
-        # When --changed is passed, only get the changed files
+    if args.changed or args.changed_direct or args.changed_with_deps:
+        # When --changed* is passed, only get the changed files
         changed = changed_files(args.branch)
 
         # If any base test file(s) changed, there's no need to filter out components
@@ -210,8 +224,25 @@ def main():
         # Get all component files
         files = get_all_component_files()
 
-    for c in get_components(files, args.changed):
-        print(c)
+    if args.changed_with_deps:
+        # Return JSON with both directly changed and all changed components
+        import json
+
+        directly_changed = get_components(files, False)
+        all_changed = get_components(files, True)
+        output = {
+            "directly_changed": directly_changed,
+            "all_changed": all_changed,
+        }
+        print(json.dumps(output))
+    elif args.changed_direct:
+        # Return only directly changed components (without dependencies)
+        for c in get_components(files, False):
+            print(c)
+    else:
+        # Return all changed components (with dependencies) - default behavior
+        for c in get_components(files, args.changed):
+            print(c)
 
 
 if __name__ == "__main__":

script/split_components_for_ci.py

@@ -56,6 +56,7 @@ def create_intelligent_batches(
     components: list[str],
     tests_dir: Path,
     batch_size: int = 40,
+    directly_changed: set[str] | None = None,
 ) -> list[list[str]]:
     """Create batches optimized for component grouping.
 
@@ -63,6 +64,7 @@ def create_intelligent_batches(
         components: List of component names to batch
         tests_dir: Path to tests/components directory
         batch_size: Target size for each batch
+        directly_changed: Set of directly changed components (for logging only)
 
     Returns:
         List of component batches (lists of component names)
@@ -94,10 +96,17 @@ def create_intelligent_batches(
 
     for component in components_with_tests:
         # Components that can't be grouped get unique signatures
-        # This includes both manually curated ISOLATED_COMPONENTS and
-        # automatically detected non_groupable components
+        # This includes:
+        # - Manually curated ISOLATED_COMPONENTS
+        # - Automatically detected non_groupable components
+        # - Directly changed components (passed via --isolate in CI)
         # These can share a batch/runner but won't be grouped/merged
-        if component in ISOLATED_COMPONENTS or component in non_groupable:
+        is_isolated = (
+            component in ISOLATED_COMPONENTS
+            or component in non_groupable
+            or (directly_changed and component in directly_changed)
+        )
+        if is_isolated:
             signature_groups[f"isolated_{component}"].append(component)
             continue
 
@@ -187,6 +196,10 @@ def main() -> int:
         default=Path("tests/components"),
         help="Path to tests/components directory",
     )
+    parser.add_argument(
+        "--directly-changed",
+        help="JSON array of directly changed component names (for logging only)",
+    )
     parser.add_argument(
         "--output",
         "-o",
@@ -208,11 +221,21 @@ def main() -> int:
         print("Components must be a JSON array", file=sys.stderr)
         return 1
 
+    # Parse directly changed components list from JSON (if provided)
+    directly_changed = None
+    if args.directly_changed:
+        try:
+            directly_changed = set(json.loads(args.directly_changed))
+        except json.JSONDecodeError as e:
+            print(f"Error parsing directly-changed JSON: {e}", file=sys.stderr)
+            return 1
+
     # Create intelligent batches
     batches = create_intelligent_batches(
         components=components,
         tests_dir=args.tests_dir,
         batch_size=args.batch_size,
+        directly_changed=directly_changed,
     )
 
     # Convert batches to space-separated strings for CI
@@ -238,13 +261,37 @@ def main() -> int:
     isolated_count = sum(
         1
         for comp in all_batched_components
-        if comp in ISOLATED_COMPONENTS or comp in non_groupable
+        if comp in ISOLATED_COMPONENTS
+        or comp in non_groupable
+        or (directly_changed and comp in directly_changed)
     )
     groupable_count = actual_components - isolated_count
 
     print("\n=== Intelligent Batch Summary ===", file=sys.stderr)
     print(f"Total components requested: {len(components)}", file=sys.stderr)
     print(f"Components with test files: {actual_components}", file=sys.stderr)
+
+    # Show breakdown of directly changed vs dependencies
+    if directly_changed:
+        direct_count = sum(
+            1 for comp in all_batched_components if comp in directly_changed
+        )
+        dep_count = actual_components - direct_count
+        direct_comps = [
+            comp for comp in all_batched_components if comp in directly_changed
+        ]
+        dep_comps = [
+            comp for comp in all_batched_components if comp not in directly_changed
+        ]
+        print(
+            f"  - Direct changes: {direct_count} ({', '.join(sorted(direct_comps))})",
+            file=sys.stderr,
+        )
+        print(
+            f"  - Dependencies: {dep_count} ({', '.join(sorted(dep_comps))})",
+            file=sys.stderr,
+        )
+
     print(f"  - Groupable (weight=1): {groupable_count}", file=sys.stderr)
     print(f"  - Isolated (weight=10): {isolated_count}", file=sys.stderr)
     if actual_components < len(components):

script/test_build_components.py

@@ -365,6 +365,7 @@ def run_grouped_component_tests(
     build_dir: Path,
     esphome_command: str,
     continue_on_fail: bool,
+    additional_isolated: set[str] | None = None,
 ) -> tuple[set[tuple[str, str]], list[str], list[str], dict[str, str]]:
     """Run grouped component tests.
 
@@ -376,6 +377,7 @@ def run_grouped_component_tests(
         build_dir: Path to build directory
         esphome_command: ESPHome command (config/compile)
         continue_on_fail: Whether to continue on failure
+        additional_isolated: Additional components to treat as isolated (not grouped)
 
     Returns:
         Tuple of (tested_components, passed_tests, failed_tests, failed_commands)
@@ -397,6 +399,17 @@ def run_grouped_component_tests(
     # Track why components can't be grouped (for detailed output)
     non_groupable_reasons = {}
 
+    # Merge additional isolated components with predefined ones
+    # ISOLATED COMPONENTS are tested individually WITHOUT --testing-mode
+    # This is critical because:
+    # - Grouped tests use --testing-mode which disables pin conflict checks and other validation
+    # - These checks are disabled to allow config merging (multiple components in one build)
+    # - For directly changed components (via --isolate), we need full validation to catch issues
+    # - Dependencies are safe to group since they weren't modified in the PR
+    all_isolated = set(ISOLATED_COMPONENTS.keys())
+    if additional_isolated:
+        all_isolated.update(additional_isolated)
+
     # Group by (platform, bus_signature)
     for component, platforms in component_buses.items():
         if component not in all_tests:
@@ -404,7 +417,7 @@ def run_grouped_component_tests(
 
         # Skip components that must be tested in isolation
         # These are shown separately and should not be in non_groupable_reasons
-        if component in ISOLATED_COMPONENTS:
+        if component in all_isolated:
             continue
 
         # Skip base bus components (these test the bus platforms themselves)
@@ -453,15 +466,28 @@ def run_grouped_component_tests(
     print("\nGrouping Plan:")
     print("-" * 80)
 
-    # Show isolated components (must test individually due to known issues)
-    isolated_in_tests = [c for c in ISOLATED_COMPONENTS if c in all_tests]
+    # Show isolated components (must test individually due to known issues or direct changes)
+    isolated_in_tests = [c for c in all_isolated if c in all_tests]
     if isolated_in_tests:
-        print(
-            f"\n⚠ {len(isolated_in_tests)} components must be tested in isolation (known build issues):"
-        )
-        for comp in sorted(isolated_in_tests):
-            reason = ISOLATED_COMPONENTS[comp]
-            print(f"  - {comp}: {reason}")
+        predefined_isolated = [c for c in isolated_in_tests if c in ISOLATED_COMPONENTS]
+        additional_in_tests = [
+            c for c in isolated_in_tests if c in (additional_isolated or set())
+        ]
+
+        if predefined_isolated:
+            print(
+                f"\n⚠ {len(predefined_isolated)} components must be tested in isolation (known build issues):"
+            )
+            for comp in sorted(predefined_isolated):
+                reason = ISOLATED_COMPONENTS[comp]
+                print(f"  - {comp}: {reason}")
+
+        if additional_in_tests:
+            print(
+                f"\n✓ {len(additional_in_tests)} components tested in isolation (directly changed in PR):"
+            )
+            for comp in sorted(additional_in_tests):
+                print(f"  - {comp}")
 
     # Show base bus components (test the bus platform implementations)
     base_bus_in_tests = [c for c in BASE_BUS_COMPONENTS if c in all_tests]
@@ -733,6 +759,7 @@ def test_components(
     esphome_command: str,
     continue_on_fail: bool,
     enable_grouping: bool = True,
+    isolated_components: set[str] | None = None,
 ) -> int:
     """Test components with optional intelligent grouping.
 
@@ -742,6 +769,10 @@ def test_components(
         esphome_command: ESPHome command (config/compile)
         continue_on_fail: Whether to continue on failure
         enable_grouping: Whether to enable component grouping
+        isolated_components: Set of component names to test in isolation (not grouped).
+            These are tested WITHOUT --testing-mode to enable full validation
+            (pin conflicts, etc). This is used in CI for directly changed components
+            to catch issues that would be missed with --testing-mode.
 
     Returns:
         Exit code (0 for success, 1 for failure)
@@ -788,6 +819,7 @@ def test_components(
             build_dir=build_dir,
             esphome_command=esphome_command,
             continue_on_fail=continue_on_fail,
+            additional_isolated=isolated_components,
         )
 
     # Then run individual tests for components not in groups
@@ -912,18 +944,30 @@ def main() -> int:
         action="store_true",
         help="Disable component grouping (test each component individually)",
     )
+    parser.add_argument(
+        "--isolate",
+        help="Comma-separated list of components to test in isolation (not grouped with others). "
+        "These are tested WITHOUT --testing-mode to enable full validation. "
+        "Used in CI for directly changed components to catch pin conflicts and other issues.",
+    )
 
     args = parser.parse_args()
 
     # Parse component patterns
     component_patterns = [p.strip() for p in args.components.split(",")]
 
+    # Parse isolated components
+    isolated_components = None
+    if args.isolate:
+        isolated_components = {c.strip() for c in args.isolate.split(",") if c.strip()}
+
     return test_components(
         component_patterns=component_patterns,
         platform_filter=args.target,
         esphome_command=args.esphome_command,
         continue_on_fail=args.continue_on_fail,
         enable_grouping=not args.no_grouping,
+        isolated_components=isolated_components,
     )
 
 

tests/script/test_determine_jobs.py

@@ -73,9 +73,11 @@ def test_main_all_tests_should_run(
     mock_should_run_clang_format.return_value = True
     mock_should_run_python_linters.return_value = True
 
-    # Mock list-components.py output
+    # Mock list-components.py output (now returns JSON with --changed-with-deps)
     mock_result = Mock()
-    mock_result.stdout = "wifi\napi\nsensor\n"
+    mock_result.stdout = json.dumps(
+        {"directly_changed": ["wifi", "api"], "all_changed": ["wifi", "api", "sensor"]}
+    )
     mock_subprocess_run.return_value = mock_result
 
     # Run main function with mocked argv
@@ -116,7 +118,7 @@ def test_main_no_tests_should_run(
 
     # Mock empty list-components.py output
     mock_result = Mock()
-    mock_result.stdout = ""
+    mock_result.stdout = json.dumps({"directly_changed": [], "all_changed": []})
     mock_subprocess_run.return_value = mock_result
 
     # Run main function with mocked argv
@@ -177,7 +179,9 @@ def test_main_with_branch_argument(
 
     # Mock list-components.py output
     mock_result = Mock()
-    mock_result.stdout = "mqtt\n"
+    mock_result.stdout = json.dumps(
+        {"directly_changed": ["mqtt"], "all_changed": ["mqtt"]}
+    )
     mock_subprocess_run.return_value = mock_result
 
     with patch("sys.argv", ["script.py", "-b", "main"]):
@@ -192,7 +196,7 @@ def test_main_with_branch_argument(
     # Check that list-components.py was called with branch
     mock_subprocess_run.assert_called_once()
     call_args = mock_subprocess_run.call_args[0][0]
-    assert "--changed" in call_args
+    assert "--changed-with-deps" in call_args
     assert "-b" in call_args
     assert "main" in call_args
 
@@ -411,7 +415,12 @@ def test_main_filters_components_without_tests(
     # Mock list-components.py output with 3 components
     # wifi: has tests, sensor: has tests, airthings_ble: no tests
     mock_result = Mock()
-    mock_result.stdout = "wifi\nsensor\nairthings_ble\n"
+    mock_result.stdout = json.dumps(
+        {
+            "directly_changed": ["wifi", "sensor"],
+            "all_changed": ["wifi", "sensor", "airthings_ble"],
+        }
+    )
     mock_subprocess_run.return_value = mock_result
 
     # Create test directory structure
@@ -436,6 +445,8 @@ def test_main_filters_components_without_tests(
         patch.object(determine_jobs, "root_path", str(tmp_path)),
         patch("sys.argv", ["determine-jobs.py"]),
     ):
+        # Clear the cache since we're mocking root_path
+        determine_jobs._component_has_tests.cache_clear()
         determine_jobs.main()
 
     # Check output