diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0363b5afdf..2b0449474b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -178,6 +178,7 @@ jobs: python-linters: ${{ steps.determine.outputs.python-linters }} changed-components: ${{ steps.determine.outputs.changed-components }} changed-components-with-tests: ${{ steps.determine.outputs.changed-components-with-tests }} + directly-changed-components-with-tests: ${{ steps.determine.outputs.directly-changed-components-with-tests }} component-test-count: ${{ steps.determine.outputs.component-test-count }} steps: - name: Check out code from GitHub @@ -206,6 +207,7 @@ jobs: echo "python-linters=$(echo "$output" | jq -r '.python_linters')" >> $GITHUB_OUTPUT echo "changed-components=$(echo "$output" | jq -c '.changed_components')" >> $GITHUB_OUTPUT echo "changed-components-with-tests=$(echo "$output" | jq -c '.changed_components_with_tests')" >> $GITHUB_OUTPUT + echo "directly-changed-components-with-tests=$(echo "$output" | jq -c '.directly_changed_components_with_tests')" >> $GITHUB_OUTPUT echo "component-test-count=$(echo "$output" | jq -r '.component_test_count')" >> $GITHUB_OUTPUT integration-tests: @@ -358,48 +360,13 @@ jobs: # yamllint disable-line rule:line-length if: always() - test-build-components: - name: Component test ${{ matrix.file }} - runs-on: ubuntu-24.04 - needs: - - common - - determine-jobs - if: github.event_name == 'pull_request' && fromJSON(needs.determine-jobs.outputs.component-test-count) > 0 && fromJSON(needs.determine-jobs.outputs.component-test-count) < 100 - strategy: - fail-fast: false - max-parallel: 2 - matrix: - file: ${{ fromJson(needs.determine-jobs.outputs.changed-components-with-tests) }} - steps: - - name: Cache apt packages - uses: awalsh128/cache-apt-pkgs-action@acb598e5ddbc6f68a970c5da0688d2f3a9f04d05 # v1.5.3 - with: - packages: libsdl2-dev - version: 1.0 - - - name: Check out code from GitHub - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 - - name: Restore Python - uses: ./.github/actions/restore-python - with: - python-version: ${{ env.DEFAULT_PYTHON }} - cache-key: ${{ needs.common.outputs.cache-key }} - - name: Validate config for ${{ matrix.file }} - run: | - . venv/bin/activate - python3 script/test_build_components.py -e config -c ${{ matrix.file }} - - name: Compile config for ${{ matrix.file }} - run: | - . venv/bin/activate - python3 script/test_build_components.py -e compile -c ${{ matrix.file }} - test-build-components-splitter: name: Split components for intelligent grouping (40 weighted per batch) runs-on: ubuntu-24.04 needs: - common - determine-jobs - if: github.event_name == 'pull_request' && fromJSON(needs.determine-jobs.outputs.component-test-count) >= 100 + if: github.event_name == 'pull_request' && fromJSON(needs.determine-jobs.outputs.component-test-count) > 0 outputs: matrix: ${{ steps.split.outputs.components }} steps: @@ -417,9 +384,10 @@ jobs: # Use intelligent splitter that groups components with same bus configs components='${{ needs.determine-jobs.outputs.changed-components-with-tests }}' + directly_changed='${{ needs.determine-jobs.outputs.directly-changed-components-with-tests }}' echo "Splitting components intelligently..." - output=$(python3 script/split_components_for_ci.py --components "$components" --batch-size 40 --output github) + output=$(python3 script/split_components_for_ci.py --components "$components" --directly-changed "$directly_changed" --batch-size 40 --output github) echo "$output" >> $GITHUB_OUTPUT @@ -430,7 +398,7 @@ jobs: - common - determine-jobs - test-build-components-splitter - if: github.event_name == 'pull_request' && fromJSON(needs.determine-jobs.outputs.component-test-count) >= 100 + if: github.event_name == 'pull_request' && fromJSON(needs.determine-jobs.outputs.component-test-count) > 0 strategy: fail-fast: false max-parallel: ${{ (github.base_ref == 'beta' || github.base_ref == 'release') && 8 || 4 }} @@ -477,18 +445,34 @@ jobs: # Convert space-separated components to comma-separated for Python script components_csv=$(echo "${{ matrix.components }}" | tr ' ' ',') - echo "Testing components: $components_csv" + # Only isolate directly changed components when targeting dev branch + # For beta/release branches, group everything for faster CI + # + # WHY ISOLATE DIRECTLY CHANGED COMPONENTS? + # - Isolated tests run WITHOUT --testing-mode, enabling full validation + # - This catches pin conflicts and other issues in directly changed code + # - Grouped tests use --testing-mode to allow config merging (disables some checks) + # - Dependencies are safe to group since they weren't modified in this PR + if [ "${{ github.base_ref }}" = "beta" ] || [ "${{ github.base_ref }}" = "release" ]; then + directly_changed_csv="" + echo "Testing components: $components_csv" + echo "Target branch: ${{ github.base_ref }} - grouping all components" + else + directly_changed_csv=$(echo '${{ needs.determine-jobs.outputs.directly-changed-components-with-tests }}' | jq -r 'join(",")') + echo "Testing components: $components_csv" + echo "Target branch: ${{ github.base_ref }} - isolating directly changed components: $directly_changed_csv" + fi echo "" - # Run config validation with grouping - python3 script/test_build_components.py -e config -c "$components_csv" -f + # Run config validation with grouping and isolation + python3 script/test_build_components.py -e config -c "$components_csv" -f --isolate "$directly_changed_csv" echo "" echo "Config validation passed! Starting compilation..." echo "" - # Run compilation with grouping - python3 script/test_build_components.py -e compile -c "$components_csv" -f + # Run compilation with grouping and isolation + python3 script/test_build_components.py -e compile -c "$components_csv" -f --isolate "$directly_changed_csv" pre-commit-ci-lite: name: pre-commit.ci lite @@ -521,7 +505,6 @@ jobs: - integration-tests - clang-tidy - determine-jobs - - test-build-components - test-build-components-splitter - test-build-components-split - pre-commit-ci-lite diff --git a/script/determine-jobs.py b/script/determine-jobs.py index a078fd8f9b..b000ecee3b 100755 --- a/script/determine-jobs.py +++ b/script/determine-jobs.py @@ -31,6 +31,7 @@ Options: from __future__ import annotations import argparse +from functools import cache import json import os from pathlib import Path @@ -45,7 +46,6 @@ from helpers import ( changed_files, get_all_dependencies, get_components_from_integration_fixtures, - parse_list_components_output, root_path, ) @@ -212,6 +212,24 @@ def _any_changed_file_endswith(branch: str | None, extensions: tuple[str, ...]) return any(file.endswith(extensions) for file in changed_files(branch)) +@cache +def _component_has_tests(component: str) -> bool: + """Check if a component has test files. + + Cached to avoid repeated filesystem operations for the same component. + + Args: + component: Component name to check + + Returns: + True if the component has test YAML files + """ + tests_dir = Path(root_path) / "tests" / "components" / component + if not tests_dir.exists(): + return False + return any(tests_dir.glob("test.*.yaml")) + + def main() -> None: """Main function that determines which CI jobs to run.""" parser = argparse.ArgumentParser( @@ -228,23 +246,37 @@ def main() -> None: run_clang_format = should_run_clang_format(args.branch) run_python_linters = should_run_python_linters(args.branch) - # Get changed components using list-components.py for exact compatibility + # Get both directly changed and all changed components (with dependencies) in one call script_path = Path(__file__).parent / "list-components.py" - cmd = [sys.executable, str(script_path), "--changed"] + cmd = [sys.executable, str(script_path), "--changed-with-deps"] if args.branch: cmd.extend(["-b", args.branch]) result = subprocess.run(cmd, capture_output=True, text=True, check=True) - changed_components = parse_list_components_output(result.stdout) + component_data = json.loads(result.stdout) + directly_changed_components = component_data["directly_changed"] + changed_components = component_data["all_changed"] # Filter to only components that have test files # Components without tests shouldn't generate CI test jobs - tests_dir = Path(root_path) / "tests" / "components" changed_components_with_tests = [ + component for component in changed_components if _component_has_tests(component) + ] + + # Get directly changed components with tests (for isolated testing) + # These will be tested WITHOUT --testing-mode in CI to enable full validation + # (pin conflicts, etc.) since they contain the actual changes being reviewed + directly_changed_with_tests = [ component - for component in changed_components - if (component_test_dir := tests_dir / component).exists() - and any(component_test_dir.glob("test.*.yaml")) + for component in directly_changed_components + if _component_has_tests(component) + ] + + # Get dependency-only components (for grouped testing) + dependency_only_components = [ + component + for component in changed_components_with_tests + if component not in directly_changed_components ] # Build output @@ -255,7 +287,11 @@ def main() -> None: "python_linters": run_python_linters, "changed_components": changed_components, "changed_components_with_tests": changed_components_with_tests, + "directly_changed_components_with_tests": directly_changed_with_tests, + "dependency_only_components_with_tests": dependency_only_components, "component_test_count": len(changed_components_with_tests), + "directly_changed_count": len(directly_changed_with_tests), + "dependency_only_count": len(dependency_only_components), } # Output as JSON diff --git a/script/list-components.py b/script/list-components.py index 9ab1cdd852..dffff6801a 100755 --- a/script/list-components.py +++ b/script/list-components.py @@ -185,18 +185,32 @@ def main(): "-c", "--changed", action="store_true", - help="List all components required for testing based on changes", + help="List all components required for testing based on changes (includes dependencies)", + ) + parser.add_argument( + "--changed-direct", + action="store_true", + help="List only directly changed components (without dependencies)", + ) + parser.add_argument( + "--changed-with-deps", + action="store_true", + help="Output JSON with both directly changed and all changed components", ) parser.add_argument( "-b", "--branch", help="Branch to compare changed files against" ) args = parser.parse_args() - if args.branch and not args.changed: - parser.error("--branch requires --changed") + if args.branch and not ( + args.changed or args.changed_direct or args.changed_with_deps + ): + parser.error( + "--branch requires --changed, --changed-direct, or --changed-with-deps" + ) - if args.changed: - # When --changed is passed, only get the changed files + if args.changed or args.changed_direct or args.changed_with_deps: + # When --changed* is passed, only get the changed files changed = changed_files(args.branch) # If any base test file(s) changed, there's no need to filter out components @@ -210,8 +224,25 @@ def main(): # Get all component files files = get_all_component_files() - for c in get_components(files, args.changed): - print(c) + if args.changed_with_deps: + # Return JSON with both directly changed and all changed components + import json + + directly_changed = get_components(files, False) + all_changed = get_components(files, True) + output = { + "directly_changed": directly_changed, + "all_changed": all_changed, + } + print(json.dumps(output)) + elif args.changed_direct: + # Return only directly changed components (without dependencies) + for c in get_components(files, False): + print(c) + else: + # Return all changed components (with dependencies) - default behavior + for c in get_components(files, args.changed): + print(c) if __name__ == "__main__": diff --git a/script/split_components_for_ci.py b/script/split_components_for_ci.py index ad2261499c..9730db4988 100755 --- a/script/split_components_for_ci.py +++ b/script/split_components_for_ci.py @@ -56,6 +56,7 @@ def create_intelligent_batches( components: list[str], tests_dir: Path, batch_size: int = 40, + directly_changed: set[str] | None = None, ) -> list[list[str]]: """Create batches optimized for component grouping. @@ -63,6 +64,7 @@ def create_intelligent_batches( components: List of component names to batch tests_dir: Path to tests/components directory batch_size: Target size for each batch + directly_changed: Set of directly changed components (for logging only) Returns: List of component batches (lists of component names) @@ -94,10 +96,17 @@ def create_intelligent_batches( for component in components_with_tests: # Components that can't be grouped get unique signatures - # This includes both manually curated ISOLATED_COMPONENTS and - # automatically detected non_groupable components + # This includes: + # - Manually curated ISOLATED_COMPONENTS + # - Automatically detected non_groupable components + # - Directly changed components (passed via --isolate in CI) # These can share a batch/runner but won't be grouped/merged - if component in ISOLATED_COMPONENTS or component in non_groupable: + is_isolated = ( + component in ISOLATED_COMPONENTS + or component in non_groupable + or (directly_changed and component in directly_changed) + ) + if is_isolated: signature_groups[f"isolated_{component}"].append(component) continue @@ -187,6 +196,10 @@ def main() -> int: default=Path("tests/components"), help="Path to tests/components directory", ) + parser.add_argument( + "--directly-changed", + help="JSON array of directly changed component names (for logging only)", + ) parser.add_argument( "--output", "-o", @@ -208,11 +221,21 @@ def main() -> int: print("Components must be a JSON array", file=sys.stderr) return 1 + # Parse directly changed components list from JSON (if provided) + directly_changed = None + if args.directly_changed: + try: + directly_changed = set(json.loads(args.directly_changed)) + except json.JSONDecodeError as e: + print(f"Error parsing directly-changed JSON: {e}", file=sys.stderr) + return 1 + # Create intelligent batches batches = create_intelligent_batches( components=components, tests_dir=args.tests_dir, batch_size=args.batch_size, + directly_changed=directly_changed, ) # Convert batches to space-separated strings for CI @@ -238,13 +261,37 @@ def main() -> int: isolated_count = sum( 1 for comp in all_batched_components - if comp in ISOLATED_COMPONENTS or comp in non_groupable + if comp in ISOLATED_COMPONENTS + or comp in non_groupable + or (directly_changed and comp in directly_changed) ) groupable_count = actual_components - isolated_count print("\n=== Intelligent Batch Summary ===", file=sys.stderr) print(f"Total components requested: {len(components)}", file=sys.stderr) print(f"Components with test files: {actual_components}", file=sys.stderr) + + # Show breakdown of directly changed vs dependencies + if directly_changed: + direct_count = sum( + 1 for comp in all_batched_components if comp in directly_changed + ) + dep_count = actual_components - direct_count + direct_comps = [ + comp for comp in all_batched_components if comp in directly_changed + ] + dep_comps = [ + comp for comp in all_batched_components if comp not in directly_changed + ] + print( + f" - Direct changes: {direct_count} ({', '.join(sorted(direct_comps))})", + file=sys.stderr, + ) + print( + f" - Dependencies: {dep_count} ({', '.join(sorted(dep_comps))})", + file=sys.stderr, + ) + print(f" - Groupable (weight=1): {groupable_count}", file=sys.stderr) print(f" - Isolated (weight=10): {isolated_count}", file=sys.stderr) if actual_components < len(components): diff --git a/script/test_build_components.py b/script/test_build_components.py index e27bd695c1..14fc10977c 100755 --- a/script/test_build_components.py +++ b/script/test_build_components.py @@ -365,6 +365,7 @@ def run_grouped_component_tests( build_dir: Path, esphome_command: str, continue_on_fail: bool, + additional_isolated: set[str] | None = None, ) -> tuple[set[tuple[str, str]], list[str], list[str], dict[str, str]]: """Run grouped component tests. @@ -376,6 +377,7 @@ def run_grouped_component_tests( build_dir: Path to build directory esphome_command: ESPHome command (config/compile) continue_on_fail: Whether to continue on failure + additional_isolated: Additional components to treat as isolated (not grouped) Returns: Tuple of (tested_components, passed_tests, failed_tests, failed_commands) @@ -397,6 +399,17 @@ def run_grouped_component_tests( # Track why components can't be grouped (for detailed output) non_groupable_reasons = {} + # Merge additional isolated components with predefined ones + # ISOLATED COMPONENTS are tested individually WITHOUT --testing-mode + # This is critical because: + # - Grouped tests use --testing-mode which disables pin conflict checks and other validation + # - These checks are disabled to allow config merging (multiple components in one build) + # - For directly changed components (via --isolate), we need full validation to catch issues + # - Dependencies are safe to group since they weren't modified in the PR + all_isolated = set(ISOLATED_COMPONENTS.keys()) + if additional_isolated: + all_isolated.update(additional_isolated) + # Group by (platform, bus_signature) for component, platforms in component_buses.items(): if component not in all_tests: @@ -404,7 +417,7 @@ def run_grouped_component_tests( # Skip components that must be tested in isolation # These are shown separately and should not be in non_groupable_reasons - if component in ISOLATED_COMPONENTS: + if component in all_isolated: continue # Skip base bus components (these test the bus platforms themselves) @@ -453,15 +466,28 @@ def run_grouped_component_tests( print("\nGrouping Plan:") print("-" * 80) - # Show isolated components (must test individually due to known issues) - isolated_in_tests = [c for c in ISOLATED_COMPONENTS if c in all_tests] + # Show isolated components (must test individually due to known issues or direct changes) + isolated_in_tests = [c for c in all_isolated if c in all_tests] if isolated_in_tests: - print( - f"\n⚠ {len(isolated_in_tests)} components must be tested in isolation (known build issues):" - ) - for comp in sorted(isolated_in_tests): - reason = ISOLATED_COMPONENTS[comp] - print(f" - {comp}: {reason}") + predefined_isolated = [c for c in isolated_in_tests if c in ISOLATED_COMPONENTS] + additional_in_tests = [ + c for c in isolated_in_tests if c in (additional_isolated or set()) + ] + + if predefined_isolated: + print( + f"\n⚠ {len(predefined_isolated)} components must be tested in isolation (known build issues):" + ) + for comp in sorted(predefined_isolated): + reason = ISOLATED_COMPONENTS[comp] + print(f" - {comp}: {reason}") + + if additional_in_tests: + print( + f"\n✓ {len(additional_in_tests)} components tested in isolation (directly changed in PR):" + ) + for comp in sorted(additional_in_tests): + print(f" - {comp}") # Show base bus components (test the bus platform implementations) base_bus_in_tests = [c for c in BASE_BUS_COMPONENTS if c in all_tests] @@ -733,6 +759,7 @@ def test_components( esphome_command: str, continue_on_fail: bool, enable_grouping: bool = True, + isolated_components: set[str] | None = None, ) -> int: """Test components with optional intelligent grouping. @@ -742,6 +769,10 @@ def test_components( esphome_command: ESPHome command (config/compile) continue_on_fail: Whether to continue on failure enable_grouping: Whether to enable component grouping + isolated_components: Set of component names to test in isolation (not grouped). + These are tested WITHOUT --testing-mode to enable full validation + (pin conflicts, etc). This is used in CI for directly changed components + to catch issues that would be missed with --testing-mode. Returns: Exit code (0 for success, 1 for failure) @@ -788,6 +819,7 @@ def test_components( build_dir=build_dir, esphome_command=esphome_command, continue_on_fail=continue_on_fail, + additional_isolated=isolated_components, ) # Then run individual tests for components not in groups @@ -912,18 +944,30 @@ def main() -> int: action="store_true", help="Disable component grouping (test each component individually)", ) + parser.add_argument( + "--isolate", + help="Comma-separated list of components to test in isolation (not grouped with others). " + "These are tested WITHOUT --testing-mode to enable full validation. " + "Used in CI for directly changed components to catch pin conflicts and other issues.", + ) args = parser.parse_args() # Parse component patterns component_patterns = [p.strip() for p in args.components.split(",")] + # Parse isolated components + isolated_components = None + if args.isolate: + isolated_components = {c.strip() for c in args.isolate.split(",") if c.strip()} + return test_components( component_patterns=component_patterns, platform_filter=args.target, esphome_command=args.esphome_command, continue_on_fail=args.continue_on_fail, enable_grouping=not args.no_grouping, + isolated_components=isolated_components, ) diff --git a/tests/script/test_determine_jobs.py b/tests/script/test_determine_jobs.py index 5d8746f434..0559d116be 100644 --- a/tests/script/test_determine_jobs.py +++ b/tests/script/test_determine_jobs.py @@ -73,9 +73,11 @@ def test_main_all_tests_should_run( mock_should_run_clang_format.return_value = True mock_should_run_python_linters.return_value = True - # Mock list-components.py output + # Mock list-components.py output (now returns JSON with --changed-with-deps) mock_result = Mock() - mock_result.stdout = "wifi\napi\nsensor\n" + mock_result.stdout = json.dumps( + {"directly_changed": ["wifi", "api"], "all_changed": ["wifi", "api", "sensor"]} + ) mock_subprocess_run.return_value = mock_result # Run main function with mocked argv @@ -116,7 +118,7 @@ def test_main_no_tests_should_run( # Mock empty list-components.py output mock_result = Mock() - mock_result.stdout = "" + mock_result.stdout = json.dumps({"directly_changed": [], "all_changed": []}) mock_subprocess_run.return_value = mock_result # Run main function with mocked argv @@ -177,7 +179,9 @@ def test_main_with_branch_argument( # Mock list-components.py output mock_result = Mock() - mock_result.stdout = "mqtt\n" + mock_result.stdout = json.dumps( + {"directly_changed": ["mqtt"], "all_changed": ["mqtt"]} + ) mock_subprocess_run.return_value = mock_result with patch("sys.argv", ["script.py", "-b", "main"]): @@ -192,7 +196,7 @@ def test_main_with_branch_argument( # Check that list-components.py was called with branch mock_subprocess_run.assert_called_once() call_args = mock_subprocess_run.call_args[0][0] - assert "--changed" in call_args + assert "--changed-with-deps" in call_args assert "-b" in call_args assert "main" in call_args @@ -411,7 +415,12 @@ def test_main_filters_components_without_tests( # Mock list-components.py output with 3 components # wifi: has tests, sensor: has tests, airthings_ble: no tests mock_result = Mock() - mock_result.stdout = "wifi\nsensor\nairthings_ble\n" + mock_result.stdout = json.dumps( + { + "directly_changed": ["wifi", "sensor"], + "all_changed": ["wifi", "sensor", "airthings_ble"], + } + ) mock_subprocess_run.return_value = mock_result # Create test directory structure @@ -436,6 +445,8 @@ def test_main_filters_components_without_tests( patch.object(determine_jobs, "root_path", str(tmp_path)), patch("sys.argv", ["determine-jobs.py"]), ): + # Clear the cache since we're mocking root_path + determine_jobs._component_has_tests.cache_clear() determine_jobs.main() # Check output