From 495bb4200c16c6f1e8da9d4ddd917bcf9c08d63f Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Wed, 8 Oct 2025 15:50:49 -1000 Subject: [PATCH] fix --- .github/workflows/ci.yml | 59 +++++++---- script/split_components_for_ci.py | 170 ++++++++++++++++++++++++++++++ script/test_build_components.py | 63 +++++++++-- 3 files changed, 265 insertions(+), 27 deletions(-) create mode 100755 script/split_components_for_ci.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1d7043c888..923e1a739d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -381,17 +381,17 @@ jobs: with: python-version: ${{ env.DEFAULT_PYTHON }} cache-key: ${{ needs.common.outputs.cache-key }} - - name: test_build_components -e config -c ${{ matrix.file }} + - name: Validate config for ${{ matrix.file }} run: | . venv/bin/activate - ./script/test_build_components -e config -c ${{ matrix.file }} - - name: test_build_components -e compile -c ${{ matrix.file }} + python3 script/test_build_components.py -e config -c ${{ matrix.file }} + - name: Compile config for ${{ matrix.file }} run: | . venv/bin/activate - ./script/test_build_components -e compile -c ${{ matrix.file }} + python3 script/test_build_components.py -e compile -c ${{ matrix.file }} test-build-components-splitter: - name: Split components for testing into 10 components per group + name: Split components for intelligent grouping (10 per batch) runs-on: ubuntu-24.04 needs: - common @@ -402,14 +402,26 @@ jobs: steps: - name: Check out code from GitHub uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 - - name: Split components into groups of 10 + - name: Restore Python + uses: ./.github/actions/restore-python + with: + python-version: ${{ env.DEFAULT_PYTHON }} + cache-key: ${{ needs.common.outputs.cache-key }} + - name: Split components intelligently based on bus configurations id: split run: | - components=$(echo '${{ needs.determine-jobs.outputs.changed-components }}' | jq -c '.[]' | shuf | jq -s -c '[_nwise(10) | join(" ")]') - echo "components=$components" >> $GITHUB_OUTPUT + . venv/bin/activate + + # Use intelligent splitter that groups components with same bus configs + components='${{ needs.determine-jobs.outputs.changed-components }}' + + echo "Splitting components intelligently..." + output=$(python3 script/split_components_for_ci.py --components "$components" --batch-size 10 --output github) + + echo "$output" >> $GITHUB_OUTPUT test-build-components-split: - name: Test split components + name: Test components with intelligent grouping runs-on: ubuntu-24.04 needs: - common @@ -437,20 +449,27 @@ jobs: with: python-version: ${{ env.DEFAULT_PYTHON }} cache-key: ${{ needs.common.outputs.cache-key }} - - name: Validate config + - name: Validate and compile components with intelligent grouping run: | . venv/bin/activate - for component in ${{ matrix.components }}; do - ./script/test_build_components -e config -c $component - done - - name: Compile config - run: | - . venv/bin/activate - mkdir build_cache + mkdir -p build_cache export PLATFORMIO_BUILD_CACHE_DIR=$PWD/build_cache - for component in ${{ matrix.components }}; do - ./script/test_build_components -e compile -c $component - done + + # Convert space-separated components to comma-separated for Python script + components_csv=$(echo "${{ matrix.components }}" | tr ' ' ',') + + echo "Testing components: $components_csv" + echo "" + + # Run config validation with grouping + python3 script/test_build_components.py -e config -c "$components_csv" -f + + echo "" + echo "Config validation passed! Starting compilation..." + echo "" + + # Run compilation with grouping + python3 script/test_build_components.py -e compile -c "$components_csv" -f pre-commit-ci-lite: name: pre-commit.ci lite diff --git a/script/split_components_for_ci.py b/script/split_components_for_ci.py new file mode 100755 index 0000000000..df6951729c --- /dev/null +++ b/script/split_components_for_ci.py @@ -0,0 +1,170 @@ +#!/usr/bin/env python3 +"""Split components into batches with intelligent grouping. + +This script analyzes components to identify which ones share common bus configurations +and intelligently groups them into batches to maximize the efficiency of the +component grouping system in CI. + +Components with the same bus signature are placed in the same batch whenever possible, +allowing the test_build_components.py script to merge them into single builds. +""" + +from __future__ import annotations + +import argparse +from collections import defaultdict +import json +from pathlib import Path +import sys + +# Add esphome to path +sys.path.insert(0, str(Path(__file__).parent.parent)) + +from script.analyze_component_buses import ( + analyze_all_components, + create_grouping_signature, +) + + +def create_intelligent_batches( + components: list[str], + tests_dir: Path, + batch_size: int = 10, +) -> list[list[str]]: + """Create batches optimized for component grouping. + + Args: + components: List of component names to batch + tests_dir: Path to tests/components directory + batch_size: Target size for each batch + + Returns: + List of component batches (lists of component names) + """ + # Analyze all components to get their bus signatures + component_buses = analyze_all_components(tests_dir) + + # Group components by their bus signature + # Key: (platform, signature), Value: list of components + signature_groups: dict[tuple[str, str], list[str]] = defaultdict(list) + + for component in components: + if component not in component_buses: + # Component has no bus configs, put in special group + signature_groups[("none", "none")].append(component) + continue + + # Group by platform and signature + comp_platforms = component_buses[component] + for platform, buses in comp_platforms.items(): + if buses: + signature = create_grouping_signature({platform: buses}, platform) + signature_groups[(platform, signature)].append(component) + break # Only use first platform for grouping + else: + # No buses found + signature_groups[("none", "none")].append(component) + + # Create batches by grouping components with same signature + batches = [] + current_batch = [] + + # Sort signature groups by size (largest first) for better packing + sorted_groups = sorted( + signature_groups.items(), + key=lambda x: len(x[1]), + reverse=True, + ) + + for (platform, signature), group_components in sorted_groups: + # Add components from this signature group to batches + for component in sorted(group_components): # Sort for determinism + current_batch.append(component) + + if len(current_batch) >= batch_size: + batches.append(current_batch) + current_batch = [] + + # Add remaining components + if current_batch: + batches.append(current_batch) + + return batches + + +def main() -> int: + """Main entry point.""" + parser = argparse.ArgumentParser( + description="Split components into intelligent batches for CI testing" + ) + parser.add_argument( + "--components", + "-c", + required=True, + help="JSON array of component names", + ) + parser.add_argument( + "--batch-size", + "-b", + type=int, + default=10, + help="Target batch size (default: 10)", + ) + parser.add_argument( + "--tests-dir", + type=Path, + default=Path("tests/components"), + help="Path to tests/components directory", + ) + parser.add_argument( + "--output", + "-o", + choices=["json", "github"], + default="github", + help="Output format (json or github for GitHub Actions)", + ) + + args = parser.parse_args() + + # Parse component list from JSON + try: + components = json.loads(args.components) + except json.JSONDecodeError as e: + print(f"Error parsing components JSON: {e}", file=sys.stderr) + return 1 + + if not isinstance(components, list): + print("Components must be a JSON array", file=sys.stderr) + return 1 + + # Create intelligent batches + batches = create_intelligent_batches( + components=components, + tests_dir=args.tests_dir, + batch_size=args.batch_size, + ) + + # Convert batches to space-separated strings for CI + batch_strings = [" ".join(batch) for batch in batches] + + if args.output == "json": + # Output as JSON array + print(json.dumps(batch_strings)) + else: + # Output for GitHub Actions (set output) + output_json = json.dumps(batch_strings) + print(f"components={output_json}") + + # Print summary to stderr so it shows in CI logs + print("\n=== Intelligent Batch Summary ===", file=sys.stderr) + print(f"Total components: {len(components)}", file=sys.stderr) + print(f"Number of batches: {len(batches)}", file=sys.stderr) + print(f"Batch size target: {args.batch_size}", file=sys.stderr) + print(f"Average batch size: {len(components) / len(batches):.1f}", file=sys.stderr) + print(file=sys.stderr) + + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/script/test_build_components.py b/script/test_build_components.py index 2ad37ce925..da81465107 100755 --- a/script/test_build_components.py +++ b/script/test_build_components.py @@ -325,13 +325,15 @@ def test_components( print(f"Found {len(all_tests)} components to test") # Group components by platform and bus signature + grouped_components: dict[tuple[str, str], list[str]] = defaultdict(list) + if enable_grouping: - print("Analyzing components for grouping opportunities...") + print("\n" + "=" * 80) + print("Analyzing components for intelligent grouping...") + print("=" * 80) component_buses = analyze_all_components(tests_dir) # Group by (platform, bus_signature) - grouped_components: dict[tuple[str, str], list[str]] = defaultdict(list) - for component, platforms in component_buses.items(): if component not in all_tests: continue @@ -346,11 +348,58 @@ def test_components( signature = create_grouping_signature({platform: buses}, platform) grouped_components[(platform, signature)].append(component) - # Print grouping analysis - total_grouped = sum( - len(comps) for comps in grouped_components.values() if len(comps) > 1 + # Print detailed grouping plan + print("\nGrouping Plan:") + print("-" * 80) + + groups_to_test = [] + individual_tests = [] + + for (platform, signature), components in sorted(grouped_components.items()): + if len(components) > 1: + groups_to_test.append((platform, signature, components)) + elif len(components) == 1: + individual_tests.extend(components) + + # Add components without grouping signatures + for component in all_tests: + if ( + component not in [c for _, _, comps in groups_to_test for c in comps] + and component not in individual_tests + ): + individual_tests.append(component) + + if groups_to_test: + print(f"\n✓ {len(groups_to_test)} groups will be tested together:") + for platform, signature, components in groups_to_test: + component_list = ", ".join(sorted(components)) + print(f" [{platform}] [{signature}]: {component_list}") + print( + f" → {len(components)} components in 1 build (saves {len(components) - 1} builds)" + ) + + if individual_tests: + print( + f"\n○ {len(individual_tests)} components will be tested individually:" + ) + for comp in sorted(individual_tests[:10]): + print(f" - {comp}") + if len(individual_tests) > 10: + print(f" ... and {len(individual_tests) - 10} more") + + total_grouped = sum(len(comps) for _, _, comps in groups_to_test) + total_builds_without_grouping = len(all_tests) + total_builds_with_grouping = len(groups_to_test) + len(individual_tests) + builds_saved = total_builds_without_grouping - total_builds_with_grouping + + print(f"\n{'=' * 80}") + print(f"Summary: {total_builds_with_grouping} builds total") + print(f" • {len(groups_to_test)} grouped builds ({total_grouped} components)") + print(f" • {len(individual_tests)} individual builds") + print( + f" • Saves {builds_saved} builds ({builds_saved / total_builds_without_grouping * 100:.1f}% reduction)" ) - print(f"Grouping analysis: {total_grouped} components can be grouped") + print("=" * 80 + "\n") # Run tests failed_tests = []