[ci] Merge components with different buses to reduce CI time (#11251)

Co-authored-by: Jesse Hills <3060199+jesserockz@users.noreply.github.com>

script/split_components_for_ci.py

@@ -22,9 +22,11 @@ sys.path.insert(0, str(Path(__file__).parent.parent))
 
 from script.analyze_component_buses import (
     ISOLATED_COMPONENTS,
+    ISOLATED_SIGNATURE_PREFIX,
     NO_BUSES_SIGNATURE,
     analyze_all_components,
     create_grouping_signature,
+    merge_compatible_bus_groups,
 )
 
 # Weighting for batch creation
@@ -33,6 +35,10 @@ from script.analyze_component_buses import (
 ISOLATED_WEIGHT = 10
 GROUPABLE_WEIGHT = 1
 
+# Platform used for batching (platform-agnostic batching)
+# Batches are split across CI runners and each runner tests all platforms
+ALL_PLATFORMS = "all"
+
 
 def has_test_files(component_name: str, tests_dir: Path) -> bool:
     """Check if a component has test files.
@@ -57,7 +63,7 @@ def create_intelligent_batches(
     tests_dir: Path,
     batch_size: int = 40,
     directly_changed: set[str] | None = None,
-) -> list[list[str]]:
+) -> tuple[list[list[str]], dict[tuple[str, str], list[str]]]:
     """Create batches optimized for component grouping.
 
     Args:
@@ -67,7 +73,9 @@ def create_intelligent_batches(
         directly_changed: Set of directly changed components (for logging only)
 
     Returns:
-        List of component batches (lists of component names)
+        Tuple of (batches, signature_groups) where:
+        - batches: List of component batches (lists of component names)
+        - signature_groups: Dict mapping (platform, signature) to component lists
     """
     # Filter out components without test files
     # Platform components like 'climate' and 'climate_ir' don't have test files
@@ -91,8 +99,9 @@ def create_intelligent_batches(
 
     # Group components by their bus signature ONLY (ignore platform)
     # All platforms will be tested by test_build_components.py for each batch
-    # Key: signature, Value: list of components
-    signature_groups: dict[str, list[str]] = defaultdict(list)
+    # Key: (platform, signature), Value: list of components
+    # We use ALL_PLATFORMS since batching is platform-agnostic
+    signature_groups: dict[tuple[str, str], list[str]] = defaultdict(list)
 
     for component in components_with_tests:
         # Components that can't be grouped get unique signatures
@@ -107,7 +116,9 @@ def create_intelligent_batches(
             or (directly_changed and component in directly_changed)
         )
         if is_isolated:
-            signature_groups[f"isolated_{component}"].append(component)
+            signature_groups[
+                (ALL_PLATFORMS, f"{ISOLATED_SIGNATURE_PREFIX}{component}")
+            ].append(component)
             continue
 
         # Get signature from any platform (they should all have the same buses)
@@ -117,11 +128,17 @@ def create_intelligent_batches(
             if buses:
                 signature = create_grouping_signature({platform: buses}, platform)
                 # Group by signature only - platform doesn't matter for batching
-                signature_groups[signature].append(component)
+                # Use ALL_PLATFORMS since we're batching across all platforms
+                signature_groups[(ALL_PLATFORMS, signature)].append(component)
                 break  # Only use first platform for grouping
         else:
             # No buses found for any platform - can be grouped together
-            signature_groups[NO_BUSES_SIGNATURE].append(component)
+            signature_groups[(ALL_PLATFORMS, NO_BUSES_SIGNATURE)].append(component)
+
+    # Merge compatible bus groups (cross-bus optimization)
+    # This allows components with different buses (ble + uart) to be batched together
+    # improving the efficiency of test_build_components.py grouping
+    signature_groups = merge_compatible_bus_groups(signature_groups)
 
     # Create batches by keeping signature groups together
     # Components with the same signature stay in the same batches
@@ -132,8 +149,8 @@ def create_intelligent_batches(
     # 2. Sort groupable signatures by size (largest first)
     # 3. "no_buses" components CAN be grouped together
     def sort_key(item):
-        signature, components = item
-        is_isolated = signature.startswith("isolated_")
+        (_platform, signature), components = item
+        is_isolated = signature.startswith(ISOLATED_SIGNATURE_PREFIX)
         # Put "isolated_*" last (1), groupable first (0)
         # Within each category, sort by size (largest first)
         return (is_isolated, -len(components))
@@ -149,8 +166,8 @@ def create_intelligent_batches(
     current_batch = []
     current_weight = 0
 
-    for signature, group_components in sorted_groups:
-        is_isolated = signature.startswith("isolated_")
+    for (_platform, signature), group_components in sorted_groups:
+        is_isolated = signature.startswith(ISOLATED_SIGNATURE_PREFIX)
         weight_per_component = ISOLATED_WEIGHT if is_isolated else GROUPABLE_WEIGHT
 
         for component in group_components:
@@ -169,7 +186,7 @@ def create_intelligent_batches(
     if current_batch:
         batches.append(current_batch)
 
-    return batches
+    return batches, signature_groups
 
 
 def main() -> int:
@@ -231,7 +248,7 @@ def main() -> int:
             return 1
 
     # Create intelligent batches
-    batches = create_intelligent_batches(
+    batches, signature_groups = create_intelligent_batches(
         components=components,
         tests_dir=args.tests_dir,
         batch_size=args.batch_size,
@@ -256,6 +273,58 @@ def main() -> int:
     # Re-analyze to get isolated component counts for summary
     _, non_groupable, _ = analyze_all_components(args.tests_dir)
 
+    # Show grouping details
+    print("\n=== Component Grouping Details ===", file=sys.stderr)
+    # Sort groups by signature for readability
+    groupable_groups = []
+    isolated_groups = []
+    for (platform, signature), group_comps in sorted(signature_groups.items()):
+        if signature.startswith(ISOLATED_SIGNATURE_PREFIX):
+            isolated_groups.append((signature, group_comps))
+        else:
+            groupable_groups.append((signature, group_comps))
+
+    if groupable_groups:
+        print(
+            f"\nGroupable signatures ({len(groupable_groups)} merged groups after cross-bus optimization):",
+            file=sys.stderr,
+        )
+        for signature, group_comps in sorted(
+            groupable_groups, key=lambda x: (-len(x[1]), x[0])
+        ):
+            # Check if this is a merged signature (contains +)
+            is_merged = "+" in signature and signature != NO_BUSES_SIGNATURE
+            # Special handling for no_buses components
+            if signature == NO_BUSES_SIGNATURE:
+                print(
+                    f"  [{signature}]: {len(group_comps)} components (used as fillers across batches)",
+                    file=sys.stderr,
+                )
+            else:
+                merge_indicator = " [MERGED]" if is_merged else ""
+                print(
+                    f"  [{signature}]{merge_indicator}: {len(group_comps)} components",
+                    file=sys.stderr,
+                )
+            # Show first few components as examples
+            examples = ", ".join(sorted(group_comps)[:8])
+            if len(group_comps) > 8:
+                examples += f", ... (+{len(group_comps) - 8} more)"
+            print(f"    → {examples}", file=sys.stderr)
+
+    if isolated_groups:
+        print(
+            f"\nIsolated components ({len(isolated_groups)} components - tested individually):",
+            file=sys.stderr,
+        )
+        isolated_names = sorted(
+            [comp for _, comps in isolated_groups for comp in comps]
+        )
+        # Group isolated components for compact display
+        for i in range(0, len(isolated_names), 10):
+            chunk = isolated_names[i : i + 10]
+            print(f"  {', '.join(chunk)}", file=sys.stderr)
+
     # Count isolated vs groupable components
     all_batched_components = [comp for batch in batches for comp in batch]
     isolated_count = sum(