[ci] Fix component batching for beta/release branches (3-4 → 40 per batch) (#11759)

script/determine-jobs.py

@@ -756,11 +756,27 @@ def main() -> None:
     component_test_batches: list[str]
     if changed_components_with_tests:
         tests_dir = Path(root_path) / ESPHOME_TESTS_COMPONENTS_PATH
+
+        # For beta/release branches, group all components for faster CI
+        # (no isolation, all components are groupable)
+        target_branch = get_target_branch()
+        is_release_branch = target_branch and (
+            target_branch.startswith("release") or target_branch.startswith("beta")
+        )
+
+        if is_release_branch:
+            # For beta/release: Don't isolate any components - group everything
+            # This allows components to be merged into single builds
+            batch_directly_changed = set()  # Empty set - no isolation
+        else:
+            # Normal PR: only directly changed components are isolated
+            batch_directly_changed = directly_changed_with_tests
+
         batches, _ = create_intelligent_batches(
             components=changed_components_with_tests,
             tests_dir=tests_dir,
             batch_size=COMPONENT_TEST_BATCH_SIZE,
-            directly_changed=directly_changed_with_tests,
+            directly_changed=batch_directly_changed,
         )
         # Convert batches to space-separated strings for CI matrix
         component_test_batches = [" ".join(batch) for batch in batches]

tests/script/test_determine_jobs.py

@@ -19,6 +19,8 @@ sys.path.insert(0, script_dir)
 # Import helpers module for patching
 import helpers  # noqa: E402
 
+import script.helpers  # noqa: E402
+
 spec = importlib.util.spec_from_file_location(
     "determine_jobs", os.path.join(script_dir, "determine-jobs.py")
 )
@@ -132,6 +134,16 @@ def test_main_all_tests_should_run(
                 ["wifi", "api"] if not deps else ["wifi", "api", "sensor"]
             ),
         ),
+        patch.object(
+            determine_jobs,
+            "detect_memory_impact_config",
+            return_value={"should_run": "false"},
+        ),
+        patch.object(
+            determine_jobs,
+            "create_intelligent_batches",
+            return_value=([["wifi", "api", "sensor"]], {}),
+        ),
     ):
         determine_jobs.main()
 
@@ -203,6 +215,16 @@ def test_main_no_tests_should_run(
         patch.object(
             determine_jobs, "get_components_with_dependencies", return_value=[]
         ),
+        patch.object(
+            determine_jobs,
+            "detect_memory_impact_config",
+            return_value={"should_run": "false"},
+        ),
+        patch.object(
+            determine_jobs,
+            "create_intelligent_batches",
+            return_value=([], {}),
+        ),
     ):
         determine_jobs.main()
 
@@ -266,6 +288,16 @@ def test_main_with_branch_argument(
         patch.object(
             determine_jobs, "get_components_with_dependencies", return_value=["mqtt"]
         ),
+        patch.object(
+            determine_jobs,
+            "detect_memory_impact_config",
+            return_value={"should_run": "false"},
+        ),
+        patch.object(
+            determine_jobs,
+            "create_intelligent_batches",
+            return_value=([["mqtt"]], {}),
+        ),
     ):
         determine_jobs.main()
 
@@ -571,6 +603,11 @@ def test_main_filters_components_without_tests(
             ),
         ),
         patch.object(determine_jobs, "changed_files", return_value=[]),
+        patch.object(
+            determine_jobs,
+            "detect_memory_impact_config",
+            return_value={"should_run": "false"},
+        ),
     ):
         # Clear the cache since we're mocking root_path
         determine_jobs.main()
@@ -670,6 +707,11 @@ def test_main_detects_components_with_variant_tests(
             ),
         ),
         patch.object(determine_jobs, "changed_files", return_value=[]),
+        patch.object(
+            determine_jobs,
+            "detect_memory_impact_config",
+            return_value={"should_run": "false"},
+        ),
     ):
         # Clear the cache since we're mocking root_path
         determine_jobs.main()
@@ -1124,6 +1166,16 @@ def test_main_core_files_changed_still_detects_components(
                 else ["select", "api", "bluetooth_proxy", "logger"]
             ),
         ),
+        patch.object(
+            determine_jobs,
+            "detect_memory_impact_config",
+            return_value={"should_run": "false"},
+        ),
+        patch.object(
+            determine_jobs,
+            "create_intelligent_batches",
+            return_value=([["select", "api", "bluetooth_proxy", "logger"]], {}),
+        ),
     ):
         determine_jobs.main()
 
@@ -1367,3 +1419,99 @@ def test_detect_memory_impact_config_runs_at_component_limit(tmp_path: Path) ->
     # Memory impact should run at exactly 40 components (at limit but not over)
     assert result["should_run"] == "true"
     assert len(result["components"]) == 40
+
+
+def test_component_batching_beta_branch_40_per_batch(
+    tmp_path: Path,
+    mock_should_run_integration_tests: Mock,
+    mock_should_run_clang_tidy: Mock,
+    mock_should_run_clang_format: Mock,
+    mock_should_run_python_linters: Mock,
+    mock_changed_files: Mock,
+    mock_determine_cpp_unit_tests: Mock,
+    capsys: pytest.CaptureFixture[str],
+) -> None:
+    """Test that beta/release branches create batches with 40 actual components each.
+
+    For beta/release branches, all components should be groupable (not isolated),
+    and each batch should contain 40 actual components with weight 1 each.
+    This matches the original behavior before consolidation.
+    """
+    # Create 120 test components with test files
+    component_names = [f"comp_{i:03d}" for i in range(120)]
+    tests_dir = tmp_path / "tests" / "components"
+
+    for comp in component_names:
+        comp_dir = tests_dir / comp
+        comp_dir.mkdir(parents=True)
+        (comp_dir / "test.esp32-idf.yaml").write_text(f"# Test for {comp}")
+
+    # Setup mocks
+    mock_should_run_integration_tests.return_value = False
+    mock_should_run_clang_tidy.return_value = False
+    mock_should_run_clang_format.return_value = False
+    mock_should_run_python_linters.return_value = False
+    mock_determine_cpp_unit_tests.return_value = (False, [])
+
+    # Mock changed_files to return all component files
+    changed_files = [
+        f"esphome/components/{comp}/{comp}.cpp" for comp in component_names
+    ]
+    mock_changed_files.return_value = changed_files
+
+    # Run main function with beta branch
+    # Don't mock create_intelligent_batches - that's what we're testing!
+    with (
+        patch("sys.argv", ["determine-jobs.py", "--branch", "beta"]),
+        patch.object(determine_jobs, "root_path", str(tmp_path)),
+        patch.object(helpers, "root_path", str(tmp_path)),
+        patch.object(script.helpers, "root_path", str(tmp_path)),
+        patch.object(determine_jobs, "get_target_branch", return_value="beta"),
+        patch.object(determine_jobs, "_is_clang_tidy_full_scan", return_value=False),
+        patch.object(
+            determine_jobs,
+            "get_changed_components",
+            return_value=component_names,
+        ),
+        patch.object(
+            determine_jobs,
+            "filter_component_and_test_files",
+            side_effect=lambda f: f.startswith("esphome/components/"),
+        ),
+        patch.object(
+            determine_jobs,
+            "get_components_with_dependencies",
+            side_effect=lambda files, deps: component_names,
+        ),
+        patch.object(
+            determine_jobs,
+            "detect_memory_impact_config",
+            return_value={"should_run": "false"},
+        ),
+    ):
+        determine_jobs.main()
+
+    # Check output
+    captured = capsys.readouterr()
+    output = json.loads(captured.out)
+
+    # Verify batches are present and properly sized
+    assert "component_test_batches" in output
+    batches = output["component_test_batches"]
+
+    # Should have 3 batches (120 components / 40 per batch = 3)
+    assert len(batches) == 3, f"Expected 3 batches, got {len(batches)}"
+
+    # Each batch should have approximately 40 components (all weight=1, groupable)
+    for i, batch_str in enumerate(batches):
+        batch_components = batch_str.split()
+        assert len(batch_components) == 40, (
+            f"Batch {i} should have 40 components, got {len(batch_components)}"
+        )
+
+    # Verify all 120 components are in batches
+    all_components = []
+    for batch_str in batches:
+        all_components.extend(batch_str.split())
+    assert len(all_components) == 120
+    assert set(all_components) == set(component_names)