sascha/esphome
1
0
Fork 0
mirror of https://github.com/esphome/esphome.git synced 2025-10-20 02:33:50 +01:00
Code Issues Releases Wiki Activity

[ci] Merge components with different buses to reduce CI time (#11251)

Browse Source 
Co-authored-by: Jesse Hills <3060199+jesserockz@users.noreply.github.com>
This commit is contained in:
J. Nick Koston
2025-10-15 17:36:03 -10:00
committed by GitHub
parent f2e0a412db
commit 14d76e9e4e
78 changed files with 954 additions and 266 deletions
Download Patch File Download Diff File Expand all files Collapse all files

136
script/merge_component_configs.py
View File

@@ -16,6 +16,7 @@ The merger handles:
from __future__ import annotations
import argparse
from functools import lru_cache
from pathlib import Path
import re
import sys
@@ -28,6 +29,10 @@ from esphome import yaml_util
from esphome.config_helpers import merge_config
from script.analyze_component_buses import PACKAGE_DEPENDENCIES, get_common_bus_packages
# Prefix for dependency markers in package tracking
# Used to mark packages that are included transitively (e.g., uart via modbus)
DEPENDENCY_MARKER_PREFIX = "_dep_"
def load_yaml_file(yaml_file: Path) -> dict:
"""Load YAML file using ESPHome's YAML loader.
@@ -44,6 +49,34 @@ def load_yaml_file(yaml_file: Path) -> dict:
return yaml_util.load_yaml(yaml_file)
@lru_cache(maxsize=256)
def get_component_packages(
component_name: str, platform: str, tests_dir_str: str
) -> dict:
"""Get packages dict from a component's test file with caching.
This function is cached to avoid re-loading and re-parsing the same file
multiple times when extracting packages during cross-bus merging.
Args:
component_name: Name of the component
platform: Platform name (e.g., "esp32-idf")
tests_dir_str: String path to tests/components directory (must be string for cache hashability)
Returns:
Dictionary with 'packages' key containing the raw packages dict from the YAML,
or empty dict if no packages section exists
"""
tests_dir = Path(tests_dir_str)
test_file = tests_dir / component_name / f"test.{platform}.yaml"
comp_data = load_yaml_file(test_file)
if "packages" not in comp_data or not isinstance(comp_data["packages"], dict):
return {}
return comp_data["packages"]
def extract_packages_from_yaml(data: dict) -> dict[str, str]:
"""Extract COMMON BUS package includes from parsed YAML.
@@ -82,7 +115,7 @@ def extract_packages_from_yaml(data: dict) -> dict[str, str]:
if dep not in common_bus_packages:
continue
# Mark as included via dependency
packages[f"_dep_{dep}"] = f"(included via {name})"
packages[f"{DEPENDENCY_MARKER_PREFIX}{dep}"] = f"(included via {name})"
return packages
@@ -195,6 +228,9 @@ def merge_component_configs(
# Start with empty config
merged_config_data = {}
# Convert tests_dir to string for caching
tests_dir_str = str(tests_dir)
# Process each component
for comp_name in component_names:
comp_dir = tests_dir / comp_name
@@ -206,26 +242,29 @@ def merge_component_configs(
# Load the component's test file
comp_data = load_yaml_file(test_file)
# Validate packages are compatible
# Components with no packages (no_buses) can merge with any group
# Merge packages from all components (cross-bus merging)
# Components can have different packages (e.g., one with ble, another with uart)
# as long as they don't conflict (checked by are_buses_compatible before calling this)
comp_packages = extract_packages_from_yaml(comp_data)
if all_packages is None:
# First component - set the baseline
all_packages = comp_packages
elif not comp_packages:
# This component has no packages (no_buses) - it can merge with any group
pass
elif not all_packages:
# Previous components had no packages, but this one does - adopt these packages
all_packages = comp_packages
elif comp_packages != all_packages:
# Both have packages but they differ - this is an error
raise ValueError(
f"Component {comp_name} has different packages than previous components. "
f"Expected: {all_packages}, Got: {comp_packages}. "
f"All components must use the same common bus configs to be merged."
)
# First component - initialize package dict
all_packages = comp_packages if comp_packages else {}
elif comp_packages:
# Merge packages - combine all unique package types
# If both have the same package type, verify they're identical
for pkg_name, pkg_config in comp_packages.items():
if pkg_name in all_packages:
# Same package type - verify config matches
if all_packages[pkg_name] != pkg_config:
raise ValueError(
f"Component {comp_name} has conflicting config for package '{pkg_name}'. "
f"Expected: {all_packages[pkg_name]}, Got: {pkg_config}. "
f"Components with conflicting bus configs cannot be merged."
)
else:
# New package type - add it
all_packages[pkg_name] = pkg_config
# Handle $component_dir by replacing with absolute path
# This allows components that use local file references to be grouped
@@ -287,26 +326,51 @@ def merge_component_configs(
# merge_config handles list merging with ID-based deduplication automatically
merged_config_data = merge_config(merged_config_data, comp_data)
# Add packages back (only once, since they're identical)
# IMPORTANT: Only re-add common bus packages (spi, i2c, uart, etc.)
# Add merged packages back (union of all component packages)
# IMPORTANT: Only include common bus packages (spi, i2c, uart, etc.)
# Do NOT re-add component-specific packages as they contain unprefixed $component_dir refs
if all_packages:
first_comp_data = load_yaml_file(
tests_dir / component_names[0] / f"test.{platform}.yaml"
)
if "packages" in first_comp_data and isinstance(
first_comp_data["packages"], dict
):
# Filter to only include common bus packages
# Only dict format can contain common bus packages
common_bus_packages = get_common_bus_packages()
filtered_packages = {
name: value
for name, value in first_comp_data["packages"].items()
if name in common_bus_packages
}
if filtered_packages:
merged_config_data["packages"] = filtered_packages
# Build packages dict from merged all_packages
# all_packages is a dict mapping package_name -> str(package_value)
# We need to reconstruct the actual package values by loading them from any component
# Since packages with the same name must have identical configs (verified above),
# we can load the package value from the first component that has each package
common_bus_packages = get_common_bus_packages()
merged_packages: dict[str, Any] = {}
# Collect packages that are included as dependencies
# If modbus is present, uart is included via modbus.packages.uart
packages_to_skip: set[str] = set()
for pkg_name in all_packages:
if pkg_name.startswith(DEPENDENCY_MARKER_PREFIX):
# Extract the actual package name (remove _dep_ prefix)
dep_name = pkg_name[len(DEPENDENCY_MARKER_PREFIX) :]
packages_to_skip.add(dep_name)
for pkg_name in all_packages:
# Skip dependency markers
if pkg_name.startswith(DEPENDENCY_MARKER_PREFIX):
continue
# Skip non-common-bus packages
if pkg_name not in common_bus_packages:
continue
# Skip packages that are included as dependencies of other packages
# This prevents duplicate definitions (e.g., uart via modbus + uart separately)
if pkg_name in packages_to_skip:
continue
# Find a component that has this package and extract its value
# Uses cached lookup to avoid re-loading the same files
for comp_name in component_names:
comp_packages = get_component_packages(
comp_name, platform, tests_dir_str
)
if pkg_name in comp_packages:
merged_packages[pkg_name] = comp_packages[pkg_name]
break
if merged_packages:
merged_config_data["packages"] = merged_packages
# Deduplicate items with same ID (keeps first occurrence)
merged_config_data = deduplicate_by_id(merged_config_data)