sascha/esphome
1
0
Fork 0
mirror of https://github.com/esphome/esphome.git synced 2025-10-20 18:53:47 +01:00
Code Issues Releases Wiki Activity

merge

Browse Source
This commit is contained in:
J. Nick Koston
2025-10-17 14:26:44 -10:00
parent 843f590db4
commit f011c44130
6 changed files with 345 additions and 124 deletions
Download Patch File Download Diff File Expand all files Collapse all files

99
script/ci_memory_impact_comment.py
View File

@@ -24,6 +24,57 @@ from esphome.analyze_memory import MemoryAnalyzer # noqa: E402
COMMENT_MARKER = "<!-- esphome-memory-impact-analysis -->"
def get_platform_toolchain(platform: str) -> tuple[str | None, str | None]:
"""Get platform-specific objdump and readelf paths.
Args:
platform: Platform name (e.g., "esp8266-ard", "esp32-idf", "esp32-c3-idf")
Returns:
Tuple of (objdump_path, readelf_path) or (None, None) if not found/supported
"""
from pathlib import Path
home = Path.home()
platformio_packages = home / ".platformio" / "packages"
# Map platform to toolchain
toolchain = None
prefix = None
if "esp8266" in platform:
toolchain = "toolchain-xtensa"
prefix = "xtensa-lx106-elf"
elif "esp32-c" in platform or "esp32-h" in platform or "esp32-p4" in platform:
# RISC-V variants (C2, C3, C5, C6, H2, P4)
toolchain = "toolchain-riscv32-esp"
prefix = "riscv32-esp-elf"
elif "esp32" in platform:
# Xtensa variants (original, S2, S3)
toolchain = "toolchain-xtensa-esp-elf"
if "s2" in platform:
prefix = "xtensa-esp32s2-elf"
elif "s3" in platform:
prefix = "xtensa-esp32s3-elf"
else:
prefix = "xtensa-esp32-elf"
else:
# Other platforms (RP2040, LibreTiny, etc.) - not supported
print(f"Platform {platform} not supported for ELF analysis", file=sys.stderr)
return None, None
toolchain_path = platformio_packages / toolchain / "bin"
objdump_path = toolchain_path / f"{prefix}-objdump"
readelf_path = toolchain_path / f"{prefix}-readelf"
if objdump_path.exists() and readelf_path.exists():
print(f"Using {platform} toolchain: {prefix}", file=sys.stderr)
return str(objdump_path), str(readelf_path)
print(f"Warning: Toolchain not found at {toolchain_path}", file=sys.stderr)
return None, None
def format_bytes(bytes_value: int) -> str:
"""Format bytes value with comma separators.
@@ -314,7 +365,7 @@ def create_detailed_breakdown_table(
def create_comment_body(
component: str,
components: list[str],
platform: str,
target_ram: int,
target_flash: int,
@@ -328,7 +379,7 @@ def create_comment_body(
"""Create the comment body with memory impact analysis.
Args:
component: Component name
components: List of component names (merged config)
platform: Platform name
target_ram: RAM usage in target branch
target_flash: Flash usage in target branch
@@ -374,10 +425,18 @@ def create_comment_body(
else:
print("No ELF files provided, skipping detailed analysis", file=sys.stderr)
# Format components list
if len(components) == 1:
components_str = f"`{components[0]}`"
config_note = "a representative test configuration"
else:
components_str = ", ".join(f"`{c}`" for c in sorted(components))
config_note = f"a merged configuration with {len(components)} components"
return f"""{COMMENT_MARKER}
## Memory Impact Analysis
**Component:** `{component}`
**Components:** {components_str}
**Platform:** `{platform}`
| Metric | Target Branch | This PR | Change |
@@ -386,7 +445,7 @@ def create_comment_body(
| **Flash** | {format_bytes(target_flash)} | {format_bytes(pr_flash)} | {flash_change} |
{component_breakdown}{symbol_changes}
---
*This analysis runs automatically when a single component changes. Memory usage is measured from a representative test configuration.*
*This analysis runs automatically when components change. Memory usage is measured from {config_note}.*
"""
@@ -537,7 +596,11 @@ def main() -> int:
description="Post or update PR comment with memory impact analysis"
)
parser.add_argument("--pr-number", required=True, help="PR number")
parser.add_argument("--component", required=True, help="Component name")
parser.add_argument(
"--components",
required=True,
help='JSON array of component names (e.g., \'["api", "wifi"]\')',
)
parser.add_argument("--platform", required=True, help="Platform name")
parser.add_argument(
"--target-ram", type=int, required=True, help="Target branch RAM usage"
@@ -560,9 +623,29 @@ def main() -> int:
args = parser.parse_args()
# Parse components from JSON
try:
components = json.loads(args.components)
if not isinstance(components, list):
print("Error: --components must be a JSON array", file=sys.stderr)
sys.exit(1)
except json.JSONDecodeError as e:
print(f"Error parsing --components JSON: {e}", file=sys.stderr)
sys.exit(1)
# Detect platform-specific toolchain paths
objdump_path = args.objdump_path
readelf_path = args.readelf_path
if not objdump_path or not readelf_path:
# Auto-detect based on platform
objdump_path, readelf_path = get_platform_toolchain(args.platform)
# Create comment body
# Note: ELF files (if provided) are from the final build when test_build_components
# runs multiple builds. Memory totals (RAM/Flash) are already summed across all builds.
comment_body = create_comment_body(
component=args.component,
components=components,
platform=args.platform,
target_ram=args.target_ram,
target_flash=args.target_flash,
@@ -570,8 +653,8 @@ def main() -> int:
pr_flash=args.pr_flash,
target_elf=args.target_elf,
pr_elf=args.pr_elf,
objdump_path=args.objdump_path,
readelf_path=args.readelf_path,
objdump_path=objdump_path,
readelf_path=readelf_path,
)
# Post or update comment

40
script/ci_memory_impact_extract.py
View File

@@ -28,27 +28,36 @@ from script.ci_helpers import write_github_output
def extract_from_compile_output(output_text: str) -> tuple[int | None, int | None]:
"""Extract memory usage from PlatformIO compile output.
Supports multiple builds (for component groups or isolated components).
When test_build_components.py creates multiple builds, this sums the
memory usage across all builds.
Looks for lines like:
RAM: [==== ] 36.1% (used 29548 bytes from 81920 bytes)
Flash: [=== ] 34.0% (used 348511 bytes from 1023984 bytes)
Args:
output_text: Compile output text
output_text: Compile output text (may contain multiple builds)
Returns:
Tuple of (ram_bytes, flash_bytes) or (None, None) if not found
Tuple of (total_ram_bytes, total_flash_bytes) or (None, None) if not found
"""
ram_match = re.search(
# Find all RAM and Flash matches (may be multiple builds)
ram_matches = re.findall(
r"RAM:\s+\[.*?\]\s+\d+\.\d+%\s+\(used\s+(\d+)\s+bytes", output_text
)
flash_match = re.search(
flash_matches = re.findall(
r"Flash:\s+\[.*?\]\s+\d+\.\d+%\s+\(used\s+(\d+)\s+bytes", output_text
)
if ram_match and flash_match:
return int(ram_match.group(1)), int(flash_match.group(1))
if not ram_matches or not flash_matches:
return None, None
return None, None
# Sum all builds (handles multiple component groups)
total_ram = sum(int(match) for match in ram_matches)
total_flash = sum(int(match) for match in flash_matches)
return total_ram, total_flash
def main() -> int:
@@ -83,8 +92,21 @@ def main() -> int:
)
return 1
print(f"RAM: {ram_bytes} bytes", file=sys.stderr)
print(f"Flash: {flash_bytes} bytes", file=sys.stderr)
# Count how many builds were found
num_builds = len(
re.findall(
r"RAM:\s+\[.*?\]\s+\d+\.\d+%\s+\(used\s+(\d+)\s+bytes", compile_output
)
)
if num_builds > 1:
print(
f"Found {num_builds} builds - summing memory usage across all builds",
file=sys.stderr,
)
print(f"Total RAM: {ram_bytes} bytes", file=sys.stderr)
print(f"Total Flash: {flash_bytes} bytes", file=sys.stderr)
if args.output_env:
# Output to GitHub Actions

98
script/determine-jobs.py
View File

@@ -237,14 +237,14 @@ def _component_has_tests(component: str) -> bool:
return any(tests_dir.glob("test.*.yaml"))
def detect_single_component_for_memory_impact(
def detect_memory_impact_config(
branch: str | None = None,
) -> dict[str, Any]:
"""Detect if exactly one component changed for memory impact analysis.
"""Determine memory impact analysis configuration.
This analyzes the actual changed files (not dependencies) to determine if
exactly one component has been modified. This is different from the
changed_components list which includes all dependencies.
Always runs memory impact analysis when there are changed components,
building a merged configuration with all changed components (like
test_build_components.py does) to get comprehensive memory analysis.
Args:
branch: Branch to compare against
@@ -252,37 +252,25 @@ def detect_single_component_for_memory_impact(
Returns:
Dictionary with memory impact analysis parameters:
- should_run: "true" or "false"
- component: component name (if should_run is true)
- test_file: test file name (if should_run is true)
- platform: platform name (if should_run is true)
- components: list of component names to analyze
- platform: platform name for the merged build
- use_merged_config: "true" (always use merged config)
"""
# Platform preference order for memory impact analysis
# Ordered by production relevance and memory constraint importance
# Prefer ESP8266 for memory impact as it's the most constrained platform
PLATFORM_PREFERENCE = [
"esp8266-ard", # ESP8266 Arduino (most memory constrained - best for impact analysis)
"esp32-idf", # Primary ESP32 IDF platform
"esp32-c3-idf", # ESP32-C3 IDF
"esp32-c6-idf", # ESP32-C6 IDF
"esp32-s2-idf", # ESP32-S2 IDF
"esp32-s3-idf", # ESP32-S3 IDF
"esp32-c2-idf", # ESP32-C2 IDF
"esp32-c5-idf", # ESP32-C5 IDF
"esp32-h2-idf", # ESP32-H2 IDF
"esp32-p4-idf", # ESP32-P4 IDF
"esp8266-ard", # ESP8266 Arduino (memory constrained)
"esp32-ard", # ESP32 Arduino
"esp32-c3-ard", # ESP32-C3 Arduino
"esp32-s2-ard", # ESP32-S2 Arduino
"esp32-s3-ard", # ESP32-S3 Arduino
"bk72xx-ard", # BK72xx Arduino
"rp2040-ard", # RP2040 Arduino
"nrf52-adafruit", # nRF52 Adafruit
"host", # Host platform (development/testing)
]
# Get actually changed files (not dependencies)
files = changed_files(branch)
# Find all changed components (excluding core)
# Find all changed components (excluding core and base bus components)
changed_component_set = set()
for file in files:
@@ -291,49 +279,53 @@ def detect_single_component_for_memory_impact(
if len(parts) >= 3:
component = parts[2]
# Skip base bus components as they're used across many builds
if component not in ["i2c", "spi", "uart", "modbus"]:
if component not in ["i2c", "spi", "uart", "modbus", "canbus"]:
changed_component_set.add(component)
# Only proceed if exactly one component changed
if len(changed_component_set) != 1:
# If no components changed, don't run memory impact
if not changed_component_set:
return {"should_run": "false"}
component = list(changed_component_set)[0]
# Find components that have tests on the preferred platform
components_with_tests = []
selected_platform = None
# Find a test configuration for this component
tests_dir = Path(root_path) / "tests" / "components" / component
for component in sorted(changed_component_set):
tests_dir = Path(root_path) / "tests" / "components" / component
if not tests_dir.exists():
continue
if not tests_dir.exists():
return {"should_run": "false"}
# Look for test files on preferred platforms
test_files = list(tests_dir.glob("test.*.yaml"))
if not test_files:
continue
# Look for test files
test_files = list(tests_dir.glob("test.*.yaml"))
if not test_files:
return {"should_run": "false"}
# Try each preferred platform in order
for preferred_platform in PLATFORM_PREFERENCE:
# Check if component has tests for any preferred platform
for test_file in test_files:
parts = test_file.stem.split(".")
if len(parts) >= 2:
platform = parts[1]
if platform == preferred_platform:
return {
"should_run": "true",
"component": component,
"test_file": test_file.name,
"platform": platform,
}
if platform in PLATFORM_PREFERENCE:
components_with_tests.append(component)
# Select the most preferred platform across all components
if selected_platform is None or PLATFORM_PREFERENCE.index(
platform
) < PLATFORM_PREFERENCE.index(selected_platform):
selected_platform = platform
break
# If no components have tests, don't run memory impact
if not components_with_tests:
return {"should_run": "false"}
# Use the most preferred platform found, or fall back to esp8266-ard
platform = selected_platform or "esp8266-ard"
# Fall back to first test file
test_file = test_files[0]
parts = test_file.stem.split(".")
platform = parts[1] if len(parts) >= 2 else "esp32-idf"
return {
"should_run": "true",
"component": component,
"test_file": test_file.name,
"components": components_with_tests,
"platform": platform,
"use_merged_config": "true",
}
@@ -386,8 +378,8 @@ def main() -> None:
if component not in directly_changed_components
]
# Detect single component change for memory impact analysis
memory_impact = detect_single_component_for_memory_impact(args.branch)
# Detect components for memory impact analysis (merged config)
memory_impact = detect_memory_impact_config(args.branch)
# Build output
output: dict[str, Any] = {