From c4eeed7f7e28f5590ea5eaab365a01bcede20fb0 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Tue, 14 Oct 2025 13:05:02 -1000 Subject: [PATCH 01/76] [ci] Automatic Flash/RAM impact analysis --- .github/workflows/memory-impact.yml | 150 +++++++++++++++++ script/ci_helpers.py | 23 +++ script/ci_memory_impact_comment.py | 244 ++++++++++++++++++++++++++++ script/ci_memory_impact_detector.py | 134 +++++++++++++++ script/ci_memory_impact_extract.py | 104 ++++++++++++ 5 files changed, 655 insertions(+) create mode 100644 .github/workflows/memory-impact.yml create mode 100755 script/ci_helpers.py create mode 100755 script/ci_memory_impact_comment.py create mode 100755 script/ci_memory_impact_detector.py create mode 100755 script/ci_memory_impact_extract.py diff --git a/.github/workflows/memory-impact.yml b/.github/workflows/memory-impact.yml new file mode 100644 index 0000000000..dff73e6cd7 --- /dev/null +++ b/.github/workflows/memory-impact.yml @@ -0,0 +1,150 @@ +--- +name: Memory Impact Analysis + +on: + pull_request: + paths: + - "esphome/components/**" + - "esphome/core/**" + +permissions: + contents: read + pull-requests: write + +env: + DEFAULT_PYTHON: "3.11" + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number }} + cancel-in-progress: true + +jobs: + detect-single-component: + name: Detect single component change + runs-on: ubuntu-24.04 + outputs: + should_run: ${{ steps.detect.outputs.should_run }} + component: ${{ steps.detect.outputs.component }} + test_file: ${{ steps.detect.outputs.test_file }} + platform: ${{ steps.detect.outputs.platform }} + steps: + - name: Check out code from GitHub + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + with: + fetch-depth: 0 + - name: Set up Python ${{ env.DEFAULT_PYTHON }} + uses: actions/setup-python@e797f83bcb11b83ae66e0230d6156d7c80228e7c # v6.0.0 + with: + python-version: ${{ env.DEFAULT_PYTHON }} + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install PyYAML + - name: Detect single component change + id: detect + run: | + python script/ci_memory_impact_detector.py + + build-target-branch: + name: Build target branch + runs-on: ubuntu-24.04 + needs: detect-single-component + if: needs.detect-single-component.outputs.should_run == 'true' + outputs: + ram_usage: ${{ steps.extract.outputs.ram_usage }} + flash_usage: ${{ steps.extract.outputs.flash_usage }} + steps: + - name: Check out target branch + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + with: + ref: ${{ github.base_ref }} + - name: Set up Python ${{ env.DEFAULT_PYTHON }} + uses: actions/setup-python@e797f83bcb11b83ae66e0230d6156d7c80228e7c # v6.0.0 + with: + python-version: ${{ env.DEFAULT_PYTHON }} + - name: Install ESPHome + run: | + pip install -e . + - name: Cache platformio + uses: actions/cache/restore@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 + with: + path: ~/.platformio + key: platformio-memory-${{ needs.detect-single-component.outputs.platform }}-${{ hashFiles('platformio.ini') }} + - name: Compile test configuration and extract memory usage + id: extract + run: | + component="${{ needs.detect-single-component.outputs.component }}" + platform="${{ needs.detect-single-component.outputs.platform }}" + test_file="${{ needs.detect-single-component.outputs.test_file }}" + + echo "Compiling $component for $platform using $test_file" + python script/test_build_components.py -e compile -c "$component" -t "$platform" --no-grouping 2>&1 | \ + python script/ci_memory_impact_extract.py --output-env + + build-pr-branch: + name: Build PR branch + runs-on: ubuntu-24.04 + needs: detect-single-component + if: needs.detect-single-component.outputs.should_run == 'true' + outputs: + ram_usage: ${{ steps.extract.outputs.ram_usage }} + flash_usage: ${{ steps.extract.outputs.flash_usage }} + steps: + - name: Check out PR branch + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + - name: Set up Python ${{ env.DEFAULT_PYTHON }} + uses: actions/setup-python@e797f83bcb11b83ae66e0230d6156d7c80228e7c # v6.0.0 + with: + python-version: ${{ env.DEFAULT_PYTHON }} + - name: Install ESPHome + run: | + pip install -e . + - name: Cache platformio + uses: actions/cache/restore@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 + with: + path: ~/.platformio + key: platformio-memory-${{ needs.detect-single-component.outputs.platform }}-${{ hashFiles('platformio.ini') }} + - name: Compile test configuration and extract memory usage + id: extract + run: | + component="${{ needs.detect-single-component.outputs.component }}" + platform="${{ needs.detect-single-component.outputs.platform }}" + test_file="${{ needs.detect-single-component.outputs.test_file }}" + + echo "Compiling $component for $platform using $test_file" + python script/test_build_components.py -e compile -c "$component" -t "$platform" --no-grouping 2>&1 | \ + python script/ci_memory_impact_extract.py --output-env + + comment-results: + name: Comment memory impact + runs-on: ubuntu-24.04 + needs: + - detect-single-component + - build-target-branch + - build-pr-branch + if: needs.detect-single-component.outputs.should_run == 'true' + steps: + - name: Check out code + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + - name: Set up Python ${{ env.DEFAULT_PYTHON }} + uses: actions/setup-python@e797f83bcb11b83ae66e0230d6156d7c80228e7c # v6.0.0 + with: + python-version: ${{ env.DEFAULT_PYTHON }} + - name: Post or update PR comment + env: + GH_TOKEN: ${{ github.token }} + COMPONENT: ${{ needs.detect-single-component.outputs.component }} + PLATFORM: ${{ needs.detect-single-component.outputs.platform }} + TARGET_RAM: ${{ needs.build-target-branch.outputs.ram_usage }} + TARGET_FLASH: ${{ needs.build-target-branch.outputs.flash_usage }} + PR_RAM: ${{ needs.build-pr-branch.outputs.ram_usage }} + PR_FLASH: ${{ needs.build-pr-branch.outputs.flash_usage }} + run: | + python script/ci_memory_impact_comment.py \ + --pr-number "${{ github.event.pull_request.number }}" \ + --component "$COMPONENT" \ + --platform "$PLATFORM" \ + --target-ram "$TARGET_RAM" \ + --target-flash "$TARGET_FLASH" \ + --pr-ram "$PR_RAM" \ + --pr-flash "$PR_FLASH" diff --git a/script/ci_helpers.py b/script/ci_helpers.py new file mode 100755 index 0000000000..48b0e4bbfe --- /dev/null +++ b/script/ci_helpers.py @@ -0,0 +1,23 @@ +"""Common helper functions for CI scripts.""" + +from __future__ import annotations + +import os + + +def write_github_output(outputs: dict[str, str | int]) -> None: + """Write multiple outputs to GITHUB_OUTPUT or stdout. + + When running in GitHub Actions, writes to the GITHUB_OUTPUT file. + When running locally, writes to stdout for debugging. + + Args: + outputs: Dictionary of key-value pairs to write + """ + github_output = os.environ.get("GITHUB_OUTPUT") + if github_output: + with open(github_output, "a", encoding="utf-8") as f: + f.writelines(f"{key}={value}\n" for key, value in outputs.items()) + else: + for key, value in outputs.items(): + print(f"{key}={value}") diff --git a/script/ci_memory_impact_comment.py b/script/ci_memory_impact_comment.py new file mode 100755 index 0000000000..69f703bd78 --- /dev/null +++ b/script/ci_memory_impact_comment.py @@ -0,0 +1,244 @@ +#!/usr/bin/env python3 +"""Post or update a PR comment with memory impact analysis results. + +This script creates or updates a GitHub PR comment with memory usage changes. +It uses the GitHub CLI (gh) to manage comments and maintains a single comment +that gets updated on subsequent runs. +""" + +from __future__ import annotations + +import argparse +import json +import subprocess +import sys + +# Comment marker to identify our memory impact comments +COMMENT_MARKER = "" + + +def format_bytes(bytes_value: int) -> str: + """Format bytes value with appropriate unit. + + Args: + bytes_value: Number of bytes + + Returns: + Formatted string (e.g., "1.5 KB", "256 bytes") + """ + if bytes_value < 1024: + return f"{bytes_value} bytes" + if bytes_value < 1024 * 1024: + return f"{bytes_value / 1024:.2f} KB" + return f"{bytes_value / (1024 * 1024):.2f} MB" + + +def format_change(before: int, after: int) -> str: + """Format memory change with delta and percentage. + + Args: + before: Memory usage before change + after: Memory usage after change + + Returns: + Formatted string with delta and percentage + """ + delta = after - before + percentage = 0.0 if before == 0 else (delta / before) * 100 + + # Format delta with sign + delta_str = f"+{format_bytes(delta)}" if delta >= 0 else format_bytes(delta) + + # Format percentage with sign + if percentage > 0: + pct_str = f"+{percentage:.2f}%" + elif percentage < 0: + pct_str = f"{percentage:.2f}%" + else: + pct_str = "0.00%" + + # Add emoji indicator + if delta > 0: + emoji = "📈" + elif delta < 0: + emoji = "📉" + else: + emoji = "➡️" + + return f"{emoji} {delta_str} ({pct_str})" + + +def create_comment_body( + component: str, + platform: str, + target_ram: int, + target_flash: int, + pr_ram: int, + pr_flash: int, +) -> str: + """Create the comment body with memory impact analysis. + + Args: + component: Component name + platform: Platform name + target_ram: RAM usage in target branch + target_flash: Flash usage in target branch + pr_ram: RAM usage in PR branch + pr_flash: Flash usage in PR branch + + Returns: + Formatted comment body + """ + ram_change = format_change(target_ram, pr_ram) + flash_change = format_change(target_flash, pr_flash) + + return f"""{COMMENT_MARKER} +## Memory Impact Analysis + +**Component:** `{component}` +**Platform:** `{platform}` + +| Metric | Target Branch | This PR | Change | +|--------|--------------|---------|--------| +| **RAM** | {format_bytes(target_ram)} | {format_bytes(pr_ram)} | {ram_change} | +| **Flash** | {format_bytes(target_flash)} | {format_bytes(pr_flash)} | {flash_change} | + +--- +*This analysis runs automatically when a single component changes. Memory usage is measured from a representative test configuration.* +""" + + +def find_existing_comment(pr_number: str) -> str | None: + """Find existing memory impact comment on the PR. + + Args: + pr_number: PR number + + Returns: + Comment ID if found, None otherwise + """ + try: + # List all comments on the PR + result = subprocess.run( + [ + "gh", + "pr", + "view", + pr_number, + "--json", + "comments", + "--jq", + ".comments[]", + ], + capture_output=True, + text=True, + check=True, + ) + + # Parse comments and look for our marker + for line in result.stdout.strip().split("\n"): + if not line: + continue + + try: + comment = json.loads(line) + if COMMENT_MARKER in comment.get("body", ""): + return str(comment["id"]) + except json.JSONDecodeError: + continue + + return None + + except subprocess.CalledProcessError as e: + print(f"Error finding existing comment: {e}", file=sys.stderr) + return None + + +def post_or_update_comment(pr_number: str, comment_body: str) -> bool: + """Post a new comment or update existing one. + + Args: + pr_number: PR number + comment_body: Comment body text + + Returns: + True if successful, False otherwise + """ + # Look for existing comment + existing_comment_id = find_existing_comment(pr_number) + + try: + if existing_comment_id: + # Update existing comment + print(f"Updating existing comment {existing_comment_id}", file=sys.stderr) + subprocess.run( + [ + "gh", + "api", + f"/repos/{{owner}}/{{repo}}/issues/comments/{existing_comment_id}", + "-X", + "PATCH", + "-f", + f"body={comment_body}", + ], + check=True, + capture_output=True, + ) + else: + # Post new comment + print("Posting new comment", file=sys.stderr) + subprocess.run( + ["gh", "pr", "comment", pr_number, "--body", comment_body], + check=True, + capture_output=True, + ) + + print("Comment posted/updated successfully", file=sys.stderr) + return True + + except subprocess.CalledProcessError as e: + print(f"Error posting/updating comment: {e}", file=sys.stderr) + if e.stderr: + print(f"stderr: {e.stderr.decode()}", file=sys.stderr) + return False + + +def main() -> int: + """Main entry point.""" + parser = argparse.ArgumentParser( + description="Post or update PR comment with memory impact analysis" + ) + parser.add_argument("--pr-number", required=True, help="PR number") + parser.add_argument("--component", required=True, help="Component name") + parser.add_argument("--platform", required=True, help="Platform name") + parser.add_argument( + "--target-ram", type=int, required=True, help="Target branch RAM usage" + ) + parser.add_argument( + "--target-flash", type=int, required=True, help="Target branch flash usage" + ) + parser.add_argument("--pr-ram", type=int, required=True, help="PR branch RAM usage") + parser.add_argument( + "--pr-flash", type=int, required=True, help="PR branch flash usage" + ) + + args = parser.parse_args() + + # Create comment body + comment_body = create_comment_body( + component=args.component, + platform=args.platform, + target_ram=args.target_ram, + target_flash=args.target_flash, + pr_ram=args.pr_ram, + pr_flash=args.pr_flash, + ) + + # Post or update comment + success = post_or_update_comment(args.pr_number, comment_body) + + return 0 if success else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/script/ci_memory_impact_detector.py b/script/ci_memory_impact_detector.py new file mode 100755 index 0000000000..8c3045ab00 --- /dev/null +++ b/script/ci_memory_impact_detector.py @@ -0,0 +1,134 @@ +#!/usr/bin/env python3 +"""Detect if a PR changes exactly one component for memory impact analysis. + +This script is used by the CI workflow to determine if a PR should trigger +memory impact analysis. The analysis only runs when: +1. Exactly one component has changed (not counting core changes) +2. The component has at least one test configuration + +The script outputs GitHub Actions environment variables to control the workflow. +""" + +from __future__ import annotations + +from pathlib import Path +import sys + +# Add esphome to path +sys.path.insert(0, str(Path(__file__).parent.parent)) + +# pylint: disable=wrong-import-position +from script.ci_helpers import write_github_output +from script.helpers import ESPHOME_COMPONENTS_PATH, changed_files + +# Platform preference order for memory impact analysis +# Ordered by production relevance and memory constraint importance +PLATFORM_PREFERENCE = [ + "esp32-idf", # Primary ESP32 IDF platform + "esp32-c3-idf", # ESP32-C3 IDF + "esp32-c6-idf", # ESP32-C6 IDF + "esp32-s2-idf", # ESP32-S2 IDF + "esp32-s3-idf", # ESP32-S3 IDF + "esp32-c2-idf", # ESP32-C2 IDF + "esp32-c5-idf", # ESP32-C5 IDF + "esp32-h2-idf", # ESP32-H2 IDF + "esp32-p4-idf", # ESP32-P4 IDF + "esp8266-ard", # ESP8266 Arduino (memory constrained) + "esp32-ard", # ESP32 Arduino + "esp32-c3-ard", # ESP32-C3 Arduino + "esp32-s2-ard", # ESP32-S2 Arduino + "esp32-s3-ard", # ESP32-S3 Arduino + "bk72xx-ard", # BK72xx Arduino + "rp2040-ard", # RP2040 Arduino + "nrf52-adafruit", # nRF52 Adafruit + "host", # Host platform (development/testing) +] + + +def find_test_for_component(component: str) -> tuple[str | None, str | None]: + """Find a test configuration for the given component. + + Prefers platforms based on PLATFORM_PREFERENCE order. + + Args: + component: Component name + + Returns: + Tuple of (test_file_name, platform) or (None, None) if no test found + """ + tests_dir = Path(__file__).parent.parent / "tests" / "components" / component + + if not tests_dir.exists(): + return None, None + + # Look for test files + test_files = list(tests_dir.glob("test.*.yaml")) + if not test_files: + return None, None + + # Try each preferred platform in order + for preferred_platform in PLATFORM_PREFERENCE: + for test_file in test_files: + parts = test_file.stem.split(".") + if len(parts) >= 2: + platform = parts[1] + if platform == preferred_platform: + return test_file.name, platform + + # Fall back to first test file + test_file = test_files[0] + parts = test_file.stem.split(".") + platform = parts[1] if len(parts) >= 2 else "esp32-idf" + return test_file.name, platform + + +def detect_single_component_change() -> None: + """Detect if exactly one component changed and output GitHub Actions variables.""" + files = changed_files() + + # Find all changed components (excluding core) + changed_components = set() + + for file in files: + if file.startswith(ESPHOME_COMPONENTS_PATH): + parts = file.split("/") + if len(parts) >= 3: + component = parts[2] + # Skip base bus components as they're used across many builds + if component not in ["i2c", "spi", "uart", "modbus"]: + changed_components.add(component) + + # Only proceed if exactly one component changed + if len(changed_components) != 1: + print( + f"Found {len(changed_components)} component(s) changed, skipping memory analysis" + ) + write_github_output({"should_run": "false"}) + return + + component = list(changed_components)[0] + print(f"Detected single component change: {component}") + + # Find a test configuration for this component + test_file, platform = find_test_for_component(component) + + if not test_file: + print(f"No test configuration found for {component}, skipping memory analysis") + write_github_output({"should_run": "false"}) + return + + print(f"Found test: {test_file} for platform: {platform}") + print("Memory impact analysis will run") + + write_github_output( + { + "should_run": "true", + "component": component, + "test_file": test_file, + "platform": platform, + } + ) + + +if __name__ == "__main__": + detect_single_component_change() diff --git a/script/ci_memory_impact_extract.py b/script/ci_memory_impact_extract.py new file mode 100755 index 0000000000..9ddd39096f --- /dev/null +++ b/script/ci_memory_impact_extract.py @@ -0,0 +1,104 @@ +#!/usr/bin/env python3 +"""Extract memory usage statistics from ESPHome build output. + +This script parses the PlatformIO build output to extract RAM and flash +usage statistics for a compiled component. It's used by the CI workflow to +compare memory usage between branches. + +The script reads compile output from stdin and looks for the standard +PlatformIO output format: + RAM: [==== ] 36.1% (used 29548 bytes from 81920 bytes) + Flash: [=== ] 34.0% (used 348511 bytes from 1023984 bytes) +""" + +from __future__ import annotations + +import argparse +from pathlib import Path +import re +import sys + +# Add esphome to path +sys.path.insert(0, str(Path(__file__).parent.parent)) + +# pylint: disable=wrong-import-position +from script.ci_helpers import write_github_output + + +def extract_from_compile_output(output_text: str) -> tuple[int | None, int | None]: + """Extract memory usage from PlatformIO compile output. + + Looks for lines like: + RAM: [==== ] 36.1% (used 29548 bytes from 81920 bytes) + Flash: [=== ] 34.0% (used 348511 bytes from 1023984 bytes) + + Args: + output_text: Compile output text + + Returns: + Tuple of (ram_bytes, flash_bytes) or (None, None) if not found + """ + ram_match = re.search( + r"RAM:\s+\[.*?\]\s+\d+\.\d+%\s+\(used\s+(\d+)\s+bytes", output_text + ) + flash_match = re.search( + r"Flash:\s+\[.*?\]\s+\d+\.\d+%\s+\(used\s+(\d+)\s+bytes", output_text + ) + + if ram_match and flash_match: + return int(ram_match.group(1)), int(flash_match.group(1)) + + return None, None + + +def main() -> int: + """Main entry point.""" + parser = argparse.ArgumentParser( + description="Extract memory usage from ESPHome build output" + ) + parser.add_argument( + "--output-env", + action="store_true", + help="Output to GITHUB_OUTPUT environment file", + ) + + args = parser.parse_args() + + # Read compile output from stdin + compile_output = sys.stdin.read() + + # Extract memory usage + ram_bytes, flash_bytes = extract_from_compile_output(compile_output) + + if ram_bytes is None or flash_bytes is None: + print("Failed to extract memory usage from compile output", file=sys.stderr) + print("Expected lines like:", file=sys.stderr) + print( + " RAM: [==== ] 36.1% (used 29548 bytes from 81920 bytes)", + file=sys.stderr, + ) + print( + " Flash: [=== ] 34.0% (used 348511 bytes from 1023984 bytes)", + file=sys.stderr, + ) + return 1 + + print(f"RAM: {ram_bytes} bytes", file=sys.stderr) + print(f"Flash: {flash_bytes} bytes", file=sys.stderr) + + if args.output_env: + # Output to GitHub Actions + write_github_output( + { + "ram_usage": ram_bytes, + "flash_usage": flash_bytes, + } + ) + else: + print(f"{ram_bytes},{flash_bytes}") + + return 0 + + +if __name__ == "__main__": + sys.exit(main()) From 25a6202bb9cc86e2ed8258f17ff98a476dbda2cc Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Tue, 14 Oct 2025 13:09:01 -1000 Subject: [PATCH 02/76] [ci] Automatic Flash/RAM impact analysis --- .github/workflows/ci.yml | 117 ++++++++++++++++++++++ .github/workflows/memory-impact.yml | 150 ---------------------------- script/ci_memory_impact_detector.py | 134 ------------------------- script/determine-jobs.py | 99 +++++++++++++++++- 4 files changed, 215 insertions(+), 285 deletions(-) delete mode 100644 .github/workflows/memory-impact.yml delete mode 100755 script/ci_memory_impact_detector.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0363b5afdf..7a731a1b02 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -179,6 +179,7 @@ jobs: changed-components: ${{ steps.determine.outputs.changed-components }} changed-components-with-tests: ${{ steps.determine.outputs.changed-components-with-tests }} component-test-count: ${{ steps.determine.outputs.component-test-count }} + memory_impact: ${{ steps.determine.outputs.memory-impact }} steps: - name: Check out code from GitHub uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 @@ -207,6 +208,7 @@ jobs: echo "changed-components=$(echo "$output" | jq -c '.changed_components')" >> $GITHUB_OUTPUT echo "changed-components-with-tests=$(echo "$output" | jq -c '.changed_components_with_tests')" >> $GITHUB_OUTPUT echo "component-test-count=$(echo "$output" | jq -r '.component_test_count')" >> $GITHUB_OUTPUT + echo "memory-impact=$(echo "$output" | jq -c '.memory_impact')" >> $GITHUB_OUTPUT integration-tests: name: Run integration tests @@ -510,6 +512,118 @@ jobs: - uses: pre-commit-ci/lite-action@5d6cc0eb514c891a40562a58a8e71576c5c7fb43 # v1.1.0 if: always() + memory-impact-target-branch: + name: Build target branch for memory impact + runs-on: ubuntu-24.04 + needs: + - common + - determine-jobs + if: github.event_name == 'pull_request' && fromJSON(needs.determine-jobs.outputs.memory_impact).should_run == 'true' + outputs: + ram_usage: ${{ steps.extract.outputs.ram_usage }} + flash_usage: ${{ steps.extract.outputs.flash_usage }} + steps: + - name: Check out target branch + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + with: + ref: ${{ github.base_ref }} + - name: Restore Python + uses: ./.github/actions/restore-python + with: + python-version: ${{ env.DEFAULT_PYTHON }} + cache-key: ${{ needs.common.outputs.cache-key }} + - name: Cache platformio + uses: actions/cache/restore@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 + with: + path: ~/.platformio + key: platformio-memory-${{ fromJSON(needs.determine-jobs.outputs.memory_impact).platform }}-${{ hashFiles('platformio.ini') }} + - name: Compile test configuration and extract memory usage + id: extract + run: | + . venv/bin/activate + component="${{ fromJSON(needs.determine-jobs.outputs.memory_impact).component }}" + platform="${{ fromJSON(needs.determine-jobs.outputs.memory_impact).platform }}" + test_file="${{ fromJSON(needs.determine-jobs.outputs.memory_impact).test_file }}" + + echo "Compiling $component for $platform using $test_file" + python script/test_build_components.py -e compile -c "$component" -t "$platform" --no-grouping 2>&1 | \ + python script/ci_memory_impact_extract.py --output-env + + memory-impact-pr-branch: + name: Build PR branch for memory impact + runs-on: ubuntu-24.04 + needs: + - common + - determine-jobs + if: github.event_name == 'pull_request' && fromJSON(needs.determine-jobs.outputs.memory_impact).should_run == 'true' + outputs: + ram_usage: ${{ steps.extract.outputs.ram_usage }} + flash_usage: ${{ steps.extract.outputs.flash_usage }} + steps: + - name: Check out PR branch + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + - name: Restore Python + uses: ./.github/actions/restore-python + with: + python-version: ${{ env.DEFAULT_PYTHON }} + cache-key: ${{ needs.common.outputs.cache-key }} + - name: Cache platformio + uses: actions/cache/restore@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 + with: + path: ~/.platformio + key: platformio-memory-${{ fromJSON(needs.determine-jobs.outputs.memory_impact).platform }}-${{ hashFiles('platformio.ini') }} + - name: Compile test configuration and extract memory usage + id: extract + run: | + . venv/bin/activate + component="${{ fromJSON(needs.determine-jobs.outputs.memory_impact).component }}" + platform="${{ fromJSON(needs.determine-jobs.outputs.memory_impact).platform }}" + test_file="${{ fromJSON(needs.determine-jobs.outputs.memory_impact).test_file }}" + + echo "Compiling $component for $platform using $test_file" + python script/test_build_components.py -e compile -c "$component" -t "$platform" --no-grouping 2>&1 | \ + python script/ci_memory_impact_extract.py --output-env + + memory-impact-comment: + name: Comment memory impact + runs-on: ubuntu-24.04 + needs: + - common + - determine-jobs + - memory-impact-target-branch + - memory-impact-pr-branch + if: github.event_name == 'pull_request' && fromJSON(needs.determine-jobs.outputs.memory_impact).should_run == 'true' + permissions: + contents: read + pull-requests: write + steps: + - name: Check out code + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + - name: Restore Python + uses: ./.github/actions/restore-python + with: + python-version: ${{ env.DEFAULT_PYTHON }} + cache-key: ${{ needs.common.outputs.cache-key }} + - name: Post or update PR comment + env: + GH_TOKEN: ${{ github.token }} + COMPONENT: ${{ fromJSON(needs.determine-jobs.outputs.memory_impact).component }} + PLATFORM: ${{ fromJSON(needs.determine-jobs.outputs.memory_impact).platform }} + TARGET_RAM: ${{ needs.memory-impact-target-branch.outputs.ram_usage }} + TARGET_FLASH: ${{ needs.memory-impact-target-branch.outputs.flash_usage }} + PR_RAM: ${{ needs.memory-impact-pr-branch.outputs.ram_usage }} + PR_FLASH: ${{ needs.memory-impact-pr-branch.outputs.flash_usage }} + run: | + . venv/bin/activate + python script/ci_memory_impact_comment.py \ + --pr-number "${{ github.event.pull_request.number }}" \ + --component "$COMPONENT" \ + --platform "$PLATFORM" \ + --target-ram "$TARGET_RAM" \ + --target-flash "$TARGET_FLASH" \ + --pr-ram "$PR_RAM" \ + --pr-flash "$PR_FLASH" + ci-status: name: CI Status runs-on: ubuntu-24.04 @@ -525,6 +639,9 @@ jobs: - test-build-components-splitter - test-build-components-split - pre-commit-ci-lite + - memory-impact-target-branch + - memory-impact-pr-branch + - memory-impact-comment if: always() steps: - name: Success diff --git a/.github/workflows/memory-impact.yml b/.github/workflows/memory-impact.yml deleted file mode 100644 index dff73e6cd7..0000000000 --- a/.github/workflows/memory-impact.yml +++ /dev/null @@ -1,150 +0,0 @@ ---- -name: Memory Impact Analysis - -on: - pull_request: - paths: - - "esphome/components/**" - - "esphome/core/**" - -permissions: - contents: read - pull-requests: write - -env: - DEFAULT_PYTHON: "3.11" - -concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number }} - cancel-in-progress: true - -jobs: - detect-single-component: - name: Detect single component change - runs-on: ubuntu-24.04 - outputs: - should_run: ${{ steps.detect.outputs.should_run }} - component: ${{ steps.detect.outputs.component }} - test_file: ${{ steps.detect.outputs.test_file }} - platform: ${{ steps.detect.outputs.platform }} - steps: - - name: Check out code from GitHub - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 - with: - fetch-depth: 0 - - name: Set up Python ${{ env.DEFAULT_PYTHON }} - uses: actions/setup-python@e797f83bcb11b83ae66e0230d6156d7c80228e7c # v6.0.0 - with: - python-version: ${{ env.DEFAULT_PYTHON }} - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install PyYAML - - name: Detect single component change - id: detect - run: | - python script/ci_memory_impact_detector.py - - build-target-branch: - name: Build target branch - runs-on: ubuntu-24.04 - needs: detect-single-component - if: needs.detect-single-component.outputs.should_run == 'true' - outputs: - ram_usage: ${{ steps.extract.outputs.ram_usage }} - flash_usage: ${{ steps.extract.outputs.flash_usage }} - steps: - - name: Check out target branch - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 - with: - ref: ${{ github.base_ref }} - - name: Set up Python ${{ env.DEFAULT_PYTHON }} - uses: actions/setup-python@e797f83bcb11b83ae66e0230d6156d7c80228e7c # v6.0.0 - with: - python-version: ${{ env.DEFAULT_PYTHON }} - - name: Install ESPHome - run: | - pip install -e . - - name: Cache platformio - uses: actions/cache/restore@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 - with: - path: ~/.platformio - key: platformio-memory-${{ needs.detect-single-component.outputs.platform }}-${{ hashFiles('platformio.ini') }} - - name: Compile test configuration and extract memory usage - id: extract - run: | - component="${{ needs.detect-single-component.outputs.component }}" - platform="${{ needs.detect-single-component.outputs.platform }}" - test_file="${{ needs.detect-single-component.outputs.test_file }}" - - echo "Compiling $component for $platform using $test_file" - python script/test_build_components.py -e compile -c "$component" -t "$platform" --no-grouping 2>&1 | \ - python script/ci_memory_impact_extract.py --output-env - - build-pr-branch: - name: Build PR branch - runs-on: ubuntu-24.04 - needs: detect-single-component - if: needs.detect-single-component.outputs.should_run == 'true' - outputs: - ram_usage: ${{ steps.extract.outputs.ram_usage }} - flash_usage: ${{ steps.extract.outputs.flash_usage }} - steps: - - name: Check out PR branch - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 - - name: Set up Python ${{ env.DEFAULT_PYTHON }} - uses: actions/setup-python@e797f83bcb11b83ae66e0230d6156d7c80228e7c # v6.0.0 - with: - python-version: ${{ env.DEFAULT_PYTHON }} - - name: Install ESPHome - run: | - pip install -e . - - name: Cache platformio - uses: actions/cache/restore@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 - with: - path: ~/.platformio - key: platformio-memory-${{ needs.detect-single-component.outputs.platform }}-${{ hashFiles('platformio.ini') }} - - name: Compile test configuration and extract memory usage - id: extract - run: | - component="${{ needs.detect-single-component.outputs.component }}" - platform="${{ needs.detect-single-component.outputs.platform }}" - test_file="${{ needs.detect-single-component.outputs.test_file }}" - - echo "Compiling $component for $platform using $test_file" - python script/test_build_components.py -e compile -c "$component" -t "$platform" --no-grouping 2>&1 | \ - python script/ci_memory_impact_extract.py --output-env - - comment-results: - name: Comment memory impact - runs-on: ubuntu-24.04 - needs: - - detect-single-component - - build-target-branch - - build-pr-branch - if: needs.detect-single-component.outputs.should_run == 'true' - steps: - - name: Check out code - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 - - name: Set up Python ${{ env.DEFAULT_PYTHON }} - uses: actions/setup-python@e797f83bcb11b83ae66e0230d6156d7c80228e7c # v6.0.0 - with: - python-version: ${{ env.DEFAULT_PYTHON }} - - name: Post or update PR comment - env: - GH_TOKEN: ${{ github.token }} - COMPONENT: ${{ needs.detect-single-component.outputs.component }} - PLATFORM: ${{ needs.detect-single-component.outputs.platform }} - TARGET_RAM: ${{ needs.build-target-branch.outputs.ram_usage }} - TARGET_FLASH: ${{ needs.build-target-branch.outputs.flash_usage }} - PR_RAM: ${{ needs.build-pr-branch.outputs.ram_usage }} - PR_FLASH: ${{ needs.build-pr-branch.outputs.flash_usage }} - run: | - python script/ci_memory_impact_comment.py \ - --pr-number "${{ github.event.pull_request.number }}" \ - --component "$COMPONENT" \ - --platform "$PLATFORM" \ - --target-ram "$TARGET_RAM" \ - --target-flash "$TARGET_FLASH" \ - --pr-ram "$PR_RAM" \ - --pr-flash "$PR_FLASH" diff --git a/script/ci_memory_impact_detector.py b/script/ci_memory_impact_detector.py deleted file mode 100755 index 8c3045ab00..0000000000 --- a/script/ci_memory_impact_detector.py +++ /dev/null @@ -1,134 +0,0 @@ -#!/usr/bin/env python3 -"""Detect if a PR changes exactly one component for memory impact analysis. - -This script is used by the CI workflow to determine if a PR should trigger -memory impact analysis. The analysis only runs when: -1. Exactly one component has changed (not counting core changes) -2. The component has at least one test configuration - -The script outputs GitHub Actions environment variables to control the workflow. -""" - -from __future__ import annotations - -from pathlib import Path -import sys - -# Add esphome to path -sys.path.insert(0, str(Path(__file__).parent.parent)) - -# pylint: disable=wrong-import-position -from script.ci_helpers import write_github_output -from script.helpers import ESPHOME_COMPONENTS_PATH, changed_files - -# Platform preference order for memory impact analysis -# Ordered by production relevance and memory constraint importance -PLATFORM_PREFERENCE = [ - "esp32-idf", # Primary ESP32 IDF platform - "esp32-c3-idf", # ESP32-C3 IDF - "esp32-c6-idf", # ESP32-C6 IDF - "esp32-s2-idf", # ESP32-S2 IDF - "esp32-s3-idf", # ESP32-S3 IDF - "esp32-c2-idf", # ESP32-C2 IDF - "esp32-c5-idf", # ESP32-C5 IDF - "esp32-h2-idf", # ESP32-H2 IDF - "esp32-p4-idf", # ESP32-P4 IDF - "esp8266-ard", # ESP8266 Arduino (memory constrained) - "esp32-ard", # ESP32 Arduino - "esp32-c3-ard", # ESP32-C3 Arduino - "esp32-s2-ard", # ESP32-S2 Arduino - "esp32-s3-ard", # ESP32-S3 Arduino - "bk72xx-ard", # BK72xx Arduino - "rp2040-ard", # RP2040 Arduino - "nrf52-adafruit", # nRF52 Adafruit - "host", # Host platform (development/testing) -] - - -def find_test_for_component(component: str) -> tuple[str | None, str | None]: - """Find a test configuration for the given component. - - Prefers platforms based on PLATFORM_PREFERENCE order. - - Args: - component: Component name - - Returns: - Tuple of (test_file_name, platform) or (None, None) if no test found - """ - tests_dir = Path(__file__).parent.parent / "tests" / "components" / component - - if not tests_dir.exists(): - return None, None - - # Look for test files - test_files = list(tests_dir.glob("test.*.yaml")) - if not test_files: - return None, None - - # Try each preferred platform in order - for preferred_platform in PLATFORM_PREFERENCE: - for test_file in test_files: - parts = test_file.stem.split(".") - if len(parts) >= 2: - platform = parts[1] - if platform == preferred_platform: - return test_file.name, platform - - # Fall back to first test file - test_file = test_files[0] - parts = test_file.stem.split(".") - platform = parts[1] if len(parts) >= 2 else "esp32-idf" - return test_file.name, platform - - -def detect_single_component_change() -> None: - """Detect if exactly one component changed and output GitHub Actions variables.""" - files = changed_files() - - # Find all changed components (excluding core) - changed_components = set() - - for file in files: - if file.startswith(ESPHOME_COMPONENTS_PATH): - parts = file.split("/") - if len(parts) >= 3: - component = parts[2] - # Skip base bus components as they're used across many builds - if component not in ["i2c", "spi", "uart", "modbus"]: - changed_components.add(component) - - # Only proceed if exactly one component changed - if len(changed_components) != 1: - print( - f"Found {len(changed_components)} component(s) changed, skipping memory analysis" - ) - write_github_output({"should_run": "false"}) - return - - component = list(changed_components)[0] - print(f"Detected single component change: {component}") - - # Find a test configuration for this component - test_file, platform = find_test_for_component(component) - - if not test_file: - print(f"No test configuration found for {component}, skipping memory analysis") - write_github_output({"should_run": "false"}) - return - - print(f"Found test: {test_file} for platform: {platform}") - print("Memory impact analysis will run") - - write_github_output( - { - "should_run": "true", - "component": component, - "test_file": test_file, - "platform": platform, - } - ) - - -if __name__ == "__main__": - detect_single_component_change() diff --git a/script/determine-jobs.py b/script/determine-jobs.py index a078fd8f9b..78fd32c3f4 100755 --- a/script/determine-jobs.py +++ b/script/determine-jobs.py @@ -10,7 +10,13 @@ what files have changed. It outputs JSON with the following structure: "clang_format": true/false, "python_linters": true/false, "changed_components": ["component1", "component2", ...], - "component_test_count": 5 + "component_test_count": 5, + "memory_impact": { + "should_run": "true/false", + "component": "component_name", + "test_file": "test.esp32-idf.yaml", + "platform": "esp32-idf" + } } The CI workflow uses this information to: @@ -20,6 +26,7 @@ The CI workflow uses this information to: - Skip or run Python linters (ruff, flake8, pylint, pyupgrade) - Determine which components to test individually - Decide how to split component tests (if there are many) +- Run memory impact analysis when exactly one component changes Usage: python script/determine-jobs.py [-b BRANCH] @@ -212,6 +219,92 @@ def _any_changed_file_endswith(branch: str | None, extensions: tuple[str, ...]) return any(file.endswith(extensions) for file in changed_files(branch)) +def detect_single_component_for_memory_impact( + changed_components: list[str], +) -> dict[str, Any]: + """Detect if exactly one component changed for memory impact analysis. + + Args: + changed_components: List of changed component names + + Returns: + Dictionary with memory impact analysis parameters: + - should_run: "true" or "false" + - component: component name (if should_run is true) + - test_file: test file name (if should_run is true) + - platform: platform name (if should_run is true) + """ + # Platform preference order for memory impact analysis + # Ordered by production relevance and memory constraint importance + PLATFORM_PREFERENCE = [ + "esp32-idf", # Primary ESP32 IDF platform + "esp32-c3-idf", # ESP32-C3 IDF + "esp32-c6-idf", # ESP32-C6 IDF + "esp32-s2-idf", # ESP32-S2 IDF + "esp32-s3-idf", # ESP32-S3 IDF + "esp32-c2-idf", # ESP32-C2 IDF + "esp32-c5-idf", # ESP32-C5 IDF + "esp32-h2-idf", # ESP32-H2 IDF + "esp32-p4-idf", # ESP32-P4 IDF + "esp8266-ard", # ESP8266 Arduino (memory constrained) + "esp32-ard", # ESP32 Arduino + "esp32-c3-ard", # ESP32-C3 Arduino + "esp32-s2-ard", # ESP32-S2 Arduino + "esp32-s3-ard", # ESP32-S3 Arduino + "bk72xx-ard", # BK72xx Arduino + "rp2040-ard", # RP2040 Arduino + "nrf52-adafruit", # nRF52 Adafruit + "host", # Host platform (development/testing) + ] + + # Skip base bus components as they're used across many builds + filtered_components = [ + c for c in changed_components if c not in ["i2c", "spi", "uart", "modbus"] + ] + + # Only proceed if exactly one component changed + if len(filtered_components) != 1: + return {"should_run": "false"} + + component = filtered_components[0] + + # Find a test configuration for this component + tests_dir = Path(root_path) / "tests" / "components" / component + + if not tests_dir.exists(): + return {"should_run": "false"} + + # Look for test files + test_files = list(tests_dir.glob("test.*.yaml")) + if not test_files: + return {"should_run": "false"} + + # Try each preferred platform in order + for preferred_platform in PLATFORM_PREFERENCE: + for test_file in test_files: + parts = test_file.stem.split(".") + if len(parts) >= 2: + platform = parts[1] + if platform == preferred_platform: + return { + "should_run": "true", + "component": component, + "test_file": test_file.name, + "platform": platform, + } + + # Fall back to first test file + test_file = test_files[0] + parts = test_file.stem.split(".") + platform = parts[1] if len(parts) >= 2 else "esp32-idf" + return { + "should_run": "true", + "component": component, + "test_file": test_file.name, + "platform": platform, + } + + def main() -> None: """Main function that determines which CI jobs to run.""" parser = argparse.ArgumentParser( @@ -247,6 +340,9 @@ def main() -> None: and any(component_test_dir.glob("test.*.yaml")) ] + # Detect single component change for memory impact analysis + memory_impact = detect_single_component_for_memory_impact(changed_components) + # Build output output: dict[str, Any] = { "integration_tests": run_integration, @@ -256,6 +352,7 @@ def main() -> None: "changed_components": changed_components, "changed_components_with_tests": changed_components_with_tests, "component_test_count": len(changed_components_with_tests), + "memory_impact": memory_impact, } # Output as JSON From 3bb95a190dc379cd5c7fcbf4b7b0d96f88e84126 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Tue, 14 Oct 2025 13:15:44 -1000 Subject: [PATCH 03/76] fix --- script/determine-jobs.py | 32 +++++++++++++++++++++++--------- 1 file changed, 23 insertions(+), 9 deletions(-) diff --git a/script/determine-jobs.py b/script/determine-jobs.py index 78fd32c3f4..ea43ed71ca 100755 --- a/script/determine-jobs.py +++ b/script/determine-jobs.py @@ -220,12 +220,16 @@ def _any_changed_file_endswith(branch: str | None, extensions: tuple[str, ...]) def detect_single_component_for_memory_impact( - changed_components: list[str], + branch: str | None = None, ) -> dict[str, Any]: """Detect if exactly one component changed for memory impact analysis. + This analyzes the actual changed files (not dependencies) to determine if + exactly one component has been modified. This is different from the + changed_components list which includes all dependencies. + Args: - changed_components: List of changed component names + branch: Branch to compare against Returns: Dictionary with memory impact analysis parameters: @@ -257,16 +261,26 @@ def detect_single_component_for_memory_impact( "host", # Host platform (development/testing) ] - # Skip base bus components as they're used across many builds - filtered_components = [ - c for c in changed_components if c not in ["i2c", "spi", "uart", "modbus"] - ] + # Get actually changed files (not dependencies) + files = changed_files(branch) + + # Find all changed components (excluding core) + changed_component_set = set() + + for file in files: + if file.startswith(ESPHOME_COMPONENTS_PATH): + parts = file.split("/") + if len(parts) >= 3: + component = parts[2] + # Skip base bus components as they're used across many builds + if component not in ["i2c", "spi", "uart", "modbus"]: + changed_component_set.add(component) # Only proceed if exactly one component changed - if len(filtered_components) != 1: + if len(changed_component_set) != 1: return {"should_run": "false"} - component = filtered_components[0] + component = list(changed_component_set)[0] # Find a test configuration for this component tests_dir = Path(root_path) / "tests" / "components" / component @@ -341,7 +355,7 @@ def main() -> None: ] # Detect single component change for memory impact analysis - memory_impact = detect_single_component_for_memory_impact(changed_components) + memory_impact = detect_single_component_for_memory_impact(args.branch) # Build output output: dict[str, Any] = { From daa39a489d9d762d31d636c98613f87cf2a5e298 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Tue, 14 Oct 2025 13:20:31 -1000 Subject: [PATCH 04/76] fix tests --- tests/script/test_determine_jobs.py | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/tests/script/test_determine_jobs.py b/tests/script/test_determine_jobs.py index 5d8746f434..9c8b8d39af 100644 --- a/tests/script/test_determine_jobs.py +++ b/tests/script/test_determine_jobs.py @@ -59,12 +59,22 @@ def mock_subprocess_run() -> Generator[Mock, None, None]: yield mock +@pytest.fixture +def mock_changed_files() -> Generator[Mock, None, None]: + """Mock changed_files for memory impact detection.""" + with patch.object(determine_jobs, "changed_files") as mock: + # Default to empty list + mock.return_value = [] + yield mock + + def test_main_all_tests_should_run( mock_should_run_integration_tests: Mock, mock_should_run_clang_tidy: Mock, mock_should_run_clang_format: Mock, mock_should_run_python_linters: Mock, mock_subprocess_run: Mock, + mock_changed_files: Mock, capsys: pytest.CaptureFixture[str], ) -> None: """Test when all tests should run.""" @@ -98,6 +108,9 @@ def test_main_all_tests_should_run( assert output["component_test_count"] == len( output["changed_components_with_tests"] ) + # memory_impact should be present + assert "memory_impact" in output + assert output["memory_impact"]["should_run"] == "false" # No files changed def test_main_no_tests_should_run( @@ -106,6 +119,7 @@ def test_main_no_tests_should_run( mock_should_run_clang_format: Mock, mock_should_run_python_linters: Mock, mock_subprocess_run: Mock, + mock_changed_files: Mock, capsys: pytest.CaptureFixture[str], ) -> None: """Test when no tests should run.""" @@ -134,6 +148,9 @@ def test_main_no_tests_should_run( assert output["changed_components"] == [] assert output["changed_components_with_tests"] == [] assert output["component_test_count"] == 0 + # memory_impact should be present + assert "memory_impact" in output + assert output["memory_impact"]["should_run"] == "false" def test_main_list_components_fails( @@ -167,6 +184,7 @@ def test_main_with_branch_argument( mock_should_run_clang_format: Mock, mock_should_run_python_linters: Mock, mock_subprocess_run: Mock, + mock_changed_files: Mock, capsys: pytest.CaptureFixture[str], ) -> None: """Test with branch argument.""" @@ -212,6 +230,9 @@ def test_main_with_branch_argument( assert output["component_test_count"] == len( output["changed_components_with_tests"] ) + # memory_impact should be present + assert "memory_impact" in output + assert output["memory_impact"]["should_run"] == "false" def test_should_run_integration_tests( @@ -399,6 +420,7 @@ def test_main_filters_components_without_tests( mock_should_run_clang_format: Mock, mock_should_run_python_linters: Mock, mock_subprocess_run: Mock, + mock_changed_files: Mock, capsys: pytest.CaptureFixture[str], tmp_path: Path, ) -> None: @@ -448,3 +470,6 @@ def test_main_filters_components_without_tests( assert set(output["changed_components_with_tests"]) == {"wifi", "sensor"} # component_test_count should be based on components with tests assert output["component_test_count"] == 2 + # memory_impact should be present + assert "memory_impact" in output + assert output["memory_impact"]["should_run"] == "false" From 5da589abd0db74c20ee33addbc95688d68b5054a Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Tue, 14 Oct 2025 13:27:13 -1000 Subject: [PATCH 05/76] fix --- tests/script/test_determine_jobs.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/tests/script/test_determine_jobs.py b/tests/script/test_determine_jobs.py index 9c8b8d39af..65eef4f785 100644 --- a/tests/script/test_determine_jobs.py +++ b/tests/script/test_determine_jobs.py @@ -76,8 +76,12 @@ def test_main_all_tests_should_run( mock_subprocess_run: Mock, mock_changed_files: Mock, capsys: pytest.CaptureFixture[str], + monkeypatch: pytest.MonkeyPatch, ) -> None: """Test when all tests should run.""" + # Ensure we're not in GITHUB_ACTIONS mode for this test + monkeypatch.delenv("GITHUB_ACTIONS", raising=False) + mock_should_run_integration_tests.return_value = True mock_should_run_clang_tidy.return_value = True mock_should_run_clang_format.return_value = True @@ -121,8 +125,12 @@ def test_main_no_tests_should_run( mock_subprocess_run: Mock, mock_changed_files: Mock, capsys: pytest.CaptureFixture[str], + monkeypatch: pytest.MonkeyPatch, ) -> None: """Test when no tests should run.""" + # Ensure we're not in GITHUB_ACTIONS mode for this test + monkeypatch.delenv("GITHUB_ACTIONS", raising=False) + mock_should_run_integration_tests.return_value = False mock_should_run_clang_tidy.return_value = False mock_should_run_clang_format.return_value = False @@ -186,8 +194,12 @@ def test_main_with_branch_argument( mock_subprocess_run: Mock, mock_changed_files: Mock, capsys: pytest.CaptureFixture[str], + monkeypatch: pytest.MonkeyPatch, ) -> None: """Test with branch argument.""" + # Ensure we're not in GITHUB_ACTIONS mode for this test + monkeypatch.delenv("GITHUB_ACTIONS", raising=False) + mock_should_run_integration_tests.return_value = False mock_should_run_clang_tidy.return_value = True mock_should_run_clang_format.return_value = False @@ -423,8 +435,12 @@ def test_main_filters_components_without_tests( mock_changed_files: Mock, capsys: pytest.CaptureFixture[str], tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, ) -> None: """Test that components without test files are filtered out.""" + # Ensure we're not in GITHUB_ACTIONS mode for this test + monkeypatch.delenv("GITHUB_ACTIONS", raising=False) + mock_should_run_integration_tests.return_value = False mock_should_run_clang_tidy.return_value = False mock_should_run_clang_format.return_value = False From 11f5f7683c51bb28d3c2349230564882847d82e3 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Tue, 14 Oct 2025 13:32:21 -1000 Subject: [PATCH 06/76] tidy --- script/ci_memory_impact_comment.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/script/ci_memory_impact_comment.py b/script/ci_memory_impact_comment.py index 69f703bd78..af8449aa99 100755 --- a/script/ci_memory_impact_comment.py +++ b/script/ci_memory_impact_comment.py @@ -115,10 +115,10 @@ def find_existing_comment(pr_number: str) -> str | None: pr_number: PR number Returns: - Comment ID if found, None otherwise + Comment numeric ID (databaseId) if found, None otherwise """ try: - # List all comments on the PR + # List all comments on the PR with both id (node ID) and databaseId (numeric ID) result = subprocess.run( [ "gh", @@ -128,7 +128,7 @@ def find_existing_comment(pr_number: str) -> str | None: "--json", "comments", "--jq", - ".comments[]", + ".comments[] | {id, databaseId, body}", ], capture_output=True, text=True, @@ -143,7 +143,8 @@ def find_existing_comment(pr_number: str) -> str | None: try: comment = json.loads(line) if COMMENT_MARKER in comment.get("body", ""): - return str(comment["id"]) + # Return the numeric databaseId, not the node ID + return str(comment["databaseId"]) except json.JSONDecodeError: continue From 7b6acd3c002d2dc9ecd258f7d8f1f4dcabd6cf3d Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Tue, 14 Oct 2025 13:33:31 -1000 Subject: [PATCH 07/76] tidy --- script/ci_memory_impact_comment.py | 34 +++++++++++++----------------- 1 file changed, 15 insertions(+), 19 deletions(-) diff --git a/script/ci_memory_impact_comment.py b/script/ci_memory_impact_comment.py index af8449aa99..da962efb11 100755 --- a/script/ci_memory_impact_comment.py +++ b/script/ci_memory_impact_comment.py @@ -18,27 +18,23 @@ COMMENT_MARKER = "" def format_bytes(bytes_value: int) -> str: - """Format bytes value with appropriate unit. + """Format bytes value with comma separators. Args: bytes_value: Number of bytes Returns: - Formatted string (e.g., "1.5 KB", "256 bytes") + Formatted string with comma separators (e.g., "1,234 bytes") """ - if bytes_value < 1024: - return f"{bytes_value} bytes" - if bytes_value < 1024 * 1024: - return f"{bytes_value / 1024:.2f} KB" - return f"{bytes_value / (1024 * 1024):.2f} MB" + return f"{bytes_value:,} bytes" def format_change(before: int, after: int) -> str: """Format memory change with delta and percentage. Args: - before: Memory usage before change - after: Memory usage after change + before: Memory usage before change (in bytes) + after: Memory usage after change (in bytes) Returns: Formatted string with delta and percentage @@ -46,8 +42,16 @@ def format_change(before: int, after: int) -> str: delta = after - before percentage = 0.0 if before == 0 else (delta / before) * 100 - # Format delta with sign - delta_str = f"+{format_bytes(delta)}" if delta >= 0 else format_bytes(delta) + # Format delta with sign and always show in bytes for precision + if delta > 0: + delta_str = f"+{delta:,} bytes" + emoji = "📈" + elif delta < 0: + delta_str = f"{delta:,} bytes" + emoji = "📉" + else: + delta_str = "+0 bytes" + emoji = "➡️" # Format percentage with sign if percentage > 0: @@ -57,14 +61,6 @@ def format_change(before: int, after: int) -> str: else: pct_str = "0.00%" - # Add emoji indicator - if delta > 0: - emoji = "📈" - elif delta < 0: - emoji = "📉" - else: - emoji = "➡️" - return f"{emoji} {delta_str} ({pct_str})" From 354f46f7c0962867727cf296babec08cc7c99a61 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Tue, 14 Oct 2025 13:38:41 -1000 Subject: [PATCH 08/76] debug --- script/ci_memory_impact_comment.py | 63 +++++++++++++++++++++++++----- 1 file changed, 54 insertions(+), 9 deletions(-) diff --git a/script/ci_memory_impact_comment.py b/script/ci_memory_impact_comment.py index da962efb11..804e369efc 100755 --- a/script/ci_memory_impact_comment.py +++ b/script/ci_memory_impact_comment.py @@ -114,6 +114,10 @@ def find_existing_comment(pr_number: str) -> str | None: Comment numeric ID (databaseId) if found, None otherwise """ try: + print( + f"DEBUG: Looking for existing comment on PR #{pr_number}", file=sys.stderr + ) + # List all comments on the PR with both id (node ID) and databaseId (numeric ID) result = subprocess.run( [ @@ -131,23 +135,46 @@ def find_existing_comment(pr_number: str) -> str | None: check=True, ) + print(f"DEBUG: gh pr view output:\n{result.stdout}", file=sys.stderr) + # Parse comments and look for our marker + comment_count = 0 for line in result.stdout.strip().split("\n"): if not line: continue try: comment = json.loads(line) - if COMMENT_MARKER in comment.get("body", ""): + comment_count += 1 + print( + f"DEBUG: Checking comment {comment_count}: id={comment.get('id')}, databaseId={comment.get('databaseId')}", + file=sys.stderr, + ) + + body = comment.get("body", "") + if COMMENT_MARKER in body: + database_id = str(comment["databaseId"]) + print( + f"DEBUG: Found existing comment with databaseId={database_id}", + file=sys.stderr, + ) # Return the numeric databaseId, not the node ID - return str(comment["databaseId"]) - except json.JSONDecodeError: + return database_id + print("DEBUG: Comment does not contain marker", file=sys.stderr) + except json.JSONDecodeError as e: + print(f"DEBUG: JSON decode error: {e}", file=sys.stderr) continue + print( + f"DEBUG: No existing comment found (checked {comment_count} comments)", + file=sys.stderr, + ) return None except subprocess.CalledProcessError as e: print(f"Error finding existing comment: {e}", file=sys.stderr) + if e.stderr: + print(f"stderr: {e.stderr.decode()}", file=sys.stderr) return None @@ -165,10 +192,13 @@ def post_or_update_comment(pr_number: str, comment_body: str) -> bool: existing_comment_id = find_existing_comment(pr_number) try: - if existing_comment_id: + if existing_comment_id and existing_comment_id != "None": # Update existing comment - print(f"Updating existing comment {existing_comment_id}", file=sys.stderr) - subprocess.run( + print( + f"DEBUG: Updating existing comment {existing_comment_id}", + file=sys.stderr, + ) + result = subprocess.run( [ "gh", "api", @@ -180,15 +210,22 @@ def post_or_update_comment(pr_number: str, comment_body: str) -> bool: ], check=True, capture_output=True, + text=True, ) + print(f"DEBUG: Update response: {result.stdout}", file=sys.stderr) else: # Post new comment - print("Posting new comment", file=sys.stderr) - subprocess.run( + print( + f"DEBUG: Posting new comment (existing_comment_id={existing_comment_id})", + file=sys.stderr, + ) + result = subprocess.run( ["gh", "pr", "comment", pr_number, "--body", comment_body], check=True, capture_output=True, + text=True, ) + print(f"DEBUG: Post response: {result.stdout}", file=sys.stderr) print("Comment posted/updated successfully", file=sys.stderr) return True @@ -196,7 +233,15 @@ def post_or_update_comment(pr_number: str, comment_body: str) -> bool: except subprocess.CalledProcessError as e: print(f"Error posting/updating comment: {e}", file=sys.stderr) if e.stderr: - print(f"stderr: {e.stderr.decode()}", file=sys.stderr) + print( + f"stderr: {e.stderr.decode() if isinstance(e.stderr, bytes) else e.stderr}", + file=sys.stderr, + ) + if e.stdout: + print( + f"stdout: {e.stdout.decode() if isinstance(e.stdout, bytes) else e.stdout}", + file=sys.stderr, + ) return False From 8e6ee2bed18a1fd5ca5ddd818d59f71a5f86561e Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Tue, 14 Oct 2025 13:43:58 -1000 Subject: [PATCH 09/76] debug --- script/ci_memory_impact_comment.py | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/script/ci_memory_impact_comment.py b/script/ci_memory_impact_comment.py index 804e369efc..e3e70d601f 100755 --- a/script/ci_memory_impact_comment.py +++ b/script/ci_memory_impact_comment.py @@ -111,31 +111,31 @@ def find_existing_comment(pr_number: str) -> str | None: pr_number: PR number Returns: - Comment numeric ID (databaseId) if found, None otherwise + Comment numeric ID if found, None otherwise """ try: print( f"DEBUG: Looking for existing comment on PR #{pr_number}", file=sys.stderr ) - # List all comments on the PR with both id (node ID) and databaseId (numeric ID) + # Use gh api to get comments directly - this returns the numeric id field result = subprocess.run( [ "gh", - "pr", - "view", - pr_number, - "--json", - "comments", + "api", + f"/repos/{{owner}}/{{repo}}/issues/{pr_number}/comments", "--jq", - ".comments[] | {id, databaseId, body}", + ".[] | {id, body}", ], capture_output=True, text=True, check=True, ) - print(f"DEBUG: gh pr view output:\n{result.stdout}", file=sys.stderr) + print( + f"DEBUG: gh api comments output (first 500 chars):\n{result.stdout[:500]}", + file=sys.stderr, + ) # Parse comments and look for our marker comment_count = 0 @@ -146,20 +146,20 @@ def find_existing_comment(pr_number: str) -> str | None: try: comment = json.loads(line) comment_count += 1 + comment_id = comment.get("id") print( - f"DEBUG: Checking comment {comment_count}: id={comment.get('id')}, databaseId={comment.get('databaseId')}", + f"DEBUG: Checking comment {comment_count}: id={comment_id}", file=sys.stderr, ) body = comment.get("body", "") if COMMENT_MARKER in body: - database_id = str(comment["databaseId"]) print( - f"DEBUG: Found existing comment with databaseId={database_id}", + f"DEBUG: Found existing comment with id={comment_id}", file=sys.stderr, ) - # Return the numeric databaseId, not the node ID - return database_id + # Return the numeric id + return str(comment_id) print("DEBUG: Comment does not contain marker", file=sys.stderr) except json.JSONDecodeError as e: print(f"DEBUG: JSON decode error: {e}", file=sys.stderr) From acfa325f23c9df37471392889866cf29eb2c7d83 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Fri, 17 Oct 2025 13:22:01 -1000 Subject: [PATCH 10/76] merge --- esphome/analyze_memory.py | 1630 +++++++++++++++++++++++++++++++++++++ 1 file changed, 1630 insertions(+) create mode 100644 esphome/analyze_memory.py diff --git a/esphome/analyze_memory.py b/esphome/analyze_memory.py new file mode 100644 index 0000000000..70c324b33f --- /dev/null +++ b/esphome/analyze_memory.py @@ -0,0 +1,1630 @@ +"""Memory usage analyzer for ESPHome compiled binaries.""" + +from collections import defaultdict +import json +import logging +from pathlib import Path +import re +import subprocess + +_LOGGER = logging.getLogger(__name__) + +# Pattern to extract ESPHome component namespaces dynamically +ESPHOME_COMPONENT_PATTERN = re.compile(r"esphome::([a-zA-Z0-9_]+)::") + +# Component identification rules +# Symbol patterns: patterns found in raw symbol names +SYMBOL_PATTERNS = { + "freertos": [ + "vTask", + "xTask", + "xQueue", + "pvPort", + "vPort", + "uxTask", + "pcTask", + "prvTimerTask", + "prvAddNewTaskToReadyList", + "pxReadyTasksLists", + "prvAddCurrentTaskToDelayedList", + "xEventGroupWaitBits", + "xRingbufferSendFromISR", + "prvSendItemDoneNoSplit", + "prvReceiveGeneric", + "prvSendAcquireGeneric", + "prvCopyItemAllowSplit", + "xEventGroup", + "xRingbuffer", + "prvSend", + "prvReceive", + "prvCopy", + "xPort", + "ulTaskGenericNotifyTake", + "prvIdleTask", + "prvInitialiseNewTask", + "prvIsYieldRequiredSMP", + "prvGetItemByteBuf", + "prvInitializeNewRingbuffer", + "prvAcquireItemNoSplit", + "prvNotifyQueueSetContainer", + "ucStaticTimerQueueStorage", + "eTaskGetState", + "main_task", + "do_system_init_fn", + "xSemaphoreCreateGenericWithCaps", + "vListInsert", + "uxListRemove", + "vRingbufferReturnItem", + "vRingbufferReturnItemFromISR", + "prvCheckItemFitsByteBuffer", + "prvGetCurMaxSizeAllowSplit", + "tick_hook", + "sys_sem_new", + "sys_arch_mbox_fetch", + "sys_arch_sem_wait", + "prvDeleteTCB", + "vQueueDeleteWithCaps", + "vRingbufferDeleteWithCaps", + "vSemaphoreDeleteWithCaps", + "prvCheckItemAvail", + "prvCheckTaskCanBeScheduledSMP", + "prvGetCurMaxSizeNoSplit", + "prvResetNextTaskUnblockTime", + "prvReturnItemByteBuf", + "vApplicationStackOverflowHook", + "vApplicationGetIdleTaskMemory", + "sys_init", + "sys_mbox_new", + "sys_arch_mbox_tryfetch", + ], + "xtensa": ["xt_", "_xt_", "xPortEnterCriticalTimeout"], + "heap": ["heap_", "multi_heap"], + "spi_flash": ["spi_flash"], + "rtc": ["rtc_", "rtcio_ll_"], + "gpio_driver": ["gpio_", "pins"], + "uart_driver": ["uart", "_uart", "UART"], + "timer": ["timer_", "esp_timer"], + "peripherals": ["periph_", "periman"], + "network_stack": [ + "vj_compress", + "raw_sendto", + "raw_input", + "etharp_", + "icmp_input", + "socket_ipv6", + "ip_napt", + "socket_ipv4_multicast", + "socket_ipv6_multicast", + "netconn_", + "recv_raw", + "accept_function", + "netconn_recv_data", + "netconn_accept", + "netconn_write_vectors_partly", + "netconn_drain", + "raw_connect", + "raw_bind", + "icmp_send_response", + "sockets", + "icmp_dest_unreach", + "inet_chksum_pseudo", + "alloc_socket", + "done_socket", + "set_global_fd_sets", + "inet_chksum_pbuf", + "tryget_socket_unconn_locked", + "tryget_socket_unconn", + "cs_create_ctrl_sock", + "netbuf_alloc", + ], + "ipv6_stack": ["nd6_", "ip6_", "mld6_", "icmp6_", "icmp6_input"], + "wifi_stack": [ + "ieee80211", + "hostap", + "sta_", + "ap_", + "scan_", + "wifi_", + "wpa_", + "wps_", + "esp_wifi", + "cnx_", + "wpa3_", + "sae_", + "wDev_", + "ic_", + "mac_", + "esf_buf", + "gWpaSm", + "sm_WPA", + "eapol_", + "owe_", + "wifiLowLevelInit", + "s_do_mapping", + "gScanStruct", + "ppSearchTxframe", + "ppMapWaitTxq", + "ppFillAMPDUBar", + "ppCheckTxConnTrafficIdle", + "ppCalTkipMic", + ], + "bluetooth": ["bt_", "ble_", "l2c_", "gatt_", "gap_", "hci_", "BT_init"], + "wifi_bt_coex": ["coex"], + "bluetooth_rom": ["r_ble", "r_lld", "r_llc", "r_llm"], + "bluedroid_bt": [ + "bluedroid", + "btc_", + "bta_", + "btm_", + "btu_", + "BTM_", + "GATT", + "L2CA_", + "smp_", + "gatts_", + "attp_", + "l2cu_", + "l2cb", + "smp_cb", + "BTA_GATTC_", + "SMP_", + "BTU_", + "BTA_Dm", + "GAP_Ble", + "BT_tx_if", + "host_recv_pkt_cb", + "saved_local_oob_data", + "string_to_bdaddr", + "string_is_bdaddr", + "CalConnectParamTimeout", + "transmit_fragment", + "transmit_data", + "event_command_ready", + "read_command_complete_header", + "parse_read_local_extended_features_response", + "parse_read_local_version_info_response", + "should_request_high", + "btdm_wakeup_request", + "BTA_SetAttributeValue", + "BTA_EnableBluetooth", + "transmit_command_futured", + "transmit_command", + "get_waiting_command", + "make_command", + "transmit_downward", + "host_recv_adv_packet", + "copy_extra_byte_in_db", + "parse_read_local_supported_commands_response", + ], + "crypto_math": [ + "ecp_", + "bignum_", + "mpi_", + "sswu", + "modp", + "dragonfly_", + "gcm_mult", + "__multiply", + "quorem", + "__mdiff", + "__lshift", + "__mprec_tens", + "ECC_", + "multiprecision_", + "mix_sub_columns", + "sbox", + "gfm2_sbox", + "gfm3_sbox", + "curve_p256", + "curve", + "p_256_init_curve", + "shift_sub_rows", + "rshift", + ], + "hw_crypto": ["esp_aes", "esp_sha", "esp_rsa", "esp_bignum", "esp_mpi"], + "libc": [ + "printf", + "scanf", + "malloc", + "free", + "memcpy", + "memset", + "strcpy", + "strlen", + "_dtoa", + "_fopen", + "__sfvwrite_r", + "qsort", + "__sf", + "__sflush_r", + "__srefill_r", + "_impure_data", + "_reclaim_reent", + "_open_r", + "strncpy", + "_strtod_l", + "__gethex", + "__hexnan", + "_setenv_r", + "_tzset_unlocked_r", + "__tzcalc_limits", + "select", + "scalbnf", + "strtof", + "strtof_l", + "__d2b", + "__b2d", + "__s2b", + "_Balloc", + "__multadd", + "__lo0bits", + "__atexit0", + "__smakebuf_r", + "__swhatbuf_r", + "_sungetc_r", + "_close_r", + "_link_r", + "_unsetenv_r", + "_rename_r", + "__month_lengths", + "tzinfo", + "__ratio", + "__hi0bits", + "__ulp", + "__any_on", + "__copybits", + "L_shift", + "_fcntl_r", + "_lseek_r", + "_read_r", + "_write_r", + "_unlink_r", + "_fstat_r", + "access", + "fsync", + "tcsetattr", + "tcgetattr", + "tcflush", + "tcdrain", + "__ssrefill_r", + "_stat_r", + "__hexdig_fun", + "__mcmp", + "_fwalk_sglue", + "__fpclassifyf", + "_setlocale_r", + "_mbrtowc_r", + "fcntl", + "__match", + "_lock_close", + "__c$", + "__func__$", + "__FUNCTION__$", + "DAYS_IN_MONTH", + "_DAYS_BEFORE_MONTH", + "CSWTCH$", + "dst$", + "sulp", + ], + "string_ops": ["strcmp", "strncmp", "strchr", "strstr", "strtok", "strdup"], + "memory_alloc": ["malloc", "calloc", "realloc", "free", "_sbrk"], + "file_io": [ + "fread", + "fwrite", + "fopen", + "fclose", + "fseek", + "ftell", + "fflush", + "s_fd_table", + ], + "string_formatting": [ + "snprintf", + "vsnprintf", + "sprintf", + "vsprintf", + "sscanf", + "vsscanf", + ], + "cpp_anonymous": ["_GLOBAL__N_", "n$"], + "cpp_runtime": ["__cxx", "_ZN", "_ZL", "_ZSt", "__gxx_personality", "_Z16"], + "exception_handling": ["__cxa_", "_Unwind_", "__gcc_personality", "uw_frame_state"], + "static_init": ["_GLOBAL__sub_I_"], + "mdns_lib": ["mdns"], + "phy_radio": [ + "phy_", + "rf_", + "chip_", + "register_chipv7", + "pbus_", + "bb_", + "fe_", + "rfcal_", + "ram_rfcal", + "tx_pwctrl", + "rx_chan", + "set_rx_gain", + "set_chan", + "agc_reg", + "ram_txiq", + "ram_txdc", + "ram_gen_rx_gain", + "rx_11b_opt", + "set_rx_sense", + "set_rx_gain_cal", + "set_chan_dig_gain", + "tx_pwctrl_init_cal", + "rfcal_txiq", + "set_tx_gain_table", + "correct_rfpll_offset", + "pll_correct_dcap", + "txiq_cal_init", + "pwdet_sar", + "pwdet_sar2_init", + "ram_iq_est_enable", + "ram_rfpll_set_freq", + "ant_wifirx_cfg", + "ant_btrx_cfg", + "force_txrxoff", + "force_txrx_off", + "tx_paon_set", + "opt_11b_resart", + "rfpll_1p2_opt", + "ram_dc_iq_est", + "ram_start_tx_tone", + "ram_en_pwdet", + "ram_cbw2040_cfg", + "rxdc_est_min", + "i2cmst_reg_init", + "temprature_sens_read", + "ram_restart_cal", + "ram_write_gain_mem", + "ram_wait_rfpll_cal_end", + "txcal_debuge_mode", + "ant_wifitx_cfg", + "reg_init_begin", + ], + "wifi_phy_pp": ["pp_", "ppT", "ppR", "ppP", "ppInstall", "ppCalTxAMPDULength"], + "wifi_lmac": ["lmac"], + "wifi_device": ["wdev", "wDev_"], + "power_mgmt": [ + "pm_", + "sleep", + "rtc_sleep", + "light_sleep", + "deep_sleep", + "power_down", + "g_pm", + ], + "memory_mgmt": [ + "mem_", + "memory_", + "tlsf_", + "memp_", + "pbuf_", + "pbuf_alloc", + "pbuf_copy_partial_pbuf", + ], + "hal_layer": ["hal_"], + "clock_mgmt": [ + "clk_", + "clock_", + "rtc_clk", + "apb_", + "cpu_freq", + "setCpuFrequencyMhz", + ], + "cache_mgmt": ["cache"], + "flash_ops": ["flash", "image_load"], + "interrupt_handlers": [ + "isr", + "interrupt", + "intr_", + "exc_", + "exception", + "port_IntStack", + ], + "wrapper_functions": ["_wrapper"], + "error_handling": ["panic", "abort", "assert", "error_", "fault"], + "authentication": ["auth"], + "ppp_protocol": ["ppp", "ipcp_", "lcp_", "chap_", "LcpEchoCheck"], + "dhcp": ["dhcp", "handle_dhcp"], + "ethernet_phy": [ + "emac_", + "eth_phy_", + "phy_tlk110", + "phy_lan87", + "phy_ip101", + "phy_rtl", + "phy_dp83", + "phy_ksz", + "lan87xx_", + "rtl8201_", + "ip101_", + "ksz80xx_", + "jl1101_", + "dp83848_", + "eth_on_state_changed", + ], + "threading": ["pthread_", "thread_", "_task_"], + "pthread": ["pthread"], + "synchronization": ["mutex", "semaphore", "spinlock", "portMUX"], + "math_lib": [ + "sin", + "cos", + "tan", + "sqrt", + "pow", + "exp", + "log", + "atan", + "asin", + "acos", + "floor", + "ceil", + "fabs", + "round", + ], + "random": ["rand", "random", "rng_", "prng"], + "time_lib": [ + "time", + "clock", + "gettimeofday", + "settimeofday", + "localtime", + "gmtime", + "mktime", + "strftime", + ], + "console_io": ["console_", "uart_tx", "uart_rx", "puts", "putchar", "getchar"], + "rom_functions": ["r_", "rom_"], + "compiler_runtime": [ + "__divdi3", + "__udivdi3", + "__moddi3", + "__muldi3", + "__ashldi3", + "__ashrdi3", + "__lshrdi3", + "__cmpdi2", + "__fixdfdi", + "__floatdidf", + ], + "libgcc": ["libgcc", "_divdi3", "_udivdi3"], + "boot_startup": ["boot", "start_cpu", "call_start", "startup", "bootloader"], + "bootloader": ["bootloader_", "esp_bootloader"], + "app_framework": ["app_", "initArduino", "setup", "loop", "Update"], + "weak_symbols": ["__weak_"], + "compiler_builtins": ["__builtin_"], + "vfs": ["vfs_", "VFS"], + "esp32_sdk": ["esp32_", "esp32c", "esp32s"], + "usb": ["usb_", "USB", "cdc_", "CDC"], + "i2c_driver": ["i2c_", "I2C"], + "i2s_driver": ["i2s_", "I2S"], + "spi_driver": ["spi_", "SPI"], + "adc_driver": ["adc_", "ADC"], + "dac_driver": ["dac_", "DAC"], + "touch_driver": ["touch_", "TOUCH"], + "pwm_driver": ["pwm_", "PWM", "ledc_", "LEDC"], + "rmt_driver": ["rmt_", "RMT"], + "pcnt_driver": ["pcnt_", "PCNT"], + "can_driver": ["can_", "CAN", "twai_", "TWAI"], + "sdmmc_driver": ["sdmmc_", "SDMMC", "sdcard", "sd_card"], + "temp_sensor": ["temp_sensor", "tsens_"], + "watchdog": ["wdt_", "WDT", "watchdog"], + "brownout": ["brownout", "bod_"], + "ulp": ["ulp_", "ULP"], + "psram": ["psram", "PSRAM", "spiram", "SPIRAM"], + "efuse": ["efuse", "EFUSE"], + "partition": ["partition", "esp_partition"], + "esp_event": ["esp_event", "event_loop", "event_callback"], + "esp_console": ["esp_console", "console_"], + "chip_specific": ["chip_", "esp_chip"], + "esp_system_utils": ["esp_system", "esp_hw", "esp_clk", "esp_sleep"], + "ipc": ["esp_ipc", "ipc_"], + "wifi_config": [ + "g_cnxMgr", + "gChmCxt", + "g_ic", + "TxRxCxt", + "s_dp", + "s_ni", + "s_reg_dump", + "packet$", + "d_mult_table", + "K", + "fcstab", + ], + "smartconfig": ["sc_ack_send"], + "rc_calibration": ["rc_cal", "rcUpdate"], + "noise_floor": ["noise_check"], + "rf_calibration": [ + "set_rx_sense", + "set_rx_gain_cal", + "set_chan_dig_gain", + "tx_pwctrl_init_cal", + "rfcal_txiq", + "set_tx_gain_table", + "correct_rfpll_offset", + "pll_correct_dcap", + "txiq_cal_init", + "pwdet_sar", + "rx_11b_opt", + ], + "wifi_crypto": [ + "pk_use_ecparams", + "process_segments", + "ccmp_", + "rc4_", + "aria_", + "mgf_mask", + "dh_group", + "ccmp_aad_nonce", + "ccmp_encrypt", + "rc4_skip", + "aria_sb1", + "aria_sb2", + "aria_is1", + "aria_is2", + "aria_sl", + "aria_a", + ], + "radio_control": ["fsm_input", "fsm_sconfreq"], + "pbuf": [ + "pbuf_", + ], + "event_group": ["xEventGroup"], + "ringbuffer": ["xRingbuffer", "prvSend", "prvReceive", "prvCopy"], + "provisioning": ["prov_", "prov_stop_and_notify"], + "scan": ["gScanStruct"], + "port": ["xPort"], + "elf_loader": [ + "elf_add", + "elf_add_note", + "elf_add_segment", + "process_image", + "read_encoded", + "read_encoded_value", + "read_encoded_value_with_base", + "process_image_header", + ], + "socket_api": [ + "sockets", + "netconn_", + "accept_function", + "recv_raw", + "socket_ipv4_multicast", + "socket_ipv6_multicast", + ], + "igmp": ["igmp_", "igmp_send", "igmp_input"], + "icmp6": ["icmp6_"], + "arp": ["arp_table"], + "ampdu": [ + "ampdu_", + "rcAmpdu", + "trc_onAmpduOp", + "rcAmpduLowerRate", + "ampdu_dispatch_upto", + ], + "ieee802_11": ["ieee802_11_", "ieee802_11_parse_elems"], + "rate_control": ["rssi_margin", "rcGetSched", "get_rate_fcc_index"], + "nan": ["nan_dp_", "nan_dp_post_tx", "nan_dp_delete_peer"], + "channel_mgmt": ["chm_init", "chm_set_current_channel"], + "trace": ["trc_init", "trc_onAmpduOp"], + "country_code": ["country_info", "country_info_24ghz"], + "multicore": ["do_multicore_settings"], + "Update_lib": ["Update"], + "stdio": [ + "__sf", + "__sflush_r", + "__srefill_r", + "_impure_data", + "_reclaim_reent", + "_open_r", + ], + "strncpy_ops": ["strncpy"], + "math_internal": ["__mdiff", "__lshift", "__mprec_tens", "quorem"], + "character_class": ["__chclass"], + "camellia": ["camellia_", "camellia_feistel"], + "crypto_tables": ["FSb", "FSb2", "FSb3", "FSb4"], + "event_buffer": ["g_eb_list_desc", "eb_space"], + "base_node": ["base_node_", "base_node_add_handler"], + "file_descriptor": ["s_fd_table"], + "tx_delay": ["tx_delay_cfg"], + "deinit": ["deinit_functions"], + "lcp_echo": ["LcpEchoCheck"], + "raw_api": ["raw_bind", "raw_connect"], + "checksum": ["process_checksum"], + "entry_management": ["add_entry"], + "esp_ota": ["esp_ota", "ota_", "read_otadata"], + "http_server": [ + "httpd_", + "parse_url_char", + "cb_headers_complete", + "delete_entry", + "validate_structure", + "config_save", + "config_new", + "verify_url", + "cb_url", + ], + "misc_system": [ + "alarm_cbs", + "start_up", + "tokens", + "unhex", + "osi_funcs_ro", + "enum_function", + "fragment_and_dispatch", + "alarm_set", + "osi_alarm_new", + "config_set_string", + "config_update_newest_section", + "config_remove_key", + "method_strings", + "interop_match", + "interop_database", + "__state_table", + "__action_table", + "s_stub_table", + "s_context", + "s_mmu_ctx", + "s_get_bus_mask", + "hli_queue_put", + "list_remove", + "list_delete", + "lock_acquire_generic", + "is_vect_desc_usable", + "io_mode_str", + "__c$20233", + "interface", + "read_id_core", + "subscribe_idle", + "unsubscribe_idle", + "s_clkout_handle", + "lock_release_generic", + "config_set_int", + "config_get_int", + "config_get_string", + "config_has_key", + "config_remove_section", + "osi_alarm_init", + "osi_alarm_deinit", + "fixed_queue_enqueue", + "fixed_queue_dequeue", + "fixed_queue_new", + "fixed_pkt_queue_enqueue", + "fixed_pkt_queue_new", + "list_append", + "list_prepend", + "list_insert_after", + "list_contains", + "list_get_node", + "hash_function_blob", + "cb_no_body", + "cb_on_body", + "profile_tab", + "get_arg", + "trim", + "buf$", + "process_appended_hash_and_sig$constprop$0", + "uuidType", + "allocate_svc_db_buf", + "_hostname_is_ours", + "s_hli_handlers", + "tick_cb", + "idle_cb", + "input", + "entry_find", + "section_find", + "find_bucket_entry_", + "config_has_section", + "hli_queue_create", + "hli_queue_get", + "hli_c_handler", + "future_ready", + "future_await", + "future_new", + "pkt_queue_enqueue", + "pkt_queue_dequeue", + "pkt_queue_cleanup", + "pkt_queue_create", + "pkt_queue_destroy", + "fixed_pkt_queue_dequeue", + "osi_alarm_cancel", + "osi_alarm_is_active", + "osi_sem_take", + "osi_event_create", + "osi_event_bind", + "alarm_cb_handler", + "list_foreach", + "list_back", + "list_front", + "list_clear", + "fixed_queue_try_peek_first", + "translate_path", + "get_idx", + "find_key", + "init", + "end", + "start", + "set_read_value", + "copy_address_list", + "copy_and_key", + "sdk_cfg_opts", + "leftshift_onebit", + "config_section_end", + "config_section_begin", + "find_entry_and_check_all_reset", + "image_validate", + "xPendingReadyList", + "vListInitialise", + "lock_init_generic", + "ant_bttx_cfg", + "ant_dft_cfg", + "cs_send_to_ctrl_sock", + "config_llc_util_funcs_reset", + "make_set_adv_report_flow_control", + "make_set_event_mask", + "raw_new", + "raw_remove", + "BTE_InitStack", + "parse_read_local_supported_features_response", + "__math_invalidf", + "tinytens", + "__mprec_tinytens", + "__mprec_bigtens", + "vRingbufferDelete", + "vRingbufferDeleteWithCaps", + "vRingbufferReturnItem", + "vRingbufferReturnItemFromISR", + "get_acl_data_size_ble", + "get_features_ble", + "get_features_classic", + "get_acl_packet_size_ble", + "get_acl_packet_size_classic", + "supports_extended_inquiry_response", + "supports_rssi_with_inquiry_results", + "supports_interlaced_inquiry_scan", + "supports_reading_remote_extended_features", + ], + "bluetooth_ll": [ + "lld_pdu_", + "ld_acl_", + "lld_stop_ind_handler", + "lld_evt_winsize_change", + "config_lld_evt_funcs_reset", + "config_lld_funcs_reset", + "config_llm_funcs_reset", + "llm_set_long_adv_data", + "lld_retry_tx_prog", + "llc_link_sup_to_ind_handler", + "config_llc_funcs_reset", + "lld_evt_rxwin_compute", + "config_btdm_funcs_reset", + "config_ea_funcs_reset", + "llc_defalut_state_tab_reset", + "config_rwip_funcs_reset", + "ke_lmp_rx_flooding_detect", + ], +} + +# Demangled patterns: patterns found in demangled C++ names +DEMANGLED_PATTERNS = { + "gpio_driver": ["GPIO"], + "uart_driver": ["UART"], + "network_stack": [ + "lwip", + "tcp", + "udp", + "ip4", + "ip6", + "dhcp", + "dns", + "netif", + "ethernet", + "ppp", + "slip", + ], + "wifi_stack": ["NetworkInterface"], + "nimble_bt": [ + "nimble", + "NimBLE", + "ble_hs", + "ble_gap", + "ble_gatt", + "ble_att", + "ble_l2cap", + "ble_sm", + ], + "crypto": ["mbedtls", "crypto", "sha", "aes", "rsa", "ecc", "tls", "ssl"], + "cpp_stdlib": ["std::", "__gnu_cxx::", "__cxxabiv"], + "static_init": ["__static_initialization"], + "rtti": ["__type_info", "__class_type_info"], + "web_server_lib": ["AsyncWebServer", "AsyncWebHandler", "WebServer"], + "async_tcp": ["AsyncClient", "AsyncServer"], + "mdns_lib": ["mdns"], + "json_lib": [ + "ArduinoJson", + "JsonDocument", + "JsonArray", + "JsonObject", + "deserialize", + "serialize", + ], + "http_lib": ["HTTP", "http_", "Request", "Response", "Uri", "WebSocket"], + "logging": ["log", "Log", "print", "Print", "diag_"], + "authentication": ["checkDigestAuthentication"], + "libgcc": ["libgcc"], + "esp_system": ["esp_", "ESP"], + "arduino": ["arduino"], + "nvs": ["nvs_", "_ZTVN3nvs", "nvs::"], + "filesystem": ["spiffs", "vfs"], + "libc": ["newlib"], +} + + +# Get the list of actual ESPHome components by scanning the components directory +def get_esphome_components(): + """Get set of actual ESPHome components from the components directory.""" + components = set() + + # Find the components directory relative to this file + current_dir = Path(__file__).parent + components_dir = current_dir / "components" + + if components_dir.exists() and components_dir.is_dir(): + for item in components_dir.iterdir(): + if ( + item.is_dir() + and not item.name.startswith(".") + and not item.name.startswith("__") + ): + components.add(item.name) + + return components + + +# Cache the component list +ESPHOME_COMPONENTS = get_esphome_components() + + +class MemorySection: + """Represents a memory section with its symbols.""" + + def __init__(self, name: str): + self.name = name + self.symbols: list[tuple[str, int, str]] = [] # (symbol_name, size, component) + self.total_size = 0 + + +class ComponentMemory: + """Tracks memory usage for a component.""" + + def __init__(self, name: str): + self.name = name + self.text_size = 0 # Code in flash + self.rodata_size = 0 # Read-only data in flash + self.data_size = 0 # Initialized data (flash + ram) + self.bss_size = 0 # Uninitialized data (ram only) + self.symbol_count = 0 + + @property + def flash_total(self) -> int: + return self.text_size + self.rodata_size + self.data_size + + @property + def ram_total(self) -> int: + return self.data_size + self.bss_size + + +class MemoryAnalyzer: + """Analyzes memory usage from ELF files.""" + + def __init__( + self, + elf_path: str, + objdump_path: str | None = None, + readelf_path: str | None = None, + external_components: set[str] | None = None, + ): + self.elf_path = Path(elf_path) + if not self.elf_path.exists(): + raise FileNotFoundError(f"ELF file not found: {elf_path}") + + self.objdump_path = objdump_path or "objdump" + self.readelf_path = readelf_path or "readelf" + self.external_components = external_components or set() + + self.sections: dict[str, MemorySection] = {} + self.components: dict[str, ComponentMemory] = defaultdict( + lambda: ComponentMemory("") + ) + self._demangle_cache: dict[str, str] = {} + self._uncategorized_symbols: list[tuple[str, str, int]] = [] + self._esphome_core_symbols: list[ + tuple[str, str, int] + ] = [] # Track core symbols + self._component_symbols: dict[str, list[tuple[str, str, int]]] = defaultdict( + list + ) # Track symbols for all components + + def analyze(self) -> dict[str, ComponentMemory]: + """Analyze the ELF file and return component memory usage.""" + self._parse_sections() + self._parse_symbols() + self._categorize_symbols() + return dict(self.components) + + def _parse_sections(self) -> None: + """Parse section headers from ELF file.""" + try: + result = subprocess.run( + [self.readelf_path, "-S", str(self.elf_path)], + capture_output=True, + text=True, + check=True, + ) + + # Parse section headers + for line in result.stdout.splitlines(): + # Look for section entries + match = re.match( + r"\s*\[\s*\d+\]\s+([\.\w]+)\s+\w+\s+[\da-fA-F]+\s+[\da-fA-F]+\s+([\da-fA-F]+)", + line, + ) + if match: + section_name = match.group(1) + size_hex = match.group(2) + size = int(size_hex, 16) + + # Map various section names to standard categories + mapped_section = None + if ".text" in section_name or ".iram" in section_name: + mapped_section = ".text" + elif ".rodata" in section_name: + mapped_section = ".rodata" + elif ".data" in section_name and "bss" not in section_name: + mapped_section = ".data" + elif ".bss" in section_name: + mapped_section = ".bss" + + if mapped_section: + if mapped_section not in self.sections: + self.sections[mapped_section] = MemorySection( + mapped_section + ) + self.sections[mapped_section].total_size += size + + except subprocess.CalledProcessError as e: + _LOGGER.error(f"Failed to parse sections: {e}") + raise + + def _parse_symbols(self) -> None: + """Parse symbols from ELF file.""" + # Section mapping - centralizes the logic + SECTION_MAPPING = { + ".text": [".text", ".iram"], + ".rodata": [".rodata"], + ".data": [".data", ".dram"], + ".bss": [".bss"], + } + + def map_section_name(raw_section: str) -> str | None: + """Map raw section name to standard section.""" + for standard_section, patterns in SECTION_MAPPING.items(): + if any(pattern in raw_section for pattern in patterns): + return standard_section + return None + + def parse_symbol_line(line: str) -> tuple[str, str, int, str] | None: + """Parse a single symbol line from objdump output. + + Returns (section, name, size, address) or None if not a valid symbol. + Format: address l/g w/d F/O section size name + Example: 40084870 l F .iram0.text 00000000 _xt_user_exc + """ + parts = line.split() + if len(parts) < 5: + return None + + try: + # Validate and extract address + address = parts[0] + int(address, 16) + except ValueError: + return None + + # Look for F (function) or O (object) flag + if "F" not in parts and "O" not in parts: + return None + + # Find section, size, and name + for i, part in enumerate(parts): + if part.startswith("."): + section = map_section_name(part) + if section and i + 1 < len(parts): + try: + size = int(parts[i + 1], 16) + if i + 2 < len(parts) and size > 0: + name = " ".join(parts[i + 2 :]) + return (section, name, size, address) + except ValueError: + pass + break + return None + + try: + result = subprocess.run( + [self.objdump_path, "-t", str(self.elf_path)], + capture_output=True, + text=True, + check=True, + ) + + # Track seen addresses to avoid duplicates + seen_addresses: set[str] = set() + + for line in result.stdout.splitlines(): + symbol_info = parse_symbol_line(line) + if symbol_info: + section, name, size, address = symbol_info + # Skip duplicate symbols at the same address (e.g., C1/C2 constructors) + if address not in seen_addresses and section in self.sections: + self.sections[section].symbols.append((name, size, "")) + seen_addresses.add(address) + + except subprocess.CalledProcessError as e: + _LOGGER.error(f"Failed to parse symbols: {e}") + raise + + def _categorize_symbols(self) -> None: + """Categorize symbols by component.""" + # First, collect all unique symbol names for batch demangling + all_symbols = set() + for section in self.sections.values(): + for symbol_name, _, _ in section.symbols: + all_symbols.add(symbol_name) + + # Batch demangle all symbols at once + self._batch_demangle_symbols(list(all_symbols)) + + # Now categorize with cached demangled names + for section_name, section in self.sections.items(): + for symbol_name, size, _ in section.symbols: + component = self._identify_component(symbol_name) + + if component not in self.components: + self.components[component] = ComponentMemory(component) + + comp_mem = self.components[component] + comp_mem.symbol_count += 1 + + if section_name == ".text": + comp_mem.text_size += size + elif section_name == ".rodata": + comp_mem.rodata_size += size + elif section_name == ".data": + comp_mem.data_size += size + elif section_name == ".bss": + comp_mem.bss_size += size + + # Track uncategorized symbols + if component == "other" and size > 0: + demangled = self._demangle_symbol(symbol_name) + self._uncategorized_symbols.append((symbol_name, demangled, size)) + + # Track ESPHome core symbols for detailed analysis + if component == "[esphome]core" and size > 0: + demangled = self._demangle_symbol(symbol_name) + self._esphome_core_symbols.append((symbol_name, demangled, size)) + + # Track all component symbols for detailed analysis + if size > 0: + demangled = self._demangle_symbol(symbol_name) + self._component_symbols[component].append( + (symbol_name, demangled, size) + ) + + def _identify_component(self, symbol_name: str) -> str: + """Identify which component a symbol belongs to.""" + # Demangle C++ names if needed + demangled = self._demangle_symbol(symbol_name) + + # Check for special component classes first (before namespace pattern) + # This handles cases like esphome::ESPHomeOTAComponent which should map to ota + if "esphome::" in demangled: + # Check for special component classes that include component name in the class + # For example: esphome::ESPHomeOTAComponent -> ota component + for component_name in ESPHOME_COMPONENTS: + # Check various naming patterns + component_upper = component_name.upper() + component_camel = component_name.replace("_", "").title() + patterns = [ + f"esphome::{component_upper}Component", # e.g., esphome::OTAComponent + f"esphome::ESPHome{component_upper}Component", # e.g., esphome::ESPHomeOTAComponent + f"esphome::{component_camel}Component", # e.g., esphome::OtaComponent + f"esphome::ESPHome{component_camel}Component", # e.g., esphome::ESPHomeOtaComponent + ] + + if any(pattern in demangled for pattern in patterns): + return f"[esphome]{component_name}" + + # Check for ESPHome component namespaces + match = ESPHOME_COMPONENT_PATTERN.search(demangled) + if match: + component_name = match.group(1) + # Strip trailing underscore if present (e.g., switch_ -> switch) + component_name = component_name.rstrip("_") + + # Check if this is an actual component in the components directory + if component_name in ESPHOME_COMPONENTS: + return f"[esphome]{component_name}" + # Check if this is a known external component from the config + if component_name in self.external_components: + return f"[external]{component_name}" + # Everything else in esphome:: namespace is core + return "[esphome]core" + + # Check for esphome core namespace (no component namespace) + if "esphome::" in demangled: + # If no component match found, it's core + return "[esphome]core" + + # Check against symbol patterns + for component, patterns in SYMBOL_PATTERNS.items(): + if any(pattern in symbol_name for pattern in patterns): + return component + + # Check against demangled patterns + for component, patterns in DEMANGLED_PATTERNS.items(): + if any(pattern in demangled for pattern in patterns): + return component + + # Special cases that need more complex logic + + # Check if spi_flash vs spi_driver + if "spi_" in symbol_name or "SPI" in symbol_name: + if "spi_flash" in symbol_name: + return "spi_flash" + return "spi_driver" + + # libc special printf variants + if symbol_name.startswith("_") and symbol_name[1:].replace("_r", "").replace( + "v", "" + ).replace("s", "") in ["printf", "fprintf", "sprintf", "scanf"]: + return "libc" + + # Track uncategorized symbols for analysis + return "other" + + def _batch_demangle_symbols(self, symbols: list[str]) -> None: + """Batch demangle C++ symbol names for efficiency.""" + if not symbols: + return + + # Try to find the appropriate c++filt for the platform + cppfilt_cmd = "c++filt" + + # Check if we have a toolchain-specific c++filt + if self.objdump_path and self.objdump_path != "objdump": + # Replace objdump with c++filt in the path + potential_cppfilt = self.objdump_path.replace("objdump", "c++filt") + if Path(potential_cppfilt).exists(): + cppfilt_cmd = potential_cppfilt + + try: + # Send all symbols to c++filt at once + result = subprocess.run( + [cppfilt_cmd], + input="\n".join(symbols), + capture_output=True, + text=True, + check=False, + ) + if result.returncode == 0: + demangled_lines = result.stdout.strip().split("\n") + # Map original to demangled names + for original, demangled in zip(symbols, demangled_lines): + self._demangle_cache[original] = demangled + else: + # If batch fails, cache originals + for symbol in symbols: + self._demangle_cache[symbol] = symbol + except Exception: + # On error, cache originals + for symbol in symbols: + self._demangle_cache[symbol] = symbol + + def _demangle_symbol(self, symbol: str) -> str: + """Get demangled C++ symbol name from cache.""" + return self._demangle_cache.get(symbol, symbol) + + def _categorize_esphome_core_symbol(self, demangled: str) -> str: + """Categorize ESPHome core symbols into subcategories.""" + # Dictionary of patterns for core subcategories + CORE_SUBCATEGORY_PATTERNS = { + "Component Framework": ["Component"], + "Application Core": ["Application"], + "Scheduler": ["Scheduler"], + "Logging": ["Logger", "log_"], + "Preferences": ["preferences", "Preferences"], + "Synchronization": ["Mutex", "Lock"], + "Helpers": ["Helper"], + "Network Utilities": ["network", "Network"], + "Time Management": ["time", "Time"], + "String Utilities": ["str_", "string"], + "Parsing/Formatting": ["parse_", "format_"], + "Optional Types": ["optional", "Optional"], + "Callbacks": ["Callback", "callback"], + "Color Utilities": ["Color"], + "C++ Operators": ["operator"], + "Global Variables": ["global_", "_GLOBAL"], + "Setup/Loop": ["setup", "loop"], + "System Control": ["reboot", "restart"], + "GPIO Management": ["GPIO", "gpio"], + "Interrupt Handling": ["ISR", "interrupt"], + "Hooks": ["Hook", "hook"], + "Entity Base Classes": ["Entity"], + "Automation Framework": ["automation", "Automation"], + "Automation Components": ["Condition", "Action", "Trigger"], + "Lambda Support": ["lambda"], + } + + # Special patterns that need to be checked separately + if any(pattern in demangled for pattern in ["vtable", "typeinfo", "thunk"]): + return "C++ Runtime (vtables/RTTI)" + + if demangled.startswith("std::"): + return "C++ STL" + + # Check against patterns + for category, patterns in CORE_SUBCATEGORY_PATTERNS.items(): + if any(pattern in demangled for pattern in patterns): + return category + + return "Other Core" + + def generate_report(self, detailed: bool = False) -> str: + """Generate a formatted memory report.""" + components = sorted( + self.components.items(), key=lambda x: x[1].flash_total, reverse=True + ) + + # Calculate totals + total_flash = sum(c.flash_total for _, c in components) + total_ram = sum(c.ram_total for _, c in components) + + # Build report + lines = [] + + # Column width constants + COL_COMPONENT = 29 + COL_FLASH_TEXT = 14 + COL_FLASH_DATA = 14 + COL_RAM_DATA = 12 + COL_RAM_BSS = 12 + COL_TOTAL_FLASH = 15 + COL_TOTAL_RAM = 12 + COL_SEPARATOR = 3 # " | " + + # Core analysis column widths + COL_CORE_SUBCATEGORY = 30 + COL_CORE_SIZE = 12 + COL_CORE_COUNT = 6 + COL_CORE_PERCENT = 10 + + # Calculate the exact table width + table_width = ( + COL_COMPONENT + + COL_SEPARATOR + + COL_FLASH_TEXT + + COL_SEPARATOR + + COL_FLASH_DATA + + COL_SEPARATOR + + COL_RAM_DATA + + COL_SEPARATOR + + COL_RAM_BSS + + COL_SEPARATOR + + COL_TOTAL_FLASH + + COL_SEPARATOR + + COL_TOTAL_RAM + ) + + lines.append("=" * table_width) + lines.append("Component Memory Analysis".center(table_width)) + lines.append("=" * table_width) + lines.append("") + + # Main table - fixed column widths + lines.append( + f"{'Component':<{COL_COMPONENT}} | {'Flash (text)':>{COL_FLASH_TEXT}} | {'Flash (data)':>{COL_FLASH_DATA}} | {'RAM (data)':>{COL_RAM_DATA}} | {'RAM (bss)':>{COL_RAM_BSS}} | {'Total Flash':>{COL_TOTAL_FLASH}} | {'Total RAM':>{COL_TOTAL_RAM}}" + ) + lines.append( + "-" * COL_COMPONENT + + "-+-" + + "-" * COL_FLASH_TEXT + + "-+-" + + "-" * COL_FLASH_DATA + + "-+-" + + "-" * COL_RAM_DATA + + "-+-" + + "-" * COL_RAM_BSS + + "-+-" + + "-" * COL_TOTAL_FLASH + + "-+-" + + "-" * COL_TOTAL_RAM + ) + + for name, mem in components: + if mem.flash_total > 0 or mem.ram_total > 0: + flash_rodata = mem.rodata_size + mem.data_size + lines.append( + f"{name:<{COL_COMPONENT}} | {mem.text_size:>{COL_FLASH_TEXT - 2},} B | {flash_rodata:>{COL_FLASH_DATA - 2},} B | " + f"{mem.data_size:>{COL_RAM_DATA - 2},} B | {mem.bss_size:>{COL_RAM_BSS - 2},} B | " + f"{mem.flash_total:>{COL_TOTAL_FLASH - 2},} B | {mem.ram_total:>{COL_TOTAL_RAM - 2},} B" + ) + + lines.append( + "-" * COL_COMPONENT + + "-+-" + + "-" * COL_FLASH_TEXT + + "-+-" + + "-" * COL_FLASH_DATA + + "-+-" + + "-" * COL_RAM_DATA + + "-+-" + + "-" * COL_RAM_BSS + + "-+-" + + "-" * COL_TOTAL_FLASH + + "-+-" + + "-" * COL_TOTAL_RAM + ) + lines.append( + f"{'TOTAL':<{COL_COMPONENT}} | {' ':>{COL_FLASH_TEXT}} | {' ':>{COL_FLASH_DATA}} | " + f"{' ':>{COL_RAM_DATA}} | {' ':>{COL_RAM_BSS}} | " + f"{total_flash:>{COL_TOTAL_FLASH - 2},} B | {total_ram:>{COL_TOTAL_RAM - 2},} B" + ) + + # Top consumers + lines.append("") + lines.append("Top Flash Consumers:") + for i, (name, mem) in enumerate(components[:25]): + if mem.flash_total > 0: + percentage = ( + (mem.flash_total / total_flash * 100) if total_flash > 0 else 0 + ) + lines.append( + f"{i + 1}. {name} ({mem.flash_total:,} B) - {percentage:.1f}% of analyzed flash" + ) + + lines.append("") + lines.append("Top RAM Consumers:") + ram_components = sorted(components, key=lambda x: x[1].ram_total, reverse=True) + for i, (name, mem) in enumerate(ram_components[:25]): + if mem.ram_total > 0: + percentage = (mem.ram_total / total_ram * 100) if total_ram > 0 else 0 + lines.append( + f"{i + 1}. {name} ({mem.ram_total:,} B) - {percentage:.1f}% of analyzed RAM" + ) + + lines.append("") + lines.append( + "Note: This analysis covers symbols in the ELF file. Some runtime allocations may not be included." + ) + lines.append("=" * table_width) + + # Add ESPHome core detailed analysis if there are core symbols + if self._esphome_core_symbols: + lines.append("") + lines.append("=" * table_width) + lines.append("[esphome]core Detailed Analysis".center(table_width)) + lines.append("=" * table_width) + lines.append("") + + # Group core symbols by subcategory + core_subcategories: dict[str, list[tuple[str, str, int]]] = defaultdict( + list + ) + + for symbol, demangled, size in self._esphome_core_symbols: + # Categorize based on demangled name patterns + subcategory = self._categorize_esphome_core_symbol(demangled) + core_subcategories[subcategory].append((symbol, demangled, size)) + + # Sort subcategories by total size + sorted_subcategories = sorted( + [ + (name, symbols, sum(s[2] for s in symbols)) + for name, symbols in core_subcategories.items() + ], + key=lambda x: x[2], + reverse=True, + ) + + lines.append( + f"{'Subcategory':<{COL_CORE_SUBCATEGORY}} | {'Size':>{COL_CORE_SIZE}} | " + f"{'Count':>{COL_CORE_COUNT}} | {'% of Core':>{COL_CORE_PERCENT}}" + ) + lines.append( + "-" * COL_CORE_SUBCATEGORY + + "-+-" + + "-" * COL_CORE_SIZE + + "-+-" + + "-" * COL_CORE_COUNT + + "-+-" + + "-" * COL_CORE_PERCENT + ) + + core_total = sum(size for _, _, size in self._esphome_core_symbols) + + for subcategory, symbols, total_size in sorted_subcategories: + percentage = (total_size / core_total * 100) if core_total > 0 else 0 + lines.append( + f"{subcategory:<{COL_CORE_SUBCATEGORY}} | {total_size:>{COL_CORE_SIZE - 2},} B | " + f"{len(symbols):>{COL_CORE_COUNT}} | {percentage:>{COL_CORE_PERCENT - 1}.1f}%" + ) + + # Top 10 largest core symbols + lines.append("") + lines.append("Top 10 Largest [esphome]core Symbols:") + sorted_core_symbols = sorted( + self._esphome_core_symbols, key=lambda x: x[2], reverse=True + ) + + for i, (symbol, demangled, size) in enumerate(sorted_core_symbols[:15]): + lines.append(f"{i + 1}. {demangled} ({size:,} B)") + + lines.append("=" * table_width) + + # Add detailed analysis for top ESPHome and external components + esphome_components = [ + (name, mem) + for name, mem in components + if name.startswith("[esphome]") and name != "[esphome]core" + ] + external_components = [ + (name, mem) for name, mem in components if name.startswith("[external]") + ] + + top_esphome_components = sorted( + esphome_components, key=lambda x: x[1].flash_total, reverse=True + )[:30] + + # Include all external components (they're usually important) + top_external_components = sorted( + external_components, key=lambda x: x[1].flash_total, reverse=True + ) + + # Check if API component exists and ensure it's included + api_component = None + for name, mem in components: + if name == "[esphome]api": + api_component = (name, mem) + break + + # Combine all components to analyze: top ESPHome + all external + API if not already included + components_to_analyze = list(top_esphome_components) + list( + top_external_components + ) + if api_component and api_component not in components_to_analyze: + components_to_analyze.append(api_component) + + if components_to_analyze: + for comp_name, comp_mem in components_to_analyze: + comp_symbols = self._component_symbols.get(comp_name, []) + if comp_symbols: + lines.append("") + lines.append("=" * table_width) + lines.append(f"{comp_name} Detailed Analysis".center(table_width)) + lines.append("=" * table_width) + lines.append("") + + # Sort symbols by size + sorted_symbols = sorted( + comp_symbols, key=lambda x: x[2], reverse=True + ) + + lines.append(f"Total symbols: {len(sorted_symbols)}") + lines.append(f"Total size: {comp_mem.flash_total:,} B") + lines.append("") + + # Show all symbols > 100 bytes for better visibility + large_symbols = [ + (sym, dem, size) + for sym, dem, size in sorted_symbols + if size > 100 + ] + + lines.append( + f"{comp_name} Symbols > 100 B ({len(large_symbols)} symbols):" + ) + for i, (symbol, demangled, size) in enumerate(large_symbols): + lines.append(f"{i + 1}. {demangled} ({size:,} B)") + + lines.append("=" * table_width) + + return "\n".join(lines) + + def to_json(self) -> str: + """Export analysis results as JSON.""" + data = { + "components": { + name: { + "text": mem.text_size, + "rodata": mem.rodata_size, + "data": mem.data_size, + "bss": mem.bss_size, + "flash_total": mem.flash_total, + "ram_total": mem.ram_total, + "symbol_count": mem.symbol_count, + } + for name, mem in self.components.items() + }, + "totals": { + "flash": sum(c.flash_total for c in self.components.values()), + "ram": sum(c.ram_total for c in self.components.values()), + }, + } + return json.dumps(data, indent=2) + + def dump_uncategorized_symbols(self, output_file: str | None = None) -> None: + """Dump uncategorized symbols for analysis.""" + # Sort by size descending + sorted_symbols = sorted( + self._uncategorized_symbols, key=lambda x: x[2], reverse=True + ) + + lines = ["Uncategorized Symbols Analysis", "=" * 80] + lines.append(f"Total uncategorized symbols: {len(sorted_symbols)}") + lines.append( + f"Total uncategorized size: {sum(s[2] for s in sorted_symbols):,} bytes" + ) + lines.append("") + lines.append(f"{'Size':>10} | {'Symbol':<60} | Demangled") + lines.append("-" * 10 + "-+-" + "-" * 60 + "-+-" + "-" * 40) + + for symbol, demangled, size in sorted_symbols[:100]: # Top 100 + if symbol != demangled: + lines.append(f"{size:>10,} | {symbol[:60]:<60} | {demangled[:100]}") + else: + lines.append(f"{size:>10,} | {symbol[:60]:<60} | [not demangled]") + + if len(sorted_symbols) > 100: + lines.append(f"\n... and {len(sorted_symbols) - 100} more symbols") + + content = "\n".join(lines) + + if output_file: + with open(output_file, "w") as f: + f.write(content) + else: + print(content) + + +def analyze_elf( + elf_path: str, + objdump_path: str | None = None, + readelf_path: str | None = None, + detailed: bool = False, + external_components: set[str] | None = None, +) -> str: + """Analyze an ELF file and return a memory report.""" + analyzer = MemoryAnalyzer(elf_path, objdump_path, readelf_path, external_components) + analyzer.analyze() + return analyzer.generate_report(detailed) + + +if __name__ == "__main__": + import sys + + if len(sys.argv) < 2: + print("Usage: analyze_memory.py ") + sys.exit(1) + + try: + report = analyze_elf(sys.argv[1]) + print(report) + except Exception as e: + print(f"Error: {e}") + sys.exit(1) From c7c408e6670e5223cd4c7abf1be634926f7043cf Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Fri, 17 Oct 2025 13:28:13 -1000 Subject: [PATCH 11/76] tweak --- .github/workflows/ci.yml | 36 +++++++ script/ci_memory_impact_comment.py | 147 ++++++++++++++++++++++++++++- 2 files changed, 182 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4682a05fea..d87945f8df 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -559,6 +559,24 @@ jobs: echo "Compiling $component for $platform using $test_file" python script/test_build_components.py -e compile -c "$component" -t "$platform" --no-grouping 2>&1 | \ python script/ci_memory_impact_extract.py --output-env + - name: Find and upload ELF file + run: | + # Find the most recently created .elf file in .esphome/build + elf_file=$(find ~/.esphome/build -name "*.elf" -type f -printf '%T@ %p\n' 2>/dev/null | sort -rn | head -1 | cut -d' ' -f2-) + if [ -n "$elf_file" ] && [ -f "$elf_file" ]; then + echo "Found ELF file: $elf_file" + mkdir -p ./elf-artifacts + cp "$elf_file" ./elf-artifacts/target.elf + else + echo "Warning: No ELF file found" + fi + - name: Upload ELF artifact + uses: actions/upload-artifact@ea05be8e2b5c27c5689e977ed6f65db0a051b1e5 # v4.6.0 + with: + name: memory-impact-target-elf + path: ./elf-artifacts/target.elf + if-no-files-found: warn + retention-days: 1 memory-impact-pr-branch: name: Build PR branch for memory impact @@ -594,6 +612,24 @@ jobs: echo "Compiling $component for $platform using $test_file" python script/test_build_components.py -e compile -c "$component" -t "$platform" --no-grouping 2>&1 | \ python script/ci_memory_impact_extract.py --output-env + - name: Find and upload ELF file + run: | + # Find the most recently created .elf file in .esphome/build + elf_file=$(find ~/.esphome/build -name "*.elf" -type f -printf '%T@ %p\n' 2>/dev/null | sort -rn | head -1 | cut -d' ' -f2-) + if [ -n "$elf_file" ] && [ -f "$elf_file" ]; then + echo "Found ELF file: $elf_file" + mkdir -p ./elf-artifacts + cp "$elf_file" ./elf-artifacts/pr.elf + else + echo "Warning: No ELF file found" + fi + - name: Upload ELF artifact + uses: actions/upload-artifact@ea05be8e2b5c27c5689e977ed6f65db0a051b1e5 # v4.6.0 + with: + name: memory-impact-pr-elf + path: ./elf-artifacts/pr.elf + if-no-files-found: warn + retention-days: 1 memory-impact-comment: name: Comment memory impact diff --git a/script/ci_memory_impact_comment.py b/script/ci_memory_impact_comment.py index e3e70d601f..f724a77c67 100755 --- a/script/ci_memory_impact_comment.py +++ b/script/ci_memory_impact_comment.py @@ -10,9 +10,16 @@ from __future__ import annotations import argparse import json +from pathlib import Path import subprocess import sys +# Add esphome to path for analyze_memory import +sys.path.insert(0, str(Path(__file__).parent.parent)) + +# pylint: disable=wrong-import-position +from esphome.analyze_memory import MemoryAnalyzer + # Comment marker to identify our memory impact comments COMMENT_MARKER = "" @@ -64,6 +71,105 @@ def format_change(before: int, after: int) -> str: return f"{emoji} {delta_str} ({pct_str})" +def run_detailed_analysis( + elf_path: str, objdump_path: str | None = None, readelf_path: str | None = None +) -> dict | None: + """Run detailed memory analysis on an ELF file. + + Args: + elf_path: Path to ELF file + objdump_path: Optional path to objdump tool + readelf_path: Optional path to readelf tool + + Returns: + Dictionary with component memory breakdown or None if analysis fails + """ + try: + analyzer = MemoryAnalyzer(elf_path, objdump_path, readelf_path) + components = analyzer.analyze() + + # Convert ComponentMemory objects to dictionaries + result = {} + for name, mem in components.items(): + result[name] = { + "text": mem.text_size, + "rodata": mem.rodata_size, + "data": mem.data_size, + "bss": mem.bss_size, + "flash_total": mem.flash_total, + "ram_total": mem.ram_total, + "symbol_count": mem.symbol_count, + } + return result + except Exception as e: + print(f"Warning: Failed to run detailed analysis: {e}", file=sys.stderr) + return None + + +def create_detailed_breakdown_table( + target_analysis: dict | None, pr_analysis: dict | None +) -> str: + """Create a markdown table showing detailed memory breakdown by component. + + Args: + target_analysis: Component memory breakdown for target branch + pr_analysis: Component memory breakdown for PR branch + + Returns: + Formatted markdown table + """ + if not target_analysis or not pr_analysis: + return "" + + # Combine all components from both analyses + all_components = set(target_analysis.keys()) | set(pr_analysis.keys()) + + # Filter to components that have changed or are significant + changed_components = [] + for comp in all_components: + target_mem = target_analysis.get(comp, {}) + pr_mem = pr_analysis.get(comp, {}) + + target_flash = target_mem.get("flash_total", 0) + pr_flash = pr_mem.get("flash_total", 0) + + # Include if component has changed or is significant (> 1KB) + if target_flash != pr_flash or target_flash > 1024 or pr_flash > 1024: + delta = pr_flash - target_flash + changed_components.append((comp, target_flash, pr_flash, delta)) + + if not changed_components: + return "" + + # Sort by absolute delta (largest changes first) + changed_components.sort(key=lambda x: abs(x[3]), reverse=True) + + # Build table - limit to top 20 changes + lines = [ + "", + "
", + "📊 Detailed Memory Breakdown (click to expand)", + "", + "| Component | Target Flash | PR Flash | Change |", + "|-----------|--------------|----------|--------|", + ] + + for comp, target_flash, pr_flash, delta in changed_components[:20]: + target_str = format_bytes(target_flash) + pr_str = format_bytes(pr_flash) + change_str = format_change(target_flash, pr_flash) + lines.append(f"| `{comp}` | {target_str} | {pr_str} | {change_str} |") + + if len(changed_components) > 20: + lines.append( + f"| ... | ... | ... | *({len(changed_components) - 20} more components not shown)* |" + ) + + lines.extend(["", "
", ""]) + + return "\n".join(lines) + + def create_comment_body( component: str, platform: str, @@ -71,6 +177,10 @@ def create_comment_body( target_flash: int, pr_ram: int, pr_flash: int, + target_elf: str | None = None, + pr_elf: str | None = None, + objdump_path: str | None = None, + readelf_path: str | None = None, ) -> str: """Create the comment body with memory impact analysis. @@ -81,6 +191,10 @@ def create_comment_body( target_flash: Flash usage in target branch pr_ram: RAM usage in PR branch pr_flash: Flash usage in PR branch + target_elf: Optional path to target branch ELF file + pr_elf: Optional path to PR branch ELF file + objdump_path: Optional path to objdump tool + readelf_path: Optional path to readelf tool Returns: Formatted comment body @@ -88,6 +202,25 @@ def create_comment_body( ram_change = format_change(target_ram, pr_ram) flash_change = format_change(target_flash, pr_flash) + # Run detailed analysis if ELF files are provided + target_analysis = None + pr_analysis = None + detailed_breakdown = "" + + if target_elf and pr_elf: + print( + f"Running detailed analysis on {target_elf} and {pr_elf}", file=sys.stderr + ) + target_analysis = run_detailed_analysis(target_elf, objdump_path, readelf_path) + pr_analysis = run_detailed_analysis(pr_elf, objdump_path, readelf_path) + + if target_analysis and pr_analysis: + detailed_breakdown = create_detailed_breakdown_table( + target_analysis, pr_analysis + ) + else: + print("No ELF files provided, skipping detailed analysis", file=sys.stderr) + return f"""{COMMENT_MARKER} ## Memory Impact Analysis @@ -98,7 +231,7 @@ def create_comment_body( |--------|--------------|---------|--------| | **RAM** | {format_bytes(target_ram)} | {format_bytes(pr_ram)} | {ram_change} | | **Flash** | {format_bytes(target_flash)} | {format_bytes(pr_flash)} | {flash_change} | - +{detailed_breakdown} --- *This analysis runs automatically when a single component changes. Memory usage is measured from a representative test configuration.* """ @@ -263,6 +396,14 @@ def main() -> int: parser.add_argument( "--pr-flash", type=int, required=True, help="PR branch flash usage" ) + parser.add_argument("--target-elf", help="Optional path to target branch ELF file") + parser.add_argument("--pr-elf", help="Optional path to PR branch ELF file") + parser.add_argument( + "--objdump-path", help="Optional path to objdump tool for detailed analysis" + ) + parser.add_argument( + "--readelf-path", help="Optional path to readelf tool for detailed analysis" + ) args = parser.parse_args() @@ -274,6 +415,10 @@ def main() -> int: target_flash=args.target_flash, pr_ram=args.pr_ram, pr_flash=args.pr_flash, + target_elf=args.target_elf, + pr_elf=args.pr_elf, + objdump_path=args.objdump_path, + readelf_path=args.readelf_path, ) # Post or update comment From 59848a2c8acbefc30ad944eeac11e23ca91b5824 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Fri, 17 Oct 2025 13:31:04 -1000 Subject: [PATCH 12/76] tweak --- .github/workflows/ci.yml | 71 ++++++++++-- script/ci_memory_impact_comment.py | 177 +++++++++++++++++++++++++++-- 2 files changed, 229 insertions(+), 19 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d87945f8df..d5f9bdca13 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -561,14 +561,26 @@ jobs: python script/ci_memory_impact_extract.py --output-env - name: Find and upload ELF file run: | - # Find the most recently created .elf file in .esphome/build - elf_file=$(find ~/.esphome/build -name "*.elf" -type f -printf '%T@ %p\n' 2>/dev/null | sort -rn | head -1 | cut -d' ' -f2-) + # Find the ELF file - try both common locations + elf_file="" + + # Try .esphome/build first (default location) + if [ -d ~/.esphome/build ]; then + elf_file=$(find ~/.esphome/build -name "firmware.elf" -o -name "*.elf" | head -1) + fi + + # Fallback to finding in .platformio if not found + if [ -z "$elf_file" ] && [ -d ~/.platformio ]; then + elf_file=$(find ~/.platformio -name "firmware.elf" | head -1) + fi + if [ -n "$elf_file" ] && [ -f "$elf_file" ]; then echo "Found ELF file: $elf_file" mkdir -p ./elf-artifacts cp "$elf_file" ./elf-artifacts/target.elf else - echo "Warning: No ELF file found" + echo "Warning: No ELF file found in ~/.esphome/build or ~/.platformio" + ls -la ~/.esphome/build/ || true fi - name: Upload ELF artifact uses: actions/upload-artifact@ea05be8e2b5c27c5689e977ed6f65db0a051b1e5 # v4.6.0 @@ -614,14 +626,26 @@ jobs: python script/ci_memory_impact_extract.py --output-env - name: Find and upload ELF file run: | - # Find the most recently created .elf file in .esphome/build - elf_file=$(find ~/.esphome/build -name "*.elf" -type f -printf '%T@ %p\n' 2>/dev/null | sort -rn | head -1 | cut -d' ' -f2-) + # Find the ELF file - try both common locations + elf_file="" + + # Try .esphome/build first (default location) + if [ -d ~/.esphome/build ]; then + elf_file=$(find ~/.esphome/build -name "firmware.elf" -o -name "*.elf" | head -1) + fi + + # Fallback to finding in .platformio if not found + if [ -z "$elf_file" ] && [ -d ~/.platformio ]; then + elf_file=$(find ~/.platformio -name "firmware.elf" | head -1) + fi + if [ -n "$elf_file" ] && [ -f "$elf_file" ]; then echo "Found ELF file: $elf_file" mkdir -p ./elf-artifacts cp "$elf_file" ./elf-artifacts/pr.elf else - echo "Warning: No ELF file found" + echo "Warning: No ELF file found in ~/.esphome/build or ~/.platformio" + ls -la ~/.esphome/build/ || true fi - name: Upload ELF artifact uses: actions/upload-artifact@ea05be8e2b5c27c5689e977ed6f65db0a051b1e5 # v4.6.0 @@ -651,6 +675,18 @@ jobs: with: python-version: ${{ env.DEFAULT_PYTHON }} cache-key: ${{ needs.common.outputs.cache-key }} + - name: Download target ELF artifact + uses: actions/download-artifact@1a18f44933c290e06e7167a92071e78bb20ab94a # v4.4.2 + with: + name: memory-impact-target-elf + path: ./elf-artifacts/target + continue-on-error: true + - name: Download PR ELF artifact + uses: actions/download-artifact@1a18f44933c290e06e7167a92071e78bb20ab94a # v4.4.2 + with: + name: memory-impact-pr-elf + path: ./elf-artifacts/pr + continue-on-error: true - name: Post or update PR comment env: GH_TOKEN: ${{ github.token }} @@ -662,6 +698,25 @@ jobs: PR_FLASH: ${{ needs.memory-impact-pr-branch.outputs.flash_usage }} run: | . venv/bin/activate + + # Check if ELF files exist + target_elf_arg="" + pr_elf_arg="" + + if [ -f ./elf-artifacts/target/target.elf ]; then + echo "Found target ELF file" + target_elf_arg="--target-elf ./elf-artifacts/target/target.elf" + else + echo "No target ELF file found" + fi + + if [ -f ./elf-artifacts/pr/pr.elf ]; then + echo "Found PR ELF file" + pr_elf_arg="--pr-elf ./elf-artifacts/pr/pr.elf" + else + echo "No PR ELF file found" + fi + python script/ci_memory_impact_comment.py \ --pr-number "${{ github.event.pull_request.number }}" \ --component "$COMPONENT" \ @@ -669,7 +724,9 @@ jobs: --target-ram "$TARGET_RAM" \ --target-flash "$TARGET_FLASH" \ --pr-ram "$PR_RAM" \ - --pr-flash "$PR_FLASH" + --pr-flash "$PR_FLASH" \ + $target_elf_arg \ + $pr_elf_arg ci-status: name: CI Status diff --git a/script/ci_memory_impact_comment.py b/script/ci_memory_impact_comment.py index f724a77c67..0b3bf87590 100755 --- a/script/ci_memory_impact_comment.py +++ b/script/ci_memory_impact_comment.py @@ -73,7 +73,7 @@ def format_change(before: int, after: int) -> str: def run_detailed_analysis( elf_path: str, objdump_path: str | None = None, readelf_path: str | None = None -) -> dict | None: +) -> tuple[dict | None, dict | None]: """Run detailed memory analysis on an ELF file. Args: @@ -82,16 +82,18 @@ def run_detailed_analysis( readelf_path: Optional path to readelf tool Returns: - Dictionary with component memory breakdown or None if analysis fails + Tuple of (component_breakdown, symbol_map) or (None, None) if analysis fails + component_breakdown: Dictionary with component memory breakdown + symbol_map: Dictionary mapping symbol names to their sizes """ try: analyzer = MemoryAnalyzer(elf_path, objdump_path, readelf_path) components = analyzer.analyze() # Convert ComponentMemory objects to dictionaries - result = {} + component_result = {} for name, mem in components.items(): - result[name] = { + component_result[name] = { "text": mem.text_size, "rodata": mem.rodata_size, "data": mem.data_size, @@ -100,10 +102,151 @@ def run_detailed_analysis( "ram_total": mem.ram_total, "symbol_count": mem.symbol_count, } - return result + + # Build symbol map from all sections + symbol_map = {} + for section in analyzer.sections.values(): + for symbol_name, size, _ in section.symbols: + if size > 0: # Only track non-zero sized symbols + # Demangle the symbol for better readability + demangled = analyzer._demangle_symbol(symbol_name) + symbol_map[demangled] = size + + return component_result, symbol_map except Exception as e: print(f"Warning: Failed to run detailed analysis: {e}", file=sys.stderr) - return None + import traceback + + traceback.print_exc(file=sys.stderr) + return None, None + + +def create_symbol_changes_table( + target_symbols: dict | None, pr_symbols: dict | None +) -> str: + """Create a markdown table showing symbols that changed size. + + Args: + target_symbols: Symbol name to size mapping for target branch + pr_symbols: Symbol name to size mapping for PR branch + + Returns: + Formatted markdown table + """ + if not target_symbols or not pr_symbols: + return "" + + # Find all symbols that exist in both branches or only in one + all_symbols = set(target_symbols.keys()) | set(pr_symbols.keys()) + + # Track changes + changed_symbols = [] + new_symbols = [] + removed_symbols = [] + + for symbol in all_symbols: + target_size = target_symbols.get(symbol, 0) + pr_size = pr_symbols.get(symbol, 0) + + if target_size == 0 and pr_size > 0: + # New symbol + new_symbols.append((symbol, pr_size)) + elif target_size > 0 and pr_size == 0: + # Removed symbol + removed_symbols.append((symbol, target_size)) + elif target_size != pr_size: + # Changed symbol + delta = pr_size - target_size + changed_symbols.append((symbol, target_size, pr_size, delta)) + + if not changed_symbols and not new_symbols and not removed_symbols: + return "" + + lines = [ + "", + "
", + "🔍 Symbol-Level Changes (click to expand)", + "", + ] + + # Show changed symbols (sorted by absolute delta) + if changed_symbols: + changed_symbols.sort(key=lambda x: abs(x[3]), reverse=True) + lines.extend( + [ + "### Changed Symbols", + "", + "| Symbol | Target Size | PR Size | Change |", + "|--------|-------------|---------|--------|", + ] + ) + + # Show top 30 changes + for symbol, target_size, pr_size, delta in changed_symbols[:30]: + target_str = format_bytes(target_size) + pr_str = format_bytes(pr_size) + change_str = format_change(target_size, pr_size) + # Truncate very long symbol names + display_symbol = symbol if len(symbol) <= 80 else symbol[:77] + "..." + lines.append( + f"| `{display_symbol}` | {target_str} | {pr_str} | {change_str} |" + ) + + if len(changed_symbols) > 30: + lines.append( + f"| ... | ... | ... | *({len(changed_symbols) - 30} more changed symbols not shown)* |" + ) + lines.append("") + + # Show new symbols + if new_symbols: + new_symbols.sort(key=lambda x: x[1], reverse=True) + lines.extend( + [ + "### New Symbols (top 15)", + "", + "| Symbol | Size |", + "|--------|------|", + ] + ) + + for symbol, size in new_symbols[:15]: + display_symbol = symbol if len(symbol) <= 80 else symbol[:77] + "..." + lines.append(f"| `{display_symbol}` | {format_bytes(size)} |") + + if len(new_symbols) > 15: + total_new_size = sum(s[1] for s in new_symbols) + lines.append( + f"| *{len(new_symbols) - 15} more new symbols...* | *Total: {format_bytes(total_new_size)}* |" + ) + lines.append("") + + # Show removed symbols + if removed_symbols: + removed_symbols.sort(key=lambda x: x[1], reverse=True) + lines.extend( + [ + "### Removed Symbols (top 15)", + "", + "| Symbol | Size |", + "|--------|------|", + ] + ) + + for symbol, size in removed_symbols[:15]: + display_symbol = symbol if len(symbol) <= 80 else symbol[:77] + "..." + lines.append(f"| `{display_symbol}` | {format_bytes(size)} |") + + if len(removed_symbols) > 15: + total_removed_size = sum(s[1] for s in removed_symbols) + lines.append( + f"| *{len(removed_symbols) - 15} more removed symbols...* | *Total: {format_bytes(total_removed_size)}* |" + ) + lines.append("") + + lines.extend(["
", ""]) + + return "\n".join(lines) def create_detailed_breakdown_table( @@ -148,7 +291,7 @@ def create_detailed_breakdown_table( lines = [ "", "
", - "📊 Detailed Memory Breakdown (click to expand)", + "📊 Component Memory Breakdown (click to expand)", "", "| Component | Target Flash | PR Flash | Change |", "|-----------|--------------|----------|--------|", @@ -205,19 +348,29 @@ def create_comment_body( # Run detailed analysis if ELF files are provided target_analysis = None pr_analysis = None - detailed_breakdown = "" + target_symbols = None + pr_symbols = None + component_breakdown = "" + symbol_changes = "" if target_elf and pr_elf: print( f"Running detailed analysis on {target_elf} and {pr_elf}", file=sys.stderr ) - target_analysis = run_detailed_analysis(target_elf, objdump_path, readelf_path) - pr_analysis = run_detailed_analysis(pr_elf, objdump_path, readelf_path) + target_analysis, target_symbols = run_detailed_analysis( + target_elf, objdump_path, readelf_path + ) + pr_analysis, pr_symbols = run_detailed_analysis( + pr_elf, objdump_path, readelf_path + ) if target_analysis and pr_analysis: - detailed_breakdown = create_detailed_breakdown_table( + component_breakdown = create_detailed_breakdown_table( target_analysis, pr_analysis ) + + if target_symbols and pr_symbols: + symbol_changes = create_symbol_changes_table(target_symbols, pr_symbols) else: print("No ELF files provided, skipping detailed analysis", file=sys.stderr) @@ -231,7 +384,7 @@ def create_comment_body( |--------|--------------|---------|--------| | **RAM** | {format_bytes(target_ram)} | {format_bytes(pr_ram)} | {ram_change} | | **Flash** | {format_bytes(target_flash)} | {format_bytes(pr_flash)} | {flash_change} | -{detailed_breakdown} +{component_breakdown}{symbol_changes} --- *This analysis runs automatically when a single component changes. Memory usage is measured from a representative test configuration.* """ From 9d081795e8b64451cbe1533638f7dd5501435c3b Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Fri, 17 Oct 2025 13:41:55 -1000 Subject: [PATCH 13/76] relo --- .github/workflows/ci.yml | 4 +- .../__init__.py} | 859 +----------------- esphome/analyze_memory/const.py | 857 +++++++++++++++++ script/ci_memory_impact_comment.py | 2 +- 4 files changed, 864 insertions(+), 858 deletions(-) rename esphome/{analyze_memory.py => analyze_memory/__init__.py} (56%) create mode 100644 esphome/analyze_memory/const.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d5f9bdca13..6fa8150b93 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -583,7 +583,7 @@ jobs: ls -la ~/.esphome/build/ || true fi - name: Upload ELF artifact - uses: actions/upload-artifact@ea05be8e2b5c27c5689e977ed6f65db0a051b1e5 # v4.6.0 + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 with: name: memory-impact-target-elf path: ./elf-artifacts/target.elf @@ -648,7 +648,7 @@ jobs: ls -la ~/.esphome/build/ || true fi - name: Upload ELF artifact - uses: actions/upload-artifact@ea05be8e2b5c27c5689e977ed6f65db0a051b1e5 # v4.6.0 + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 with: name: memory-impact-pr-elf path: ./elf-artifacts/pr.elf diff --git a/esphome/analyze_memory.py b/esphome/analyze_memory/__init__.py similarity index 56% rename from esphome/analyze_memory.py rename to esphome/analyze_memory/__init__.py index 70c324b33f..c6fdb1028d 100644 --- a/esphome/analyze_memory.py +++ b/esphome/analyze_memory/__init__.py @@ -7,862 +7,10 @@ from pathlib import Path import re import subprocess +from .const import DEMANGLED_PATTERNS, ESPHOME_COMPONENT_PATTERN, SYMBOL_PATTERNS + _LOGGER = logging.getLogger(__name__) -# Pattern to extract ESPHome component namespaces dynamically -ESPHOME_COMPONENT_PATTERN = re.compile(r"esphome::([a-zA-Z0-9_]+)::") - -# Component identification rules -# Symbol patterns: patterns found in raw symbol names -SYMBOL_PATTERNS = { - "freertos": [ - "vTask", - "xTask", - "xQueue", - "pvPort", - "vPort", - "uxTask", - "pcTask", - "prvTimerTask", - "prvAddNewTaskToReadyList", - "pxReadyTasksLists", - "prvAddCurrentTaskToDelayedList", - "xEventGroupWaitBits", - "xRingbufferSendFromISR", - "prvSendItemDoneNoSplit", - "prvReceiveGeneric", - "prvSendAcquireGeneric", - "prvCopyItemAllowSplit", - "xEventGroup", - "xRingbuffer", - "prvSend", - "prvReceive", - "prvCopy", - "xPort", - "ulTaskGenericNotifyTake", - "prvIdleTask", - "prvInitialiseNewTask", - "prvIsYieldRequiredSMP", - "prvGetItemByteBuf", - "prvInitializeNewRingbuffer", - "prvAcquireItemNoSplit", - "prvNotifyQueueSetContainer", - "ucStaticTimerQueueStorage", - "eTaskGetState", - "main_task", - "do_system_init_fn", - "xSemaphoreCreateGenericWithCaps", - "vListInsert", - "uxListRemove", - "vRingbufferReturnItem", - "vRingbufferReturnItemFromISR", - "prvCheckItemFitsByteBuffer", - "prvGetCurMaxSizeAllowSplit", - "tick_hook", - "sys_sem_new", - "sys_arch_mbox_fetch", - "sys_arch_sem_wait", - "prvDeleteTCB", - "vQueueDeleteWithCaps", - "vRingbufferDeleteWithCaps", - "vSemaphoreDeleteWithCaps", - "prvCheckItemAvail", - "prvCheckTaskCanBeScheduledSMP", - "prvGetCurMaxSizeNoSplit", - "prvResetNextTaskUnblockTime", - "prvReturnItemByteBuf", - "vApplicationStackOverflowHook", - "vApplicationGetIdleTaskMemory", - "sys_init", - "sys_mbox_new", - "sys_arch_mbox_tryfetch", - ], - "xtensa": ["xt_", "_xt_", "xPortEnterCriticalTimeout"], - "heap": ["heap_", "multi_heap"], - "spi_flash": ["spi_flash"], - "rtc": ["rtc_", "rtcio_ll_"], - "gpio_driver": ["gpio_", "pins"], - "uart_driver": ["uart", "_uart", "UART"], - "timer": ["timer_", "esp_timer"], - "peripherals": ["periph_", "periman"], - "network_stack": [ - "vj_compress", - "raw_sendto", - "raw_input", - "etharp_", - "icmp_input", - "socket_ipv6", - "ip_napt", - "socket_ipv4_multicast", - "socket_ipv6_multicast", - "netconn_", - "recv_raw", - "accept_function", - "netconn_recv_data", - "netconn_accept", - "netconn_write_vectors_partly", - "netconn_drain", - "raw_connect", - "raw_bind", - "icmp_send_response", - "sockets", - "icmp_dest_unreach", - "inet_chksum_pseudo", - "alloc_socket", - "done_socket", - "set_global_fd_sets", - "inet_chksum_pbuf", - "tryget_socket_unconn_locked", - "tryget_socket_unconn", - "cs_create_ctrl_sock", - "netbuf_alloc", - ], - "ipv6_stack": ["nd6_", "ip6_", "mld6_", "icmp6_", "icmp6_input"], - "wifi_stack": [ - "ieee80211", - "hostap", - "sta_", - "ap_", - "scan_", - "wifi_", - "wpa_", - "wps_", - "esp_wifi", - "cnx_", - "wpa3_", - "sae_", - "wDev_", - "ic_", - "mac_", - "esf_buf", - "gWpaSm", - "sm_WPA", - "eapol_", - "owe_", - "wifiLowLevelInit", - "s_do_mapping", - "gScanStruct", - "ppSearchTxframe", - "ppMapWaitTxq", - "ppFillAMPDUBar", - "ppCheckTxConnTrafficIdle", - "ppCalTkipMic", - ], - "bluetooth": ["bt_", "ble_", "l2c_", "gatt_", "gap_", "hci_", "BT_init"], - "wifi_bt_coex": ["coex"], - "bluetooth_rom": ["r_ble", "r_lld", "r_llc", "r_llm"], - "bluedroid_bt": [ - "bluedroid", - "btc_", - "bta_", - "btm_", - "btu_", - "BTM_", - "GATT", - "L2CA_", - "smp_", - "gatts_", - "attp_", - "l2cu_", - "l2cb", - "smp_cb", - "BTA_GATTC_", - "SMP_", - "BTU_", - "BTA_Dm", - "GAP_Ble", - "BT_tx_if", - "host_recv_pkt_cb", - "saved_local_oob_data", - "string_to_bdaddr", - "string_is_bdaddr", - "CalConnectParamTimeout", - "transmit_fragment", - "transmit_data", - "event_command_ready", - "read_command_complete_header", - "parse_read_local_extended_features_response", - "parse_read_local_version_info_response", - "should_request_high", - "btdm_wakeup_request", - "BTA_SetAttributeValue", - "BTA_EnableBluetooth", - "transmit_command_futured", - "transmit_command", - "get_waiting_command", - "make_command", - "transmit_downward", - "host_recv_adv_packet", - "copy_extra_byte_in_db", - "parse_read_local_supported_commands_response", - ], - "crypto_math": [ - "ecp_", - "bignum_", - "mpi_", - "sswu", - "modp", - "dragonfly_", - "gcm_mult", - "__multiply", - "quorem", - "__mdiff", - "__lshift", - "__mprec_tens", - "ECC_", - "multiprecision_", - "mix_sub_columns", - "sbox", - "gfm2_sbox", - "gfm3_sbox", - "curve_p256", - "curve", - "p_256_init_curve", - "shift_sub_rows", - "rshift", - ], - "hw_crypto": ["esp_aes", "esp_sha", "esp_rsa", "esp_bignum", "esp_mpi"], - "libc": [ - "printf", - "scanf", - "malloc", - "free", - "memcpy", - "memset", - "strcpy", - "strlen", - "_dtoa", - "_fopen", - "__sfvwrite_r", - "qsort", - "__sf", - "__sflush_r", - "__srefill_r", - "_impure_data", - "_reclaim_reent", - "_open_r", - "strncpy", - "_strtod_l", - "__gethex", - "__hexnan", - "_setenv_r", - "_tzset_unlocked_r", - "__tzcalc_limits", - "select", - "scalbnf", - "strtof", - "strtof_l", - "__d2b", - "__b2d", - "__s2b", - "_Balloc", - "__multadd", - "__lo0bits", - "__atexit0", - "__smakebuf_r", - "__swhatbuf_r", - "_sungetc_r", - "_close_r", - "_link_r", - "_unsetenv_r", - "_rename_r", - "__month_lengths", - "tzinfo", - "__ratio", - "__hi0bits", - "__ulp", - "__any_on", - "__copybits", - "L_shift", - "_fcntl_r", - "_lseek_r", - "_read_r", - "_write_r", - "_unlink_r", - "_fstat_r", - "access", - "fsync", - "tcsetattr", - "tcgetattr", - "tcflush", - "tcdrain", - "__ssrefill_r", - "_stat_r", - "__hexdig_fun", - "__mcmp", - "_fwalk_sglue", - "__fpclassifyf", - "_setlocale_r", - "_mbrtowc_r", - "fcntl", - "__match", - "_lock_close", - "__c$", - "__func__$", - "__FUNCTION__$", - "DAYS_IN_MONTH", - "_DAYS_BEFORE_MONTH", - "CSWTCH$", - "dst$", - "sulp", - ], - "string_ops": ["strcmp", "strncmp", "strchr", "strstr", "strtok", "strdup"], - "memory_alloc": ["malloc", "calloc", "realloc", "free", "_sbrk"], - "file_io": [ - "fread", - "fwrite", - "fopen", - "fclose", - "fseek", - "ftell", - "fflush", - "s_fd_table", - ], - "string_formatting": [ - "snprintf", - "vsnprintf", - "sprintf", - "vsprintf", - "sscanf", - "vsscanf", - ], - "cpp_anonymous": ["_GLOBAL__N_", "n$"], - "cpp_runtime": ["__cxx", "_ZN", "_ZL", "_ZSt", "__gxx_personality", "_Z16"], - "exception_handling": ["__cxa_", "_Unwind_", "__gcc_personality", "uw_frame_state"], - "static_init": ["_GLOBAL__sub_I_"], - "mdns_lib": ["mdns"], - "phy_radio": [ - "phy_", - "rf_", - "chip_", - "register_chipv7", - "pbus_", - "bb_", - "fe_", - "rfcal_", - "ram_rfcal", - "tx_pwctrl", - "rx_chan", - "set_rx_gain", - "set_chan", - "agc_reg", - "ram_txiq", - "ram_txdc", - "ram_gen_rx_gain", - "rx_11b_opt", - "set_rx_sense", - "set_rx_gain_cal", - "set_chan_dig_gain", - "tx_pwctrl_init_cal", - "rfcal_txiq", - "set_tx_gain_table", - "correct_rfpll_offset", - "pll_correct_dcap", - "txiq_cal_init", - "pwdet_sar", - "pwdet_sar2_init", - "ram_iq_est_enable", - "ram_rfpll_set_freq", - "ant_wifirx_cfg", - "ant_btrx_cfg", - "force_txrxoff", - "force_txrx_off", - "tx_paon_set", - "opt_11b_resart", - "rfpll_1p2_opt", - "ram_dc_iq_est", - "ram_start_tx_tone", - "ram_en_pwdet", - "ram_cbw2040_cfg", - "rxdc_est_min", - "i2cmst_reg_init", - "temprature_sens_read", - "ram_restart_cal", - "ram_write_gain_mem", - "ram_wait_rfpll_cal_end", - "txcal_debuge_mode", - "ant_wifitx_cfg", - "reg_init_begin", - ], - "wifi_phy_pp": ["pp_", "ppT", "ppR", "ppP", "ppInstall", "ppCalTxAMPDULength"], - "wifi_lmac": ["lmac"], - "wifi_device": ["wdev", "wDev_"], - "power_mgmt": [ - "pm_", - "sleep", - "rtc_sleep", - "light_sleep", - "deep_sleep", - "power_down", - "g_pm", - ], - "memory_mgmt": [ - "mem_", - "memory_", - "tlsf_", - "memp_", - "pbuf_", - "pbuf_alloc", - "pbuf_copy_partial_pbuf", - ], - "hal_layer": ["hal_"], - "clock_mgmt": [ - "clk_", - "clock_", - "rtc_clk", - "apb_", - "cpu_freq", - "setCpuFrequencyMhz", - ], - "cache_mgmt": ["cache"], - "flash_ops": ["flash", "image_load"], - "interrupt_handlers": [ - "isr", - "interrupt", - "intr_", - "exc_", - "exception", - "port_IntStack", - ], - "wrapper_functions": ["_wrapper"], - "error_handling": ["panic", "abort", "assert", "error_", "fault"], - "authentication": ["auth"], - "ppp_protocol": ["ppp", "ipcp_", "lcp_", "chap_", "LcpEchoCheck"], - "dhcp": ["dhcp", "handle_dhcp"], - "ethernet_phy": [ - "emac_", - "eth_phy_", - "phy_tlk110", - "phy_lan87", - "phy_ip101", - "phy_rtl", - "phy_dp83", - "phy_ksz", - "lan87xx_", - "rtl8201_", - "ip101_", - "ksz80xx_", - "jl1101_", - "dp83848_", - "eth_on_state_changed", - ], - "threading": ["pthread_", "thread_", "_task_"], - "pthread": ["pthread"], - "synchronization": ["mutex", "semaphore", "spinlock", "portMUX"], - "math_lib": [ - "sin", - "cos", - "tan", - "sqrt", - "pow", - "exp", - "log", - "atan", - "asin", - "acos", - "floor", - "ceil", - "fabs", - "round", - ], - "random": ["rand", "random", "rng_", "prng"], - "time_lib": [ - "time", - "clock", - "gettimeofday", - "settimeofday", - "localtime", - "gmtime", - "mktime", - "strftime", - ], - "console_io": ["console_", "uart_tx", "uart_rx", "puts", "putchar", "getchar"], - "rom_functions": ["r_", "rom_"], - "compiler_runtime": [ - "__divdi3", - "__udivdi3", - "__moddi3", - "__muldi3", - "__ashldi3", - "__ashrdi3", - "__lshrdi3", - "__cmpdi2", - "__fixdfdi", - "__floatdidf", - ], - "libgcc": ["libgcc", "_divdi3", "_udivdi3"], - "boot_startup": ["boot", "start_cpu", "call_start", "startup", "bootloader"], - "bootloader": ["bootloader_", "esp_bootloader"], - "app_framework": ["app_", "initArduino", "setup", "loop", "Update"], - "weak_symbols": ["__weak_"], - "compiler_builtins": ["__builtin_"], - "vfs": ["vfs_", "VFS"], - "esp32_sdk": ["esp32_", "esp32c", "esp32s"], - "usb": ["usb_", "USB", "cdc_", "CDC"], - "i2c_driver": ["i2c_", "I2C"], - "i2s_driver": ["i2s_", "I2S"], - "spi_driver": ["spi_", "SPI"], - "adc_driver": ["adc_", "ADC"], - "dac_driver": ["dac_", "DAC"], - "touch_driver": ["touch_", "TOUCH"], - "pwm_driver": ["pwm_", "PWM", "ledc_", "LEDC"], - "rmt_driver": ["rmt_", "RMT"], - "pcnt_driver": ["pcnt_", "PCNT"], - "can_driver": ["can_", "CAN", "twai_", "TWAI"], - "sdmmc_driver": ["sdmmc_", "SDMMC", "sdcard", "sd_card"], - "temp_sensor": ["temp_sensor", "tsens_"], - "watchdog": ["wdt_", "WDT", "watchdog"], - "brownout": ["brownout", "bod_"], - "ulp": ["ulp_", "ULP"], - "psram": ["psram", "PSRAM", "spiram", "SPIRAM"], - "efuse": ["efuse", "EFUSE"], - "partition": ["partition", "esp_partition"], - "esp_event": ["esp_event", "event_loop", "event_callback"], - "esp_console": ["esp_console", "console_"], - "chip_specific": ["chip_", "esp_chip"], - "esp_system_utils": ["esp_system", "esp_hw", "esp_clk", "esp_sleep"], - "ipc": ["esp_ipc", "ipc_"], - "wifi_config": [ - "g_cnxMgr", - "gChmCxt", - "g_ic", - "TxRxCxt", - "s_dp", - "s_ni", - "s_reg_dump", - "packet$", - "d_mult_table", - "K", - "fcstab", - ], - "smartconfig": ["sc_ack_send"], - "rc_calibration": ["rc_cal", "rcUpdate"], - "noise_floor": ["noise_check"], - "rf_calibration": [ - "set_rx_sense", - "set_rx_gain_cal", - "set_chan_dig_gain", - "tx_pwctrl_init_cal", - "rfcal_txiq", - "set_tx_gain_table", - "correct_rfpll_offset", - "pll_correct_dcap", - "txiq_cal_init", - "pwdet_sar", - "rx_11b_opt", - ], - "wifi_crypto": [ - "pk_use_ecparams", - "process_segments", - "ccmp_", - "rc4_", - "aria_", - "mgf_mask", - "dh_group", - "ccmp_aad_nonce", - "ccmp_encrypt", - "rc4_skip", - "aria_sb1", - "aria_sb2", - "aria_is1", - "aria_is2", - "aria_sl", - "aria_a", - ], - "radio_control": ["fsm_input", "fsm_sconfreq"], - "pbuf": [ - "pbuf_", - ], - "event_group": ["xEventGroup"], - "ringbuffer": ["xRingbuffer", "prvSend", "prvReceive", "prvCopy"], - "provisioning": ["prov_", "prov_stop_and_notify"], - "scan": ["gScanStruct"], - "port": ["xPort"], - "elf_loader": [ - "elf_add", - "elf_add_note", - "elf_add_segment", - "process_image", - "read_encoded", - "read_encoded_value", - "read_encoded_value_with_base", - "process_image_header", - ], - "socket_api": [ - "sockets", - "netconn_", - "accept_function", - "recv_raw", - "socket_ipv4_multicast", - "socket_ipv6_multicast", - ], - "igmp": ["igmp_", "igmp_send", "igmp_input"], - "icmp6": ["icmp6_"], - "arp": ["arp_table"], - "ampdu": [ - "ampdu_", - "rcAmpdu", - "trc_onAmpduOp", - "rcAmpduLowerRate", - "ampdu_dispatch_upto", - ], - "ieee802_11": ["ieee802_11_", "ieee802_11_parse_elems"], - "rate_control": ["rssi_margin", "rcGetSched", "get_rate_fcc_index"], - "nan": ["nan_dp_", "nan_dp_post_tx", "nan_dp_delete_peer"], - "channel_mgmt": ["chm_init", "chm_set_current_channel"], - "trace": ["trc_init", "trc_onAmpduOp"], - "country_code": ["country_info", "country_info_24ghz"], - "multicore": ["do_multicore_settings"], - "Update_lib": ["Update"], - "stdio": [ - "__sf", - "__sflush_r", - "__srefill_r", - "_impure_data", - "_reclaim_reent", - "_open_r", - ], - "strncpy_ops": ["strncpy"], - "math_internal": ["__mdiff", "__lshift", "__mprec_tens", "quorem"], - "character_class": ["__chclass"], - "camellia": ["camellia_", "camellia_feistel"], - "crypto_tables": ["FSb", "FSb2", "FSb3", "FSb4"], - "event_buffer": ["g_eb_list_desc", "eb_space"], - "base_node": ["base_node_", "base_node_add_handler"], - "file_descriptor": ["s_fd_table"], - "tx_delay": ["tx_delay_cfg"], - "deinit": ["deinit_functions"], - "lcp_echo": ["LcpEchoCheck"], - "raw_api": ["raw_bind", "raw_connect"], - "checksum": ["process_checksum"], - "entry_management": ["add_entry"], - "esp_ota": ["esp_ota", "ota_", "read_otadata"], - "http_server": [ - "httpd_", - "parse_url_char", - "cb_headers_complete", - "delete_entry", - "validate_structure", - "config_save", - "config_new", - "verify_url", - "cb_url", - ], - "misc_system": [ - "alarm_cbs", - "start_up", - "tokens", - "unhex", - "osi_funcs_ro", - "enum_function", - "fragment_and_dispatch", - "alarm_set", - "osi_alarm_new", - "config_set_string", - "config_update_newest_section", - "config_remove_key", - "method_strings", - "interop_match", - "interop_database", - "__state_table", - "__action_table", - "s_stub_table", - "s_context", - "s_mmu_ctx", - "s_get_bus_mask", - "hli_queue_put", - "list_remove", - "list_delete", - "lock_acquire_generic", - "is_vect_desc_usable", - "io_mode_str", - "__c$20233", - "interface", - "read_id_core", - "subscribe_idle", - "unsubscribe_idle", - "s_clkout_handle", - "lock_release_generic", - "config_set_int", - "config_get_int", - "config_get_string", - "config_has_key", - "config_remove_section", - "osi_alarm_init", - "osi_alarm_deinit", - "fixed_queue_enqueue", - "fixed_queue_dequeue", - "fixed_queue_new", - "fixed_pkt_queue_enqueue", - "fixed_pkt_queue_new", - "list_append", - "list_prepend", - "list_insert_after", - "list_contains", - "list_get_node", - "hash_function_blob", - "cb_no_body", - "cb_on_body", - "profile_tab", - "get_arg", - "trim", - "buf$", - "process_appended_hash_and_sig$constprop$0", - "uuidType", - "allocate_svc_db_buf", - "_hostname_is_ours", - "s_hli_handlers", - "tick_cb", - "idle_cb", - "input", - "entry_find", - "section_find", - "find_bucket_entry_", - "config_has_section", - "hli_queue_create", - "hli_queue_get", - "hli_c_handler", - "future_ready", - "future_await", - "future_new", - "pkt_queue_enqueue", - "pkt_queue_dequeue", - "pkt_queue_cleanup", - "pkt_queue_create", - "pkt_queue_destroy", - "fixed_pkt_queue_dequeue", - "osi_alarm_cancel", - "osi_alarm_is_active", - "osi_sem_take", - "osi_event_create", - "osi_event_bind", - "alarm_cb_handler", - "list_foreach", - "list_back", - "list_front", - "list_clear", - "fixed_queue_try_peek_first", - "translate_path", - "get_idx", - "find_key", - "init", - "end", - "start", - "set_read_value", - "copy_address_list", - "copy_and_key", - "sdk_cfg_opts", - "leftshift_onebit", - "config_section_end", - "config_section_begin", - "find_entry_and_check_all_reset", - "image_validate", - "xPendingReadyList", - "vListInitialise", - "lock_init_generic", - "ant_bttx_cfg", - "ant_dft_cfg", - "cs_send_to_ctrl_sock", - "config_llc_util_funcs_reset", - "make_set_adv_report_flow_control", - "make_set_event_mask", - "raw_new", - "raw_remove", - "BTE_InitStack", - "parse_read_local_supported_features_response", - "__math_invalidf", - "tinytens", - "__mprec_tinytens", - "__mprec_bigtens", - "vRingbufferDelete", - "vRingbufferDeleteWithCaps", - "vRingbufferReturnItem", - "vRingbufferReturnItemFromISR", - "get_acl_data_size_ble", - "get_features_ble", - "get_features_classic", - "get_acl_packet_size_ble", - "get_acl_packet_size_classic", - "supports_extended_inquiry_response", - "supports_rssi_with_inquiry_results", - "supports_interlaced_inquiry_scan", - "supports_reading_remote_extended_features", - ], - "bluetooth_ll": [ - "lld_pdu_", - "ld_acl_", - "lld_stop_ind_handler", - "lld_evt_winsize_change", - "config_lld_evt_funcs_reset", - "config_lld_funcs_reset", - "config_llm_funcs_reset", - "llm_set_long_adv_data", - "lld_retry_tx_prog", - "llc_link_sup_to_ind_handler", - "config_llc_funcs_reset", - "lld_evt_rxwin_compute", - "config_btdm_funcs_reset", - "config_ea_funcs_reset", - "llc_defalut_state_tab_reset", - "config_rwip_funcs_reset", - "ke_lmp_rx_flooding_detect", - ], -} - -# Demangled patterns: patterns found in demangled C++ names -DEMANGLED_PATTERNS = { - "gpio_driver": ["GPIO"], - "uart_driver": ["UART"], - "network_stack": [ - "lwip", - "tcp", - "udp", - "ip4", - "ip6", - "dhcp", - "dns", - "netif", - "ethernet", - "ppp", - "slip", - ], - "wifi_stack": ["NetworkInterface"], - "nimble_bt": [ - "nimble", - "NimBLE", - "ble_hs", - "ble_gap", - "ble_gatt", - "ble_att", - "ble_l2cap", - "ble_sm", - ], - "crypto": ["mbedtls", "crypto", "sha", "aes", "rsa", "ecc", "tls", "ssl"], - "cpp_stdlib": ["std::", "__gnu_cxx::", "__cxxabiv"], - "static_init": ["__static_initialization"], - "rtti": ["__type_info", "__class_type_info"], - "web_server_lib": ["AsyncWebServer", "AsyncWebHandler", "WebServer"], - "async_tcp": ["AsyncClient", "AsyncServer"], - "mdns_lib": ["mdns"], - "json_lib": [ - "ArduinoJson", - "JsonDocument", - "JsonArray", - "JsonObject", - "deserialize", - "serialize", - ], - "http_lib": ["HTTP", "http_", "Request", "Response", "Uri", "WebSocket"], - "logging": ["log", "Log", "print", "Print", "diag_"], - "authentication": ["checkDigestAuthentication"], - "libgcc": ["libgcc"], - "esp_system": ["esp_", "ESP"], - "arduino": ["arduino"], - "nvs": ["nvs_", "_ZTVN3nvs", "nvs::"], - "filesystem": ["spiffs", "vfs"], - "libc": ["newlib"], -} - # Get the list of actual ESPHome components by scanning the components directory def get_esphome_components(): @@ -870,7 +18,8 @@ def get_esphome_components(): components = set() # Find the components directory relative to this file - current_dir = Path(__file__).parent + # Go up two levels from analyze_memory/__init__.py to esphome/ + current_dir = Path(__file__).parent.parent components_dir = current_dir / "components" if components_dir.exists() and components_dir.is_dir(): diff --git a/esphome/analyze_memory/const.py b/esphome/analyze_memory/const.py new file mode 100644 index 0000000000..68cd957090 --- /dev/null +++ b/esphome/analyze_memory/const.py @@ -0,0 +1,857 @@ +"""Constants for memory analysis symbol pattern matching.""" + +import re + +# Pattern to extract ESPHome component namespaces dynamically +ESPHOME_COMPONENT_PATTERN = re.compile(r"esphome::([a-zA-Z0-9_]+)::") + +# Component identification rules +# Symbol patterns: patterns found in raw symbol names +SYMBOL_PATTERNS = { + "freertos": [ + "vTask", + "xTask", + "xQueue", + "pvPort", + "vPort", + "uxTask", + "pcTask", + "prvTimerTask", + "prvAddNewTaskToReadyList", + "pxReadyTasksLists", + "prvAddCurrentTaskToDelayedList", + "xEventGroupWaitBits", + "xRingbufferSendFromISR", + "prvSendItemDoneNoSplit", + "prvReceiveGeneric", + "prvSendAcquireGeneric", + "prvCopyItemAllowSplit", + "xEventGroup", + "xRingbuffer", + "prvSend", + "prvReceive", + "prvCopy", + "xPort", + "ulTaskGenericNotifyTake", + "prvIdleTask", + "prvInitialiseNewTask", + "prvIsYieldRequiredSMP", + "prvGetItemByteBuf", + "prvInitializeNewRingbuffer", + "prvAcquireItemNoSplit", + "prvNotifyQueueSetContainer", + "ucStaticTimerQueueStorage", + "eTaskGetState", + "main_task", + "do_system_init_fn", + "xSemaphoreCreateGenericWithCaps", + "vListInsert", + "uxListRemove", + "vRingbufferReturnItem", + "vRingbufferReturnItemFromISR", + "prvCheckItemFitsByteBuffer", + "prvGetCurMaxSizeAllowSplit", + "tick_hook", + "sys_sem_new", + "sys_arch_mbox_fetch", + "sys_arch_sem_wait", + "prvDeleteTCB", + "vQueueDeleteWithCaps", + "vRingbufferDeleteWithCaps", + "vSemaphoreDeleteWithCaps", + "prvCheckItemAvail", + "prvCheckTaskCanBeScheduledSMP", + "prvGetCurMaxSizeNoSplit", + "prvResetNextTaskUnblockTime", + "prvReturnItemByteBuf", + "vApplicationStackOverflowHook", + "vApplicationGetIdleTaskMemory", + "sys_init", + "sys_mbox_new", + "sys_arch_mbox_tryfetch", + ], + "xtensa": ["xt_", "_xt_", "xPortEnterCriticalTimeout"], + "heap": ["heap_", "multi_heap"], + "spi_flash": ["spi_flash"], + "rtc": ["rtc_", "rtcio_ll_"], + "gpio_driver": ["gpio_", "pins"], + "uart_driver": ["uart", "_uart", "UART"], + "timer": ["timer_", "esp_timer"], + "peripherals": ["periph_", "periman"], + "network_stack": [ + "vj_compress", + "raw_sendto", + "raw_input", + "etharp_", + "icmp_input", + "socket_ipv6", + "ip_napt", + "socket_ipv4_multicast", + "socket_ipv6_multicast", + "netconn_", + "recv_raw", + "accept_function", + "netconn_recv_data", + "netconn_accept", + "netconn_write_vectors_partly", + "netconn_drain", + "raw_connect", + "raw_bind", + "icmp_send_response", + "sockets", + "icmp_dest_unreach", + "inet_chksum_pseudo", + "alloc_socket", + "done_socket", + "set_global_fd_sets", + "inet_chksum_pbuf", + "tryget_socket_unconn_locked", + "tryget_socket_unconn", + "cs_create_ctrl_sock", + "netbuf_alloc", + ], + "ipv6_stack": ["nd6_", "ip6_", "mld6_", "icmp6_", "icmp6_input"], + "wifi_stack": [ + "ieee80211", + "hostap", + "sta_", + "ap_", + "scan_", + "wifi_", + "wpa_", + "wps_", + "esp_wifi", + "cnx_", + "wpa3_", + "sae_", + "wDev_", + "ic_", + "mac_", + "esf_buf", + "gWpaSm", + "sm_WPA", + "eapol_", + "owe_", + "wifiLowLevelInit", + "s_do_mapping", + "gScanStruct", + "ppSearchTxframe", + "ppMapWaitTxq", + "ppFillAMPDUBar", + "ppCheckTxConnTrafficIdle", + "ppCalTkipMic", + ], + "bluetooth": ["bt_", "ble_", "l2c_", "gatt_", "gap_", "hci_", "BT_init"], + "wifi_bt_coex": ["coex"], + "bluetooth_rom": ["r_ble", "r_lld", "r_llc", "r_llm"], + "bluedroid_bt": [ + "bluedroid", + "btc_", + "bta_", + "btm_", + "btu_", + "BTM_", + "GATT", + "L2CA_", + "smp_", + "gatts_", + "attp_", + "l2cu_", + "l2cb", + "smp_cb", + "BTA_GATTC_", + "SMP_", + "BTU_", + "BTA_Dm", + "GAP_Ble", + "BT_tx_if", + "host_recv_pkt_cb", + "saved_local_oob_data", + "string_to_bdaddr", + "string_is_bdaddr", + "CalConnectParamTimeout", + "transmit_fragment", + "transmit_data", + "event_command_ready", + "read_command_complete_header", + "parse_read_local_extended_features_response", + "parse_read_local_version_info_response", + "should_request_high", + "btdm_wakeup_request", + "BTA_SetAttributeValue", + "BTA_EnableBluetooth", + "transmit_command_futured", + "transmit_command", + "get_waiting_command", + "make_command", + "transmit_downward", + "host_recv_adv_packet", + "copy_extra_byte_in_db", + "parse_read_local_supported_commands_response", + ], + "crypto_math": [ + "ecp_", + "bignum_", + "mpi_", + "sswu", + "modp", + "dragonfly_", + "gcm_mult", + "__multiply", + "quorem", + "__mdiff", + "__lshift", + "__mprec_tens", + "ECC_", + "multiprecision_", + "mix_sub_columns", + "sbox", + "gfm2_sbox", + "gfm3_sbox", + "curve_p256", + "curve", + "p_256_init_curve", + "shift_sub_rows", + "rshift", + ], + "hw_crypto": ["esp_aes", "esp_sha", "esp_rsa", "esp_bignum", "esp_mpi"], + "libc": [ + "printf", + "scanf", + "malloc", + "free", + "memcpy", + "memset", + "strcpy", + "strlen", + "_dtoa", + "_fopen", + "__sfvwrite_r", + "qsort", + "__sf", + "__sflush_r", + "__srefill_r", + "_impure_data", + "_reclaim_reent", + "_open_r", + "strncpy", + "_strtod_l", + "__gethex", + "__hexnan", + "_setenv_r", + "_tzset_unlocked_r", + "__tzcalc_limits", + "select", + "scalbnf", + "strtof", + "strtof_l", + "__d2b", + "__b2d", + "__s2b", + "_Balloc", + "__multadd", + "__lo0bits", + "__atexit0", + "__smakebuf_r", + "__swhatbuf_r", + "_sungetc_r", + "_close_r", + "_link_r", + "_unsetenv_r", + "_rename_r", + "__month_lengths", + "tzinfo", + "__ratio", + "__hi0bits", + "__ulp", + "__any_on", + "__copybits", + "L_shift", + "_fcntl_r", + "_lseek_r", + "_read_r", + "_write_r", + "_unlink_r", + "_fstat_r", + "access", + "fsync", + "tcsetattr", + "tcgetattr", + "tcflush", + "tcdrain", + "__ssrefill_r", + "_stat_r", + "__hexdig_fun", + "__mcmp", + "_fwalk_sglue", + "__fpclassifyf", + "_setlocale_r", + "_mbrtowc_r", + "fcntl", + "__match", + "_lock_close", + "__c$", + "__func__$", + "__FUNCTION__$", + "DAYS_IN_MONTH", + "_DAYS_BEFORE_MONTH", + "CSWTCH$", + "dst$", + "sulp", + ], + "string_ops": ["strcmp", "strncmp", "strchr", "strstr", "strtok", "strdup"], + "memory_alloc": ["malloc", "calloc", "realloc", "free", "_sbrk"], + "file_io": [ + "fread", + "fwrite", + "fopen", + "fclose", + "fseek", + "ftell", + "fflush", + "s_fd_table", + ], + "string_formatting": [ + "snprintf", + "vsnprintf", + "sprintf", + "vsprintf", + "sscanf", + "vsscanf", + ], + "cpp_anonymous": ["_GLOBAL__N_", "n$"], + "cpp_runtime": ["__cxx", "_ZN", "_ZL", "_ZSt", "__gxx_personality", "_Z16"], + "exception_handling": ["__cxa_", "_Unwind_", "__gcc_personality", "uw_frame_state"], + "static_init": ["_GLOBAL__sub_I_"], + "mdns_lib": ["mdns"], + "phy_radio": [ + "phy_", + "rf_", + "chip_", + "register_chipv7", + "pbus_", + "bb_", + "fe_", + "rfcal_", + "ram_rfcal", + "tx_pwctrl", + "rx_chan", + "set_rx_gain", + "set_chan", + "agc_reg", + "ram_txiq", + "ram_txdc", + "ram_gen_rx_gain", + "rx_11b_opt", + "set_rx_sense", + "set_rx_gain_cal", + "set_chan_dig_gain", + "tx_pwctrl_init_cal", + "rfcal_txiq", + "set_tx_gain_table", + "correct_rfpll_offset", + "pll_correct_dcap", + "txiq_cal_init", + "pwdet_sar", + "pwdet_sar2_init", + "ram_iq_est_enable", + "ram_rfpll_set_freq", + "ant_wifirx_cfg", + "ant_btrx_cfg", + "force_txrxoff", + "force_txrx_off", + "tx_paon_set", + "opt_11b_resart", + "rfpll_1p2_opt", + "ram_dc_iq_est", + "ram_start_tx_tone", + "ram_en_pwdet", + "ram_cbw2040_cfg", + "rxdc_est_min", + "i2cmst_reg_init", + "temprature_sens_read", + "ram_restart_cal", + "ram_write_gain_mem", + "ram_wait_rfpll_cal_end", + "txcal_debuge_mode", + "ant_wifitx_cfg", + "reg_init_begin", + ], + "wifi_phy_pp": ["pp_", "ppT", "ppR", "ppP", "ppInstall", "ppCalTxAMPDULength"], + "wifi_lmac": ["lmac"], + "wifi_device": ["wdev", "wDev_"], + "power_mgmt": [ + "pm_", + "sleep", + "rtc_sleep", + "light_sleep", + "deep_sleep", + "power_down", + "g_pm", + ], + "memory_mgmt": [ + "mem_", + "memory_", + "tlsf_", + "memp_", + "pbuf_", + "pbuf_alloc", + "pbuf_copy_partial_pbuf", + ], + "hal_layer": ["hal_"], + "clock_mgmt": [ + "clk_", + "clock_", + "rtc_clk", + "apb_", + "cpu_freq", + "setCpuFrequencyMhz", + ], + "cache_mgmt": ["cache"], + "flash_ops": ["flash", "image_load"], + "interrupt_handlers": [ + "isr", + "interrupt", + "intr_", + "exc_", + "exception", + "port_IntStack", + ], + "wrapper_functions": ["_wrapper"], + "error_handling": ["panic", "abort", "assert", "error_", "fault"], + "authentication": ["auth"], + "ppp_protocol": ["ppp", "ipcp_", "lcp_", "chap_", "LcpEchoCheck"], + "dhcp": ["dhcp", "handle_dhcp"], + "ethernet_phy": [ + "emac_", + "eth_phy_", + "phy_tlk110", + "phy_lan87", + "phy_ip101", + "phy_rtl", + "phy_dp83", + "phy_ksz", + "lan87xx_", + "rtl8201_", + "ip101_", + "ksz80xx_", + "jl1101_", + "dp83848_", + "eth_on_state_changed", + ], + "threading": ["pthread_", "thread_", "_task_"], + "pthread": ["pthread"], + "synchronization": ["mutex", "semaphore", "spinlock", "portMUX"], + "math_lib": [ + "sin", + "cos", + "tan", + "sqrt", + "pow", + "exp", + "log", + "atan", + "asin", + "acos", + "floor", + "ceil", + "fabs", + "round", + ], + "random": ["rand", "random", "rng_", "prng"], + "time_lib": [ + "time", + "clock", + "gettimeofday", + "settimeofday", + "localtime", + "gmtime", + "mktime", + "strftime", + ], + "console_io": ["console_", "uart_tx", "uart_rx", "puts", "putchar", "getchar"], + "rom_functions": ["r_", "rom_"], + "compiler_runtime": [ + "__divdi3", + "__udivdi3", + "__moddi3", + "__muldi3", + "__ashldi3", + "__ashrdi3", + "__lshrdi3", + "__cmpdi2", + "__fixdfdi", + "__floatdidf", + ], + "libgcc": ["libgcc", "_divdi3", "_udivdi3"], + "boot_startup": ["boot", "start_cpu", "call_start", "startup", "bootloader"], + "bootloader": ["bootloader_", "esp_bootloader"], + "app_framework": ["app_", "initArduino", "setup", "loop", "Update"], + "weak_symbols": ["__weak_"], + "compiler_builtins": ["__builtin_"], + "vfs": ["vfs_", "VFS"], + "esp32_sdk": ["esp32_", "esp32c", "esp32s"], + "usb": ["usb_", "USB", "cdc_", "CDC"], + "i2c_driver": ["i2c_", "I2C"], + "i2s_driver": ["i2s_", "I2S"], + "spi_driver": ["spi_", "SPI"], + "adc_driver": ["adc_", "ADC"], + "dac_driver": ["dac_", "DAC"], + "touch_driver": ["touch_", "TOUCH"], + "pwm_driver": ["pwm_", "PWM", "ledc_", "LEDC"], + "rmt_driver": ["rmt_", "RMT"], + "pcnt_driver": ["pcnt_", "PCNT"], + "can_driver": ["can_", "CAN", "twai_", "TWAI"], + "sdmmc_driver": ["sdmmc_", "SDMMC", "sdcard", "sd_card"], + "temp_sensor": ["temp_sensor", "tsens_"], + "watchdog": ["wdt_", "WDT", "watchdog"], + "brownout": ["brownout", "bod_"], + "ulp": ["ulp_", "ULP"], + "psram": ["psram", "PSRAM", "spiram", "SPIRAM"], + "efuse": ["efuse", "EFUSE"], + "partition": ["partition", "esp_partition"], + "esp_event": ["esp_event", "event_loop", "event_callback"], + "esp_console": ["esp_console", "console_"], + "chip_specific": ["chip_", "esp_chip"], + "esp_system_utils": ["esp_system", "esp_hw", "esp_clk", "esp_sleep"], + "ipc": ["esp_ipc", "ipc_"], + "wifi_config": [ + "g_cnxMgr", + "gChmCxt", + "g_ic", + "TxRxCxt", + "s_dp", + "s_ni", + "s_reg_dump", + "packet$", + "d_mult_table", + "K", + "fcstab", + ], + "smartconfig": ["sc_ack_send"], + "rc_calibration": ["rc_cal", "rcUpdate"], + "noise_floor": ["noise_check"], + "rf_calibration": [ + "set_rx_sense", + "set_rx_gain_cal", + "set_chan_dig_gain", + "tx_pwctrl_init_cal", + "rfcal_txiq", + "set_tx_gain_table", + "correct_rfpll_offset", + "pll_correct_dcap", + "txiq_cal_init", + "pwdet_sar", + "rx_11b_opt", + ], + "wifi_crypto": [ + "pk_use_ecparams", + "process_segments", + "ccmp_", + "rc4_", + "aria_", + "mgf_mask", + "dh_group", + "ccmp_aad_nonce", + "ccmp_encrypt", + "rc4_skip", + "aria_sb1", + "aria_sb2", + "aria_is1", + "aria_is2", + "aria_sl", + "aria_a", + ], + "radio_control": ["fsm_input", "fsm_sconfreq"], + "pbuf": [ + "pbuf_", + ], + "event_group": ["xEventGroup"], + "ringbuffer": ["xRingbuffer", "prvSend", "prvReceive", "prvCopy"], + "provisioning": ["prov_", "prov_stop_and_notify"], + "scan": ["gScanStruct"], + "port": ["xPort"], + "elf_loader": [ + "elf_add", + "elf_add_note", + "elf_add_segment", + "process_image", + "read_encoded", + "read_encoded_value", + "read_encoded_value_with_base", + "process_image_header", + ], + "socket_api": [ + "sockets", + "netconn_", + "accept_function", + "recv_raw", + "socket_ipv4_multicast", + "socket_ipv6_multicast", + ], + "igmp": ["igmp_", "igmp_send", "igmp_input"], + "icmp6": ["icmp6_"], + "arp": ["arp_table"], + "ampdu": [ + "ampdu_", + "rcAmpdu", + "trc_onAmpduOp", + "rcAmpduLowerRate", + "ampdu_dispatch_upto", + ], + "ieee802_11": ["ieee802_11_", "ieee802_11_parse_elems"], + "rate_control": ["rssi_margin", "rcGetSched", "get_rate_fcc_index"], + "nan": ["nan_dp_", "nan_dp_post_tx", "nan_dp_delete_peer"], + "channel_mgmt": ["chm_init", "chm_set_current_channel"], + "trace": ["trc_init", "trc_onAmpduOp"], + "country_code": ["country_info", "country_info_24ghz"], + "multicore": ["do_multicore_settings"], + "Update_lib": ["Update"], + "stdio": [ + "__sf", + "__sflush_r", + "__srefill_r", + "_impure_data", + "_reclaim_reent", + "_open_r", + ], + "strncpy_ops": ["strncpy"], + "math_internal": ["__mdiff", "__lshift", "__mprec_tens", "quorem"], + "character_class": ["__chclass"], + "camellia": ["camellia_", "camellia_feistel"], + "crypto_tables": ["FSb", "FSb2", "FSb3", "FSb4"], + "event_buffer": ["g_eb_list_desc", "eb_space"], + "base_node": ["base_node_", "base_node_add_handler"], + "file_descriptor": ["s_fd_table"], + "tx_delay": ["tx_delay_cfg"], + "deinit": ["deinit_functions"], + "lcp_echo": ["LcpEchoCheck"], + "raw_api": ["raw_bind", "raw_connect"], + "checksum": ["process_checksum"], + "entry_management": ["add_entry"], + "esp_ota": ["esp_ota", "ota_", "read_otadata"], + "http_server": [ + "httpd_", + "parse_url_char", + "cb_headers_complete", + "delete_entry", + "validate_structure", + "config_save", + "config_new", + "verify_url", + "cb_url", + ], + "misc_system": [ + "alarm_cbs", + "start_up", + "tokens", + "unhex", + "osi_funcs_ro", + "enum_function", + "fragment_and_dispatch", + "alarm_set", + "osi_alarm_new", + "config_set_string", + "config_update_newest_section", + "config_remove_key", + "method_strings", + "interop_match", + "interop_database", + "__state_table", + "__action_table", + "s_stub_table", + "s_context", + "s_mmu_ctx", + "s_get_bus_mask", + "hli_queue_put", + "list_remove", + "list_delete", + "lock_acquire_generic", + "is_vect_desc_usable", + "io_mode_str", + "__c$20233", + "interface", + "read_id_core", + "subscribe_idle", + "unsubscribe_idle", + "s_clkout_handle", + "lock_release_generic", + "config_set_int", + "config_get_int", + "config_get_string", + "config_has_key", + "config_remove_section", + "osi_alarm_init", + "osi_alarm_deinit", + "fixed_queue_enqueue", + "fixed_queue_dequeue", + "fixed_queue_new", + "fixed_pkt_queue_enqueue", + "fixed_pkt_queue_new", + "list_append", + "list_prepend", + "list_insert_after", + "list_contains", + "list_get_node", + "hash_function_blob", + "cb_no_body", + "cb_on_body", + "profile_tab", + "get_arg", + "trim", + "buf$", + "process_appended_hash_and_sig$constprop$0", + "uuidType", + "allocate_svc_db_buf", + "_hostname_is_ours", + "s_hli_handlers", + "tick_cb", + "idle_cb", + "input", + "entry_find", + "section_find", + "find_bucket_entry_", + "config_has_section", + "hli_queue_create", + "hli_queue_get", + "hli_c_handler", + "future_ready", + "future_await", + "future_new", + "pkt_queue_enqueue", + "pkt_queue_dequeue", + "pkt_queue_cleanup", + "pkt_queue_create", + "pkt_queue_destroy", + "fixed_pkt_queue_dequeue", + "osi_alarm_cancel", + "osi_alarm_is_active", + "osi_sem_take", + "osi_event_create", + "osi_event_bind", + "alarm_cb_handler", + "list_foreach", + "list_back", + "list_front", + "list_clear", + "fixed_queue_try_peek_first", + "translate_path", + "get_idx", + "find_key", + "init", + "end", + "start", + "set_read_value", + "copy_address_list", + "copy_and_key", + "sdk_cfg_opts", + "leftshift_onebit", + "config_section_end", + "config_section_begin", + "find_entry_and_check_all_reset", + "image_validate", + "xPendingReadyList", + "vListInitialise", + "lock_init_generic", + "ant_bttx_cfg", + "ant_dft_cfg", + "cs_send_to_ctrl_sock", + "config_llc_util_funcs_reset", + "make_set_adv_report_flow_control", + "make_set_event_mask", + "raw_new", + "raw_remove", + "BTE_InitStack", + "parse_read_local_supported_features_response", + "__math_invalidf", + "tinytens", + "__mprec_tinytens", + "__mprec_bigtens", + "vRingbufferDelete", + "vRingbufferDeleteWithCaps", + "vRingbufferReturnItem", + "vRingbufferReturnItemFromISR", + "get_acl_data_size_ble", + "get_features_ble", + "get_features_classic", + "get_acl_packet_size_ble", + "get_acl_packet_size_classic", + "supports_extended_inquiry_response", + "supports_rssi_with_inquiry_results", + "supports_interlaced_inquiry_scan", + "supports_reading_remote_extended_features", + ], + "bluetooth_ll": [ + "lld_pdu_", + "ld_acl_", + "lld_stop_ind_handler", + "lld_evt_winsize_change", + "config_lld_evt_funcs_reset", + "config_lld_funcs_reset", + "config_llm_funcs_reset", + "llm_set_long_adv_data", + "lld_retry_tx_prog", + "llc_link_sup_to_ind_handler", + "config_llc_funcs_reset", + "lld_evt_rxwin_compute", + "config_btdm_funcs_reset", + "config_ea_funcs_reset", + "llc_defalut_state_tab_reset", + "config_rwip_funcs_reset", + "ke_lmp_rx_flooding_detect", + ], +} + +# Demangled patterns: patterns found in demangled C++ names +DEMANGLED_PATTERNS = { + "gpio_driver": ["GPIO"], + "uart_driver": ["UART"], + "network_stack": [ + "lwip", + "tcp", + "udp", + "ip4", + "ip6", + "dhcp", + "dns", + "netif", + "ethernet", + "ppp", + "slip", + ], + "wifi_stack": ["NetworkInterface"], + "nimble_bt": [ + "nimble", + "NimBLE", + "ble_hs", + "ble_gap", + "ble_gatt", + "ble_att", + "ble_l2cap", + "ble_sm", + ], + "crypto": ["mbedtls", "crypto", "sha", "aes", "rsa", "ecc", "tls", "ssl"], + "cpp_stdlib": ["std::", "__gnu_cxx::", "__cxxabiv"], + "static_init": ["__static_initialization"], + "rtti": ["__type_info", "__class_type_info"], + "web_server_lib": ["AsyncWebServer", "AsyncWebHandler", "WebServer"], + "async_tcp": ["AsyncClient", "AsyncServer"], + "mdns_lib": ["mdns"], + "json_lib": [ + "ArduinoJson", + "JsonDocument", + "JsonArray", + "JsonObject", + "deserialize", + "serialize", + ], + "http_lib": ["HTTP", "http_", "Request", "Response", "Uri", "WebSocket"], + "logging": ["log", "Log", "print", "Print", "diag_"], + "authentication": ["checkDigestAuthentication"], + "libgcc": ["libgcc"], + "esp_system": ["esp_", "ESP"], + "arduino": ["arduino"], + "nvs": ["nvs_", "_ZTVN3nvs", "nvs::"], + "filesystem": ["spiffs", "vfs"], + "libc": ["newlib"], +} diff --git a/script/ci_memory_impact_comment.py b/script/ci_memory_impact_comment.py index 0b3bf87590..0f65e4fbbd 100755 --- a/script/ci_memory_impact_comment.py +++ b/script/ci_memory_impact_comment.py @@ -18,7 +18,7 @@ import sys sys.path.insert(0, str(Path(__file__).parent.parent)) # pylint: disable=wrong-import-position -from esphome.analyze_memory import MemoryAnalyzer +from esphome.analyze_memory import MemoryAnalyzer # noqa: E402 # Comment marker to identify our memory impact comments COMMENT_MARKER = "" From 6d2c700c438e63fc6e2f8dc0da69ceb9790fdec3 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Fri, 17 Oct 2025 13:43:05 -1000 Subject: [PATCH 14/76] relo --- esphome/analyze_memory/__init__.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/esphome/analyze_memory/__init__.py b/esphome/analyze_memory/__init__.py index c6fdb1028d..b85b1d5765 100644 --- a/esphome/analyze_memory/__init__.py +++ b/esphome/analyze_memory/__init__.py @@ -1,6 +1,7 @@ """Memory usage analyzer for ESPHome compiled binaries.""" from collections import defaultdict +from functools import cache import json import logging from pathlib import Path @@ -13,6 +14,7 @@ _LOGGER = logging.getLogger(__name__) # Get the list of actual ESPHome components by scanning the components directory +@cache def get_esphome_components(): """Get set of actual ESPHome components from the components directory.""" components = set() @@ -34,10 +36,6 @@ def get_esphome_components(): return components -# Cache the component list -ESPHOME_COMPONENTS = get_esphome_components() - - class MemorySection: """Represents a memory section with its symbols.""" @@ -285,7 +283,7 @@ class MemoryAnalyzer: if "esphome::" in demangled: # Check for special component classes that include component name in the class # For example: esphome::ESPHomeOTAComponent -> ota component - for component_name in ESPHOME_COMPONENTS: + for component_name in get_esphome_components(): # Check various naming patterns component_upper = component_name.upper() component_camel = component_name.replace("_", "").title() @@ -307,7 +305,7 @@ class MemoryAnalyzer: component_name = component_name.rstrip("_") # Check if this is an actual component in the components directory - if component_name in ESPHOME_COMPONENTS: + if component_name in get_esphome_components(): return f"[esphome]{component_name}" # Check if this is a known external component from the config if component_name in self.external_components: From 256d3b119b907e4551dcfeb0e01122facc61fd3a Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Fri, 17 Oct 2025 13:44:30 -1000 Subject: [PATCH 15/76] relo --- esphome/analyze_memory/__init__.py | 27 ++++++++++++++++----------- 1 file changed, 16 insertions(+), 11 deletions(-) diff --git a/esphome/analyze_memory/__init__.py b/esphome/analyze_memory/__init__.py index b85b1d5765..050bc011a8 100644 --- a/esphome/analyze_memory/__init__.py +++ b/esphome/analyze_memory/__init__.py @@ -1,6 +1,7 @@ """Memory usage analyzer for ESPHome compiled binaries.""" from collections import defaultdict +from dataclasses import dataclass, field from functools import cache import json import logging @@ -36,32 +37,36 @@ def get_esphome_components(): return components +@dataclass class MemorySection: """Represents a memory section with its symbols.""" - def __init__(self, name: str): - self.name = name - self.symbols: list[tuple[str, int, str]] = [] # (symbol_name, size, component) - self.total_size = 0 + name: str + symbols: list[tuple[str, int, str]] = field( + default_factory=list + ) # (symbol_name, size, component) + total_size: int = 0 +@dataclass class ComponentMemory: """Tracks memory usage for a component.""" - def __init__(self, name: str): - self.name = name - self.text_size = 0 # Code in flash - self.rodata_size = 0 # Read-only data in flash - self.data_size = 0 # Initialized data (flash + ram) - self.bss_size = 0 # Uninitialized data (ram only) - self.symbol_count = 0 + name: str + text_size: int = 0 # Code in flash + rodata_size: int = 0 # Read-only data in flash + data_size: int = 0 # Initialized data (flash + ram) + bss_size: int = 0 # Uninitialized data (ram only) + symbol_count: int = 0 @property def flash_total(self) -> int: + """Total flash usage (text + rodata + data).""" return self.text_size + self.rodata_size + self.data_size @property def ram_total(self) -> int: + """Total RAM usage (data + bss).""" return self.data_size + self.bss_size From 5049c7227d6cb2935c767fab1697c5494f9d5947 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Fri, 17 Oct 2025 13:50:15 -1000 Subject: [PATCH 16/76] reduce --- esphome/analyze_memory/__init__.py | 69 ++++++++++++------------------ esphome/analyze_memory/const.py | 28 ++++++++++++ 2 files changed, 56 insertions(+), 41 deletions(-) diff --git a/esphome/analyze_memory/__init__.py b/esphome/analyze_memory/__init__.py index 050bc011a8..63002d848d 100644 --- a/esphome/analyze_memory/__init__.py +++ b/esphome/analyze_memory/__init__.py @@ -9,7 +9,12 @@ from pathlib import Path import re import subprocess -from .const import DEMANGLED_PATTERNS, ESPHOME_COMPONENT_PATTERN, SYMBOL_PATTERNS +from .const import ( + CORE_SUBCATEGORY_PATTERNS, + DEMANGLED_PATTERNS, + ESPHOME_COMPONENT_PATTERN, + SYMBOL_PATTERNS, +) _LOGGER = logging.getLogger(__name__) @@ -37,6 +42,26 @@ def get_esphome_components(): return components +@cache +def get_component_class_patterns(component_name: str) -> list[str]: + """Generate component class name patterns for symbol matching. + + Args: + component_name: The component name (e.g., "ota", "wifi", "api") + + Returns: + List of pattern strings to match against demangled symbols + """ + component_upper = component_name.upper() + component_camel = component_name.replace("_", "").title() + return [ + f"esphome::{component_upper}Component", # e.g., esphome::OTAComponent + f"esphome::ESPHome{component_upper}Component", # e.g., esphome::ESPHomeOTAComponent + f"esphome::{component_camel}Component", # e.g., esphome::OtaComponent + f"esphome::ESPHome{component_camel}Component", # e.g., esphome::ESPHomeOtaComponent + ] + + @dataclass class MemorySection: """Represents a memory section with its symbols.""" @@ -289,16 +314,7 @@ class MemoryAnalyzer: # Check for special component classes that include component name in the class # For example: esphome::ESPHomeOTAComponent -> ota component for component_name in get_esphome_components(): - # Check various naming patterns - component_upper = component_name.upper() - component_camel = component_name.replace("_", "").title() - patterns = [ - f"esphome::{component_upper}Component", # e.g., esphome::OTAComponent - f"esphome::ESPHome{component_upper}Component", # e.g., esphome::ESPHomeOTAComponent - f"esphome::{component_camel}Component", # e.g., esphome::OtaComponent - f"esphome::ESPHome{component_camel}Component", # e.g., esphome::ESPHomeOtaComponent - ] - + patterns = get_component_class_patterns(component_name) if any(pattern in demangled for pattern in patterns): return f"[esphome]{component_name}" @@ -394,35 +410,6 @@ class MemoryAnalyzer: def _categorize_esphome_core_symbol(self, demangled: str) -> str: """Categorize ESPHome core symbols into subcategories.""" - # Dictionary of patterns for core subcategories - CORE_SUBCATEGORY_PATTERNS = { - "Component Framework": ["Component"], - "Application Core": ["Application"], - "Scheduler": ["Scheduler"], - "Logging": ["Logger", "log_"], - "Preferences": ["preferences", "Preferences"], - "Synchronization": ["Mutex", "Lock"], - "Helpers": ["Helper"], - "Network Utilities": ["network", "Network"], - "Time Management": ["time", "Time"], - "String Utilities": ["str_", "string"], - "Parsing/Formatting": ["parse_", "format_"], - "Optional Types": ["optional", "Optional"], - "Callbacks": ["Callback", "callback"], - "Color Utilities": ["Color"], - "C++ Operators": ["operator"], - "Global Variables": ["global_", "_GLOBAL"], - "Setup/Loop": ["setup", "loop"], - "System Control": ["reboot", "restart"], - "GPIO Management": ["GPIO", "gpio"], - "Interrupt Handling": ["ISR", "interrupt"], - "Hooks": ["Hook", "hook"], - "Entity Base Classes": ["Entity"], - "Automation Framework": ["automation", "Automation"], - "Automation Components": ["Condition", "Action", "Trigger"], - "Lambda Support": ["lambda"], - } - # Special patterns that need to be checked separately if any(pattern in demangled for pattern in ["vtable", "typeinfo", "thunk"]): return "C++ Runtime (vtables/RTTI)" @@ -430,7 +417,7 @@ class MemoryAnalyzer: if demangled.startswith("std::"): return "C++ STL" - # Check against patterns + # Check against patterns from const.py for category, patterns in CORE_SUBCATEGORY_PATTERNS.items(): if any(pattern in demangled for pattern in patterns): return category diff --git a/esphome/analyze_memory/const.py b/esphome/analyze_memory/const.py index 68cd957090..df37c0b2cd 100644 --- a/esphome/analyze_memory/const.py +++ b/esphome/analyze_memory/const.py @@ -855,3 +855,31 @@ DEMANGLED_PATTERNS = { "filesystem": ["spiffs", "vfs"], "libc": ["newlib"], } + +# Patterns for categorizing ESPHome core symbols into subcategories +CORE_SUBCATEGORY_PATTERNS = { + "Component Framework": ["Component"], + "Application Core": ["Application"], + "Scheduler": ["Scheduler"], + "Component Iterator": ["ComponentIterator"], + "Helper Functions": ["Helpers", "helpers"], + "Preferences/Storage": ["Preferences", "ESPPreferences"], + "I/O Utilities": ["HighFrequencyLoopRequester"], + "String Utilities": ["str_"], + "Bit Utilities": ["reverse_bits"], + "Data Conversion": ["convert_"], + "Network Utilities": ["network", "IPAddress"], + "API Protocol": ["api::"], + "WiFi Manager": ["wifi::"], + "MQTT Client": ["mqtt::"], + "Logger": ["logger::"], + "OTA Updates": ["ota::"], + "Web Server": ["web_server::"], + "Time Management": ["time::"], + "Sensor Framework": ["sensor::"], + "Binary Sensor": ["binary_sensor::"], + "Switch Framework": ["switch_::"], + "Light Framework": ["light::"], + "Climate Framework": ["climate::"], + "Cover Framework": ["cover::"], +} From 43c62297e84d91b581a22f34d9f1b3196cac7ebd Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Fri, 17 Oct 2025 13:56:31 -1000 Subject: [PATCH 17/76] merge --- esphome/analyze_memory/__init__.py | 327 +--------------------------- esphome/analyze_memory/__main__.py | 6 + esphome/analyze_memory/cli.py | 338 +++++++++++++++++++++++++++++ 3 files changed, 350 insertions(+), 321 deletions(-) create mode 100644 esphome/analyze_memory/__main__.py create mode 100644 esphome/analyze_memory/cli.py diff --git a/esphome/analyze_memory/__init__.py b/esphome/analyze_memory/__init__.py index 63002d848d..9c35965b74 100644 --- a/esphome/analyze_memory/__init__.py +++ b/esphome/analyze_memory/__init__.py @@ -174,7 +174,7 @@ class MemoryAnalyzer: self.sections[mapped_section].total_size += size except subprocess.CalledProcessError as e: - _LOGGER.error(f"Failed to parse sections: {e}") + _LOGGER.error("Failed to parse sections: %s", e) raise def _parse_symbols(self) -> None: @@ -252,7 +252,7 @@ class MemoryAnalyzer: seen_addresses.add(address) except subprocess.CalledProcessError as e: - _LOGGER.error(f"Failed to parse symbols: {e}") + _LOGGER.error("Failed to parse symbols: %s", e) raise def _categorize_symbols(self) -> None: @@ -399,8 +399,9 @@ class MemoryAnalyzer: # If batch fails, cache originals for symbol in symbols: self._demangle_cache[symbol] = symbol - except Exception: + except (subprocess.SubprocessError, OSError, UnicodeDecodeError) as e: # On error, cache originals + _LOGGER.debug("Failed to batch demangle symbols: %s", e) for symbol in symbols: self._demangle_cache[symbol] = symbol @@ -424,267 +425,6 @@ class MemoryAnalyzer: return "Other Core" - def generate_report(self, detailed: bool = False) -> str: - """Generate a formatted memory report.""" - components = sorted( - self.components.items(), key=lambda x: x[1].flash_total, reverse=True - ) - - # Calculate totals - total_flash = sum(c.flash_total for _, c in components) - total_ram = sum(c.ram_total for _, c in components) - - # Build report - lines = [] - - # Column width constants - COL_COMPONENT = 29 - COL_FLASH_TEXT = 14 - COL_FLASH_DATA = 14 - COL_RAM_DATA = 12 - COL_RAM_BSS = 12 - COL_TOTAL_FLASH = 15 - COL_TOTAL_RAM = 12 - COL_SEPARATOR = 3 # " | " - - # Core analysis column widths - COL_CORE_SUBCATEGORY = 30 - COL_CORE_SIZE = 12 - COL_CORE_COUNT = 6 - COL_CORE_PERCENT = 10 - - # Calculate the exact table width - table_width = ( - COL_COMPONENT - + COL_SEPARATOR - + COL_FLASH_TEXT - + COL_SEPARATOR - + COL_FLASH_DATA - + COL_SEPARATOR - + COL_RAM_DATA - + COL_SEPARATOR - + COL_RAM_BSS - + COL_SEPARATOR - + COL_TOTAL_FLASH - + COL_SEPARATOR - + COL_TOTAL_RAM - ) - - lines.append("=" * table_width) - lines.append("Component Memory Analysis".center(table_width)) - lines.append("=" * table_width) - lines.append("") - - # Main table - fixed column widths - lines.append( - f"{'Component':<{COL_COMPONENT}} | {'Flash (text)':>{COL_FLASH_TEXT}} | {'Flash (data)':>{COL_FLASH_DATA}} | {'RAM (data)':>{COL_RAM_DATA}} | {'RAM (bss)':>{COL_RAM_BSS}} | {'Total Flash':>{COL_TOTAL_FLASH}} | {'Total RAM':>{COL_TOTAL_RAM}}" - ) - lines.append( - "-" * COL_COMPONENT - + "-+-" - + "-" * COL_FLASH_TEXT - + "-+-" - + "-" * COL_FLASH_DATA - + "-+-" - + "-" * COL_RAM_DATA - + "-+-" - + "-" * COL_RAM_BSS - + "-+-" - + "-" * COL_TOTAL_FLASH - + "-+-" - + "-" * COL_TOTAL_RAM - ) - - for name, mem in components: - if mem.flash_total > 0 or mem.ram_total > 0: - flash_rodata = mem.rodata_size + mem.data_size - lines.append( - f"{name:<{COL_COMPONENT}} | {mem.text_size:>{COL_FLASH_TEXT - 2},} B | {flash_rodata:>{COL_FLASH_DATA - 2},} B | " - f"{mem.data_size:>{COL_RAM_DATA - 2},} B | {mem.bss_size:>{COL_RAM_BSS - 2},} B | " - f"{mem.flash_total:>{COL_TOTAL_FLASH - 2},} B | {mem.ram_total:>{COL_TOTAL_RAM - 2},} B" - ) - - lines.append( - "-" * COL_COMPONENT - + "-+-" - + "-" * COL_FLASH_TEXT - + "-+-" - + "-" * COL_FLASH_DATA - + "-+-" - + "-" * COL_RAM_DATA - + "-+-" - + "-" * COL_RAM_BSS - + "-+-" - + "-" * COL_TOTAL_FLASH - + "-+-" - + "-" * COL_TOTAL_RAM - ) - lines.append( - f"{'TOTAL':<{COL_COMPONENT}} | {' ':>{COL_FLASH_TEXT}} | {' ':>{COL_FLASH_DATA}} | " - f"{' ':>{COL_RAM_DATA}} | {' ':>{COL_RAM_BSS}} | " - f"{total_flash:>{COL_TOTAL_FLASH - 2},} B | {total_ram:>{COL_TOTAL_RAM - 2},} B" - ) - - # Top consumers - lines.append("") - lines.append("Top Flash Consumers:") - for i, (name, mem) in enumerate(components[:25]): - if mem.flash_total > 0: - percentage = ( - (mem.flash_total / total_flash * 100) if total_flash > 0 else 0 - ) - lines.append( - f"{i + 1}. {name} ({mem.flash_total:,} B) - {percentage:.1f}% of analyzed flash" - ) - - lines.append("") - lines.append("Top RAM Consumers:") - ram_components = sorted(components, key=lambda x: x[1].ram_total, reverse=True) - for i, (name, mem) in enumerate(ram_components[:25]): - if mem.ram_total > 0: - percentage = (mem.ram_total / total_ram * 100) if total_ram > 0 else 0 - lines.append( - f"{i + 1}. {name} ({mem.ram_total:,} B) - {percentage:.1f}% of analyzed RAM" - ) - - lines.append("") - lines.append( - "Note: This analysis covers symbols in the ELF file. Some runtime allocations may not be included." - ) - lines.append("=" * table_width) - - # Add ESPHome core detailed analysis if there are core symbols - if self._esphome_core_symbols: - lines.append("") - lines.append("=" * table_width) - lines.append("[esphome]core Detailed Analysis".center(table_width)) - lines.append("=" * table_width) - lines.append("") - - # Group core symbols by subcategory - core_subcategories: dict[str, list[tuple[str, str, int]]] = defaultdict( - list - ) - - for symbol, demangled, size in self._esphome_core_symbols: - # Categorize based on demangled name patterns - subcategory = self._categorize_esphome_core_symbol(demangled) - core_subcategories[subcategory].append((symbol, demangled, size)) - - # Sort subcategories by total size - sorted_subcategories = sorted( - [ - (name, symbols, sum(s[2] for s in symbols)) - for name, symbols in core_subcategories.items() - ], - key=lambda x: x[2], - reverse=True, - ) - - lines.append( - f"{'Subcategory':<{COL_CORE_SUBCATEGORY}} | {'Size':>{COL_CORE_SIZE}} | " - f"{'Count':>{COL_CORE_COUNT}} | {'% of Core':>{COL_CORE_PERCENT}}" - ) - lines.append( - "-" * COL_CORE_SUBCATEGORY - + "-+-" - + "-" * COL_CORE_SIZE - + "-+-" - + "-" * COL_CORE_COUNT - + "-+-" - + "-" * COL_CORE_PERCENT - ) - - core_total = sum(size for _, _, size in self._esphome_core_symbols) - - for subcategory, symbols, total_size in sorted_subcategories: - percentage = (total_size / core_total * 100) if core_total > 0 else 0 - lines.append( - f"{subcategory:<{COL_CORE_SUBCATEGORY}} | {total_size:>{COL_CORE_SIZE - 2},} B | " - f"{len(symbols):>{COL_CORE_COUNT}} | {percentage:>{COL_CORE_PERCENT - 1}.1f}%" - ) - - # Top 10 largest core symbols - lines.append("") - lines.append("Top 10 Largest [esphome]core Symbols:") - sorted_core_symbols = sorted( - self._esphome_core_symbols, key=lambda x: x[2], reverse=True - ) - - for i, (symbol, demangled, size) in enumerate(sorted_core_symbols[:15]): - lines.append(f"{i + 1}. {demangled} ({size:,} B)") - - lines.append("=" * table_width) - - # Add detailed analysis for top ESPHome and external components - esphome_components = [ - (name, mem) - for name, mem in components - if name.startswith("[esphome]") and name != "[esphome]core" - ] - external_components = [ - (name, mem) for name, mem in components if name.startswith("[external]") - ] - - top_esphome_components = sorted( - esphome_components, key=lambda x: x[1].flash_total, reverse=True - )[:30] - - # Include all external components (they're usually important) - top_external_components = sorted( - external_components, key=lambda x: x[1].flash_total, reverse=True - ) - - # Check if API component exists and ensure it's included - api_component = None - for name, mem in components: - if name == "[esphome]api": - api_component = (name, mem) - break - - # Combine all components to analyze: top ESPHome + all external + API if not already included - components_to_analyze = list(top_esphome_components) + list( - top_external_components - ) - if api_component and api_component not in components_to_analyze: - components_to_analyze.append(api_component) - - if components_to_analyze: - for comp_name, comp_mem in components_to_analyze: - comp_symbols = self._component_symbols.get(comp_name, []) - if comp_symbols: - lines.append("") - lines.append("=" * table_width) - lines.append(f"{comp_name} Detailed Analysis".center(table_width)) - lines.append("=" * table_width) - lines.append("") - - # Sort symbols by size - sorted_symbols = sorted( - comp_symbols, key=lambda x: x[2], reverse=True - ) - - lines.append(f"Total symbols: {len(sorted_symbols)}") - lines.append(f"Total size: {comp_mem.flash_total:,} B") - lines.append("") - - # Show all symbols > 100 bytes for better visibility - large_symbols = [ - (sym, dem, size) - for sym, dem, size in sorted_symbols - if size > 100 - ] - - lines.append( - f"{comp_name} Symbols > 100 B ({len(large_symbols)} symbols):" - ) - for i, (symbol, demangled, size) in enumerate(large_symbols): - lines.append(f"{i + 1}. {demangled} ({size:,} B)") - - lines.append("=" * table_width) - - return "\n".join(lines) - def to_json(self) -> str: """Export analysis results as JSON.""" data = { @@ -707,63 +447,8 @@ class MemoryAnalyzer: } return json.dumps(data, indent=2) - def dump_uncategorized_symbols(self, output_file: str | None = None) -> None: - """Dump uncategorized symbols for analysis.""" - # Sort by size descending - sorted_symbols = sorted( - self._uncategorized_symbols, key=lambda x: x[2], reverse=True - ) - - lines = ["Uncategorized Symbols Analysis", "=" * 80] - lines.append(f"Total uncategorized symbols: {len(sorted_symbols)}") - lines.append( - f"Total uncategorized size: {sum(s[2] for s in sorted_symbols):,} bytes" - ) - lines.append("") - lines.append(f"{'Size':>10} | {'Symbol':<60} | Demangled") - lines.append("-" * 10 + "-+-" + "-" * 60 + "-+-" + "-" * 40) - - for symbol, demangled, size in sorted_symbols[:100]: # Top 100 - if symbol != demangled: - lines.append(f"{size:>10,} | {symbol[:60]:<60} | {demangled[:100]}") - else: - lines.append(f"{size:>10,} | {symbol[:60]:<60} | [not demangled]") - - if len(sorted_symbols) > 100: - lines.append(f"\n... and {len(sorted_symbols) - 100} more symbols") - - content = "\n".join(lines) - - if output_file: - with open(output_file, "w") as f: - f.write(content) - else: - print(content) - - -def analyze_elf( - elf_path: str, - objdump_path: str | None = None, - readelf_path: str | None = None, - detailed: bool = False, - external_components: set[str] | None = None, -) -> str: - """Analyze an ELF file and return a memory report.""" - analyzer = MemoryAnalyzer(elf_path, objdump_path, readelf_path, external_components) - analyzer.analyze() - return analyzer.generate_report(detailed) - if __name__ == "__main__": - import sys + from .cli import main - if len(sys.argv) < 2: - print("Usage: analyze_memory.py ") - sys.exit(1) - - try: - report = analyze_elf(sys.argv[1]) - print(report) - except Exception as e: - print(f"Error: {e}") - sys.exit(1) + main() diff --git a/esphome/analyze_memory/__main__.py b/esphome/analyze_memory/__main__.py new file mode 100644 index 0000000000..aa772c3ad4 --- /dev/null +++ b/esphome/analyze_memory/__main__.py @@ -0,0 +1,6 @@ +"""Main entry point for running the memory analyzer as a module.""" + +from .cli import main + +if __name__ == "__main__": + main() diff --git a/esphome/analyze_memory/cli.py b/esphome/analyze_memory/cli.py new file mode 100644 index 0000000000..ffce04bb6e --- /dev/null +++ b/esphome/analyze_memory/cli.py @@ -0,0 +1,338 @@ +"""CLI interface for memory analysis with report generation.""" + +from collections import defaultdict +import subprocess +import sys + +from . import MemoryAnalyzer + + +class MemoryAnalyzerCLI(MemoryAnalyzer): + """Memory analyzer with CLI-specific report generation.""" + + def generate_report(self, detailed: bool = False) -> str: + """Generate a formatted memory report.""" + components = sorted( + self.components.items(), key=lambda x: x[1].flash_total, reverse=True + ) + + # Calculate totals + total_flash = sum(c.flash_total for _, c in components) + total_ram = sum(c.ram_total for _, c in components) + + # Build report + lines = [] + + # Column width constants + COL_COMPONENT = 29 + COL_FLASH_TEXT = 14 + COL_FLASH_DATA = 14 + COL_RAM_DATA = 12 + COL_RAM_BSS = 12 + COL_TOTAL_FLASH = 15 + COL_TOTAL_RAM = 12 + COL_SEPARATOR = 3 # " | " + + # Core analysis column widths + COL_CORE_SUBCATEGORY = 30 + COL_CORE_SIZE = 12 + COL_CORE_COUNT = 6 + COL_CORE_PERCENT = 10 + + # Calculate the exact table width + table_width = ( + COL_COMPONENT + + COL_SEPARATOR + + COL_FLASH_TEXT + + COL_SEPARATOR + + COL_FLASH_DATA + + COL_SEPARATOR + + COL_RAM_DATA + + COL_SEPARATOR + + COL_RAM_BSS + + COL_SEPARATOR + + COL_TOTAL_FLASH + + COL_SEPARATOR + + COL_TOTAL_RAM + ) + + lines.append("=" * table_width) + lines.append("Component Memory Analysis".center(table_width)) + lines.append("=" * table_width) + lines.append("") + + # Main table - fixed column widths + lines.append( + f"{'Component':<{COL_COMPONENT}} | {'Flash (text)':>{COL_FLASH_TEXT}} | {'Flash (data)':>{COL_FLASH_DATA}} | {'RAM (data)':>{COL_RAM_DATA}} | {'RAM (bss)':>{COL_RAM_BSS}} | {'Total Flash':>{COL_TOTAL_FLASH}} | {'Total RAM':>{COL_TOTAL_RAM}}" + ) + lines.append( + "-" * COL_COMPONENT + + "-+-" + + "-" * COL_FLASH_TEXT + + "-+-" + + "-" * COL_FLASH_DATA + + "-+-" + + "-" * COL_RAM_DATA + + "-+-" + + "-" * COL_RAM_BSS + + "-+-" + + "-" * COL_TOTAL_FLASH + + "-+-" + + "-" * COL_TOTAL_RAM + ) + + for name, mem in components: + if mem.flash_total > 0 or mem.ram_total > 0: + flash_rodata = mem.rodata_size + mem.data_size + lines.append( + f"{name:<{COL_COMPONENT}} | {mem.text_size:>{COL_FLASH_TEXT - 2},} B | {flash_rodata:>{COL_FLASH_DATA - 2},} B | " + f"{mem.data_size:>{COL_RAM_DATA - 2},} B | {mem.bss_size:>{COL_RAM_BSS - 2},} B | " + f"{mem.flash_total:>{COL_TOTAL_FLASH - 2},} B | {mem.ram_total:>{COL_TOTAL_RAM - 2},} B" + ) + + lines.append( + "-" * COL_COMPONENT + + "-+-" + + "-" * COL_FLASH_TEXT + + "-+-" + + "-" * COL_FLASH_DATA + + "-+-" + + "-" * COL_RAM_DATA + + "-+-" + + "-" * COL_RAM_BSS + + "-+-" + + "-" * COL_TOTAL_FLASH + + "-+-" + + "-" * COL_TOTAL_RAM + ) + lines.append( + f"{'TOTAL':<{COL_COMPONENT}} | {' ':>{COL_FLASH_TEXT}} | {' ':>{COL_FLASH_DATA}} | " + f"{' ':>{COL_RAM_DATA}} | {' ':>{COL_RAM_BSS}} | " + f"{total_flash:>{COL_TOTAL_FLASH - 2},} B | {total_ram:>{COL_TOTAL_RAM - 2},} B" + ) + + # Top consumers + lines.append("") + lines.append("Top Flash Consumers:") + for i, (name, mem) in enumerate(components[:25]): + if mem.flash_total > 0: + percentage = ( + (mem.flash_total / total_flash * 100) if total_flash > 0 else 0 + ) + lines.append( + f"{i + 1}. {name} ({mem.flash_total:,} B) - {percentage:.1f}% of analyzed flash" + ) + + lines.append("") + lines.append("Top RAM Consumers:") + ram_components = sorted(components, key=lambda x: x[1].ram_total, reverse=True) + for i, (name, mem) in enumerate(ram_components[:25]): + if mem.ram_total > 0: + percentage = (mem.ram_total / total_ram * 100) if total_ram > 0 else 0 + lines.append( + f"{i + 1}. {name} ({mem.ram_total:,} B) - {percentage:.1f}% of analyzed RAM" + ) + + lines.append("") + lines.append( + "Note: This analysis covers symbols in the ELF file. Some runtime allocations may not be included." + ) + lines.append("=" * table_width) + + # Add ESPHome core detailed analysis if there are core symbols + if self._esphome_core_symbols: + lines.append("") + lines.append("=" * table_width) + lines.append("[esphome]core Detailed Analysis".center(table_width)) + lines.append("=" * table_width) + lines.append("") + + # Group core symbols by subcategory + core_subcategories: dict[str, list[tuple[str, str, int]]] = defaultdict( + list + ) + + for symbol, demangled, size in self._esphome_core_symbols: + # Categorize based on demangled name patterns + subcategory = self._categorize_esphome_core_symbol(demangled) + core_subcategories[subcategory].append((symbol, demangled, size)) + + # Sort subcategories by total size + sorted_subcategories = sorted( + [ + (name, symbols, sum(s[2] for s in symbols)) + for name, symbols in core_subcategories.items() + ], + key=lambda x: x[2], + reverse=True, + ) + + lines.append( + f"{'Subcategory':<{COL_CORE_SUBCATEGORY}} | {'Size':>{COL_CORE_SIZE}} | " + f"{'Count':>{COL_CORE_COUNT}} | {'% of Core':>{COL_CORE_PERCENT}}" + ) + lines.append( + "-" * COL_CORE_SUBCATEGORY + + "-+-" + + "-" * COL_CORE_SIZE + + "-+-" + + "-" * COL_CORE_COUNT + + "-+-" + + "-" * COL_CORE_PERCENT + ) + + core_total = sum(size for _, _, size in self._esphome_core_symbols) + + for subcategory, symbols, total_size in sorted_subcategories: + percentage = (total_size / core_total * 100) if core_total > 0 else 0 + lines.append( + f"{subcategory:<{COL_CORE_SUBCATEGORY}} | {total_size:>{COL_CORE_SIZE - 2},} B | " + f"{len(symbols):>{COL_CORE_COUNT}} | {percentage:>{COL_CORE_PERCENT - 1}.1f}%" + ) + + # Top 10 largest core symbols + lines.append("") + lines.append("Top 10 Largest [esphome]core Symbols:") + sorted_core_symbols = sorted( + self._esphome_core_symbols, key=lambda x: x[2], reverse=True + ) + + for i, (symbol, demangled, size) in enumerate(sorted_core_symbols[:15]): + lines.append(f"{i + 1}. {demangled} ({size:,} B)") + + lines.append("=" * table_width) + + # Add detailed analysis for top ESPHome and external components + esphome_components = [ + (name, mem) + for name, mem in components + if name.startswith("[esphome]") and name != "[esphome]core" + ] + external_components = [ + (name, mem) for name, mem in components if name.startswith("[external]") + ] + + top_esphome_components = sorted( + esphome_components, key=lambda x: x[1].flash_total, reverse=True + )[:30] + + # Include all external components (they're usually important) + top_external_components = sorted( + external_components, key=lambda x: x[1].flash_total, reverse=True + ) + + # Check if API component exists and ensure it's included + api_component = None + for name, mem in components: + if name == "[esphome]api": + api_component = (name, mem) + break + + # Combine all components to analyze: top ESPHome + all external + API if not already included + components_to_analyze = list(top_esphome_components) + list( + top_external_components + ) + if api_component and api_component not in components_to_analyze: + components_to_analyze.append(api_component) + + if components_to_analyze: + for comp_name, comp_mem in components_to_analyze: + comp_symbols = self._component_symbols.get(comp_name, []) + if comp_symbols: + lines.append("") + lines.append("=" * table_width) + lines.append(f"{comp_name} Detailed Analysis".center(table_width)) + lines.append("=" * table_width) + lines.append("") + + # Sort symbols by size + sorted_symbols = sorted( + comp_symbols, key=lambda x: x[2], reverse=True + ) + + lines.append(f"Total symbols: {len(sorted_symbols)}") + lines.append(f"Total size: {comp_mem.flash_total:,} B") + lines.append("") + + # Show all symbols > 100 bytes for better visibility + large_symbols = [ + (sym, dem, size) + for sym, dem, size in sorted_symbols + if size > 100 + ] + + lines.append( + f"{comp_name} Symbols > 100 B ({len(large_symbols)} symbols):" + ) + for i, (symbol, demangled, size) in enumerate(large_symbols): + lines.append(f"{i + 1}. {demangled} ({size:,} B)") + + lines.append("=" * table_width) + + return "\n".join(lines) + + def dump_uncategorized_symbols(self, output_file: str | None = None) -> None: + """Dump uncategorized symbols for analysis.""" + # Sort by size descending + sorted_symbols = sorted( + self._uncategorized_symbols, key=lambda x: x[2], reverse=True + ) + + lines = ["Uncategorized Symbols Analysis", "=" * 80] + lines.append(f"Total uncategorized symbols: {len(sorted_symbols)}") + lines.append( + f"Total uncategorized size: {sum(s[2] for s in sorted_symbols):,} bytes" + ) + lines.append("") + lines.append(f"{'Size':>10} | {'Symbol':<60} | Demangled") + lines.append("-" * 10 + "-+-" + "-" * 60 + "-+-" + "-" * 40) + + for symbol, demangled, size in sorted_symbols[:100]: # Top 100 + if symbol != demangled: + lines.append(f"{size:>10,} | {symbol[:60]:<60} | {demangled[:100]}") + else: + lines.append(f"{size:>10,} | {symbol[:60]:<60} | [not demangled]") + + if len(sorted_symbols) > 100: + lines.append(f"\n... and {len(sorted_symbols) - 100} more symbols") + + content = "\n".join(lines) + + if output_file: + with open(output_file, "w", encoding="utf-8") as f: + f.write(content) + else: + print(content) + + +def analyze_elf( + elf_path: str, + objdump_path: str | None = None, + readelf_path: str | None = None, + detailed: bool = False, + external_components: set[str] | None = None, +) -> str: + """Analyze an ELF file and return a memory report.""" + analyzer = MemoryAnalyzerCLI( + elf_path, objdump_path, readelf_path, external_components + ) + analyzer.analyze() + return analyzer.generate_report(detailed) + + +def main(): + """CLI entrypoint for memory analysis.""" + if len(sys.argv) < 2: + print("Usage: analyze_memory.py ") + sys.exit(1) + + try: + report = analyze_elf(sys.argv[1]) + print(report) + except (subprocess.CalledProcessError, FileNotFoundError, OSError) as e: + print(f"Error: {e}") + sys.exit(1) + + +if __name__ == "__main__": + main() From 7879df4dd19036928a0a67ad05416182300dbdb4 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Fri, 17 Oct 2025 13:57:57 -1000 Subject: [PATCH 18/76] merge --- esphome/analyze_memory/__init__.py | 28 ++++++++++------------------ esphome/analyze_memory/const.py | 9 +++++++++ 2 files changed, 19 insertions(+), 18 deletions(-) diff --git a/esphome/analyze_memory/__init__.py b/esphome/analyze_memory/__init__.py index 9c35965b74..8cacc1b513 100644 --- a/esphome/analyze_memory/__init__.py +++ b/esphome/analyze_memory/__init__.py @@ -13,6 +13,7 @@ from .const import ( CORE_SUBCATEGORY_PATTERNS, DEMANGLED_PATTERNS, ESPHOME_COMPONENT_PATTERN, + SECTION_MAPPING, SYMBOL_PATTERNS, ) @@ -23,23 +24,21 @@ _LOGGER = logging.getLogger(__name__) @cache def get_esphome_components(): """Get set of actual ESPHome components from the components directory.""" - components = set() - # Find the components directory relative to this file # Go up two levels from analyze_memory/__init__.py to esphome/ current_dir = Path(__file__).parent.parent components_dir = current_dir / "components" - if components_dir.exists() and components_dir.is_dir(): - for item in components_dir.iterdir(): - if ( - item.is_dir() - and not item.name.startswith(".") - and not item.name.startswith("__") - ): - components.add(item.name) + if not components_dir.exists() or not components_dir.is_dir(): + return frozenset() - return components + return frozenset( + item.name + for item in components_dir.iterdir() + if item.is_dir() + and not item.name.startswith(".") + and not item.name.startswith("__") + ) @cache @@ -179,13 +178,6 @@ class MemoryAnalyzer: def _parse_symbols(self) -> None: """Parse symbols from ELF file.""" - # Section mapping - centralizes the logic - SECTION_MAPPING = { - ".text": [".text", ".iram"], - ".rodata": [".rodata"], - ".data": [".data", ".dram"], - ".bss": [".bss"], - } def map_section_name(raw_section: str) -> str | None: """Map raw section name to standard section.""" diff --git a/esphome/analyze_memory/const.py b/esphome/analyze_memory/const.py index df37c0b2cd..8543c6ec2b 100644 --- a/esphome/analyze_memory/const.py +++ b/esphome/analyze_memory/const.py @@ -5,6 +5,15 @@ import re # Pattern to extract ESPHome component namespaces dynamically ESPHOME_COMPONENT_PATTERN = re.compile(r"esphome::([a-zA-Z0-9_]+)::") +# Section mapping for ELF file sections +# Maps standard section names to their various platform-specific variants +SECTION_MAPPING = { + ".text": frozenset([".text", ".iram"]), + ".rodata": frozenset([".rodata"]), + ".data": frozenset([".data", ".dram"]), + ".bss": frozenset([".bss"]), +} + # Component identification rules # Symbol patterns: patterns found in raw symbol names SYMBOL_PATTERNS = { From a78a7dfa4e835a084adac775f443b0b75a8f704c Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Fri, 17 Oct 2025 13:58:59 -1000 Subject: [PATCH 19/76] merge --- esphome/analyze_memory/__init__.py | 37 +++++++++++++++--------------- 1 file changed, 18 insertions(+), 19 deletions(-) diff --git a/esphome/analyze_memory/__init__.py b/esphome/analyze_memory/__init__.py index 8cacc1b513..6d70232448 100644 --- a/esphome/analyze_memory/__init__.py +++ b/esphome/analyze_memory/__init__.py @@ -20,6 +20,21 @@ from .const import ( _LOGGER = logging.getLogger(__name__) +def _map_section_name(raw_section: str) -> str | None: + """Map raw section name to standard section. + + Args: + raw_section: Raw section name from ELF file (e.g., ".iram0.text", ".rodata.str1.1") + + Returns: + Standard section name (".text", ".rodata", ".data", ".bss") or None + """ + for standard_section, patterns in SECTION_MAPPING.items(): + if any(pattern in raw_section for pattern in patterns): + return standard_section + return None + + # Get the list of actual ESPHome components by scanning the components directory @cache def get_esphome_components(): @@ -154,17 +169,8 @@ class MemoryAnalyzer: size_hex = match.group(2) size = int(size_hex, 16) - # Map various section names to standard categories - mapped_section = None - if ".text" in section_name or ".iram" in section_name: - mapped_section = ".text" - elif ".rodata" in section_name: - mapped_section = ".rodata" - elif ".data" in section_name and "bss" not in section_name: - mapped_section = ".data" - elif ".bss" in section_name: - mapped_section = ".bss" - + # Map to standard section name + mapped_section = _map_section_name(section_name) if mapped_section: if mapped_section not in self.sections: self.sections[mapped_section] = MemorySection( @@ -179,13 +185,6 @@ class MemoryAnalyzer: def _parse_symbols(self) -> None: """Parse symbols from ELF file.""" - def map_section_name(raw_section: str) -> str | None: - """Map raw section name to standard section.""" - for standard_section, patterns in SECTION_MAPPING.items(): - if any(pattern in raw_section for pattern in patterns): - return standard_section - return None - def parse_symbol_line(line: str) -> tuple[str, str, int, str] | None: """Parse a single symbol line from objdump output. @@ -211,7 +210,7 @@ class MemoryAnalyzer: # Find section, size, and name for i, part in enumerate(parts): if part.startswith("."): - section = map_section_name(part) + section = _map_section_name(part) if section and i + 1 < len(parts): try: size = int(parts[i + 1], 16) From a5d6e39b2f8c834c1e82098b753ddb8c4df1a56c Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Fri, 17 Oct 2025 14:01:07 -1000 Subject: [PATCH 20/76] merge --- esphome/analyze_memory/__init__.py | 75 ++++++------------------------ 1 file changed, 13 insertions(+), 62 deletions(-) diff --git a/esphome/analyze_memory/__init__.py b/esphome/analyze_memory/__init__.py index 6d70232448..11e5b64f7d 100644 --- a/esphome/analyze_memory/__init__.py +++ b/esphome/analyze_memory/__init__.py @@ -13,28 +13,13 @@ from .const import ( CORE_SUBCATEGORY_PATTERNS, DEMANGLED_PATTERNS, ESPHOME_COMPONENT_PATTERN, - SECTION_MAPPING, SYMBOL_PATTERNS, ) +from .helpers import map_section_name, parse_symbol_line _LOGGER = logging.getLogger(__name__) -def _map_section_name(raw_section: str) -> str | None: - """Map raw section name to standard section. - - Args: - raw_section: Raw section name from ELF file (e.g., ".iram0.text", ".rodata.str1.1") - - Returns: - Standard section name (".text", ".rodata", ".data", ".bss") or None - """ - for standard_section, patterns in SECTION_MAPPING.items(): - if any(pattern in raw_section for pattern in patterns): - return standard_section - return None - - # Get the list of actual ESPHome components by scanning the components directory @cache def get_esphome_components(): @@ -170,7 +155,7 @@ class MemoryAnalyzer: size = int(size_hex, 16) # Map to standard section name - mapped_section = _map_section_name(section_name) + mapped_section = map_section_name(section_name) if mapped_section: if mapped_section not in self.sections: self.sections[mapped_section] = MemorySection( @@ -184,44 +169,6 @@ class MemoryAnalyzer: def _parse_symbols(self) -> None: """Parse symbols from ELF file.""" - - def parse_symbol_line(line: str) -> tuple[str, str, int, str] | None: - """Parse a single symbol line from objdump output. - - Returns (section, name, size, address) or None if not a valid symbol. - Format: address l/g w/d F/O section size name - Example: 40084870 l F .iram0.text 00000000 _xt_user_exc - """ - parts = line.split() - if len(parts) < 5: - return None - - try: - # Validate and extract address - address = parts[0] - int(address, 16) - except ValueError: - return None - - # Look for F (function) or O (object) flag - if "F" not in parts and "O" not in parts: - return None - - # Find section, size, and name - for i, part in enumerate(parts): - if part.startswith("."): - section = _map_section_name(part) - if section and i + 1 < len(parts): - try: - size = int(parts[i + 1], 16) - if i + 2 < len(parts) and size > 0: - name = " ".join(parts[i + 2 :]) - return (section, name, size, address) - except ValueError: - pass - break - return None - try: result = subprocess.run( [self.objdump_path, "-t", str(self.elf_path)], @@ -234,13 +181,17 @@ class MemoryAnalyzer: seen_addresses: set[str] = set() for line in result.stdout.splitlines(): - symbol_info = parse_symbol_line(line) - if symbol_info: - section, name, size, address = symbol_info - # Skip duplicate symbols at the same address (e.g., C1/C2 constructors) - if address not in seen_addresses and section in self.sections: - self.sections[section].symbols.append((name, size, "")) - seen_addresses.add(address) + if not (symbol_info := parse_symbol_line(line)): + continue + + section, name, size, address = symbol_info + + # Skip duplicate symbols at the same address (e.g., C1/C2 constructors) + if address in seen_addresses or section not in self.sections: + continue + + self.sections[section].symbols.append((name, size, "")) + seen_addresses.add(address) except subprocess.CalledProcessError as e: _LOGGER.error("Failed to parse symbols: %s", e) From 79aafe2cd51dc31877b800d8ea989518e304b1cd Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Fri, 17 Oct 2025 14:01:21 -1000 Subject: [PATCH 21/76] merge --- esphome/analyze_memory/helpers.py | 72 +++++++++++++++++++++++++++++++ 1 file changed, 72 insertions(+) create mode 100644 esphome/analyze_memory/helpers.py diff --git a/esphome/analyze_memory/helpers.py b/esphome/analyze_memory/helpers.py new file mode 100644 index 0000000000..c529aad52a --- /dev/null +++ b/esphome/analyze_memory/helpers.py @@ -0,0 +1,72 @@ +"""Helper functions for memory analysis.""" + +from .const import SECTION_MAPPING + + +def map_section_name(raw_section: str) -> str | None: + """Map raw section name to standard section. + + Args: + raw_section: Raw section name from ELF file (e.g., ".iram0.text", ".rodata.str1.1") + + Returns: + Standard section name (".text", ".rodata", ".data", ".bss") or None + """ + for standard_section, patterns in SECTION_MAPPING.items(): + if any(pattern in raw_section for pattern in patterns): + return standard_section + return None + + +def parse_symbol_line(line: str) -> tuple[str, str, int, str] | None: + """Parse a single symbol line from objdump output. + + Args: + line: Line from objdump -t output + + Returns: + Tuple of (section, name, size, address) or None if not a valid symbol. + Format: address l/g w/d F/O section size name + Example: 40084870 l F .iram0.text 00000000 _xt_user_exc + """ + parts = line.split() + if len(parts) < 5: + return None + + try: + # Validate and extract address + address = parts[0] + int(address, 16) + except ValueError: + return None + + # Look for F (function) or O (object) flag + if "F" not in parts and "O" not in parts: + return None + + # Find section, size, and name + for i, part in enumerate(parts): + if not part.startswith("."): + continue + + section = map_section_name(part) + if not section: + break + + # Need at least size field after section + if i + 1 >= len(parts): + break + + try: + size = int(parts[i + 1], 16) + except ValueError: + break + + # Need symbol name and non-zero size + if i + 2 >= len(parts) or size == 0: + break + + name = " ".join(parts[i + 2 :]) + return (section, name, size, address) + + return None From 86c12079b415fb9b8784fc601023c72f65c4eaf5 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Fri, 17 Oct 2025 14:05:24 -1000 Subject: [PATCH 22/76] merge --- esphome/analyze_memory/__init__.py | 12 +- esphome/analyze_memory/cli.py | 186 ++++++++++++++--------------- esphome/analyze_memory/const.py | 9 ++ 3 files changed, 104 insertions(+), 103 deletions(-) diff --git a/esphome/analyze_memory/__init__.py b/esphome/analyze_memory/__init__.py index 11e5b64f7d..2a3955144c 100644 --- a/esphome/analyze_memory/__init__.py +++ b/esphome/analyze_memory/__init__.py @@ -13,6 +13,7 @@ from .const import ( CORE_SUBCATEGORY_PATTERNS, DEMANGLED_PATTERNS, ESPHOME_COMPONENT_PATTERN, + SECTION_TO_ATTR, SYMBOL_PATTERNS, ) from .helpers import map_section_name, parse_symbol_line @@ -219,14 +220,9 @@ class MemoryAnalyzer: comp_mem = self.components[component] comp_mem.symbol_count += 1 - if section_name == ".text": - comp_mem.text_size += size - elif section_name == ".rodata": - comp_mem.rodata_size += size - elif section_name == ".data": - comp_mem.data_size += size - elif section_name == ".bss": - comp_mem.bss_size += size + # Update the appropriate size attribute based on section + if attr_name := SECTION_TO_ATTR.get(section_name): + setattr(comp_mem, attr_name, getattr(comp_mem, attr_name) + size) # Track uncategorized symbols if component == "other" and size > 0: diff --git a/esphome/analyze_memory/cli.py b/esphome/analyze_memory/cli.py index ffce04bb6e..07d0a9320e 100644 --- a/esphome/analyze_memory/cli.py +++ b/esphome/analyze_memory/cli.py @@ -10,6 +10,69 @@ from . import MemoryAnalyzer class MemoryAnalyzerCLI(MemoryAnalyzer): """Memory analyzer with CLI-specific report generation.""" + # Column width constants + COL_COMPONENT: int = 29 + COL_FLASH_TEXT: int = 14 + COL_FLASH_DATA: int = 14 + COL_RAM_DATA: int = 12 + COL_RAM_BSS: int = 12 + COL_TOTAL_FLASH: int = 15 + COL_TOTAL_RAM: int = 12 + COL_SEPARATOR: int = 3 # " | " + + # Core analysis column widths + COL_CORE_SUBCATEGORY: int = 30 + COL_CORE_SIZE: int = 12 + COL_CORE_COUNT: int = 6 + COL_CORE_PERCENT: int = 10 + + # Calculate table width once at class level + TABLE_WIDTH: int = ( + COL_COMPONENT + + COL_SEPARATOR + + COL_FLASH_TEXT + + COL_SEPARATOR + + COL_FLASH_DATA + + COL_SEPARATOR + + COL_RAM_DATA + + COL_SEPARATOR + + COL_RAM_BSS + + COL_SEPARATOR + + COL_TOTAL_FLASH + + COL_SEPARATOR + + COL_TOTAL_RAM + ) + + @staticmethod + def _make_separator_line(*widths: int) -> str: + """Create a separator line with given column widths. + + Args: + widths: Column widths to create separators for + + Returns: + Separator line like "----+---------+-----" + """ + return "-+-".join("-" * width for width in widths) + + # Pre-computed separator lines + MAIN_TABLE_SEPARATOR: str = _make_separator_line( + COL_COMPONENT, + COL_FLASH_TEXT, + COL_FLASH_DATA, + COL_RAM_DATA, + COL_RAM_BSS, + COL_TOTAL_FLASH, + COL_TOTAL_RAM, + ) + + CORE_TABLE_SEPARATOR: str = _make_separator_line( + COL_CORE_SUBCATEGORY, + COL_CORE_SIZE, + COL_CORE_COUNT, + COL_CORE_PERCENT, + ) + def generate_report(self, detailed: bool = False) -> str: """Generate a formatted memory report.""" components = sorted( @@ -23,92 +86,31 @@ class MemoryAnalyzerCLI(MemoryAnalyzer): # Build report lines = [] - # Column width constants - COL_COMPONENT = 29 - COL_FLASH_TEXT = 14 - COL_FLASH_DATA = 14 - COL_RAM_DATA = 12 - COL_RAM_BSS = 12 - COL_TOTAL_FLASH = 15 - COL_TOTAL_RAM = 12 - COL_SEPARATOR = 3 # " | " - - # Core analysis column widths - COL_CORE_SUBCATEGORY = 30 - COL_CORE_SIZE = 12 - COL_CORE_COUNT = 6 - COL_CORE_PERCENT = 10 - - # Calculate the exact table width - table_width = ( - COL_COMPONENT - + COL_SEPARATOR - + COL_FLASH_TEXT - + COL_SEPARATOR - + COL_FLASH_DATA - + COL_SEPARATOR - + COL_RAM_DATA - + COL_SEPARATOR - + COL_RAM_BSS - + COL_SEPARATOR - + COL_TOTAL_FLASH - + COL_SEPARATOR - + COL_TOTAL_RAM - ) - - lines.append("=" * table_width) - lines.append("Component Memory Analysis".center(table_width)) - lines.append("=" * table_width) + lines.append("=" * self.TABLE_WIDTH) + lines.append("Component Memory Analysis".center(self.TABLE_WIDTH)) + lines.append("=" * self.TABLE_WIDTH) lines.append("") # Main table - fixed column widths lines.append( - f"{'Component':<{COL_COMPONENT}} | {'Flash (text)':>{COL_FLASH_TEXT}} | {'Flash (data)':>{COL_FLASH_DATA}} | {'RAM (data)':>{COL_RAM_DATA}} | {'RAM (bss)':>{COL_RAM_BSS}} | {'Total Flash':>{COL_TOTAL_FLASH}} | {'Total RAM':>{COL_TOTAL_RAM}}" - ) - lines.append( - "-" * COL_COMPONENT - + "-+-" - + "-" * COL_FLASH_TEXT - + "-+-" - + "-" * COL_FLASH_DATA - + "-+-" - + "-" * COL_RAM_DATA - + "-+-" - + "-" * COL_RAM_BSS - + "-+-" - + "-" * COL_TOTAL_FLASH - + "-+-" - + "-" * COL_TOTAL_RAM + f"{'Component':<{self.COL_COMPONENT}} | {'Flash (text)':>{self.COL_FLASH_TEXT}} | {'Flash (data)':>{self.COL_FLASH_DATA}} | {'RAM (data)':>{self.COL_RAM_DATA}} | {'RAM (bss)':>{self.COL_RAM_BSS}} | {'Total Flash':>{self.COL_TOTAL_FLASH}} | {'Total RAM':>{self.COL_TOTAL_RAM}}" ) + lines.append(self.MAIN_TABLE_SEPARATOR) for name, mem in components: if mem.flash_total > 0 or mem.ram_total > 0: flash_rodata = mem.rodata_size + mem.data_size lines.append( - f"{name:<{COL_COMPONENT}} | {mem.text_size:>{COL_FLASH_TEXT - 2},} B | {flash_rodata:>{COL_FLASH_DATA - 2},} B | " - f"{mem.data_size:>{COL_RAM_DATA - 2},} B | {mem.bss_size:>{COL_RAM_BSS - 2},} B | " - f"{mem.flash_total:>{COL_TOTAL_FLASH - 2},} B | {mem.ram_total:>{COL_TOTAL_RAM - 2},} B" + f"{name:<{self.COL_COMPONENT}} | {mem.text_size:>{self.COL_FLASH_TEXT - 2},} B | {flash_rodata:>{self.COL_FLASH_DATA - 2},} B | " + f"{mem.data_size:>{self.COL_RAM_DATA - 2},} B | {mem.bss_size:>{self.COL_RAM_BSS - 2},} B | " + f"{mem.flash_total:>{self.COL_TOTAL_FLASH - 2},} B | {mem.ram_total:>{self.COL_TOTAL_RAM - 2},} B" ) + lines.append(self.MAIN_TABLE_SEPARATOR) lines.append( - "-" * COL_COMPONENT - + "-+-" - + "-" * COL_FLASH_TEXT - + "-+-" - + "-" * COL_FLASH_DATA - + "-+-" - + "-" * COL_RAM_DATA - + "-+-" - + "-" * COL_RAM_BSS - + "-+-" - + "-" * COL_TOTAL_FLASH - + "-+-" - + "-" * COL_TOTAL_RAM - ) - lines.append( - f"{'TOTAL':<{COL_COMPONENT}} | {' ':>{COL_FLASH_TEXT}} | {' ':>{COL_FLASH_DATA}} | " - f"{' ':>{COL_RAM_DATA}} | {' ':>{COL_RAM_BSS}} | " - f"{total_flash:>{COL_TOTAL_FLASH - 2},} B | {total_ram:>{COL_TOTAL_RAM - 2},} B" + f"{'TOTAL':<{self.COL_COMPONENT}} | {' ':>{self.COL_FLASH_TEXT}} | {' ':>{self.COL_FLASH_DATA}} | " + f"{' ':>{self.COL_RAM_DATA}} | {' ':>{self.COL_RAM_BSS}} | " + f"{total_flash:>{self.COL_TOTAL_FLASH - 2},} B | {total_ram:>{self.COL_TOTAL_RAM - 2},} B" ) # Top consumers @@ -137,14 +139,14 @@ class MemoryAnalyzerCLI(MemoryAnalyzer): lines.append( "Note: This analysis covers symbols in the ELF file. Some runtime allocations may not be included." ) - lines.append("=" * table_width) + lines.append("=" * self.TABLE_WIDTH) # Add ESPHome core detailed analysis if there are core symbols if self._esphome_core_symbols: lines.append("") - lines.append("=" * table_width) - lines.append("[esphome]core Detailed Analysis".center(table_width)) - lines.append("=" * table_width) + lines.append("=" * self.TABLE_WIDTH) + lines.append("[esphome]core Detailed Analysis".center(self.TABLE_WIDTH)) + lines.append("=" * self.TABLE_WIDTH) lines.append("") # Group core symbols by subcategory @@ -168,26 +170,18 @@ class MemoryAnalyzerCLI(MemoryAnalyzer): ) lines.append( - f"{'Subcategory':<{COL_CORE_SUBCATEGORY}} | {'Size':>{COL_CORE_SIZE}} | " - f"{'Count':>{COL_CORE_COUNT}} | {'% of Core':>{COL_CORE_PERCENT}}" - ) - lines.append( - "-" * COL_CORE_SUBCATEGORY - + "-+-" - + "-" * COL_CORE_SIZE - + "-+-" - + "-" * COL_CORE_COUNT - + "-+-" - + "-" * COL_CORE_PERCENT + f"{'Subcategory':<{self.COL_CORE_SUBCATEGORY}} | {'Size':>{self.COL_CORE_SIZE}} | " + f"{'Count':>{self.COL_CORE_COUNT}} | {'% of Core':>{self.COL_CORE_PERCENT}}" ) + lines.append(self.CORE_TABLE_SEPARATOR) core_total = sum(size for _, _, size in self._esphome_core_symbols) for subcategory, symbols, total_size in sorted_subcategories: percentage = (total_size / core_total * 100) if core_total > 0 else 0 lines.append( - f"{subcategory:<{COL_CORE_SUBCATEGORY}} | {total_size:>{COL_CORE_SIZE - 2},} B | " - f"{len(symbols):>{COL_CORE_COUNT}} | {percentage:>{COL_CORE_PERCENT - 1}.1f}%" + f"{subcategory:<{self.COL_CORE_SUBCATEGORY}} | {total_size:>{self.COL_CORE_SIZE - 2},} B | " + f"{len(symbols):>{self.COL_CORE_COUNT}} | {percentage:>{self.COL_CORE_PERCENT - 1}.1f}%" ) # Top 10 largest core symbols @@ -200,7 +194,7 @@ class MemoryAnalyzerCLI(MemoryAnalyzer): for i, (symbol, demangled, size) in enumerate(sorted_core_symbols[:15]): lines.append(f"{i + 1}. {demangled} ({size:,} B)") - lines.append("=" * table_width) + lines.append("=" * self.TABLE_WIDTH) # Add detailed analysis for top ESPHome and external components esphome_components = [ @@ -240,9 +234,11 @@ class MemoryAnalyzerCLI(MemoryAnalyzer): comp_symbols = self._component_symbols.get(comp_name, []) if comp_symbols: lines.append("") - lines.append("=" * table_width) - lines.append(f"{comp_name} Detailed Analysis".center(table_width)) - lines.append("=" * table_width) + lines.append("=" * self.TABLE_WIDTH) + lines.append( + f"{comp_name} Detailed Analysis".center(self.TABLE_WIDTH) + ) + lines.append("=" * self.TABLE_WIDTH) lines.append("") # Sort symbols by size @@ -267,7 +263,7 @@ class MemoryAnalyzerCLI(MemoryAnalyzer): for i, (symbol, demangled, size) in enumerate(large_symbols): lines.append(f"{i + 1}. {demangled} ({size:,} B)") - lines.append("=" * table_width) + lines.append("=" * self.TABLE_WIDTH) return "\n".join(lines) diff --git a/esphome/analyze_memory/const.py b/esphome/analyze_memory/const.py index 8543c6ec2b..c60b70aeec 100644 --- a/esphome/analyze_memory/const.py +++ b/esphome/analyze_memory/const.py @@ -14,6 +14,15 @@ SECTION_MAPPING = { ".bss": frozenset([".bss"]), } +# Section to ComponentMemory attribute mapping +# Maps section names to the attribute name in ComponentMemory dataclass +SECTION_TO_ATTR = { + ".text": "text_size", + ".rodata": "rodata_size", + ".data": "data_size", + ".bss": "bss_size", +} + # Component identification rules # Symbol patterns: patterns found in raw symbol names SYMBOL_PATTERNS = { From 25fe4a1476b00d6d42588c66f29d52c7f078d33d Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Fri, 17 Oct 2025 14:09:08 -1000 Subject: [PATCH 23/76] merge --- esphome/analyze_memory/__init__.py | 93 ++++++++++-------------------- esphome/analyze_memory/cli.py | 58 +++++++++---------- esphome/analyze_memory/helpers.py | 44 ++++++++++++++ 3 files changed, 100 insertions(+), 95 deletions(-) diff --git a/esphome/analyze_memory/__init__.py b/esphome/analyze_memory/__init__.py index 2a3955144c..b76cb4ec3f 100644 --- a/esphome/analyze_memory/__init__.py +++ b/esphome/analyze_memory/__init__.py @@ -2,7 +2,6 @@ from collections import defaultdict from dataclasses import dataclass, field -from functools import cache import json import logging from pathlib import Path @@ -16,52 +15,16 @@ from .const import ( SECTION_TO_ATTR, SYMBOL_PATTERNS, ) -from .helpers import map_section_name, parse_symbol_line +from .helpers import ( + get_component_class_patterns, + get_esphome_components, + map_section_name, + parse_symbol_line, +) _LOGGER = logging.getLogger(__name__) -# Get the list of actual ESPHome components by scanning the components directory -@cache -def get_esphome_components(): - """Get set of actual ESPHome components from the components directory.""" - # Find the components directory relative to this file - # Go up two levels from analyze_memory/__init__.py to esphome/ - current_dir = Path(__file__).parent.parent - components_dir = current_dir / "components" - - if not components_dir.exists() or not components_dir.is_dir(): - return frozenset() - - return frozenset( - item.name - for item in components_dir.iterdir() - if item.is_dir() - and not item.name.startswith(".") - and not item.name.startswith("__") - ) - - -@cache -def get_component_class_patterns(component_name: str) -> list[str]: - """Generate component class name patterns for symbol matching. - - Args: - component_name: The component name (e.g., "ota", "wifi", "api") - - Returns: - List of pattern strings to match against demangled symbols - """ - component_upper = component_name.upper() - component_camel = component_name.replace("_", "").title() - return [ - f"esphome::{component_upper}Component", # e.g., esphome::OTAComponent - f"esphome::ESPHome{component_upper}Component", # e.g., esphome::ESPHomeOTAComponent - f"esphome::{component_camel}Component", # e.g., esphome::OtaComponent - f"esphome::ESPHome{component_camel}Component", # e.g., esphome::ESPHomeOtaComponent - ] - - @dataclass class MemorySection: """Represents a memory section with its symbols.""" @@ -146,23 +109,26 @@ class MemoryAnalyzer: # Parse section headers for line in result.stdout.splitlines(): # Look for section entries - match = re.match( - r"\s*\[\s*\d+\]\s+([\.\w]+)\s+\w+\s+[\da-fA-F]+\s+[\da-fA-F]+\s+([\da-fA-F]+)", - line, - ) - if match: - section_name = match.group(1) - size_hex = match.group(2) - size = int(size_hex, 16) + if not ( + match := re.match( + r"\s*\[\s*\d+\]\s+([\.\w]+)\s+\w+\s+[\da-fA-F]+\s+[\da-fA-F]+\s+([\da-fA-F]+)", + line, + ) + ): + continue - # Map to standard section name - mapped_section = map_section_name(section_name) - if mapped_section: - if mapped_section not in self.sections: - self.sections[mapped_section] = MemorySection( - mapped_section - ) - self.sections[mapped_section].total_size += size + section_name = match.group(1) + size_hex = match.group(2) + size = int(size_hex, 16) + + # Map to standard section name + mapped_section = map_section_name(section_name) + if not mapped_section: + continue + + if mapped_section not in self.sections: + self.sections[mapped_section] = MemorySection(mapped_section) + self.sections[mapped_section].total_size += size except subprocess.CalledProcessError as e: _LOGGER.error("Failed to parse sections: %s", e) @@ -201,10 +167,11 @@ class MemoryAnalyzer: def _categorize_symbols(self) -> None: """Categorize symbols by component.""" # First, collect all unique symbol names for batch demangling - all_symbols = set() - for section in self.sections.values(): - for symbol_name, _, _ in section.symbols: - all_symbols.add(symbol_name) + all_symbols = { + symbol_name + for section in self.sections.values() + for symbol_name, _, _ in section.symbols + } # Batch demangle all symbols at once self._batch_demangle_symbols(list(all_symbols)) diff --git a/esphome/analyze_memory/cli.py b/esphome/analyze_memory/cli.py index 07d0a9320e..b79a5b6d55 100644 --- a/esphome/analyze_memory/cli.py +++ b/esphome/analyze_memory/cli.py @@ -231,39 +231,33 @@ class MemoryAnalyzerCLI(MemoryAnalyzer): if components_to_analyze: for comp_name, comp_mem in components_to_analyze: - comp_symbols = self._component_symbols.get(comp_name, []) - if comp_symbols: - lines.append("") - lines.append("=" * self.TABLE_WIDTH) - lines.append( - f"{comp_name} Detailed Analysis".center(self.TABLE_WIDTH) - ) - lines.append("=" * self.TABLE_WIDTH) - lines.append("") + if not (comp_symbols := self._component_symbols.get(comp_name, [])): + continue + lines.append("") + lines.append("=" * self.TABLE_WIDTH) + lines.append(f"{comp_name} Detailed Analysis".center(self.TABLE_WIDTH)) + lines.append("=" * self.TABLE_WIDTH) + lines.append("") - # Sort symbols by size - sorted_symbols = sorted( - comp_symbols, key=lambda x: x[2], reverse=True - ) + # Sort symbols by size + sorted_symbols = sorted(comp_symbols, key=lambda x: x[2], reverse=True) - lines.append(f"Total symbols: {len(sorted_symbols)}") - lines.append(f"Total size: {comp_mem.flash_total:,} B") - lines.append("") + lines.append(f"Total symbols: {len(sorted_symbols)}") + lines.append(f"Total size: {comp_mem.flash_total:,} B") + lines.append("") - # Show all symbols > 100 bytes for better visibility - large_symbols = [ - (sym, dem, size) - for sym, dem, size in sorted_symbols - if size > 100 - ] + # Show all symbols > 100 bytes for better visibility + large_symbols = [ + (sym, dem, size) for sym, dem, size in sorted_symbols if size > 100 + ] - lines.append( - f"{comp_name} Symbols > 100 B ({len(large_symbols)} symbols):" - ) - for i, (symbol, demangled, size) in enumerate(large_symbols): - lines.append(f"{i + 1}. {demangled} ({size:,} B)") + lines.append( + f"{comp_name} Symbols > 100 B ({len(large_symbols)} symbols):" + ) + for i, (symbol, demangled, size) in enumerate(large_symbols): + lines.append(f"{i + 1}. {demangled} ({size:,} B)") - lines.append("=" * self.TABLE_WIDTH) + lines.append("=" * self.TABLE_WIDTH) return "\n".join(lines) @@ -284,10 +278,10 @@ class MemoryAnalyzerCLI(MemoryAnalyzer): lines.append("-" * 10 + "-+-" + "-" * 60 + "-+-" + "-" * 40) for symbol, demangled, size in sorted_symbols[:100]: # Top 100 - if symbol != demangled: - lines.append(f"{size:>10,} | {symbol[:60]:<60} | {demangled[:100]}") - else: - lines.append(f"{size:>10,} | {symbol[:60]:<60} | [not demangled]") + demangled_display = ( + demangled[:100] if symbol != demangled else "[not demangled]" + ) + lines.append(f"{size:>10,} | {symbol[:60]:<60} | {demangled_display}") if len(sorted_symbols) > 100: lines.append(f"\n... and {len(sorted_symbols) - 100} more symbols") diff --git a/esphome/analyze_memory/helpers.py b/esphome/analyze_memory/helpers.py index c529aad52a..1b5a1c67c2 100644 --- a/esphome/analyze_memory/helpers.py +++ b/esphome/analyze_memory/helpers.py @@ -1,8 +1,52 @@ """Helper functions for memory analysis.""" +from functools import cache +from pathlib import Path + from .const import SECTION_MAPPING +# Get the list of actual ESPHome components by scanning the components directory +@cache +def get_esphome_components(): + """Get set of actual ESPHome components from the components directory.""" + # Find the components directory relative to this file + # Go up two levels from analyze_memory/helpers.py to esphome/ + current_dir = Path(__file__).parent.parent + components_dir = current_dir / "components" + + if not components_dir.exists() or not components_dir.is_dir(): + return frozenset() + + return frozenset( + item.name + for item in components_dir.iterdir() + if item.is_dir() + and not item.name.startswith(".") + and not item.name.startswith("__") + ) + + +@cache +def get_component_class_patterns(component_name: str) -> list[str]: + """Generate component class name patterns for symbol matching. + + Args: + component_name: The component name (e.g., "ota", "wifi", "api") + + Returns: + List of pattern strings to match against demangled symbols + """ + component_upper = component_name.upper() + component_camel = component_name.replace("_", "").title() + return [ + f"esphome::{component_upper}Component", # e.g., esphome::OTAComponent + f"esphome::ESPHome{component_upper}Component", # e.g., esphome::ESPHomeOTAComponent + f"esphome::{component_camel}Component", # e.g., esphome::OtaComponent + f"esphome::ESPHome{component_camel}Component", # e.g., esphome::ESPHomeOtaComponent + ] + + def map_section_name(raw_section: str) -> str | None: """Map raw section name to standard section. From 2c86ebaf7ff2fa01b817069568540b2c62b4e03f Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Fri, 17 Oct 2025 14:10:23 -1000 Subject: [PATCH 24/76] merge --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6fa8150b93..22ae046246 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -676,13 +676,13 @@ jobs: python-version: ${{ env.DEFAULT_PYTHON }} cache-key: ${{ needs.common.outputs.cache-key }} - name: Download target ELF artifact - uses: actions/download-artifact@1a18f44933c290e06e7167a92071e78bb20ab94a # v4.4.2 + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 with: name: memory-impact-target-elf path: ./elf-artifacts/target continue-on-error: true - name: Download PR ELF artifact - uses: actions/download-artifact@1a18f44933c290e06e7167a92071e78bb20ab94a # v4.4.2 + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 with: name: memory-impact-pr-elf path: ./elf-artifacts/pr From 843f590db47ba1cf349c9a21f106af64b4486cbf Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Fri, 17 Oct 2025 14:13:25 -1000 Subject: [PATCH 25/76] fix --- esphome/analyze_memory/cli.py | 34 +++++++++++++++++++++++++++++++--- 1 file changed, 31 insertions(+), 3 deletions(-) diff --git a/esphome/analyze_memory/cli.py b/esphome/analyze_memory/cli.py index b79a5b6d55..675e93ae07 100644 --- a/esphome/analyze_memory/cli.py +++ b/esphome/analyze_memory/cli.py @@ -313,14 +313,42 @@ def analyze_elf( def main(): """CLI entrypoint for memory analysis.""" if len(sys.argv) < 2: - print("Usage: analyze_memory.py ") + print( + "Usage: python -m esphome.analyze_memory [objdump_path] [readelf_path]" + ) + print("\nExample for ESP8266:") + print(" python -m esphome.analyze_memory firmware.elf \\") + print( + " ~/.platformio/packages/toolchain-xtensa/bin/xtensa-lx106-elf-objdump \\" + ) + print( + " ~/.platformio/packages/toolchain-xtensa/bin/xtensa-lx106-elf-readelf" + ) + print("\nExample for ESP32:") + print(" python -m esphome.analyze_memory firmware.elf \\") + print( + " ~/.platformio/packages/toolchain-xtensa-esp-elf/bin/xtensa-esp32-elf-objdump \\" + ) + print( + " ~/.platformio/packages/toolchain-xtensa-esp-elf/bin/xtensa-esp32-elf-readelf" + ) sys.exit(1) + elf_file = sys.argv[1] + objdump_path = sys.argv[2] if len(sys.argv) > 2 else None + readelf_path = sys.argv[3] if len(sys.argv) > 3 else None + try: - report = analyze_elf(sys.argv[1]) + report = analyze_elf(elf_file, objdump_path, readelf_path) print(report) except (subprocess.CalledProcessError, FileNotFoundError, OSError) as e: - print(f"Error: {e}") + print(f"Error: {e}", file=sys.stderr) + if "readelf" in str(e) or "objdump" in str(e): + print( + "\nHint: You need to specify the toolchain-specific tools.", + file=sys.stderr, + ) + print("See usage above for examples.", file=sys.stderr) sys.exit(1) From f011c44130c07bd873b570e65aa73195026b01f0 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Fri, 17 Oct 2025 14:26:44 -1000 Subject: [PATCH 26/76] merge --- .github/workflows/ci.yml | 122 ++++++++++++++++------------- esphome/analyze_memory/cli.py | 9 +++ esphome/platformio_api.py | 101 +++++++++++++++++++++++- script/ci_memory_impact_comment.py | 99 +++++++++++++++++++++-- script/ci_memory_impact_extract.py | 40 +++++++--- script/determine-jobs.py | 98 +++++++++++------------ 6 files changed, 345 insertions(+), 124 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 22ae046246..0842248db9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -548,45 +548,53 @@ jobs: with: path: ~/.platformio key: platformio-memory-${{ fromJSON(needs.determine-jobs.outputs.memory_impact).platform }}-${{ hashFiles('platformio.ini') }} - - name: Compile test configuration and extract memory usage + - name: Build and compile with test_build_components id: extract run: | . venv/bin/activate - component="${{ fromJSON(needs.determine-jobs.outputs.memory_impact).component }}" + components='${{ toJSON(fromJSON(needs.determine-jobs.outputs.memory_impact).components) }}' platform="${{ fromJSON(needs.determine-jobs.outputs.memory_impact).platform }}" - test_file="${{ fromJSON(needs.determine-jobs.outputs.memory_impact).test_file }}" - echo "Compiling $component for $platform using $test_file" - python script/test_build_components.py -e compile -c "$component" -t "$platform" --no-grouping 2>&1 | \ + echo "Building with test_build_components.py for $platform with components:" + echo "$components" | jq -r '.[]' | sed 's/^/ - /' + + # Use test_build_components.py which handles grouping automatically + # Pass components as comma-separated list + component_list=$(echo "$components" | jq -r 'join(",")') + + echo "Compiling with test_build_components.py..." + python script/test_build_components.py \ + -e compile \ + -c "$component_list" \ + -t "$platform" 2>&1 | \ python script/ci_memory_impact_extract.py --output-env - - name: Find and upload ELF file + - name: Find and upload final ELF file run: | - # Find the ELF file - try both common locations - elf_file="" + # Note: test_build_components.py may run multiple builds, but each overwrites + # the previous firmware.elf. The memory totals (RAM/Flash) are already summed + # by ci_memory_impact_extract.py. This ELF is from the last build and is used + # for detailed component breakdown (if available). + mkdir -p ./elf-artifacts/target - # Try .esphome/build first (default location) + # Find the most recent firmware.elf if [ -d ~/.esphome/build ]; then - elf_file=$(find ~/.esphome/build -name "firmware.elf" -o -name "*.elf" | head -1) - fi + elf_file=$(find ~/.esphome/build -name "firmware.elf" -type f -printf '%T@ %p\n' | sort -rn | head -1 | cut -d' ' -f2-) - # Fallback to finding in .platformio if not found - if [ -z "$elf_file" ] && [ -d ~/.platformio ]; then - elf_file=$(find ~/.platformio -name "firmware.elf" | head -1) - fi - - if [ -n "$elf_file" ] && [ -f "$elf_file" ]; then - echo "Found ELF file: $elf_file" - mkdir -p ./elf-artifacts - cp "$elf_file" ./elf-artifacts/target.elf + if [ -n "$elf_file" ] && [ -f "$elf_file" ]; then + echo "Found final ELF file: $elf_file" + cp "$elf_file" "./elf-artifacts/target/firmware.elf" + else + echo "Warning: No ELF file found in ~/.esphome/build" + ls -la ~/.esphome/build/ || true + fi else - echo "Warning: No ELF file found in ~/.esphome/build or ~/.platformio" - ls -la ~/.esphome/build/ || true + echo "Warning: ~/.esphome/build directory not found" fi - name: Upload ELF artifact uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 with: name: memory-impact-target-elf - path: ./elf-artifacts/target.elf + path: ./elf-artifacts/target/firmware.elf if-no-files-found: warn retention-days: 1 @@ -613,45 +621,53 @@ jobs: with: path: ~/.platformio key: platformio-memory-${{ fromJSON(needs.determine-jobs.outputs.memory_impact).platform }}-${{ hashFiles('platformio.ini') }} - - name: Compile test configuration and extract memory usage + - name: Build and compile with test_build_components id: extract run: | . venv/bin/activate - component="${{ fromJSON(needs.determine-jobs.outputs.memory_impact).component }}" + components='${{ toJSON(fromJSON(needs.determine-jobs.outputs.memory_impact).components) }}' platform="${{ fromJSON(needs.determine-jobs.outputs.memory_impact).platform }}" - test_file="${{ fromJSON(needs.determine-jobs.outputs.memory_impact).test_file }}" - echo "Compiling $component for $platform using $test_file" - python script/test_build_components.py -e compile -c "$component" -t "$platform" --no-grouping 2>&1 | \ + echo "Building with test_build_components.py for $platform with components:" + echo "$components" | jq -r '.[]' | sed 's/^/ - /' + + # Use test_build_components.py which handles grouping automatically + # Pass components as comma-separated list + component_list=$(echo "$components" | jq -r 'join(",")') + + echo "Compiling with test_build_components.py..." + python script/test_build_components.py \ + -e compile \ + -c "$component_list" \ + -t "$platform" 2>&1 | \ python script/ci_memory_impact_extract.py --output-env - - name: Find and upload ELF file + - name: Find and upload final ELF file run: | - # Find the ELF file - try both common locations - elf_file="" + # Note: test_build_components.py may run multiple builds, but each overwrites + # the previous firmware.elf. The memory totals (RAM/Flash) are already summed + # by ci_memory_impact_extract.py. This ELF is from the last build and is used + # for detailed component breakdown (if available). + mkdir -p ./elf-artifacts/pr - # Try .esphome/build first (default location) + # Find the most recent firmware.elf if [ -d ~/.esphome/build ]; then - elf_file=$(find ~/.esphome/build -name "firmware.elf" -o -name "*.elf" | head -1) - fi + elf_file=$(find ~/.esphome/build -name "firmware.elf" -type f -printf '%T@ %p\n' | sort -rn | head -1 | cut -d' ' -f2-) - # Fallback to finding in .platformio if not found - if [ -z "$elf_file" ] && [ -d ~/.platformio ]; then - elf_file=$(find ~/.platformio -name "firmware.elf" | head -1) - fi - - if [ -n "$elf_file" ] && [ -f "$elf_file" ]; then - echo "Found ELF file: $elf_file" - mkdir -p ./elf-artifacts - cp "$elf_file" ./elf-artifacts/pr.elf + if [ -n "$elf_file" ] && [ -f "$elf_file" ]; then + echo "Found final ELF file: $elf_file" + cp "$elf_file" "./elf-artifacts/pr/firmware.elf" + else + echo "Warning: No ELF file found in ~/.esphome/build" + ls -la ~/.esphome/build/ || true + fi else - echo "Warning: No ELF file found in ~/.esphome/build or ~/.platformio" - ls -la ~/.esphome/build/ || true + echo "Warning: ~/.esphome/build directory not found" fi - name: Upload ELF artifact uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 with: name: memory-impact-pr-elf - path: ./elf-artifacts/pr.elf + path: ./elf-artifacts/pr/firmware.elf if-no-files-found: warn retention-days: 1 @@ -690,7 +706,7 @@ jobs: - name: Post or update PR comment env: GH_TOKEN: ${{ github.token }} - COMPONENT: ${{ fromJSON(needs.determine-jobs.outputs.memory_impact).component }} + COMPONENTS: ${{ toJSON(fromJSON(needs.determine-jobs.outputs.memory_impact).components) }} PLATFORM: ${{ fromJSON(needs.determine-jobs.outputs.memory_impact).platform }} TARGET_RAM: ${{ needs.memory-impact-target-branch.outputs.ram_usage }} TARGET_FLASH: ${{ needs.memory-impact-target-branch.outputs.flash_usage }} @@ -699,27 +715,27 @@ jobs: run: | . venv/bin/activate - # Check if ELF files exist + # Check if ELF files exist (from final build) target_elf_arg="" pr_elf_arg="" - if [ -f ./elf-artifacts/target/target.elf ]; then + if [ -f ./elf-artifacts/target/firmware.elf ]; then echo "Found target ELF file" - target_elf_arg="--target-elf ./elf-artifacts/target/target.elf" + target_elf_arg="--target-elf ./elf-artifacts/target/firmware.elf" else echo "No target ELF file found" fi - if [ -f ./elf-artifacts/pr/pr.elf ]; then + if [ -f ./elf-artifacts/pr/firmware.elf ]; then echo "Found PR ELF file" - pr_elf_arg="--pr-elf ./elf-artifacts/pr/pr.elf" + pr_elf_arg="--pr-elf ./elf-artifacts/pr/firmware.elf" else echo "No PR ELF file found" fi python script/ci_memory_impact_comment.py \ --pr-number "${{ github.event.pull_request.number }}" \ - --component "$COMPONENT" \ + --components "$COMPONENTS" \ --platform "$PLATFORM" \ --target-ram "$TARGET_RAM" \ --target-flash "$TARGET_FLASH" \ diff --git a/esphome/analyze_memory/cli.py b/esphome/analyze_memory/cli.py index 675e93ae07..184f95ffa6 100644 --- a/esphome/analyze_memory/cli.py +++ b/esphome/analyze_memory/cli.py @@ -316,6 +316,7 @@ def main(): print( "Usage: python -m esphome.analyze_memory [objdump_path] [readelf_path]" ) + print("\nIf objdump/readelf paths are not provided, you must specify them.") print("\nExample for ESP8266:") print(" python -m esphome.analyze_memory firmware.elf \\") print( @@ -332,6 +333,14 @@ def main(): print( " ~/.platformio/packages/toolchain-xtensa-esp-elf/bin/xtensa-esp32-elf-readelf" ) + print("\nExample for ESP32-C3 (RISC-V):") + print(" python -m esphome.analyze_memory firmware.elf \\") + print( + " ~/.platformio/packages/toolchain-riscv32-esp/bin/riscv32-esp-elf-objdump \\" + ) + print( + " ~/.platformio/packages/toolchain-riscv32-esp/bin/riscv32-esp-elf-readelf" + ) sys.exit(1) elf_file = sys.argv[1] diff --git a/esphome/platformio_api.py b/esphome/platformio_api.py index 9418c1c7d3..a4b5b432fd 100644 --- a/esphome/platformio_api.py +++ b/esphome/platformio_api.py @@ -145,7 +145,16 @@ def run_compile(config, verbose): args = [] if CONF_COMPILE_PROCESS_LIMIT in config[CONF_ESPHOME]: args += [f"-j{config[CONF_ESPHOME][CONF_COMPILE_PROCESS_LIMIT]}"] - return run_platformio_cli_run(config, verbose, *args) + result = run_platformio_cli_run(config, verbose, *args) + + # Run memory analysis if enabled + if config.get(CONF_ESPHOME, {}).get("analyze_memory", False): + try: + analyze_memory_usage(config) + except Exception as e: + _LOGGER.warning("Failed to analyze memory usage: %s", e) + + return result def _run_idedata(config): @@ -374,3 +383,93 @@ class IDEData: return f"{self.cc_path[:-7]}addr2line.exe" return f"{self.cc_path[:-3]}addr2line" + + @property + def objdump_path(self) -> str: + # replace gcc at end with objdump + + # Windows + if self.cc_path.endswith(".exe"): + return f"{self.cc_path[:-7]}objdump.exe" + + return f"{self.cc_path[:-3]}objdump" + + @property + def readelf_path(self) -> str: + # replace gcc at end with readelf + + # Windows + if self.cc_path.endswith(".exe"): + return f"{self.cc_path[:-7]}readelf.exe" + + return f"{self.cc_path[:-3]}readelf" + + +def analyze_memory_usage(config: dict[str, Any]) -> None: + """Analyze memory usage by component after compilation.""" + # Lazy import to avoid overhead when not needed + from esphome.analyze_memory import MemoryAnalyzer + + idedata = get_idedata(config) + + # Get paths to tools + elf_path = idedata.firmware_elf_path + objdump_path = idedata.objdump_path + readelf_path = idedata.readelf_path + + # Debug logging + _LOGGER.debug("ELF path from idedata: %s", elf_path) + + # Check if file exists + if not Path(elf_path).exists(): + # Try alternate path + alt_path = Path(CORE.relative_build_path(".pioenvs", CORE.name, "firmware.elf")) + if alt_path.exists(): + elf_path = str(alt_path) + _LOGGER.debug("Using alternate ELF path: %s", elf_path) + else: + _LOGGER.warning("ELF file not found at %s or %s", elf_path, alt_path) + return + + # Extract external components from config + external_components = set() + + # Get the list of built-in ESPHome components + from esphome.analyze_memory import get_esphome_components + + builtin_components = get_esphome_components() + + # Special non-component keys that appear in configs + NON_COMPONENT_KEYS = { + CONF_ESPHOME, + "substitutions", + "packages", + "globals", + "<<", + } + + # Check all top-level keys in config + for key in config: + if key not in builtin_components and key not in NON_COMPONENT_KEYS: + # This is an external component + external_components.add(key) + + _LOGGER.debug("Detected external components: %s", external_components) + + # Create analyzer and run analysis + analyzer = MemoryAnalyzer(elf_path, objdump_path, readelf_path, external_components) + analyzer.analyze() + + # Generate and print report + report = analyzer.generate_report() + _LOGGER.info("\n%s", report) + + # Optionally save to file + if config.get(CONF_ESPHOME, {}).get("memory_report_file"): + report_file = Path(config[CONF_ESPHOME]["memory_report_file"]) + if report_file.suffix == ".json": + report_file.write_text(analyzer.to_json()) + _LOGGER.info("Memory report saved to %s", report_file) + else: + report_file.write_text(report) + _LOGGER.info("Memory report saved to %s", report_file) diff --git a/script/ci_memory_impact_comment.py b/script/ci_memory_impact_comment.py index 0f65e4fbbd..d31868ed1c 100755 --- a/script/ci_memory_impact_comment.py +++ b/script/ci_memory_impact_comment.py @@ -24,6 +24,57 @@ from esphome.analyze_memory import MemoryAnalyzer # noqa: E402 COMMENT_MARKER = "" +def get_platform_toolchain(platform: str) -> tuple[str | None, str | None]: + """Get platform-specific objdump and readelf paths. + + Args: + platform: Platform name (e.g., "esp8266-ard", "esp32-idf", "esp32-c3-idf") + + Returns: + Tuple of (objdump_path, readelf_path) or (None, None) if not found/supported + """ + from pathlib import Path + + home = Path.home() + platformio_packages = home / ".platformio" / "packages" + + # Map platform to toolchain + toolchain = None + prefix = None + + if "esp8266" in platform: + toolchain = "toolchain-xtensa" + prefix = "xtensa-lx106-elf" + elif "esp32-c" in platform or "esp32-h" in platform or "esp32-p4" in platform: + # RISC-V variants (C2, C3, C5, C6, H2, P4) + toolchain = "toolchain-riscv32-esp" + prefix = "riscv32-esp-elf" + elif "esp32" in platform: + # Xtensa variants (original, S2, S3) + toolchain = "toolchain-xtensa-esp-elf" + if "s2" in platform: + prefix = "xtensa-esp32s2-elf" + elif "s3" in platform: + prefix = "xtensa-esp32s3-elf" + else: + prefix = "xtensa-esp32-elf" + else: + # Other platforms (RP2040, LibreTiny, etc.) - not supported + print(f"Platform {platform} not supported for ELF analysis", file=sys.stderr) + return None, None + + toolchain_path = platformio_packages / toolchain / "bin" + objdump_path = toolchain_path / f"{prefix}-objdump" + readelf_path = toolchain_path / f"{prefix}-readelf" + + if objdump_path.exists() and readelf_path.exists(): + print(f"Using {platform} toolchain: {prefix}", file=sys.stderr) + return str(objdump_path), str(readelf_path) + + print(f"Warning: Toolchain not found at {toolchain_path}", file=sys.stderr) + return None, None + + def format_bytes(bytes_value: int) -> str: """Format bytes value with comma separators. @@ -314,7 +365,7 @@ def create_detailed_breakdown_table( def create_comment_body( - component: str, + components: list[str], platform: str, target_ram: int, target_flash: int, @@ -328,7 +379,7 @@ def create_comment_body( """Create the comment body with memory impact analysis. Args: - component: Component name + components: List of component names (merged config) platform: Platform name target_ram: RAM usage in target branch target_flash: Flash usage in target branch @@ -374,10 +425,18 @@ def create_comment_body( else: print("No ELF files provided, skipping detailed analysis", file=sys.stderr) + # Format components list + if len(components) == 1: + components_str = f"`{components[0]}`" + config_note = "a representative test configuration" + else: + components_str = ", ".join(f"`{c}`" for c in sorted(components)) + config_note = f"a merged configuration with {len(components)} components" + return f"""{COMMENT_MARKER} ## Memory Impact Analysis -**Component:** `{component}` +**Components:** {components_str} **Platform:** `{platform}` | Metric | Target Branch | This PR | Change | @@ -386,7 +445,7 @@ def create_comment_body( | **Flash** | {format_bytes(target_flash)} | {format_bytes(pr_flash)} | {flash_change} | {component_breakdown}{symbol_changes} --- -*This analysis runs automatically when a single component changes. Memory usage is measured from a representative test configuration.* +*This analysis runs automatically when components change. Memory usage is measured from {config_note}.* """ @@ -537,7 +596,11 @@ def main() -> int: description="Post or update PR comment with memory impact analysis" ) parser.add_argument("--pr-number", required=True, help="PR number") - parser.add_argument("--component", required=True, help="Component name") + parser.add_argument( + "--components", + required=True, + help='JSON array of component names (e.g., \'["api", "wifi"]\')', + ) parser.add_argument("--platform", required=True, help="Platform name") parser.add_argument( "--target-ram", type=int, required=True, help="Target branch RAM usage" @@ -560,9 +623,29 @@ def main() -> int: args = parser.parse_args() + # Parse components from JSON + try: + components = json.loads(args.components) + if not isinstance(components, list): + print("Error: --components must be a JSON array", file=sys.stderr) + sys.exit(1) + except json.JSONDecodeError as e: + print(f"Error parsing --components JSON: {e}", file=sys.stderr) + sys.exit(1) + + # Detect platform-specific toolchain paths + objdump_path = args.objdump_path + readelf_path = args.readelf_path + + if not objdump_path or not readelf_path: + # Auto-detect based on platform + objdump_path, readelf_path = get_platform_toolchain(args.platform) + # Create comment body + # Note: ELF files (if provided) are from the final build when test_build_components + # runs multiple builds. Memory totals (RAM/Flash) are already summed across all builds. comment_body = create_comment_body( - component=args.component, + components=components, platform=args.platform, target_ram=args.target_ram, target_flash=args.target_flash, @@ -570,8 +653,8 @@ def main() -> int: pr_flash=args.pr_flash, target_elf=args.target_elf, pr_elf=args.pr_elf, - objdump_path=args.objdump_path, - readelf_path=args.readelf_path, + objdump_path=objdump_path, + readelf_path=readelf_path, ) # Post or update comment diff --git a/script/ci_memory_impact_extract.py b/script/ci_memory_impact_extract.py index 9ddd39096f..1b8a994f14 100755 --- a/script/ci_memory_impact_extract.py +++ b/script/ci_memory_impact_extract.py @@ -28,27 +28,36 @@ from script.ci_helpers import write_github_output def extract_from_compile_output(output_text: str) -> tuple[int | None, int | None]: """Extract memory usage from PlatformIO compile output. + Supports multiple builds (for component groups or isolated components). + When test_build_components.py creates multiple builds, this sums the + memory usage across all builds. + Looks for lines like: RAM: [==== ] 36.1% (used 29548 bytes from 81920 bytes) Flash: [=== ] 34.0% (used 348511 bytes from 1023984 bytes) Args: - output_text: Compile output text + output_text: Compile output text (may contain multiple builds) Returns: - Tuple of (ram_bytes, flash_bytes) or (None, None) if not found + Tuple of (total_ram_bytes, total_flash_bytes) or (None, None) if not found """ - ram_match = re.search( + # Find all RAM and Flash matches (may be multiple builds) + ram_matches = re.findall( r"RAM:\s+\[.*?\]\s+\d+\.\d+%\s+\(used\s+(\d+)\s+bytes", output_text ) - flash_match = re.search( + flash_matches = re.findall( r"Flash:\s+\[.*?\]\s+\d+\.\d+%\s+\(used\s+(\d+)\s+bytes", output_text ) - if ram_match and flash_match: - return int(ram_match.group(1)), int(flash_match.group(1)) + if not ram_matches or not flash_matches: + return None, None - return None, None + # Sum all builds (handles multiple component groups) + total_ram = sum(int(match) for match in ram_matches) + total_flash = sum(int(match) for match in flash_matches) + + return total_ram, total_flash def main() -> int: @@ -83,8 +92,21 @@ def main() -> int: ) return 1 - print(f"RAM: {ram_bytes} bytes", file=sys.stderr) - print(f"Flash: {flash_bytes} bytes", file=sys.stderr) + # Count how many builds were found + num_builds = len( + re.findall( + r"RAM:\s+\[.*?\]\s+\d+\.\d+%\s+\(used\s+(\d+)\s+bytes", compile_output + ) + ) + + if num_builds > 1: + print( + f"Found {num_builds} builds - summing memory usage across all builds", + file=sys.stderr, + ) + + print(f"Total RAM: {ram_bytes} bytes", file=sys.stderr) + print(f"Total Flash: {flash_bytes} bytes", file=sys.stderr) if args.output_env: # Output to GitHub Actions diff --git a/script/determine-jobs.py b/script/determine-jobs.py index fa44941c29..56de0e77ba 100755 --- a/script/determine-jobs.py +++ b/script/determine-jobs.py @@ -237,14 +237,14 @@ def _component_has_tests(component: str) -> bool: return any(tests_dir.glob("test.*.yaml")) -def detect_single_component_for_memory_impact( +def detect_memory_impact_config( branch: str | None = None, ) -> dict[str, Any]: - """Detect if exactly one component changed for memory impact analysis. + """Determine memory impact analysis configuration. - This analyzes the actual changed files (not dependencies) to determine if - exactly one component has been modified. This is different from the - changed_components list which includes all dependencies. + Always runs memory impact analysis when there are changed components, + building a merged configuration with all changed components (like + test_build_components.py does) to get comprehensive memory analysis. Args: branch: Branch to compare against @@ -252,37 +252,25 @@ def detect_single_component_for_memory_impact( Returns: Dictionary with memory impact analysis parameters: - should_run: "true" or "false" - - component: component name (if should_run is true) - - test_file: test file name (if should_run is true) - - platform: platform name (if should_run is true) + - components: list of component names to analyze + - platform: platform name for the merged build + - use_merged_config: "true" (always use merged config) """ # Platform preference order for memory impact analysis - # Ordered by production relevance and memory constraint importance + # Prefer ESP8266 for memory impact as it's the most constrained platform PLATFORM_PREFERENCE = [ + "esp8266-ard", # ESP8266 Arduino (most memory constrained - best for impact analysis) "esp32-idf", # Primary ESP32 IDF platform "esp32-c3-idf", # ESP32-C3 IDF "esp32-c6-idf", # ESP32-C6 IDF "esp32-s2-idf", # ESP32-S2 IDF "esp32-s3-idf", # ESP32-S3 IDF - "esp32-c2-idf", # ESP32-C2 IDF - "esp32-c5-idf", # ESP32-C5 IDF - "esp32-h2-idf", # ESP32-H2 IDF - "esp32-p4-idf", # ESP32-P4 IDF - "esp8266-ard", # ESP8266 Arduino (memory constrained) - "esp32-ard", # ESP32 Arduino - "esp32-c3-ard", # ESP32-C3 Arduino - "esp32-s2-ard", # ESP32-S2 Arduino - "esp32-s3-ard", # ESP32-S3 Arduino - "bk72xx-ard", # BK72xx Arduino - "rp2040-ard", # RP2040 Arduino - "nrf52-adafruit", # nRF52 Adafruit - "host", # Host platform (development/testing) ] # Get actually changed files (not dependencies) files = changed_files(branch) - # Find all changed components (excluding core) + # Find all changed components (excluding core and base bus components) changed_component_set = set() for file in files: @@ -291,49 +279,53 @@ def detect_single_component_for_memory_impact( if len(parts) >= 3: component = parts[2] # Skip base bus components as they're used across many builds - if component not in ["i2c", "spi", "uart", "modbus"]: + if component not in ["i2c", "spi", "uart", "modbus", "canbus"]: changed_component_set.add(component) - # Only proceed if exactly one component changed - if len(changed_component_set) != 1: + # If no components changed, don't run memory impact + if not changed_component_set: return {"should_run": "false"} - component = list(changed_component_set)[0] + # Find components that have tests on the preferred platform + components_with_tests = [] + selected_platform = None - # Find a test configuration for this component - tests_dir = Path(root_path) / "tests" / "components" / component + for component in sorted(changed_component_set): + tests_dir = Path(root_path) / "tests" / "components" / component + if not tests_dir.exists(): + continue - if not tests_dir.exists(): - return {"should_run": "false"} + # Look for test files on preferred platforms + test_files = list(tests_dir.glob("test.*.yaml")) + if not test_files: + continue - # Look for test files - test_files = list(tests_dir.glob("test.*.yaml")) - if not test_files: - return {"should_run": "false"} - - # Try each preferred platform in order - for preferred_platform in PLATFORM_PREFERENCE: + # Check if component has tests for any preferred platform for test_file in test_files: parts = test_file.stem.split(".") if len(parts) >= 2: platform = parts[1] - if platform == preferred_platform: - return { - "should_run": "true", - "component": component, - "test_file": test_file.name, - "platform": platform, - } + if platform in PLATFORM_PREFERENCE: + components_with_tests.append(component) + # Select the most preferred platform across all components + if selected_platform is None or PLATFORM_PREFERENCE.index( + platform + ) < PLATFORM_PREFERENCE.index(selected_platform): + selected_platform = platform + break + + # If no components have tests, don't run memory impact + if not components_with_tests: + return {"should_run": "false"} + + # Use the most preferred platform found, or fall back to esp8266-ard + platform = selected_platform or "esp8266-ard" - # Fall back to first test file - test_file = test_files[0] - parts = test_file.stem.split(".") - platform = parts[1] if len(parts) >= 2 else "esp32-idf" return { "should_run": "true", - "component": component, - "test_file": test_file.name, + "components": components_with_tests, "platform": platform, + "use_merged_config": "true", } @@ -386,8 +378,8 @@ def main() -> None: if component not in directly_changed_components ] - # Detect single component change for memory impact analysis - memory_impact = detect_single_component_for_memory_impact(args.branch) + # Detect components for memory impact analysis (merged config) + memory_impact = detect_memory_impact_config(args.branch) # Build output output: dict[str, Any] = { From f87c969b4315a67e54237f40e50455b7f0ea2fd8 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Fri, 17 Oct 2025 14:40:45 -1000 Subject: [PATCH 27/76] tweak --- .github/workflows/ci.yml | 134 ++++++++++---------- esphome/analyze_memory/__init__.py | 73 +++++++++++ esphome/analyze_memory/cli.py | 119 ++++++++++++------ esphome/platformio_api.py | 9 +- script/ci_memory_impact_comment.py | 188 ++++++++--------------------- script/ci_memory_impact_extract.py | 112 +++++++++++++++++ 6 files changed, 381 insertions(+), 254 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0842248db9..7a4d8bf929 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -548,7 +548,7 @@ jobs: with: path: ~/.platformio key: platformio-memory-${{ fromJSON(needs.determine-jobs.outputs.memory_impact).platform }}-${{ hashFiles('platformio.ini') }} - - name: Build and compile with test_build_components + - name: Build, compile, and analyze memory id: extract run: | . venv/bin/activate @@ -563,38 +563,32 @@ jobs: component_list=$(echo "$components" | jq -r 'join(",")') echo "Compiling with test_build_components.py..." - python script/test_build_components.py \ - -e compile \ - -c "$component_list" \ - -t "$platform" 2>&1 | \ - python script/ci_memory_impact_extract.py --output-env - - name: Find and upload final ELF file - run: | - # Note: test_build_components.py may run multiple builds, but each overwrites - # the previous firmware.elf. The memory totals (RAM/Flash) are already summed - # by ci_memory_impact_extract.py. This ELF is from the last build and is used - # for detailed component breakdown (if available). - mkdir -p ./elf-artifacts/target - # Find the most recent firmware.elf - if [ -d ~/.esphome/build ]; then - elf_file=$(find ~/.esphome/build -name "firmware.elf" -type f -printf '%T@ %p\n' | sort -rn | head -1 | cut -d' ' -f2-) + # Find most recent build directory for detailed analysis + build_dir=$(find ~/.esphome/build -type d -maxdepth 1 -mindepth 1 -printf '%T@ %p\n' 2>/dev/null | sort -rn | head -1 | cut -d' ' -f2- || echo "") - if [ -n "$elf_file" ] && [ -f "$elf_file" ]; then - echo "Found final ELF file: $elf_file" - cp "$elf_file" "./elf-artifacts/target/firmware.elf" - else - echo "Warning: No ELF file found in ~/.esphome/build" - ls -la ~/.esphome/build/ || true - fi + # Run build and extract memory, with optional detailed analysis + if [ -n "$build_dir" ]; then + python script/test_build_components.py \ + -e compile \ + -c "$component_list" \ + -t "$platform" 2>&1 | \ + python script/ci_memory_impact_extract.py \ + --output-env \ + --build-dir "$build_dir" \ + --output-json memory-analysis-target.json else - echo "Warning: ~/.esphome/build directory not found" + python script/test_build_components.py \ + -e compile \ + -c "$component_list" \ + -t "$platform" 2>&1 | \ + python script/ci_memory_impact_extract.py --output-env fi - - name: Upload ELF artifact + - name: Upload memory analysis JSON uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 with: - name: memory-impact-target-elf - path: ./elf-artifacts/target/firmware.elf + name: memory-analysis-target + path: memory-analysis-target.json if-no-files-found: warn retention-days: 1 @@ -621,7 +615,7 @@ jobs: with: path: ~/.platformio key: platformio-memory-${{ fromJSON(needs.determine-jobs.outputs.memory_impact).platform }}-${{ hashFiles('platformio.ini') }} - - name: Build and compile with test_build_components + - name: Build, compile, and analyze memory id: extract run: | . venv/bin/activate @@ -636,38 +630,32 @@ jobs: component_list=$(echo "$components" | jq -r 'join(",")') echo "Compiling with test_build_components.py..." - python script/test_build_components.py \ - -e compile \ - -c "$component_list" \ - -t "$platform" 2>&1 | \ - python script/ci_memory_impact_extract.py --output-env - - name: Find and upload final ELF file - run: | - # Note: test_build_components.py may run multiple builds, but each overwrites - # the previous firmware.elf. The memory totals (RAM/Flash) are already summed - # by ci_memory_impact_extract.py. This ELF is from the last build and is used - # for detailed component breakdown (if available). - mkdir -p ./elf-artifacts/pr - # Find the most recent firmware.elf - if [ -d ~/.esphome/build ]; then - elf_file=$(find ~/.esphome/build -name "firmware.elf" -type f -printf '%T@ %p\n' | sort -rn | head -1 | cut -d' ' -f2-) + # Find most recent build directory for detailed analysis + build_dir=$(find ~/.esphome/build -type d -maxdepth 1 -mindepth 1 -printf '%T@ %p\n' 2>/dev/null | sort -rn | head -1 | cut -d' ' -f2- || echo "") - if [ -n "$elf_file" ] && [ -f "$elf_file" ]; then - echo "Found final ELF file: $elf_file" - cp "$elf_file" "./elf-artifacts/pr/firmware.elf" - else - echo "Warning: No ELF file found in ~/.esphome/build" - ls -la ~/.esphome/build/ || true - fi + # Run build and extract memory, with optional detailed analysis + if [ -n "$build_dir" ]; then + python script/test_build_components.py \ + -e compile \ + -c "$component_list" \ + -t "$platform" 2>&1 | \ + python script/ci_memory_impact_extract.py \ + --output-env \ + --build-dir "$build_dir" \ + --output-json memory-analysis-pr.json else - echo "Warning: ~/.esphome/build directory not found" + python script/test_build_components.py \ + -e compile \ + -c "$component_list" \ + -t "$platform" 2>&1 | \ + python script/ci_memory_impact_extract.py --output-env fi - - name: Upload ELF artifact + - name: Upload memory analysis JSON uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 with: - name: memory-impact-pr-elf - path: ./elf-artifacts/pr/firmware.elf + name: memory-analysis-pr + path: memory-analysis-pr.json if-no-files-found: warn retention-days: 1 @@ -691,17 +679,17 @@ jobs: with: python-version: ${{ env.DEFAULT_PYTHON }} cache-key: ${{ needs.common.outputs.cache-key }} - - name: Download target ELF artifact + - name: Download target analysis JSON uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 with: - name: memory-impact-target-elf - path: ./elf-artifacts/target + name: memory-analysis-target + path: ./memory-analysis continue-on-error: true - - name: Download PR ELF artifact + - name: Download PR analysis JSON uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 with: - name: memory-impact-pr-elf - path: ./elf-artifacts/pr + name: memory-analysis-pr + path: ./memory-analysis continue-on-error: true - name: Post or update PR comment env: @@ -715,22 +703,22 @@ jobs: run: | . venv/bin/activate - # Check if ELF files exist (from final build) - target_elf_arg="" - pr_elf_arg="" + # Check if analysis JSON files exist + target_json_arg="" + pr_json_arg="" - if [ -f ./elf-artifacts/target/firmware.elf ]; then - echo "Found target ELF file" - target_elf_arg="--target-elf ./elf-artifacts/target/firmware.elf" + if [ -f ./memory-analysis/memory-analysis-target.json ]; then + echo "Found target analysis JSON" + target_json_arg="--target-json ./memory-analysis/memory-analysis-target.json" else - echo "No target ELF file found" + echo "No target analysis JSON found" fi - if [ -f ./elf-artifacts/pr/firmware.elf ]; then - echo "Found PR ELF file" - pr_elf_arg="--pr-elf ./elf-artifacts/pr/firmware.elf" + if [ -f ./memory-analysis/memory-analysis-pr.json ]; then + echo "Found PR analysis JSON" + pr_json_arg="--pr-json ./memory-analysis/memory-analysis-pr.json" else - echo "No PR ELF file found" + echo "No PR analysis JSON found" fi python script/ci_memory_impact_comment.py \ @@ -741,8 +729,8 @@ jobs: --target-flash "$TARGET_FLASH" \ --pr-ram "$PR_RAM" \ --pr-flash "$PR_FLASH" \ - $target_elf_arg \ - $pr_elf_arg + $target_json_arg \ + $pr_json_arg ci-status: name: CI Status diff --git a/esphome/analyze_memory/__init__.py b/esphome/analyze_memory/__init__.py index b76cb4ec3f..5bd46fd01e 100644 --- a/esphome/analyze_memory/__init__.py +++ b/esphome/analyze_memory/__init__.py @@ -7,6 +7,7 @@ import logging from pathlib import Path import re import subprocess +from typing import TYPE_CHECKING from .const import ( CORE_SUBCATEGORY_PATTERNS, @@ -22,9 +23,65 @@ from .helpers import ( parse_symbol_line, ) +if TYPE_CHECKING: + from esphome.platformio_api import IDEData + _LOGGER = logging.getLogger(__name__) +def get_toolchain_for_platform(platform: str) -> tuple[str | None, str | None]: + """Get objdump and readelf paths for a given platform. + + This function auto-detects the correct toolchain based on the platform name, + using the same detection logic as PlatformIO's IDEData class. + + Args: + platform: Platform name (e.g., "esp8266-ard", "esp32-idf", "esp32-c3-idf") + + Returns: + Tuple of (objdump_path, readelf_path) or (None, None) if not found/supported + """ + home = Path.home() + platformio_packages = home / ".platformio" / "packages" + + # Map platform to toolchain and prefix (same logic as PlatformIO uses) + toolchain = None + prefix = None + + if "esp8266" in platform: + toolchain = "toolchain-xtensa" + prefix = "xtensa-lx106-elf" + elif "esp32-c" in platform or "esp32-h" in platform or "esp32-p4" in platform: + # RISC-V variants (C2, C3, C5, C6, H2, P4) + toolchain = "toolchain-riscv32-esp" + prefix = "riscv32-esp-elf" + elif "esp32" in platform: + # Xtensa variants (original, S2, S3) + toolchain = "toolchain-xtensa-esp-elf" + if "s2" in platform: + prefix = "xtensa-esp32s2-elf" + elif "s3" in platform: + prefix = "xtensa-esp32s3-elf" + else: + prefix = "xtensa-esp32-elf" + else: + # Other platforms (RP2040, LibreTiny, etc.) - not supported for ELF analysis + _LOGGER.debug("Platform %s not supported for ELF analysis", platform) + return None, None + + # Construct paths (same pattern as IDEData.objdump_path/readelf_path) + toolchain_path = platformio_packages / toolchain / "bin" + objdump_path = toolchain_path / f"{prefix}-objdump" + readelf_path = toolchain_path / f"{prefix}-readelf" + + if objdump_path.exists() and readelf_path.exists(): + _LOGGER.debug("Found %s toolchain: %s", platform, prefix) + return str(objdump_path), str(readelf_path) + + _LOGGER.warning("Toolchain not found at %s", toolchain_path) + return None, None + + @dataclass class MemorySection: """Represents a memory section with its symbols.""" @@ -67,11 +124,27 @@ class MemoryAnalyzer: objdump_path: str | None = None, readelf_path: str | None = None, external_components: set[str] | None = None, + idedata: "IDEData | None" = None, ): + """Initialize memory analyzer. + + Args: + elf_path: Path to ELF file to analyze + objdump_path: Path to objdump binary (auto-detected from idedata if not provided) + readelf_path: Path to readelf binary (auto-detected from idedata if not provided) + external_components: Set of external component names + idedata: Optional PlatformIO IDEData object to auto-detect toolchain paths + """ self.elf_path = Path(elf_path) if not self.elf_path.exists(): raise FileNotFoundError(f"ELF file not found: {elf_path}") + # Auto-detect toolchain paths from idedata if not provided + if idedata is not None and (objdump_path is None or readelf_path is None): + objdump_path = objdump_path or idedata.objdump_path + readelf_path = readelf_path or idedata.readelf_path + _LOGGER.debug("Using toolchain paths from PlatformIO idedata") + self.objdump_path = objdump_path or "objdump" self.readelf_path = readelf_path or "readelf" self.external_components = external_components or set() diff --git a/esphome/analyze_memory/cli.py b/esphome/analyze_memory/cli.py index 184f95ffa6..e8541b1621 100644 --- a/esphome/analyze_memory/cli.py +++ b/esphome/analyze_memory/cli.py @@ -1,7 +1,6 @@ """CLI interface for memory analysis with report generation.""" from collections import defaultdict -import subprocess import sys from . import MemoryAnalyzer @@ -313,51 +312,91 @@ def analyze_elf( def main(): """CLI entrypoint for memory analysis.""" if len(sys.argv) < 2: - print( - "Usage: python -m esphome.analyze_memory [objdump_path] [readelf_path]" - ) - print("\nIf objdump/readelf paths are not provided, you must specify them.") - print("\nExample for ESP8266:") - print(" python -m esphome.analyze_memory firmware.elf \\") - print( - " ~/.platformio/packages/toolchain-xtensa/bin/xtensa-lx106-elf-objdump \\" - ) - print( - " ~/.platformio/packages/toolchain-xtensa/bin/xtensa-lx106-elf-readelf" - ) - print("\nExample for ESP32:") - print(" python -m esphome.analyze_memory firmware.elf \\") - print( - " ~/.platformio/packages/toolchain-xtensa-esp-elf/bin/xtensa-esp32-elf-objdump \\" - ) - print( - " ~/.platformio/packages/toolchain-xtensa-esp-elf/bin/xtensa-esp32-elf-readelf" - ) - print("\nExample for ESP32-C3 (RISC-V):") - print(" python -m esphome.analyze_memory firmware.elf \\") - print( - " ~/.platformio/packages/toolchain-riscv32-esp/bin/riscv32-esp-elf-objdump \\" - ) - print( - " ~/.platformio/packages/toolchain-riscv32-esp/bin/riscv32-esp-elf-readelf" - ) + print("Usage: python -m esphome.analyze_memory ") + print("\nAnalyze memory usage from an ESPHome build directory.") + print("The build directory should contain firmware.elf and idedata will be") + print("loaded from ~/.esphome/.internal/idedata/.json") + print("\nExamples:") + print(" python -m esphome.analyze_memory ~/.esphome/build/my-device") + print(" python -m esphome.analyze_memory .esphome/build/my-device") + print(" python -m esphome.analyze_memory my-device # Short form") sys.exit(1) - elf_file = sys.argv[1] - objdump_path = sys.argv[2] if len(sys.argv) > 2 else None - readelf_path = sys.argv[3] if len(sys.argv) > 3 else None + build_dir = sys.argv[1] + + # Load build directory + import json + from pathlib import Path + + from esphome.platformio_api import IDEData + + build_path = Path(build_dir) + + # If no path separator in name, assume it's a device name + if "/" not in build_dir and not build_path.is_dir(): + # Try current directory first + cwd_path = Path.cwd() / ".esphome" / "build" / build_dir + if cwd_path.is_dir(): + build_path = cwd_path + print(f"Using build directory: {build_path}", file=sys.stderr) + else: + # Fall back to home directory + build_path = Path.home() / ".esphome" / "build" / build_dir + print(f"Using build directory: {build_path}", file=sys.stderr) + + if not build_path.is_dir(): + print(f"Error: {build_path} is not a directory", file=sys.stderr) + sys.exit(1) + + # Find firmware.elf + elf_file = None + for elf_candidate in [ + build_path / "firmware.elf", + build_path / ".pioenvs" / build_path.name / "firmware.elf", + ]: + if elf_candidate.exists(): + elf_file = str(elf_candidate) + break + + if not elf_file: + print(f"Error: firmware.elf not found in {build_dir}", file=sys.stderr) + sys.exit(1) + + # Find idedata.json - check current directory first, then home + device_name = build_path.name + idedata_candidates = [ + Path.cwd() / ".esphome" / "idedata" / f"{device_name}.json", + Path.home() / ".esphome" / "idedata" / f"{device_name}.json", + ] + + idedata = None + for idedata_path in idedata_candidates: + if idedata_path.exists(): + try: + with open(idedata_path, encoding="utf-8") as f: + raw_data = json.load(f) + idedata = IDEData(raw_data) + print(f"Loaded idedata from: {idedata_path}", file=sys.stderr) + break + except (json.JSONDecodeError, OSError) as e: + print(f"Warning: Failed to load idedata: {e}", file=sys.stderr) + + if not idedata: + print( + f"Warning: idedata not found (searched {idedata_candidates[0]} and {idedata_candidates[1]})", + file=sys.stderr, + ) try: - report = analyze_elf(elf_file, objdump_path, readelf_path) + analyzer = MemoryAnalyzerCLI(elf_file, idedata=idedata) + analyzer.analyze() + report = analyzer.generate_report() print(report) - except (subprocess.CalledProcessError, FileNotFoundError, OSError) as e: + except Exception as e: print(f"Error: {e}", file=sys.stderr) - if "readelf" in str(e) or "objdump" in str(e): - print( - "\nHint: You need to specify the toolchain-specific tools.", - file=sys.stderr, - ) - print("See usage above for examples.", file=sys.stderr) + import traceback + + traceback.print_exc(file=sys.stderr) sys.exit(1) diff --git a/esphome/platformio_api.py b/esphome/platformio_api.py index a4b5b432fd..065a8cf896 100644 --- a/esphome/platformio_api.py +++ b/esphome/platformio_api.py @@ -412,10 +412,8 @@ def analyze_memory_usage(config: dict[str, Any]) -> None: idedata = get_idedata(config) - # Get paths to tools + # Get ELF path elf_path = idedata.firmware_elf_path - objdump_path = idedata.objdump_path - readelf_path = idedata.readelf_path # Debug logging _LOGGER.debug("ELF path from idedata: %s", elf_path) @@ -457,7 +455,10 @@ def analyze_memory_usage(config: dict[str, Any]) -> None: _LOGGER.debug("Detected external components: %s", external_components) # Create analyzer and run analysis - analyzer = MemoryAnalyzer(elf_path, objdump_path, readelf_path, external_components) + # Pass idedata to auto-detect toolchain paths + analyzer = MemoryAnalyzer( + elf_path, external_components=external_components, idedata=idedata + ) analyzer.analyze() # Generate and print report diff --git a/script/ci_memory_impact_comment.py b/script/ci_memory_impact_comment.py index d31868ed1c..c5eb9e701f 100755 --- a/script/ci_memory_impact_comment.py +++ b/script/ci_memory_impact_comment.py @@ -18,61 +18,31 @@ import sys sys.path.insert(0, str(Path(__file__).parent.parent)) # pylint: disable=wrong-import-position -from esphome.analyze_memory import MemoryAnalyzer # noqa: E402 # Comment marker to identify our memory impact comments COMMENT_MARKER = "" -def get_platform_toolchain(platform: str) -> tuple[str | None, str | None]: - """Get platform-specific objdump and readelf paths. +def load_analysis_json(json_path: str) -> dict | None: + """Load memory analysis results from JSON file. Args: - platform: Platform name (e.g., "esp8266-ard", "esp32-idf", "esp32-c3-idf") + json_path: Path to analysis JSON file Returns: - Tuple of (objdump_path, readelf_path) or (None, None) if not found/supported + Dictionary with analysis results or None if file doesn't exist/can't be loaded """ - from pathlib import Path + json_file = Path(json_path) + if not json_file.exists(): + print(f"Analysis JSON not found: {json_path}", file=sys.stderr) + return None - home = Path.home() - platformio_packages = home / ".platformio" / "packages" - - # Map platform to toolchain - toolchain = None - prefix = None - - if "esp8266" in platform: - toolchain = "toolchain-xtensa" - prefix = "xtensa-lx106-elf" - elif "esp32-c" in platform or "esp32-h" in platform or "esp32-p4" in platform: - # RISC-V variants (C2, C3, C5, C6, H2, P4) - toolchain = "toolchain-riscv32-esp" - prefix = "riscv32-esp-elf" - elif "esp32" in platform: - # Xtensa variants (original, S2, S3) - toolchain = "toolchain-xtensa-esp-elf" - if "s2" in platform: - prefix = "xtensa-esp32s2-elf" - elif "s3" in platform: - prefix = "xtensa-esp32s3-elf" - else: - prefix = "xtensa-esp32-elf" - else: - # Other platforms (RP2040, LibreTiny, etc.) - not supported - print(f"Platform {platform} not supported for ELF analysis", file=sys.stderr) - return None, None - - toolchain_path = platformio_packages / toolchain / "bin" - objdump_path = toolchain_path / f"{prefix}-objdump" - readelf_path = toolchain_path / f"{prefix}-readelf" - - if objdump_path.exists() and readelf_path.exists(): - print(f"Using {platform} toolchain: {prefix}", file=sys.stderr) - return str(objdump_path), str(readelf_path) - - print(f"Warning: Toolchain not found at {toolchain_path}", file=sys.stderr) - return None, None + try: + with open(json_file, encoding="utf-8") as f: + return json.load(f) + except (json.JSONDecodeError, OSError) as e: + print(f"Failed to load analysis JSON: {e}", file=sys.stderr) + return None def format_bytes(bytes_value: int) -> str: @@ -122,56 +92,6 @@ def format_change(before: int, after: int) -> str: return f"{emoji} {delta_str} ({pct_str})" -def run_detailed_analysis( - elf_path: str, objdump_path: str | None = None, readelf_path: str | None = None -) -> tuple[dict | None, dict | None]: - """Run detailed memory analysis on an ELF file. - - Args: - elf_path: Path to ELF file - objdump_path: Optional path to objdump tool - readelf_path: Optional path to readelf tool - - Returns: - Tuple of (component_breakdown, symbol_map) or (None, None) if analysis fails - component_breakdown: Dictionary with component memory breakdown - symbol_map: Dictionary mapping symbol names to their sizes - """ - try: - analyzer = MemoryAnalyzer(elf_path, objdump_path, readelf_path) - components = analyzer.analyze() - - # Convert ComponentMemory objects to dictionaries - component_result = {} - for name, mem in components.items(): - component_result[name] = { - "text": mem.text_size, - "rodata": mem.rodata_size, - "data": mem.data_size, - "bss": mem.bss_size, - "flash_total": mem.flash_total, - "ram_total": mem.ram_total, - "symbol_count": mem.symbol_count, - } - - # Build symbol map from all sections - symbol_map = {} - for section in analyzer.sections.values(): - for symbol_name, size, _ in section.symbols: - if size > 0: # Only track non-zero sized symbols - # Demangle the symbol for better readability - demangled = analyzer._demangle_symbol(symbol_name) - symbol_map[demangled] = size - - return component_result, symbol_map - except Exception as e: - print(f"Warning: Failed to run detailed analysis: {e}", file=sys.stderr) - import traceback - - traceback.print_exc(file=sys.stderr) - return None, None - - def create_symbol_changes_table( target_symbols: dict | None, pr_symbols: dict | None ) -> str: @@ -371,10 +291,10 @@ def create_comment_body( target_flash: int, pr_ram: int, pr_flash: int, - target_elf: str | None = None, - pr_elf: str | None = None, - objdump_path: str | None = None, - readelf_path: str | None = None, + target_analysis: dict | None = None, + pr_analysis: dict | None = None, + target_symbols: dict | None = None, + pr_symbols: dict | None = None, ) -> str: """Create the comment body with memory impact analysis. @@ -385,10 +305,10 @@ def create_comment_body( target_flash: Flash usage in target branch pr_ram: RAM usage in PR branch pr_flash: Flash usage in PR branch - target_elf: Optional path to target branch ELF file - pr_elf: Optional path to PR branch ELF file - objdump_path: Optional path to objdump tool - readelf_path: Optional path to readelf tool + target_analysis: Optional component breakdown for target branch + pr_analysis: Optional component breakdown for PR branch + target_symbols: Optional symbol map for target branch + pr_symbols: Optional symbol map for PR branch Returns: Formatted comment body @@ -396,29 +316,14 @@ def create_comment_body( ram_change = format_change(target_ram, pr_ram) flash_change = format_change(target_flash, pr_flash) - # Run detailed analysis if ELF files are provided - target_analysis = None - pr_analysis = None - target_symbols = None - pr_symbols = None + # Use provided analysis data if available component_breakdown = "" symbol_changes = "" - if target_elf and pr_elf: - print( - f"Running detailed analysis on {target_elf} and {pr_elf}", file=sys.stderr + if target_analysis and pr_analysis: + component_breakdown = create_detailed_breakdown_table( + target_analysis, pr_analysis ) - target_analysis, target_symbols = run_detailed_analysis( - target_elf, objdump_path, readelf_path - ) - pr_analysis, pr_symbols = run_detailed_analysis( - pr_elf, objdump_path, readelf_path - ) - - if target_analysis and pr_analysis: - component_breakdown = create_detailed_breakdown_table( - target_analysis, pr_analysis - ) if target_symbols and pr_symbols: symbol_changes = create_symbol_changes_table(target_symbols, pr_symbols) @@ -612,13 +517,13 @@ def main() -> int: parser.add_argument( "--pr-flash", type=int, required=True, help="PR branch flash usage" ) - parser.add_argument("--target-elf", help="Optional path to target branch ELF file") - parser.add_argument("--pr-elf", help="Optional path to PR branch ELF file") parser.add_argument( - "--objdump-path", help="Optional path to objdump tool for detailed analysis" + "--target-json", + help="Optional path to target branch analysis JSON (for detailed analysis)", ) parser.add_argument( - "--readelf-path", help="Optional path to readelf tool for detailed analysis" + "--pr-json", + help="Optional path to PR branch analysis JSON (for detailed analysis)", ) args = parser.parse_args() @@ -633,17 +538,26 @@ def main() -> int: print(f"Error parsing --components JSON: {e}", file=sys.stderr) sys.exit(1) - # Detect platform-specific toolchain paths - objdump_path = args.objdump_path - readelf_path = args.readelf_path + # Load analysis JSON files + target_analysis = None + pr_analysis = None + target_symbols = None + pr_symbols = None - if not objdump_path or not readelf_path: - # Auto-detect based on platform - objdump_path, readelf_path = get_platform_toolchain(args.platform) + if args.target_json: + target_data = load_analysis_json(args.target_json) + if target_data and target_data.get("detailed_analysis"): + target_analysis = target_data["detailed_analysis"].get("components") + target_symbols = target_data["detailed_analysis"].get("symbols") + + if args.pr_json: + pr_data = load_analysis_json(args.pr_json) + if pr_data and pr_data.get("detailed_analysis"): + pr_analysis = pr_data["detailed_analysis"].get("components") + pr_symbols = pr_data["detailed_analysis"].get("symbols") # Create comment body - # Note: ELF files (if provided) are from the final build when test_build_components - # runs multiple builds. Memory totals (RAM/Flash) are already summed across all builds. + # Note: Memory totals (RAM/Flash) are summed across all builds if multiple were run. comment_body = create_comment_body( components=components, platform=args.platform, @@ -651,10 +565,10 @@ def main() -> int: target_flash=args.target_flash, pr_ram=args.pr_ram, pr_flash=args.pr_flash, - target_elf=args.target_elf, - pr_elf=args.pr_elf, - objdump_path=objdump_path, - readelf_path=readelf_path, + target_analysis=target_analysis, + pr_analysis=pr_analysis, + target_symbols=target_symbols, + pr_symbols=pr_symbols, ) # Post or update comment diff --git a/script/ci_memory_impact_extract.py b/script/ci_memory_impact_extract.py index 1b8a994f14..283b521860 100755 --- a/script/ci_memory_impact_extract.py +++ b/script/ci_memory_impact_extract.py @@ -9,11 +9,14 @@ The script reads compile output from stdin and looks for the standard PlatformIO output format: RAM: [==== ] 36.1% (used 29548 bytes from 81920 bytes) Flash: [=== ] 34.0% (used 348511 bytes from 1023984 bytes) + +Optionally performs detailed memory analysis if a build directory is provided. """ from __future__ import annotations import argparse +import json from pathlib import Path import re import sys @@ -60,6 +63,87 @@ def extract_from_compile_output(output_text: str) -> tuple[int | None, int | Non return total_ram, total_flash +def run_detailed_analysis(build_dir: str) -> dict | None: + """Run detailed memory analysis on build directory. + + Args: + build_dir: Path to ESPHome build directory + + Returns: + Dictionary with analysis results or None if analysis fails + """ + from esphome.analyze_memory import MemoryAnalyzer + from esphome.platformio_api import IDEData + + build_path = Path(build_dir) + if not build_path.exists(): + print(f"Build directory not found: {build_dir}", file=sys.stderr) + return None + + # Find firmware.elf + elf_path = None + for elf_candidate in [ + build_path / "firmware.elf", + build_path / ".pioenvs" / build_path.name / "firmware.elf", + ]: + if elf_candidate.exists(): + elf_path = str(elf_candidate) + break + + if not elf_path: + print(f"firmware.elf not found in {build_dir}", file=sys.stderr) + return None + + # Find idedata.json + device_name = build_path.name + idedata_path = Path.home() / ".esphome" / "idedata" / f"{device_name}.json" + + idedata = None + if idedata_path.exists(): + try: + with open(idedata_path, encoding="utf-8") as f: + raw_data = json.load(f) + idedata = IDEData(raw_data) + except (json.JSONDecodeError, OSError) as e: + print(f"Warning: Failed to load idedata: {e}", file=sys.stderr) + + try: + analyzer = MemoryAnalyzer(elf_path, idedata=idedata) + components = analyzer.analyze() + + # Convert to JSON-serializable format + result = { + "components": {}, + "symbols": {}, + } + + for name, mem in components.items(): + result["components"][name] = { + "text": mem.text_size, + "rodata": mem.rodata_size, + "data": mem.data_size, + "bss": mem.bss_size, + "flash_total": mem.flash_total, + "ram_total": mem.ram_total, + "symbol_count": mem.symbol_count, + } + + # Build symbol map + for section in analyzer.sections.values(): + for symbol_name, size, _ in section.symbols: + if size > 0: + demangled = analyzer._demangle_symbol(symbol_name) + result["symbols"][demangled] = size + + return result + except Exception as e: + print(f"Warning: Failed to run detailed analysis: {e}", file=sys.stderr) + import traceback + + traceback.print_exc(file=sys.stderr) + return None + + def main() -> int: """Main entry point.""" parser = argparse.ArgumentParser( @@ -70,6 +154,14 @@ def main() -> int: action="store_true", help="Output to GITHUB_OUTPUT environment file", ) + parser.add_argument( + "--build-dir", + help="Optional build directory for detailed memory analysis", + ) + parser.add_argument( + "--output-json", + help="Optional path to save detailed analysis JSON", + ) args = parser.parse_args() @@ -108,6 +200,26 @@ def main() -> int: print(f"Total RAM: {ram_bytes} bytes", file=sys.stderr) print(f"Total Flash: {flash_bytes} bytes", file=sys.stderr) + # Run detailed analysis if build directory provided + detailed_analysis = None + if args.build_dir: + print(f"Running detailed analysis on {args.build_dir}", file=sys.stderr) + detailed_analysis = run_detailed_analysis(args.build_dir) + + # Save JSON output if requested + if args.output_json: + output_data = { + "ram_bytes": ram_bytes, + "flash_bytes": flash_bytes, + "detailed_analysis": detailed_analysis, + } + + output_path = Path(args.output_json) + output_path.parent.mkdir(parents=True, exist_ok=True) + with open(output_path, "w", encoding="utf-8") as f: + json.dump(output_data, f, indent=2) + print(f"Saved analysis to {args.output_json}", file=sys.stderr) + if args.output_env: # Output to GitHub Actions write_github_output( From e2101f5a20bd99d105321f2ad83f3d5b89a57d08 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Fri, 17 Oct 2025 14:52:07 -1000 Subject: [PATCH 28/76] tweak --- .github/workflows/ci.yml | 56 ++++++------------- script/ci_memory_impact_extract.py | 87 ++++++++++++++++++++++-------- 2 files changed, 82 insertions(+), 61 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7a4d8bf929..440f64298b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -564,26 +564,14 @@ jobs: echo "Compiling with test_build_components.py..." - # Find most recent build directory for detailed analysis - build_dir=$(find ~/.esphome/build -type d -maxdepth 1 -mindepth 1 -printf '%T@ %p\n' 2>/dev/null | sort -rn | head -1 | cut -d' ' -f2- || echo "") - - # Run build and extract memory, with optional detailed analysis - if [ -n "$build_dir" ]; then - python script/test_build_components.py \ - -e compile \ - -c "$component_list" \ - -t "$platform" 2>&1 | \ - python script/ci_memory_impact_extract.py \ - --output-env \ - --build-dir "$build_dir" \ - --output-json memory-analysis-target.json - else - python script/test_build_components.py \ - -e compile \ - -c "$component_list" \ - -t "$platform" 2>&1 | \ - python script/ci_memory_impact_extract.py --output-env - fi + # Run build and extract memory with auto-detection of build directory for detailed analysis + python script/test_build_components.py \ + -e compile \ + -c "$component_list" \ + -t "$platform" 2>&1 | \ + python script/ci_memory_impact_extract.py \ + --output-env \ + --output-json memory-analysis-target.json - name: Upload memory analysis JSON uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 with: @@ -631,26 +619,14 @@ jobs: echo "Compiling with test_build_components.py..." - # Find most recent build directory for detailed analysis - build_dir=$(find ~/.esphome/build -type d -maxdepth 1 -mindepth 1 -printf '%T@ %p\n' 2>/dev/null | sort -rn | head -1 | cut -d' ' -f2- || echo "") - - # Run build and extract memory, with optional detailed analysis - if [ -n "$build_dir" ]; then - python script/test_build_components.py \ - -e compile \ - -c "$component_list" \ - -t "$platform" 2>&1 | \ - python script/ci_memory_impact_extract.py \ - --output-env \ - --build-dir "$build_dir" \ - --output-json memory-analysis-pr.json - else - python script/test_build_components.py \ - -e compile \ - -c "$component_list" \ - -t "$platform" 2>&1 | \ - python script/ci_memory_impact_extract.py --output-env - fi + # Run build and extract memory with auto-detection of build directory for detailed analysis + python script/test_build_components.py \ + -e compile \ + -c "$component_list" \ + -t "$platform" 2>&1 | \ + python script/ci_memory_impact_extract.py \ + --output-env \ + --output-json memory-analysis-pr.json - name: Upload memory analysis JSON uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 with: diff --git a/script/ci_memory_impact_extract.py b/script/ci_memory_impact_extract.py index 283b521860..9a9c294f2e 100755 --- a/script/ci_memory_impact_extract.py +++ b/script/ci_memory_impact_extract.py @@ -28,8 +28,10 @@ sys.path.insert(0, str(Path(__file__).parent.parent)) from script.ci_helpers import write_github_output -def extract_from_compile_output(output_text: str) -> tuple[int | None, int | None]: - """Extract memory usage from PlatformIO compile output. +def extract_from_compile_output( + output_text: str, +) -> tuple[int | None, int | None, str | None]: + """Extract memory usage and build directory from PlatformIO compile output. Supports multiple builds (for component groups or isolated components). When test_build_components.py creates multiple builds, this sums the @@ -39,11 +41,14 @@ def extract_from_compile_output(output_text: str) -> tuple[int | None, int | Non RAM: [==== ] 36.1% (used 29548 bytes from 81920 bytes) Flash: [=== ] 34.0% (used 348511 bytes from 1023984 bytes) + Also extracts build directory from lines like: + INFO Deleting /path/to/build/.esphome/build/componenttestesp8266ard/.pioenvs + Args: output_text: Compile output text (may contain multiple builds) Returns: - Tuple of (total_ram_bytes, total_flash_bytes) or (None, None) if not found + Tuple of (total_ram_bytes, total_flash_bytes, build_dir) or (None, None, None) if not found """ # Find all RAM and Flash matches (may be multiple builds) ram_matches = re.findall( @@ -54,13 +59,21 @@ def extract_from_compile_output(output_text: str) -> tuple[int | None, int | Non ) if not ram_matches or not flash_matches: - return None, None + return None, None, None # Sum all builds (handles multiple component groups) total_ram = sum(int(match) for match in ram_matches) total_flash = sum(int(match) for match in flash_matches) - return total_ram, total_flash + # Extract build directory from ESPHome's delete messages + # Look for: INFO Deleting /path/to/build/.esphome/build/componenttest.../.pioenvs + build_dir = None + if match := re.search( + r"INFO Deleting (.+/\.esphome/build/componenttest[^/]+)/\.pioenvs", output_text + ): + build_dir = match.group(1) + + return total_ram, total_flash, build_dir def run_detailed_analysis(build_dir: str) -> dict | None: @@ -94,18 +107,31 @@ def run_detailed_analysis(build_dir: str) -> dict | None: print(f"firmware.elf not found in {build_dir}", file=sys.stderr) return None - # Find idedata.json + # Find idedata.json - check multiple locations device_name = build_path.name - idedata_path = Path.home() / ".esphome" / "idedata" / f"{device_name}.json" + idedata_candidates = [ + # In .pioenvs for test builds + build_path / ".pioenvs" / device_name / "idedata.json", + # In .esphome/idedata for regular builds + Path.home() / ".esphome" / "idedata" / f"{device_name}.json", + # Check parent directories for .esphome/idedata (for test_build_components) + build_path.parent.parent.parent / "idedata" / f"{device_name}.json", + ] idedata = None - if idedata_path.exists(): - try: - with open(idedata_path, encoding="utf-8") as f: - raw_data = json.load(f) - idedata = IDEData(raw_data) - except (json.JSONDecodeError, OSError) as e: - print(f"Warning: Failed to load idedata: {e}", file=sys.stderr) + for idedata_path in idedata_candidates: + if idedata_path.exists(): + try: + with open(idedata_path, encoding="utf-8") as f: + raw_data = json.load(f) + idedata = IDEData(raw_data) + print(f"Loaded idedata from: {idedata_path}", file=sys.stderr) + break + except (json.JSONDecodeError, OSError) as e: + print( + f"Warning: Failed to load idedata from {idedata_path}: {e}", + file=sys.stderr, + ) try: analyzer = MemoryAnalyzer(elf_path, idedata=idedata) @@ -156,20 +182,26 @@ def main() -> int: ) parser.add_argument( "--build-dir", - help="Optional build directory for detailed memory analysis", + help="Optional build directory for detailed memory analysis (overrides auto-detection)", ) parser.add_argument( "--output-json", help="Optional path to save detailed analysis JSON", ) + parser.add_argument( + "--output-build-dir", + help="Optional path to write the detected build directory", + ) args = parser.parse_args() # Read compile output from stdin compile_output = sys.stdin.read() - # Extract memory usage - ram_bytes, flash_bytes = extract_from_compile_output(compile_output) + # Extract memory usage and build directory + ram_bytes, flash_bytes, detected_build_dir = extract_from_compile_output( + compile_output + ) if ram_bytes is None or flash_bytes is None: print("Failed to extract memory usage from compile output", file=sys.stderr) @@ -200,11 +232,24 @@ def main() -> int: print(f"Total RAM: {ram_bytes} bytes", file=sys.stderr) print(f"Total Flash: {flash_bytes} bytes", file=sys.stderr) - # Run detailed analysis if build directory provided + # Determine which build directory to use (explicit arg overrides auto-detection) + build_dir = args.build_dir or detected_build_dir + + if detected_build_dir: + print(f"Detected build directory: {detected_build_dir}", file=sys.stderr) + + # Write build directory to file if requested + if args.output_build_dir and build_dir: + build_dir_path = Path(args.output_build_dir) + build_dir_path.parent.mkdir(parents=True, exist_ok=True) + build_dir_path.write_text(build_dir) + print(f"Wrote build directory to {args.output_build_dir}", file=sys.stderr) + + # Run detailed analysis if build directory available detailed_analysis = None - if args.build_dir: - print(f"Running detailed analysis on {args.build_dir}", file=sys.stderr) - detailed_analysis = run_detailed_analysis(args.build_dir) + if build_dir: + print(f"Running detailed analysis on {build_dir}", file=sys.stderr) + detailed_analysis = run_detailed_analysis(build_dir) # Save JSON output if requested if args.output_json: From b0ada914bcf19b86699e61019e1977f5ea9d647d Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Fri, 17 Oct 2025 14:57:45 -1000 Subject: [PATCH 29/76] tweak --- esphome/__main__.py | 4 +++- script/ci_memory_impact_extract.py | 10 ++++------ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/esphome/__main__.py b/esphome/__main__.py index d9bdfb175b..a0b7d16ae9 100644 --- a/esphome/__main__.py +++ b/esphome/__main__.py @@ -466,7 +466,9 @@ def write_cpp_file() -> int: def compile_program(args: ArgsProtocol, config: ConfigType) -> int: from esphome import platformio_api - _LOGGER.info("Compiling app...") + # NOTE: "Build path:" format is parsed by script/ci_memory_impact_extract.py + # If you change this format, update the regex in that script as well + _LOGGER.info("Compiling app... Build path: %s", CORE.build_path) rc = platformio_api.run_compile(config, CORE.verbose) if rc != 0: return rc diff --git a/script/ci_memory_impact_extract.py b/script/ci_memory_impact_extract.py index 9a9c294f2e..97f3750950 100755 --- a/script/ci_memory_impact_extract.py +++ b/script/ci_memory_impact_extract.py @@ -65,13 +65,11 @@ def extract_from_compile_output( total_ram = sum(int(match) for match in ram_matches) total_flash = sum(int(match) for match in flash_matches) - # Extract build directory from ESPHome's delete messages - # Look for: INFO Deleting /path/to/build/.esphome/build/componenttest.../.pioenvs + # Extract build directory from ESPHome's explicit build path output + # Look for: INFO Compiling app... Build path: /path/to/build build_dir = None - if match := re.search( - r"INFO Deleting (.+/\.esphome/build/componenttest[^/]+)/\.pioenvs", output_text - ): - build_dir = match.group(1) + if match := re.search(r"Build path: (.+)", output_text): + build_dir = match.group(1).strip() return total_ram, total_flash, build_dir From e1e047c53fd5a8cf90d16ade711fb81a9360017d Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Fri, 17 Oct 2025 15:02:09 -1000 Subject: [PATCH 30/76] tweak --- .github/workflows/ci.yml | 4 ++ esphome/platformio_api.py | 82 +----------------------------- script/ci_memory_impact_extract.py | 24 ++++----- script/determine-jobs.py | 22 ++++---- 4 files changed, 29 insertions(+), 103 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 440f64298b..0935fe609c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -565,10 +565,12 @@ jobs: echo "Compiling with test_build_components.py..." # Run build and extract memory with auto-detection of build directory for detailed analysis + # Use tee to show output in CI while also piping to extraction script python script/test_build_components.py \ -e compile \ -c "$component_list" \ -t "$platform" 2>&1 | \ + tee /dev/stderr | \ python script/ci_memory_impact_extract.py \ --output-env \ --output-json memory-analysis-target.json @@ -620,10 +622,12 @@ jobs: echo "Compiling with test_build_components.py..." # Run build and extract memory with auto-detection of build directory for detailed analysis + # Use tee to show output in CI while also piping to extraction script python script/test_build_components.py \ -e compile \ -c "$component_list" \ -t "$platform" 2>&1 | \ + tee /dev/stderr | \ python script/ci_memory_impact_extract.py \ --output-env \ --output-json memory-analysis-pr.json diff --git a/esphome/platformio_api.py b/esphome/platformio_api.py index 065a8cf896..cc48562b4c 100644 --- a/esphome/platformio_api.py +++ b/esphome/platformio_api.py @@ -145,16 +145,7 @@ def run_compile(config, verbose): args = [] if CONF_COMPILE_PROCESS_LIMIT in config[CONF_ESPHOME]: args += [f"-j{config[CONF_ESPHOME][CONF_COMPILE_PROCESS_LIMIT]}"] - result = run_platformio_cli_run(config, verbose, *args) - - # Run memory analysis if enabled - if config.get(CONF_ESPHOME, {}).get("analyze_memory", False): - try: - analyze_memory_usage(config) - except Exception as e: - _LOGGER.warning("Failed to analyze memory usage: %s", e) - - return result + return run_platformio_cli_run(config, verbose, *args) def _run_idedata(config): @@ -403,74 +394,3 @@ class IDEData: return f"{self.cc_path[:-7]}readelf.exe" return f"{self.cc_path[:-3]}readelf" - - -def analyze_memory_usage(config: dict[str, Any]) -> None: - """Analyze memory usage by component after compilation.""" - # Lazy import to avoid overhead when not needed - from esphome.analyze_memory import MemoryAnalyzer - - idedata = get_idedata(config) - - # Get ELF path - elf_path = idedata.firmware_elf_path - - # Debug logging - _LOGGER.debug("ELF path from idedata: %s", elf_path) - - # Check if file exists - if not Path(elf_path).exists(): - # Try alternate path - alt_path = Path(CORE.relative_build_path(".pioenvs", CORE.name, "firmware.elf")) - if alt_path.exists(): - elf_path = str(alt_path) - _LOGGER.debug("Using alternate ELF path: %s", elf_path) - else: - _LOGGER.warning("ELF file not found at %s or %s", elf_path, alt_path) - return - - # Extract external components from config - external_components = set() - - # Get the list of built-in ESPHome components - from esphome.analyze_memory import get_esphome_components - - builtin_components = get_esphome_components() - - # Special non-component keys that appear in configs - NON_COMPONENT_KEYS = { - CONF_ESPHOME, - "substitutions", - "packages", - "globals", - "<<", - } - - # Check all top-level keys in config - for key in config: - if key not in builtin_components and key not in NON_COMPONENT_KEYS: - # This is an external component - external_components.add(key) - - _LOGGER.debug("Detected external components: %s", external_components) - - # Create analyzer and run analysis - # Pass idedata to auto-detect toolchain paths - analyzer = MemoryAnalyzer( - elf_path, external_components=external_components, idedata=idedata - ) - analyzer.analyze() - - # Generate and print report - report = analyzer.generate_report() - _LOGGER.info("\n%s", report) - - # Optionally save to file - if config.get(CONF_ESPHOME, {}).get("memory_report_file"): - report_file = Path(config[CONF_ESPHOME]["memory_report_file"]) - if report_file.suffix == ".json": - report_file.write_text(analyzer.to_json()) - _LOGGER.info("Memory report saved to %s", report_file) - else: - report_file.write_text(report) - _LOGGER.info("Memory report saved to %s", report_file) diff --git a/script/ci_memory_impact_extract.py b/script/ci_memory_impact_extract.py index 97f3750950..7b722fcfd4 100755 --- a/script/ci_memory_impact_extract.py +++ b/script/ci_memory_impact_extract.py @@ -137,21 +137,21 @@ def run_detailed_analysis(build_dir: str) -> dict | None: # Convert to JSON-serializable format result = { - "components": {}, + "components": { + name: { + "text": mem.text_size, + "rodata": mem.rodata_size, + "data": mem.data_size, + "bss": mem.bss_size, + "flash_total": mem.flash_total, + "ram_total": mem.ram_total, + "symbol_count": mem.symbol_count, + } + for name, mem in components.items() + }, "symbols": {}, } - for name, mem in components.items(): - result["components"][name] = { - "text": mem.text_size, - "rodata": mem.rodata_size, - "data": mem.data_size, - "bss": mem.bss_size, - "flash_total": mem.flash_total, - "ram_total": mem.ram_total, - "symbol_count": mem.symbol_count, - } - # Build symbol map for section in analyzer.sections.values(): for symbol_name, size, _ in section.symbols: diff --git a/script/determine-jobs.py b/script/determine-jobs.py index 56de0e77ba..bd21926c53 100755 --- a/script/determine-jobs.py +++ b/script/determine-jobs.py @@ -303,16 +303,18 @@ def detect_memory_impact_config( # Check if component has tests for any preferred platform for test_file in test_files: parts = test_file.stem.split(".") - if len(parts) >= 2: - platform = parts[1] - if platform in PLATFORM_PREFERENCE: - components_with_tests.append(component) - # Select the most preferred platform across all components - if selected_platform is None or PLATFORM_PREFERENCE.index( - platform - ) < PLATFORM_PREFERENCE.index(selected_platform): - selected_platform = platform - break + if len(parts) < 2: + continue + platform = parts[1] + if platform not in PLATFORM_PREFERENCE: + continue + components_with_tests.append(component) + # Select the most preferred platform across all components + if selected_platform is None or PLATFORM_PREFERENCE.index( + platform + ) < PLATFORM_PREFERENCE.index(selected_platform): + selected_platform = platform + break # If no components have tests, don't run memory impact if not components_with_tests: From 84316d62f9478ecb022fff58cdb6fd1cb16c6d55 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Fri, 17 Oct 2025 15:04:19 -1000 Subject: [PATCH 31/76] tweak --- esphome/analyze_memory/__init__.py | 160 +++++++++-------------------- 1 file changed, 48 insertions(+), 112 deletions(-) diff --git a/esphome/analyze_memory/__init__.py b/esphome/analyze_memory/__init__.py index 5bd46fd01e..f2a2628ad8 100644 --- a/esphome/analyze_memory/__init__.py +++ b/esphome/analyze_memory/__init__.py @@ -29,59 +29,6 @@ if TYPE_CHECKING: _LOGGER = logging.getLogger(__name__) -def get_toolchain_for_platform(platform: str) -> tuple[str | None, str | None]: - """Get objdump and readelf paths for a given platform. - - This function auto-detects the correct toolchain based on the platform name, - using the same detection logic as PlatformIO's IDEData class. - - Args: - platform: Platform name (e.g., "esp8266-ard", "esp32-idf", "esp32-c3-idf") - - Returns: - Tuple of (objdump_path, readelf_path) or (None, None) if not found/supported - """ - home = Path.home() - platformio_packages = home / ".platformio" / "packages" - - # Map platform to toolchain and prefix (same logic as PlatformIO uses) - toolchain = None - prefix = None - - if "esp8266" in platform: - toolchain = "toolchain-xtensa" - prefix = "xtensa-lx106-elf" - elif "esp32-c" in platform or "esp32-h" in platform or "esp32-p4" in platform: - # RISC-V variants (C2, C3, C5, C6, H2, P4) - toolchain = "toolchain-riscv32-esp" - prefix = "riscv32-esp-elf" - elif "esp32" in platform: - # Xtensa variants (original, S2, S3) - toolchain = "toolchain-xtensa-esp-elf" - if "s2" in platform: - prefix = "xtensa-esp32s2-elf" - elif "s3" in platform: - prefix = "xtensa-esp32s3-elf" - else: - prefix = "xtensa-esp32-elf" - else: - # Other platforms (RP2040, LibreTiny, etc.) - not supported for ELF analysis - _LOGGER.debug("Platform %s not supported for ELF analysis", platform) - return None, None - - # Construct paths (same pattern as IDEData.objdump_path/readelf_path) - toolchain_path = platformio_packages / toolchain / "bin" - objdump_path = toolchain_path / f"{prefix}-objdump" - readelf_path = toolchain_path / f"{prefix}-readelf" - - if objdump_path.exists() and readelf_path.exists(): - _LOGGER.debug("Found %s toolchain: %s", platform, prefix) - return str(objdump_path), str(readelf_path) - - _LOGGER.warning("Toolchain not found at %s", toolchain_path) - return None, None - - @dataclass class MemorySection: """Represents a memory section with its symbols.""" @@ -171,71 +118,61 @@ class MemoryAnalyzer: def _parse_sections(self) -> None: """Parse section headers from ELF file.""" - try: - result = subprocess.run( - [self.readelf_path, "-S", str(self.elf_path)], - capture_output=True, - text=True, - check=True, - ) + result = subprocess.run( + [self.readelf_path, "-S", str(self.elf_path)], + capture_output=True, + text=True, + check=True, + ) - # Parse section headers - for line in result.stdout.splitlines(): - # Look for section entries - if not ( - match := re.match( - r"\s*\[\s*\d+\]\s+([\.\w]+)\s+\w+\s+[\da-fA-F]+\s+[\da-fA-F]+\s+([\da-fA-F]+)", - line, - ) - ): - continue + # Parse section headers + for line in result.stdout.splitlines(): + # Look for section entries + if not ( + match := re.match( + r"\s*\[\s*\d+\]\s+([\.\w]+)\s+\w+\s+[\da-fA-F]+\s+[\da-fA-F]+\s+([\da-fA-F]+)", + line, + ) + ): + continue - section_name = match.group(1) - size_hex = match.group(2) - size = int(size_hex, 16) + section_name = match.group(1) + size_hex = match.group(2) + size = int(size_hex, 16) - # Map to standard section name - mapped_section = map_section_name(section_name) - if not mapped_section: - continue + # Map to standard section name + mapped_section = map_section_name(section_name) + if not mapped_section: + continue - if mapped_section not in self.sections: - self.sections[mapped_section] = MemorySection(mapped_section) - self.sections[mapped_section].total_size += size - - except subprocess.CalledProcessError as e: - _LOGGER.error("Failed to parse sections: %s", e) - raise + if mapped_section not in self.sections: + self.sections[mapped_section] = MemorySection(mapped_section) + self.sections[mapped_section].total_size += size def _parse_symbols(self) -> None: """Parse symbols from ELF file.""" - try: - result = subprocess.run( - [self.objdump_path, "-t", str(self.elf_path)], - capture_output=True, - text=True, - check=True, - ) + result = subprocess.run( + [self.objdump_path, "-t", str(self.elf_path)], + capture_output=True, + text=True, + check=True, + ) - # Track seen addresses to avoid duplicates - seen_addresses: set[str] = set() + # Track seen addresses to avoid duplicates + seen_addresses: set[str] = set() - for line in result.stdout.splitlines(): - if not (symbol_info := parse_symbol_line(line)): - continue + for line in result.stdout.splitlines(): + if not (symbol_info := parse_symbol_line(line)): + continue - section, name, size, address = symbol_info + section, name, size, address = symbol_info - # Skip duplicate symbols at the same address (e.g., C1/C2 constructors) - if address in seen_addresses or section not in self.sections: - continue + # Skip duplicate symbols at the same address (e.g., C1/C2 constructors) + if address in seen_addresses or section not in self.sections: + continue - self.sections[section].symbols.append((name, size, "")) - seen_addresses.add(address) - - except subprocess.CalledProcessError as e: - _LOGGER.error("Failed to parse symbols: %s", e) - raise + self.sections[section].symbols.append((name, size, "")) + seen_addresses.add(address) def _categorize_symbols(self) -> None: """Categorize symbols by component.""" @@ -373,15 +310,14 @@ class MemoryAnalyzer: # Map original to demangled names for original, demangled in zip(symbols, demangled_lines): self._demangle_cache[original] = demangled - else: - # If batch fails, cache originals - for symbol in symbols: - self._demangle_cache[symbol] = symbol + return except (subprocess.SubprocessError, OSError, UnicodeDecodeError) as e: # On error, cache originals _LOGGER.debug("Failed to batch demangle symbols: %s", e) - for symbol in symbols: - self._demangle_cache[symbol] = symbol + + # If demangling failed, cache originals + for symbol in symbols: + self._demangle_cache[symbol] = symbol def _demangle_symbol(self, symbol: str) -> str: """Get demangled C++ symbol name from cache.""" From 95a0c9594f3f94bdcb57ac173f6b6ffb49ed8d2b Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Fri, 17 Oct 2025 15:12:36 -1000 Subject: [PATCH 32/76] tweak --- script/ci_memory_impact_comment.py | 6 +++--- script/ci_memory_impact_extract.py | 10 ++++++++++ 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/script/ci_memory_impact_comment.py b/script/ci_memory_impact_comment.py index c5eb9e701f..140bd2f08e 100755 --- a/script/ci_memory_impact_comment.py +++ b/script/ci_memory_impact_comment.py @@ -238,7 +238,7 @@ def create_detailed_breakdown_table( # Combine all components from both analyses all_components = set(target_analysis.keys()) | set(pr_analysis.keys()) - # Filter to components that have changed or are significant + # Filter to components that have changed changed_components = [] for comp in all_components: target_mem = target_analysis.get(comp, {}) @@ -247,8 +247,8 @@ def create_detailed_breakdown_table( target_flash = target_mem.get("flash_total", 0) pr_flash = pr_mem.get("flash_total", 0) - # Include if component has changed or is significant (> 1KB) - if target_flash != pr_flash or target_flash > 1024 or pr_flash > 1024: + # Only include if component has changed + if target_flash != pr_flash: delta = pr_flash - target_flash changed_components.append((comp, target_flash, pr_flash, delta)) diff --git a/script/ci_memory_impact_extract.py b/script/ci_memory_impact_extract.py index 7b722fcfd4..96f947e12a 100755 --- a/script/ci_memory_impact_extract.py +++ b/script/ci_memory_impact_extract.py @@ -67,6 +67,7 @@ def extract_from_compile_output( # Extract build directory from ESPHome's explicit build path output # Look for: INFO Compiling app... Build path: /path/to/build + # Note: Multiple builds reuse the same build path (each overwrites the previous) build_dir = None if match := re.search(r"Build path: (.+)", output_text): build_dir = match.group(1).strip() @@ -226,6 +227,10 @@ def main() -> int: f"Found {num_builds} builds - summing memory usage across all builds", file=sys.stderr, ) + print( + "WARNING: Detailed analysis will only cover the last build", + file=sys.stderr, + ) print(f"Total RAM: {ram_bytes} bytes", file=sys.stderr) print(f"Total Flash: {flash_bytes} bytes", file=sys.stderr) @@ -235,6 +240,11 @@ def main() -> int: if detected_build_dir: print(f"Detected build directory: {detected_build_dir}", file=sys.stderr) + if num_builds > 1: + print( + f" (using last of {num_builds} builds for detailed analysis)", + file=sys.stderr, + ) # Write build directory to file if requested if args.output_build_dir and build_dir: From a9e5e4d6d223785117a4facee0ad73f8d9118b52 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Fri, 17 Oct 2025 15:14:00 -1000 Subject: [PATCH 33/76] tweak --- script/ci_memory_impact_comment.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/script/ci_memory_impact_comment.py b/script/ci_memory_impact_comment.py index 140bd2f08e..2b747629d5 100755 --- a/script/ci_memory_impact_comment.py +++ b/script/ci_memory_impact_comment.py @@ -157,8 +157,14 @@ def create_symbol_changes_table( target_str = format_bytes(target_size) pr_str = format_bytes(pr_size) change_str = format_change(target_size, pr_size) - # Truncate very long symbol names - display_symbol = symbol if len(symbol) <= 80 else symbol[:77] + "..." + # Truncate very long symbol names but show full name in title attribute + if len(symbol) <= 100: + display_symbol = symbol + else: + # Use HTML details for very long symbols + display_symbol = ( + f"
{symbol[:97]}...{symbol}
" + ) lines.append( f"| `{display_symbol}` | {target_str} | {pr_str} | {change_str} |" ) @@ -261,8 +267,8 @@ def create_detailed_breakdown_table( # Build table - limit to top 20 changes lines = [ "", - "
", - "📊 Component Memory Breakdown (click to expand)", + "
", + "📊 Component Memory Breakdown", "", "| Component | Target Flash | PR Flash | Change |", "|-----------|--------------|----------|--------|", From 62ce39e4307d8c7b085012e4e72b4586dac8ddf9 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Fri, 17 Oct 2025 15:17:15 -1000 Subject: [PATCH 34/76] fix --- esphome/analyze_memory/__init__.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/esphome/analyze_memory/__init__.py b/esphome/analyze_memory/__init__.py index f2a2628ad8..11e5933911 100644 --- a/esphome/analyze_memory/__init__.py +++ b/esphome/analyze_memory/__init__.py @@ -295,6 +295,14 @@ class MemoryAnalyzer: potential_cppfilt = self.objdump_path.replace("objdump", "c++filt") if Path(potential_cppfilt).exists(): cppfilt_cmd = potential_cppfilt + _LOGGER.warning("Using toolchain c++filt: %s", cppfilt_cmd) + else: + _LOGGER.warning( + "Toolchain c++filt not found at %s, using system c++filt", + potential_cppfilt, + ) + else: + _LOGGER.warning("Using system c++filt (objdump_path=%s)", self.objdump_path) try: # Send all symbols to c++filt at once @@ -310,6 +318,9 @@ class MemoryAnalyzer: # Map original to demangled names for original, demangled in zip(symbols, demangled_lines): self._demangle_cache[original] = demangled + # Log symbols that failed to demangle (stayed the same) + if original == demangled and original.startswith("_Z"): + _LOGGER.debug("Failed to demangle symbol: %s", original[:100]) return except (subprocess.SubprocessError, OSError, UnicodeDecodeError) as e: # On error, cache originals From daa03e5b3c70b39029f01589e1a7b996561d1513 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Fri, 17 Oct 2025 15:17:28 -1000 Subject: [PATCH 35/76] fix --- esphome/analyze_memory/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/esphome/analyze_memory/__init__.py b/esphome/analyze_memory/__init__.py index 11e5933911..af1aee66c8 100644 --- a/esphome/analyze_memory/__init__.py +++ b/esphome/analyze_memory/__init__.py @@ -320,7 +320,7 @@ class MemoryAnalyzer: self._demangle_cache[original] = demangled # Log symbols that failed to demangle (stayed the same) if original == demangled and original.startswith("_Z"): - _LOGGER.debug("Failed to demangle symbol: %s", original[:100]) + _LOGGER.debug("Failed to demangle symbol: %s", original) return except (subprocess.SubprocessError, OSError, UnicodeDecodeError) as e: # On error, cache originals From 3bc0041b948d6e97d1491c3917412e658821985a Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Fri, 17 Oct 2025 15:22:06 -1000 Subject: [PATCH 36/76] fix --- script/test_build_components.py | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/script/test_build_components.py b/script/test_build_components.py index df092c091d..07f2680799 100755 --- a/script/test_build_components.py +++ b/script/test_build_components.py @@ -82,13 +82,14 @@ def show_disk_space_if_ci(esphome_command: str) -> None: def find_component_tests( - components_dir: Path, component_pattern: str = "*" + components_dir: Path, component_pattern: str = "*", base_only: bool = False ) -> dict[str, list[Path]]: """Find all component test files. Args: components_dir: Path to tests/components directory component_pattern: Glob pattern for component names + base_only: If True, only find base test files (test.*.yaml), not variant files (test-*.yaml) Returns: Dictionary mapping component name to list of test files @@ -99,8 +100,9 @@ def find_component_tests( if not comp_dir.is_dir(): continue - # Find test files matching test.*.yaml or test-*.yaml patterns - for test_file in comp_dir.glob("test[.-]*.yaml"): + # Find test files - either base only (test.*.yaml) or all (test[.-]*.yaml) + pattern = "test.*.yaml" if base_only else "test[.-]*.yaml" + for test_file in comp_dir.glob(pattern): component_tests[comp_dir.name].append(test_file) return dict(component_tests) @@ -931,6 +933,7 @@ def test_components( continue_on_fail: bool, enable_grouping: bool = True, isolated_components: set[str] | None = None, + base_only: bool = False, ) -> int: """Test components with optional intelligent grouping. @@ -944,6 +947,7 @@ def test_components( These are tested WITHOUT --testing-mode to enable full validation (pin conflicts, etc). This is used in CI for directly changed components to catch issues that would be missed with --testing-mode. + base_only: If True, only test base test files (test.*.yaml), not variant files (test-*.yaml) Returns: Exit code (0 for success, 1 for failure) @@ -961,7 +965,7 @@ def test_components( # Find all component tests all_tests = {} for pattern in component_patterns: - all_tests.update(find_component_tests(tests_dir, pattern)) + all_tests.update(find_component_tests(tests_dir, pattern, base_only)) if not all_tests: print(f"No components found matching: {component_patterns}") @@ -1122,6 +1126,11 @@ def main() -> int: "These are tested WITHOUT --testing-mode to enable full validation. " "Used in CI for directly changed components to catch pin conflicts and other issues.", ) + parser.add_argument( + "--base-only", + action="store_true", + help="Only test base test files (test.*.yaml), not variant files (test-*.yaml)", + ) args = parser.parse_args() @@ -1140,6 +1149,7 @@ def main() -> int: continue_on_fail=args.continue_on_fail, enable_grouping=not args.no_grouping, isolated_components=isolated_components, + base_only=args.base_only, ) From 5e9b97283188df8377ec0cf692a397329b950e31 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Fri, 17 Oct 2025 15:24:49 -1000 Subject: [PATCH 37/76] fix --- .github/workflows/ci.yml | 94 +++++++++++++++++++++++++++++- script/ci_memory_impact_comment.py | 16 ++++- 2 files changed, 106 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0935fe609c..74ba831bc4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -533,23 +533,79 @@ jobs: outputs: ram_usage: ${{ steps.extract.outputs.ram_usage }} flash_usage: ${{ steps.extract.outputs.flash_usage }} + cache_hit: ${{ steps.cache-memory-analysis.outputs.cache-hit }} steps: - name: Check out target branch uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 with: ref: ${{ github.base_ref }} + + # Create cache key based on: + # 1. Target branch commit SHA + # 2. Hash of build infrastructure files (scripts and CI workflow) + # 3. Platform being tested + # 4. Component list + - name: Generate cache key + id: cache-key + run: | + # Get the commit SHA of the target branch + target_sha=$(git rev-parse HEAD) + + # Hash the build infrastructure files (all files that affect build/analysis) + infra_hash=$(cat \ + script/test_build_components.py \ + script/ci_memory_impact_extract.py \ + script/analyze_component_buses.py \ + script/merge_component_configs.py \ + script/ci_helpers.py \ + .github/workflows/ci.yml \ + | sha256sum | cut -d' ' -f1) + + # Get platform and components from job inputs + platform="${{ fromJSON(needs.determine-jobs.outputs.memory_impact).platform }}" + components='${{ toJSON(fromJSON(needs.determine-jobs.outputs.memory_impact).components) }}' + components_hash=$(echo "$components" | sha256sum | cut -d' ' -f1) + + # Combine into cache key + cache_key="memory-analysis-target-${target_sha}-${infra_hash}-${platform}-${components_hash}" + echo "cache-key=${cache_key}" >> $GITHUB_OUTPUT + echo "Cache key: ${cache_key}" + + # Try to restore cached analysis results + - name: Restore cached memory analysis + id: cache-memory-analysis + uses: actions/cache/restore@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 + with: + path: memory-analysis-target.json + key: ${{ steps.cache-key.outputs.cache-key }} + + - name: Cache status + run: | + if [ "${{ steps.cache-memory-analysis.outputs.cache-hit }}" == "true" ]; then + echo "✓ Cache hit! Using cached memory analysis results." + echo " Skipping build step to save time." + else + echo "✗ Cache miss. Will build and analyze memory usage." + fi + + # Only restore Python and build if cache miss - name: Restore Python + if: steps.cache-memory-analysis.outputs.cache-hit != 'true' uses: ./.github/actions/restore-python with: python-version: ${{ env.DEFAULT_PYTHON }} cache-key: ${{ needs.common.outputs.cache-key }} + - name: Cache platformio + if: steps.cache-memory-analysis.outputs.cache-hit != 'true' uses: actions/cache/restore@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 with: path: ~/.platformio key: platformio-memory-${{ fromJSON(needs.determine-jobs.outputs.memory_impact).platform }}-${{ hashFiles('platformio.ini') }} + - name: Build, compile, and analyze memory - id: extract + if: steps.cache-memory-analysis.outputs.cache-hit != 'true' + id: build run: | . venv/bin/activate components='${{ toJSON(fromJSON(needs.determine-jobs.outputs.memory_impact).components) }}' @@ -574,12 +630,36 @@ jobs: python script/ci_memory_impact_extract.py \ --output-env \ --output-json memory-analysis-target.json + + # Save build results to cache for future runs + - name: Save memory analysis to cache + if: steps.cache-memory-analysis.outputs.cache-hit != 'true' && steps.build.outcome == 'success' + uses: actions/cache/save@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 + with: + path: memory-analysis-target.json + key: ${{ steps.cache-key.outputs.cache-key }} + + # Extract outputs from cached or freshly built analysis + - name: Extract memory usage for outputs + id: extract + run: | + if [ -f memory-analysis-target.json ]; then + ram=$(jq -r '.ram_bytes' memory-analysis-target.json) + flash=$(jq -r '.flash_bytes' memory-analysis-target.json) + echo "ram_usage=${ram}" >> $GITHUB_OUTPUT + echo "flash_usage=${flash}" >> $GITHUB_OUTPUT + echo "RAM: ${ram} bytes, Flash: ${flash} bytes" + else + echo "Error: memory-analysis-target.json not found" + exit 1 + fi + - name: Upload memory analysis JSON uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 with: name: memory-analysis-target path: memory-analysis-target.json - if-no-files-found: warn + if-no-files-found: error retention-days: 1 memory-impact-pr-branch: @@ -680,6 +760,7 @@ jobs: TARGET_FLASH: ${{ needs.memory-impact-target-branch.outputs.flash_usage }} PR_RAM: ${{ needs.memory-impact-pr-branch.outputs.ram_usage }} PR_FLASH: ${{ needs.memory-impact-pr-branch.outputs.flash_usage }} + TARGET_CACHE_HIT: ${{ needs.memory-impact-target-branch.outputs.cache_hit }} run: | . venv/bin/activate @@ -701,6 +782,12 @@ jobs: echo "No PR analysis JSON found" fi + # Add cache flag if target was cached + cache_flag="" + if [ "$TARGET_CACHE_HIT" == "true" ]; then + cache_flag="--target-cache-hit" + fi + python script/ci_memory_impact_comment.py \ --pr-number "${{ github.event.pull_request.number }}" \ --components "$COMPONENTS" \ @@ -710,7 +797,8 @@ jobs: --pr-ram "$PR_RAM" \ --pr-flash "$PR_FLASH" \ $target_json_arg \ - $pr_json_arg + $pr_json_arg \ + $cache_flag ci-status: name: CI Status diff --git a/script/ci_memory_impact_comment.py b/script/ci_memory_impact_comment.py index 2b747629d5..055c2a9a96 100755 --- a/script/ci_memory_impact_comment.py +++ b/script/ci_memory_impact_comment.py @@ -301,6 +301,7 @@ def create_comment_body( pr_analysis: dict | None = None, target_symbols: dict | None = None, pr_symbols: dict | None = None, + target_cache_hit: bool = False, ) -> str: """Create the comment body with memory impact analysis. @@ -315,6 +316,7 @@ def create_comment_body( pr_analysis: Optional component breakdown for PR branch target_symbols: Optional symbol map for target branch pr_symbols: Optional symbol map for PR branch + target_cache_hit: Whether target branch analysis was loaded from cache Returns: Formatted comment body @@ -344,6 +346,11 @@ def create_comment_body( components_str = ", ".join(f"`{c}`" for c in sorted(components)) config_note = f"a merged configuration with {len(components)} components" + # Add cache info note if target was cached + cache_note = "" + if target_cache_hit: + cache_note = "\n\n> ⚡ Target branch analysis was loaded from cache (build skipped for faster CI)." + return f"""{COMMENT_MARKER} ## Memory Impact Analysis @@ -354,7 +361,8 @@ def create_comment_body( |--------|--------------|---------|--------| | **RAM** | {format_bytes(target_ram)} | {format_bytes(pr_ram)} | {ram_change} | | **Flash** | {format_bytes(target_flash)} | {format_bytes(pr_flash)} | {flash_change} | -{component_breakdown}{symbol_changes} +{component_breakdown}{symbol_changes}{cache_note} + --- *This analysis runs automatically when components change. Memory usage is measured from {config_note}.* """ @@ -531,6 +539,11 @@ def main() -> int: "--pr-json", help="Optional path to PR branch analysis JSON (for detailed analysis)", ) + parser.add_argument( + "--target-cache-hit", + action="store_true", + help="Indicates that target branch analysis was loaded from cache", + ) args = parser.parse_args() @@ -575,6 +588,7 @@ def main() -> int: pr_analysis=pr_analysis, target_symbols=target_symbols, pr_symbols=pr_symbols, + target_cache_hit=args.target_cache_hit, ) # Post or update comment From 922c2bcd5aa87b024b4a32671bc14596aecd63d8 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Fri, 17 Oct 2025 15:26:55 -1000 Subject: [PATCH 38/76] fix --- esphome/analyze_memory/__init__.py | 36 +++++++++++++++++++++++++----- script/ci_memory_impact_comment.py | 10 ++++----- 2 files changed, 35 insertions(+), 11 deletions(-) diff --git a/esphome/analyze_memory/__init__.py b/esphome/analyze_memory/__init__.py index af1aee66c8..cb8fb94c14 100644 --- a/esphome/analyze_memory/__init__.py +++ b/esphome/analyze_memory/__init__.py @@ -289,20 +289,26 @@ class MemoryAnalyzer: # Try to find the appropriate c++filt for the platform cppfilt_cmd = "c++filt" + _LOGGER.warning("Demangling %d symbols", len(symbols)) + _LOGGER.warning("objdump_path = %s", self.objdump_path) + # Check if we have a toolchain-specific c++filt if self.objdump_path and self.objdump_path != "objdump": # Replace objdump with c++filt in the path potential_cppfilt = self.objdump_path.replace("objdump", "c++filt") + _LOGGER.warning("Checking for toolchain c++filt at: %s", potential_cppfilt) if Path(potential_cppfilt).exists(): cppfilt_cmd = potential_cppfilt - _LOGGER.warning("Using toolchain c++filt: %s", cppfilt_cmd) + _LOGGER.warning("✓ Using toolchain c++filt: %s", cppfilt_cmd) else: _LOGGER.warning( - "Toolchain c++filt not found at %s, using system c++filt", + "✗ Toolchain c++filt not found at %s, using system c++filt", potential_cppfilt, ) else: - _LOGGER.warning("Using system c++filt (objdump_path=%s)", self.objdump_path) + _LOGGER.warning( + "✗ Using system c++filt (objdump_path=%s)", self.objdump_path + ) try: # Send all symbols to c++filt at once @@ -316,15 +322,35 @@ class MemoryAnalyzer: if result.returncode == 0: demangled_lines = result.stdout.strip().split("\n") # Map original to demangled names + failed_count = 0 for original, demangled in zip(symbols, demangled_lines): self._demangle_cache[original] = demangled # Log symbols that failed to demangle (stayed the same) if original == demangled and original.startswith("_Z"): - _LOGGER.debug("Failed to demangle symbol: %s", original) + failed_count += 1 + if failed_count <= 5: # Only log first 5 failures + _LOGGER.warning("Failed to demangle: %s", original[:100]) + + if failed_count > 0: + _LOGGER.warning( + "Failed to demangle %d/%d symbols using %s", + failed_count, + len(symbols), + cppfilt_cmd, + ) + else: + _LOGGER.warning( + "Successfully demangled all %d symbols", len(symbols) + ) return + _LOGGER.warning( + "c++filt exited with code %d: %s", + result.returncode, + result.stderr[:200] if result.stderr else "(no error output)", + ) except (subprocess.SubprocessError, OSError, UnicodeDecodeError) as e: # On error, cache originals - _LOGGER.debug("Failed to batch demangle symbols: %s", e) + _LOGGER.warning("Failed to batch demangle symbols: %s", e) # If demangling failed, cache originals for symbol in symbols: diff --git a/script/ci_memory_impact_comment.py b/script/ci_memory_impact_comment.py index 055c2a9a96..84e821cbec 100755 --- a/script/ci_memory_impact_comment.py +++ b/script/ci_memory_impact_comment.py @@ -159,14 +159,12 @@ def create_symbol_changes_table( change_str = format_change(target_size, pr_size) # Truncate very long symbol names but show full name in title attribute if len(symbol) <= 100: - display_symbol = symbol + display_symbol = f"`{symbol}`" else: - # Use HTML details for very long symbols - display_symbol = ( - f"
{symbol[:97]}...{symbol}
" - ) + # Use HTML details for very long symbols (no backticks inside HTML) + display_symbol = f"
{symbol[:97]}...{symbol}
" lines.append( - f"| `{display_symbol}` | {target_str} | {pr_str} | {change_str} |" + f"| {display_symbol} | {target_str} | {pr_str} | {change_str} |" ) if len(changed_symbols) > 30: From 57bf3f968ff417ee57b15aaab316ac1256521d30 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Fri, 17 Oct 2025 15:34:17 -1000 Subject: [PATCH 39/76] fix --- script/determine-jobs.py | 37 +++++++++++++++++++++++++++---------- 1 file changed, 27 insertions(+), 10 deletions(-) diff --git a/script/determine-jobs.py b/script/determine-jobs.py index bd21926c53..6a24c9eb01 100755 --- a/script/determine-jobs.py +++ b/script/determine-jobs.py @@ -258,9 +258,10 @@ def detect_memory_impact_config( """ # Platform preference order for memory impact analysis # Prefer ESP8266 for memory impact as it's the most constrained platform + # ESP32-IDF is preferred over ESP32-Arduino as it's faster to build and more commonly used PLATFORM_PREFERENCE = [ "esp8266-ard", # ESP8266 Arduino (most memory constrained - best for impact analysis) - "esp32-idf", # Primary ESP32 IDF platform + "esp32-idf", # ESP32 IDF platform (primary ESP32 platform, faster builds) "esp32-c3-idf", # ESP32-C3 IDF "esp32-c6-idf", # ESP32-C6 IDF "esp32-s2-idf", # ESP32-S2 IDF @@ -289,6 +290,7 @@ def detect_memory_impact_config( # Find components that have tests on the preferred platform components_with_tests = [] selected_platform = None + component_platforms = {} # Track which platforms each component has for component in sorted(changed_component_set): tests_dir = Path(root_path) / "tests" / "components" / component @@ -301,20 +303,28 @@ def detect_memory_impact_config( continue # Check if component has tests for any preferred platform + available_platforms = [] for test_file in test_files: parts = test_file.stem.split(".") if len(parts) < 2: continue platform = parts[1] - if platform not in PLATFORM_PREFERENCE: - continue - components_with_tests.append(component) - # Select the most preferred platform across all components - if selected_platform is None or PLATFORM_PREFERENCE.index( - platform - ) < PLATFORM_PREFERENCE.index(selected_platform): - selected_platform = platform - break + if platform in PLATFORM_PREFERENCE: + available_platforms.append(platform) + + if not available_platforms: + continue + + # Find the most preferred platform for this component + component_platform = min(available_platforms, key=PLATFORM_PREFERENCE.index) + component_platforms[component] = component_platform + components_with_tests.append(component) + + # Select the most preferred platform across all components + if selected_platform is None or PLATFORM_PREFERENCE.index( + component_platform + ) < PLATFORM_PREFERENCE.index(selected_platform): + selected_platform = component_platform # If no components have tests, don't run memory impact if not components_with_tests: @@ -323,6 +333,13 @@ def detect_memory_impact_config( # Use the most preferred platform found, or fall back to esp8266-ard platform = selected_platform or "esp8266-ard" + # Debug output + print("Memory impact analysis:", file=sys.stderr) + print(f" Changed components: {sorted(changed_component_set)}", file=sys.stderr) + print(f" Components with tests: {components_with_tests}", file=sys.stderr) + print(f" Component platforms: {component_platforms}", file=sys.stderr) + print(f" Selected platform: {platform}", file=sys.stderr) + return { "should_run": "true", "components": components_with_tests, From 293400ee1474ca410113f3aef7c40d539c37e4b8 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Fri, 17 Oct 2025 15:35:51 -1000 Subject: [PATCH 40/76] fix --- esphome/analyze_memory/__init__.py | 28 ++++++++++++++++++++++++---- 1 file changed, 24 insertions(+), 4 deletions(-) diff --git a/esphome/analyze_memory/__init__.py b/esphome/analyze_memory/__init__.py index cb8fb94c14..349c3da507 100644 --- a/esphome/analyze_memory/__init__.py +++ b/esphome/analyze_memory/__init__.py @@ -310,11 +310,19 @@ class MemoryAnalyzer: "✗ Using system c++filt (objdump_path=%s)", self.objdump_path ) + # Strip GCC optimization suffixes before demangling + # Suffixes like $isra$0, $part$0, $constprop$0 confuse c++filt + symbols_stripped = [] + for symbol in symbols: + # Remove GCC optimization markers + stripped = re.sub(r"\$(?:isra|part|constprop)\$\d+", "", symbol) + symbols_stripped.append(stripped) + try: # Send all symbols to c++filt at once result = subprocess.run( [cppfilt_cmd], - input="\n".join(symbols), + input="\n".join(symbols_stripped), capture_output=True, text=True, check=False, @@ -323,10 +331,22 @@ class MemoryAnalyzer: demangled_lines = result.stdout.strip().split("\n") # Map original to demangled names failed_count = 0 - for original, demangled in zip(symbols, demangled_lines): + for original, stripped, demangled in zip( + symbols, symbols_stripped, demangled_lines + ): + # If we stripped a suffix, add it back to the demangled name for clarity + if original != stripped: + # Find what was stripped + suffix_match = re.search( + r"(\$(?:isra|part|constprop)\$\d+)", original + ) + if suffix_match: + demangled = f"{demangled} [{suffix_match.group(1)}]" + self._demangle_cache[original] = demangled - # Log symbols that failed to demangle (stayed the same) - if original == demangled and original.startswith("_Z"): + + # Log symbols that failed to demangle (stayed the same as stripped version) + if stripped == demangled and stripped.startswith("_Z"): failed_count += 1 if failed_count <= 5: # Only log first 5 failures _LOGGER.warning("Failed to demangle: %s", original[:100]) From db69ce24ae1b53b6077d1cbddd105b3cba5fe9f6 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Fri, 17 Oct 2025 15:41:20 -1000 Subject: [PATCH 41/76] fix --- script/ci_memory_impact_comment.py | 42 ++++++++++++++++++++---------- 1 file changed, 28 insertions(+), 14 deletions(-) diff --git a/script/ci_memory_impact_comment.py b/script/ci_memory_impact_comment.py index 84e821cbec..60676949e8 100755 --- a/script/ci_memory_impact_comment.py +++ b/script/ci_memory_impact_comment.py @@ -92,6 +92,21 @@ def format_change(before: int, after: int) -> str: return f"{emoji} {delta_str} ({pct_str})" +def format_symbol_for_display(symbol: str) -> str: + """Format a symbol name for display in markdown table. + + Args: + symbol: Symbol name to format + + Returns: + Formatted symbol with backticks or HTML details tag for long names + """ + if len(symbol) <= 100: + return f"`{symbol}`" + # Use HTML details for very long symbols (no backticks inside HTML) + return f"
{symbol[:97]}...{symbol}
" + + def create_symbol_changes_table( target_symbols: dict | None, pr_symbols: dict | None ) -> str: @@ -157,12 +172,7 @@ def create_symbol_changes_table( target_str = format_bytes(target_size) pr_str = format_bytes(pr_size) change_str = format_change(target_size, pr_size) - # Truncate very long symbol names but show full name in title attribute - if len(symbol) <= 100: - display_symbol = f"`{symbol}`" - else: - # Use HTML details for very long symbols (no backticks inside HTML) - display_symbol = f"
{symbol[:97]}...{symbol}
" + display_symbol = format_symbol_for_display(symbol) lines.append( f"| {display_symbol} | {target_str} | {pr_str} | {change_str} |" ) @@ -186,8 +196,8 @@ def create_symbol_changes_table( ) for symbol, size in new_symbols[:15]: - display_symbol = symbol if len(symbol) <= 80 else symbol[:77] + "..." - lines.append(f"| `{display_symbol}` | {format_bytes(size)} |") + display_symbol = format_symbol_for_display(symbol) + lines.append(f"| {display_symbol} | {format_bytes(size)} |") if len(new_symbols) > 15: total_new_size = sum(s[1] for s in new_symbols) @@ -209,8 +219,8 @@ def create_symbol_changes_table( ) for symbol, size in removed_symbols[:15]: - display_symbol = symbol if len(symbol) <= 80 else symbol[:77] + "..." - lines.append(f"| `{display_symbol}` | {format_bytes(size)} |") + display_symbol = format_symbol_for_display(symbol) + lines.append(f"| {display_symbol} | {format_bytes(size)} |") if len(removed_symbols) > 15: total_removed_size = sum(s[1] for s in removed_symbols) @@ -242,7 +252,7 @@ def create_detailed_breakdown_table( # Combine all components from both analyses all_components = set(target_analysis.keys()) | set(pr_analysis.keys()) - # Filter to components that have changed + # Filter to components that have changed (ignoring noise ≤2 bytes) changed_components = [] for comp in all_components: target_mem = target_analysis.get(comp, {}) @@ -251,9 +261,9 @@ def create_detailed_breakdown_table( target_flash = target_mem.get("flash_total", 0) pr_flash = pr_mem.get("flash_total", 0) - # Only include if component has changed - if target_flash != pr_flash: - delta = pr_flash - target_flash + # Only include if component has meaningful change (>2 bytes) + delta = pr_flash - target_flash + if abs(delta) > 2: changed_components.append((comp, target_flash, pr_flash, delta)) if not changed_components: @@ -362,6 +372,10 @@ def create_comment_body( {component_breakdown}{symbol_changes}{cache_note} --- +> **Note:** This analysis measures **static RAM and Flash usage** only (compile-time allocation). +> **Dynamic memory (heap)** cannot be measured automatically. +> **⚠️ You must test this PR on a real device** to measure free heap and ensure no runtime memory issues. + *This analysis runs automatically when components change. Memory usage is measured from {config_note}.* """ From a1d6bac21a41fcbfd49a7356dc13ea0a06f6ea08 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Fri, 17 Oct 2025 15:44:36 -1000 Subject: [PATCH 42/76] preen --- esphome/analyze_memory/cli.py | 15 ++++----------- 1 file changed, 4 insertions(+), 11 deletions(-) diff --git a/esphome/analyze_memory/cli.py b/esphome/analyze_memory/cli.py index e8541b1621..7b004353ec 100644 --- a/esphome/analyze_memory/cli.py +++ b/esphome/analyze_memory/cli.py @@ -387,17 +387,10 @@ def main(): file=sys.stderr, ) - try: - analyzer = MemoryAnalyzerCLI(elf_file, idedata=idedata) - analyzer.analyze() - report = analyzer.generate_report() - print(report) - except Exception as e: - print(f"Error: {e}", file=sys.stderr) - import traceback - - traceback.print_exc(file=sys.stderr) - sys.exit(1) + analyzer = MemoryAnalyzerCLI(elf_file, idedata=idedata) + analyzer.analyze() + report = analyzer.generate_report() + print(report) if __name__ == "__main__": From 0fcae15c257772d3c1d868555fcd1cbb82856fe5 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Fri, 17 Oct 2025 15:53:03 -1000 Subject: [PATCH 43/76] preen --- script/determine-jobs.py | 28 +++++++++++++++++++++++++--- 1 file changed, 25 insertions(+), 3 deletions(-) diff --git a/script/determine-jobs.py b/script/determine-jobs.py index 6a24c9eb01..e7a9b649b0 100755 --- a/script/determine-jobs.py +++ b/script/determine-jobs.py @@ -56,6 +56,10 @@ from helpers import ( root_path, ) +# Memory impact analysis constants +MEMORY_IMPACT_FALLBACK_COMPONENT = "api" # Representative component for core changes +MEMORY_IMPACT_FALLBACK_PLATFORM = "esp32-idf" # Most representative platform + def should_run_integration_tests(branch: str | None = None) -> bool: """Determine if integration tests should run based on changed files. @@ -273,6 +277,7 @@ def detect_memory_impact_config( # Find all changed components (excluding core and base bus components) changed_component_set = set() + has_core_changes = False for file in files: if file.startswith(ESPHOME_COMPONENTS_PATH): @@ -282,9 +287,22 @@ def detect_memory_impact_config( # Skip base bus components as they're used across many builds if component not in ["i2c", "spi", "uart", "modbus", "canbus"]: changed_component_set.add(component) + elif file.startswith("esphome/"): + # Core ESPHome files changed (not component-specific) + has_core_changes = True - # If no components changed, don't run memory impact - if not changed_component_set: + # If no components changed but core changed, test representative component + force_fallback_platform = False + if not changed_component_set and has_core_changes: + print( + f"Memory impact: No components changed, but core files changed. " + f"Testing {MEMORY_IMPACT_FALLBACK_COMPONENT} component on {MEMORY_IMPACT_FALLBACK_PLATFORM}.", + file=sys.stderr, + ) + changed_component_set.add(MEMORY_IMPACT_FALLBACK_COMPONENT) + force_fallback_platform = True # Use fallback platform (most representative) + elif not changed_component_set: + # No components and no core changes return {"should_run": "false"} # Find components that have tests on the preferred platform @@ -331,7 +349,11 @@ def detect_memory_impact_config( return {"should_run": "false"} # Use the most preferred platform found, or fall back to esp8266-ard - platform = selected_platform or "esp8266-ard" + # Exception: for core changes, use fallback platform (most representative of codebase) + if force_fallback_platform: + platform = MEMORY_IMPACT_FALLBACK_PLATFORM + else: + platform = selected_platform or "esp8266-ard" # Debug output print("Memory impact analysis:", file=sys.stderr) From 71f2fb83532f12b168ff4381f5a9e3c5a984b756 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Fri, 17 Oct 2025 15:56:13 -1000 Subject: [PATCH 44/76] preen --- script/determine-jobs.py | 31 ++++++++++++++++++++++--------- 1 file changed, 22 insertions(+), 9 deletions(-) diff --git a/script/determine-jobs.py b/script/determine-jobs.py index e7a9b649b0..eb8cd5df54 100755 --- a/script/determine-jobs.py +++ b/script/determine-jobs.py @@ -38,6 +38,7 @@ Options: from __future__ import annotations import argparse +from enum import StrEnum from functools import cache import json import os @@ -56,9 +57,21 @@ from helpers import ( root_path, ) + +class Platform(StrEnum): + """Platform identifiers for memory impact analysis.""" + + ESP8266_ARD = "esp8266-ard" + ESP32_IDF = "esp32-idf" + ESP32_C3_IDF = "esp32-c3-idf" + ESP32_C6_IDF = "esp32-c6-idf" + ESP32_S2_IDF = "esp32-s2-idf" + ESP32_S3_IDF = "esp32-s3-idf" + + # Memory impact analysis constants MEMORY_IMPACT_FALLBACK_COMPONENT = "api" # Representative component for core changes -MEMORY_IMPACT_FALLBACK_PLATFORM = "esp32-idf" # Most representative platform +MEMORY_IMPACT_FALLBACK_PLATFORM = Platform.ESP32_IDF # Most representative platform def should_run_integration_tests(branch: str | None = None) -> bool: @@ -262,14 +275,14 @@ def detect_memory_impact_config( """ # Platform preference order for memory impact analysis # Prefer ESP8266 for memory impact as it's the most constrained platform - # ESP32-IDF is preferred over ESP32-Arduino as it's faster to build and more commonly used + # ESP32-IDF is preferred over ESP32-Arduino as it's the most representative of codebase PLATFORM_PREFERENCE = [ - "esp8266-ard", # ESP8266 Arduino (most memory constrained - best for impact analysis) - "esp32-idf", # ESP32 IDF platform (primary ESP32 platform, faster builds) - "esp32-c3-idf", # ESP32-C3 IDF - "esp32-c6-idf", # ESP32-C6 IDF - "esp32-s2-idf", # ESP32-S2 IDF - "esp32-s3-idf", # ESP32-S3 IDF + Platform.ESP8266_ARD, # ESP8266 Arduino (most memory constrained - best for impact analysis) + Platform.ESP32_IDF, # ESP32 IDF platform (primary ESP32 platform, most representative) + Platform.ESP32_C3_IDF, # ESP32-C3 IDF + Platform.ESP32_C6_IDF, # ESP32-C6 IDF + Platform.ESP32_S2_IDF, # ESP32-S2 IDF + Platform.ESP32_S3_IDF, # ESP32-S3 IDF ] # Get actually changed files (not dependencies) @@ -353,7 +366,7 @@ def detect_memory_impact_config( if force_fallback_platform: platform = MEMORY_IMPACT_FALLBACK_PLATFORM else: - platform = selected_platform or "esp8266-ard" + platform = selected_platform or Platform.ESP8266_ARD # Debug output print("Memory impact analysis:", file=sys.stderr) From a45e94cd06fd063b3370f5eafba9b4536b38b1c5 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Fri, 17 Oct 2025 16:02:08 -1000 Subject: [PATCH 45/76] preen --- esphome/analyze_memory/__init__.py | 34 ++++++++++++++++++++++++++---- 1 file changed, 30 insertions(+), 4 deletions(-) diff --git a/esphome/analyze_memory/__init__.py b/esphome/analyze_memory/__init__.py index 349c3da507..b8bbd68df2 100644 --- a/esphome/analyze_memory/__init__.py +++ b/esphome/analyze_memory/__init__.py @@ -310,13 +310,27 @@ class MemoryAnalyzer: "✗ Using system c++filt (objdump_path=%s)", self.objdump_path ) - # Strip GCC optimization suffixes before demangling + # Strip GCC optimization suffixes and prefixes before demangling # Suffixes like $isra$0, $part$0, $constprop$0 confuse c++filt + # Prefixes like _GLOBAL__sub_I_ need to be removed and tracked symbols_stripped = [] + symbols_prefixes = [] # Track removed prefixes for symbol in symbols: # Remove GCC optimization markers stripped = re.sub(r"\$(?:isra|part|constprop)\$\d+", "", symbol) + + # Handle GCC global constructor/initializer prefixes + # _GLOBAL__sub_I_ -> extract for demangling + prefix = "" + if stripped.startswith("_GLOBAL__sub_I_"): + prefix = "_GLOBAL__sub_I_" + stripped = stripped[len(prefix) :] + elif stripped.startswith("_GLOBAL__sub_D_"): + prefix = "_GLOBAL__sub_D_" + stripped = stripped[len(prefix) :] + symbols_stripped.append(stripped) + symbols_prefixes.append(prefix) try: # Send all symbols to c++filt at once @@ -331,11 +345,23 @@ class MemoryAnalyzer: demangled_lines = result.stdout.strip().split("\n") # Map original to demangled names failed_count = 0 - for original, stripped, demangled in zip( - symbols, symbols_stripped, demangled_lines + for original, stripped, prefix, demangled in zip( + symbols, symbols_stripped, symbols_prefixes, demangled_lines ): + # Add back any prefix that was removed + if prefix: + if demangled != stripped: + # Successfully demangled - add descriptive prefix + if prefix == "_GLOBAL__sub_I_": + demangled = f"[global constructor for: {demangled}]" + elif prefix == "_GLOBAL__sub_D_": + demangled = f"[global destructor for: {demangled}]" + else: + # Failed to demangle - restore original prefix + demangled = prefix + demangled + # If we stripped a suffix, add it back to the demangled name for clarity - if original != stripped: + if original != stripped and not prefix: # Find what was stripped suffix_match = re.search( r"(\$(?:isra|part|constprop)\$\d+)", original From 29b9073d62631fe1ec8bb57bd8e09185b9119d5e Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Fri, 17 Oct 2025 16:07:46 -1000 Subject: [PATCH 46/76] esp32 only platforms --- script/determine-jobs.py | 44 ++++++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 15 deletions(-) diff --git a/script/determine-jobs.py b/script/determine-jobs.py index eb8cd5df54..d4b46e5474 100755 --- a/script/determine-jobs.py +++ b/script/determine-jobs.py @@ -318,10 +318,9 @@ def detect_memory_impact_config( # No components and no core changes return {"should_run": "false"} - # Find components that have tests on the preferred platform + # Find components that have tests and collect their supported platforms components_with_tests = [] - selected_platform = None - component_platforms = {} # Track which platforms each component has + component_platforms_map = {} # Track which platforms each component supports for component in sorted(changed_component_set): tests_dir = Path(root_path) / "tests" / "components" / component @@ -346,33 +345,48 @@ def detect_memory_impact_config( if not available_platforms: continue - # Find the most preferred platform for this component - component_platform = min(available_platforms, key=PLATFORM_PREFERENCE.index) - component_platforms[component] = component_platform + component_platforms_map[component] = set(available_platforms) components_with_tests.append(component) - # Select the most preferred platform across all components - if selected_platform is None or PLATFORM_PREFERENCE.index( - component_platform - ) < PLATFORM_PREFERENCE.index(selected_platform): - selected_platform = component_platform - # If no components have tests, don't run memory impact if not components_with_tests: return {"should_run": "false"} - # Use the most preferred platform found, or fall back to esp8266-ard + # Find common platforms supported by ALL components + # This ensures we can build all components together in a merged config + common_platforms = set(PLATFORM_PREFERENCE) + for component, platforms in component_platforms_map.items(): + common_platforms &= platforms + + # Select the most preferred platform from the common set # Exception: for core changes, use fallback platform (most representative of codebase) if force_fallback_platform: platform = MEMORY_IMPACT_FALLBACK_PLATFORM + elif common_platforms: + # Pick the most preferred platform that all components support + platform = min(common_platforms, key=PLATFORM_PREFERENCE.index) else: - platform = selected_platform or Platform.ESP8266_ARD + # No common platform - fall back to testing each component individually + # Pick the most commonly supported platform + platform_counts = {} + for platforms in component_platforms_map.values(): + for p in platforms: + platform_counts[p] = platform_counts.get(p, 0) + 1 + # Pick the platform supported by most components, preferring earlier in PLATFORM_PREFERENCE + platform = max( + platform_counts.keys(), + key=lambda p: (platform_counts[p], -PLATFORM_PREFERENCE.index(p)), + ) # Debug output print("Memory impact analysis:", file=sys.stderr) print(f" Changed components: {sorted(changed_component_set)}", file=sys.stderr) print(f" Components with tests: {components_with_tests}", file=sys.stderr) - print(f" Component platforms: {component_platforms}", file=sys.stderr) + print( + f" Component platforms: {dict(sorted(component_platforms_map.items()))}", + file=sys.stderr, + ) + print(f" Common platforms: {sorted(common_platforms)}", file=sys.stderr) print(f" Selected platform: {platform}", file=sys.stderr) return { From f5d69a25393119f08aec1b46a2f5324435b10eaa Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Fri, 17 Oct 2025 16:11:28 -1000 Subject: [PATCH 47/76] esp32 only platforms --- esphome/analyze_memory/__init__.py | 23 ----------------------- script/determine-jobs.py | 8 +++----- script/helpers.py | 17 +++++++++++++++++ 3 files changed, 20 insertions(+), 28 deletions(-) diff --git a/esphome/analyze_memory/__init__.py b/esphome/analyze_memory/__init__.py index b8bbd68df2..07f8df8767 100644 --- a/esphome/analyze_memory/__init__.py +++ b/esphome/analyze_memory/__init__.py @@ -2,7 +2,6 @@ from collections import defaultdict from dataclasses import dataclass, field -import json import logging from pathlib import Path import re @@ -422,28 +421,6 @@ class MemoryAnalyzer: return "Other Core" - def to_json(self) -> str: - """Export analysis results as JSON.""" - data = { - "components": { - name: { - "text": mem.text_size, - "rodata": mem.rodata_size, - "data": mem.data_size, - "bss": mem.bss_size, - "flash_total": mem.flash_total, - "ram_total": mem.ram_total, - "symbol_count": mem.symbol_count, - } - for name, mem in self.components.items() - }, - "totals": { - "flash": sum(c.flash_total for c in self.components.values()), - "ram": sum(c.ram_total for c in self.components.values()), - }, - } - return json.dumps(data, indent=2) - if __name__ == "__main__": from .cli import main diff --git a/script/determine-jobs.py b/script/determine-jobs.py index d4b46e5474..befd75fb5b 100755 --- a/script/determine-jobs.py +++ b/script/determine-jobs.py @@ -54,6 +54,7 @@ from helpers import ( changed_files, get_all_dependencies, get_components_from_integration_fixtures, + parse_test_filename, root_path, ) @@ -335,11 +336,8 @@ def detect_memory_impact_config( # Check if component has tests for any preferred platform available_platforms = [] for test_file in test_files: - parts = test_file.stem.split(".") - if len(parts) < 2: - continue - platform = parts[1] - if platform in PLATFORM_PREFERENCE: + _, platform = parse_test_filename(test_file) + if platform != "all" and platform in PLATFORM_PREFERENCE: available_platforms.append(platform) if not available_platforms: diff --git a/script/helpers.py b/script/helpers.py index 61306b9489..85e568dcf8 100644 --- a/script/helpers.py +++ b/script/helpers.py @@ -46,6 +46,23 @@ def parse_list_components_output(output: str) -> list[str]: return [c.strip() for c in output.strip().split("\n") if c.strip()] +def parse_test_filename(test_file: Path) -> tuple[str, str]: + """Parse test filename to extract test name and platform. + + Test files follow the naming pattern: test..yaml or test-..yaml + + Args: + test_file: Path to test file + + Returns: + Tuple of (test_name, platform) + """ + parts = test_file.stem.split(".") + if len(parts) == 2: + return parts[0], parts[1] # test, platform + return parts[0], "all" + + def styled(color: str | tuple[str, ...], msg: str, reset: bool = True) -> str: prefix = "".join(color) if isinstance(color, tuple) else color suffix = colorama.Style.RESET_ALL if reset else "" From 3b8b2c07542e543077bbd4f8b95476d02940f602 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Fri, 17 Oct 2025 16:13:30 -1000 Subject: [PATCH 48/76] esp32 only platforms --- script/determine-jobs.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/script/determine-jobs.py b/script/determine-jobs.py index befd75fb5b..bf944886ea 100755 --- a/script/determine-jobs.py +++ b/script/determine-jobs.py @@ -275,13 +275,13 @@ def detect_memory_impact_config( - use_merged_config: "true" (always use merged config) """ # Platform preference order for memory impact analysis - # Prefer ESP8266 for memory impact as it's the most constrained platform - # ESP32-IDF is preferred over ESP32-Arduino as it's the most representative of codebase + # Prefer newer platforms first as they represent the future of ESPHome + # ESP8266 is most constrained but many new features don't support it PLATFORM_PREFERENCE = [ + Platform.ESP32_C6_IDF, # ESP32-C6 IDF (newest, supports Thread/Zigbee) Platform.ESP8266_ARD, # ESP8266 Arduino (most memory constrained - best for impact analysis) Platform.ESP32_IDF, # ESP32 IDF platform (primary ESP32 platform, most representative) Platform.ESP32_C3_IDF, # ESP32-C3 IDF - Platform.ESP32_C6_IDF, # ESP32-C6 IDF Platform.ESP32_S2_IDF, # ESP32-S2 IDF Platform.ESP32_S3_IDF, # ESP32-S3 IDF ] @@ -364,8 +364,8 @@ def detect_memory_impact_config( # Pick the most preferred platform that all components support platform = min(common_platforms, key=PLATFORM_PREFERENCE.index) else: - # No common platform - fall back to testing each component individually - # Pick the most commonly supported platform + # No common platform - pick the most commonly supported platform + # This allows testing components individually even if they can't be merged platform_counts = {} for platforms in component_platforms_map.values(): for p in platforms: From c6ecfd0c55d278e43189988a081e7908c36461a4 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Fri, 17 Oct 2025 16:15:46 -1000 Subject: [PATCH 49/76] esp32 only platforms --- esphome/analyze_memory/__init__.py | 162 +++++++++++++++++++---------- 1 file changed, 109 insertions(+), 53 deletions(-) diff --git a/esphome/analyze_memory/__init__.py b/esphome/analyze_memory/__init__.py index 07f8df8767..5ef9eab526 100644 --- a/esphome/analyze_memory/__init__.py +++ b/esphome/analyze_memory/__init__.py @@ -27,6 +27,12 @@ if TYPE_CHECKING: _LOGGER = logging.getLogger(__name__) +# GCC global constructor/destructor prefix annotations +_GCC_PREFIX_ANNOTATIONS = { + "_GLOBAL__sub_I_": "global constructor for", + "_GLOBAL__sub_D_": "global destructor for", +} + @dataclass class MemorySection: @@ -340,66 +346,116 @@ class MemoryAnalyzer: text=True, check=False, ) - if result.returncode == 0: - demangled_lines = result.stdout.strip().split("\n") - # Map original to demangled names - failed_count = 0 - for original, stripped, prefix, demangled in zip( - symbols, symbols_stripped, symbols_prefixes, demangled_lines - ): - # Add back any prefix that was removed - if prefix: - if demangled != stripped: - # Successfully demangled - add descriptive prefix - if prefix == "_GLOBAL__sub_I_": - demangled = f"[global constructor for: {demangled}]" - elif prefix == "_GLOBAL__sub_D_": - demangled = f"[global destructor for: {demangled}]" - else: - # Failed to demangle - restore original prefix - demangled = prefix + demangled + except (subprocess.SubprocessError, OSError, UnicodeDecodeError) as e: + # On error, cache originals + _LOGGER.warning("Failed to batch demangle symbols: %s", e) + for symbol in symbols: + self._demangle_cache[symbol] = symbol + return - # If we stripped a suffix, add it back to the demangled name for clarity - if original != stripped and not prefix: - # Find what was stripped - suffix_match = re.search( - r"(\$(?:isra|part|constprop)\$\d+)", original - ) - if suffix_match: - demangled = f"{demangled} [{suffix_match.group(1)}]" - - self._demangle_cache[original] = demangled - - # Log symbols that failed to demangle (stayed the same as stripped version) - if stripped == demangled and stripped.startswith("_Z"): - failed_count += 1 - if failed_count <= 5: # Only log first 5 failures - _LOGGER.warning("Failed to demangle: %s", original[:100]) - - if failed_count > 0: - _LOGGER.warning( - "Failed to demangle %d/%d symbols using %s", - failed_count, - len(symbols), - cppfilt_cmd, - ) - else: - _LOGGER.warning( - "Successfully demangled all %d symbols", len(symbols) - ) - return + if result.returncode != 0: _LOGGER.warning( "c++filt exited with code %d: %s", result.returncode, result.stderr[:200] if result.stderr else "(no error output)", ) - except (subprocess.SubprocessError, OSError, UnicodeDecodeError) as e: - # On error, cache originals - _LOGGER.warning("Failed to batch demangle symbols: %s", e) + # Cache originals on failure + for symbol in symbols: + self._demangle_cache[symbol] = symbol + return - # If demangling failed, cache originals - for symbol in symbols: - self._demangle_cache[symbol] = symbol + # Process demangled output + self._process_demangled_output( + symbols, symbols_stripped, symbols_prefixes, result.stdout, cppfilt_cmd + ) + + def _process_demangled_output( + self, + symbols: list[str], + symbols_stripped: list[str], + symbols_prefixes: list[str], + demangled_output: str, + cppfilt_cmd: str, + ) -> None: + """Process demangled symbol output and populate cache. + + Args: + symbols: Original symbol names + symbols_stripped: Stripped symbol names sent to c++filt + symbols_prefixes: Removed prefixes to restore + demangled_output: Output from c++filt + cppfilt_cmd: Path to c++filt command (for logging) + """ + demangled_lines = demangled_output.strip().split("\n") + failed_count = 0 + + for original, stripped, prefix, demangled in zip( + symbols, symbols_stripped, symbols_prefixes, demangled_lines + ): + # Add back any prefix that was removed + demangled = self._restore_symbol_prefix(prefix, stripped, demangled) + + # If we stripped a suffix, add it back to the demangled name for clarity + if original != stripped and not prefix: + demangled = self._restore_symbol_suffix(original, demangled) + + self._demangle_cache[original] = demangled + + # Log symbols that failed to demangle (stayed the same as stripped version) + if stripped == demangled and stripped.startswith("_Z"): + failed_count += 1 + if failed_count <= 5: # Only log first 5 failures + _LOGGER.warning("Failed to demangle: %s", original[:100]) + + if failed_count > 0: + _LOGGER.warning( + "Failed to demangle %d/%d symbols using %s", + failed_count, + len(symbols), + cppfilt_cmd, + ) + else: + _LOGGER.warning("Successfully demangled all %d symbols", len(symbols)) + + @staticmethod + def _restore_symbol_prefix(prefix: str, stripped: str, demangled: str) -> str: + """Restore prefix that was removed before demangling. + + Args: + prefix: Prefix that was removed (e.g., "_GLOBAL__sub_I_") + stripped: Stripped symbol name + demangled: Demangled symbol name + + Returns: + Demangled name with prefix restored/annotated + """ + if not prefix: + return demangled + + # Successfully demangled - add descriptive prefix + if demangled != stripped and ( + annotation := _GCC_PREFIX_ANNOTATIONS.get(prefix) + ): + return f"[{annotation}: {demangled}]" + + # Failed to demangle - restore original prefix + return prefix + demangled + + @staticmethod + def _restore_symbol_suffix(original: str, demangled: str) -> str: + """Restore GCC optimization suffix that was removed before demangling. + + Args: + original: Original symbol name with suffix + demangled: Demangled symbol name without suffix + + Returns: + Demangled name with suffix annotation + """ + suffix_match = re.search(r"(\$(?:isra|part|constprop)\$\d+)", original) + if suffix_match: + return f"{demangled} [{suffix_match.group(1)}]" + return demangled def _demangle_symbol(self, symbol: str) -> str: """Get demangled C++ symbol name from cache.""" From 558d4eb9ddfbd7274151046798630dd5c40e6cd7 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Fri, 17 Oct 2025 16:19:50 -1000 Subject: [PATCH 50/76] preen --- script/ci_memory_impact_extract.py | 57 +++++++++++++----------------- script/determine-jobs.py | 36 ++++++++++--------- 2 files changed, 45 insertions(+), 48 deletions(-) diff --git a/script/ci_memory_impact_extract.py b/script/ci_memory_impact_extract.py index 96f947e12a..76632ebc33 100755 --- a/script/ci_memory_impact_extract.py +++ b/script/ci_memory_impact_extract.py @@ -132,41 +132,34 @@ def run_detailed_analysis(build_dir: str) -> dict | None: file=sys.stderr, ) - try: - analyzer = MemoryAnalyzer(elf_path, idedata=idedata) - components = analyzer.analyze() + analyzer = MemoryAnalyzer(elf_path, idedata=idedata) + components = analyzer.analyze() - # Convert to JSON-serializable format - result = { - "components": { - name: { - "text": mem.text_size, - "rodata": mem.rodata_size, - "data": mem.data_size, - "bss": mem.bss_size, - "flash_total": mem.flash_total, - "ram_total": mem.ram_total, - "symbol_count": mem.symbol_count, - } - for name, mem in components.items() - }, - "symbols": {}, - } + # Convert to JSON-serializable format + result = { + "components": { + name: { + "text": mem.text_size, + "rodata": mem.rodata_size, + "data": mem.data_size, + "bss": mem.bss_size, + "flash_total": mem.flash_total, + "ram_total": mem.ram_total, + "symbol_count": mem.symbol_count, + } + for name, mem in components.items() + }, + "symbols": {}, + } - # Build symbol map - for section in analyzer.sections.values(): - for symbol_name, size, _ in section.symbols: - if size > 0: - demangled = analyzer._demangle_symbol(symbol_name) - result["symbols"][demangled] = size + # Build symbol map + for section in analyzer.sections.values(): + for symbol_name, size, _ in section.symbols: + if size > 0: + demangled = analyzer._demangle_symbol(symbol_name) + result["symbols"][demangled] = size - return result - except Exception as e: - print(f"Warning: Failed to run detailed analysis: {e}", file=sys.stderr) - import traceback - - traceback.print_exc(file=sys.stderr) - return None + return result def main() -> int: diff --git a/script/determine-jobs.py b/script/determine-jobs.py index bf944886ea..8e2c239fe2 100755 --- a/script/determine-jobs.py +++ b/script/determine-jobs.py @@ -74,6 +74,18 @@ class Platform(StrEnum): MEMORY_IMPACT_FALLBACK_COMPONENT = "api" # Representative component for core changes MEMORY_IMPACT_FALLBACK_PLATFORM = Platform.ESP32_IDF # Most representative platform +# Platform preference order for memory impact analysis +# Prefer newer platforms first as they represent the future of ESPHome +# ESP8266 is most constrained but many new features don't support it +MEMORY_IMPACT_PLATFORM_PREFERENCE = [ + Platform.ESP32_C6_IDF, # ESP32-C6 IDF (newest, supports Thread/Zigbee) + Platform.ESP8266_ARD, # ESP8266 Arduino (most memory constrained - best for impact analysis) + Platform.ESP32_IDF, # ESP32 IDF platform (primary ESP32 platform, most representative) + Platform.ESP32_C3_IDF, # ESP32-C3 IDF + Platform.ESP32_S2_IDF, # ESP32-S2 IDF + Platform.ESP32_S3_IDF, # ESP32-S3 IDF +] + def should_run_integration_tests(branch: str | None = None) -> bool: """Determine if integration tests should run based on changed files. @@ -274,17 +286,6 @@ def detect_memory_impact_config( - platform: platform name for the merged build - use_merged_config: "true" (always use merged config) """ - # Platform preference order for memory impact analysis - # Prefer newer platforms first as they represent the future of ESPHome - # ESP8266 is most constrained but many new features don't support it - PLATFORM_PREFERENCE = [ - Platform.ESP32_C6_IDF, # ESP32-C6 IDF (newest, supports Thread/Zigbee) - Platform.ESP8266_ARD, # ESP8266 Arduino (most memory constrained - best for impact analysis) - Platform.ESP32_IDF, # ESP32 IDF platform (primary ESP32 platform, most representative) - Platform.ESP32_C3_IDF, # ESP32-C3 IDF - Platform.ESP32_S2_IDF, # ESP32-S2 IDF - Platform.ESP32_S3_IDF, # ESP32-S3 IDF - ] # Get actually changed files (not dependencies) files = changed_files(branch) @@ -337,7 +338,7 @@ def detect_memory_impact_config( available_platforms = [] for test_file in test_files: _, platform = parse_test_filename(test_file) - if platform != "all" and platform in PLATFORM_PREFERENCE: + if platform != "all" and platform in MEMORY_IMPACT_PLATFORM_PREFERENCE: available_platforms.append(platform) if not available_platforms: @@ -352,7 +353,7 @@ def detect_memory_impact_config( # Find common platforms supported by ALL components # This ensures we can build all components together in a merged config - common_platforms = set(PLATFORM_PREFERENCE) + common_platforms = set(MEMORY_IMPACT_PLATFORM_PREFERENCE) for component, platforms in component_platforms_map.items(): common_platforms &= platforms @@ -362,7 +363,7 @@ def detect_memory_impact_config( platform = MEMORY_IMPACT_FALLBACK_PLATFORM elif common_platforms: # Pick the most preferred platform that all components support - platform = min(common_platforms, key=PLATFORM_PREFERENCE.index) + platform = min(common_platforms, key=MEMORY_IMPACT_PLATFORM_PREFERENCE.index) else: # No common platform - pick the most commonly supported platform # This allows testing components individually even if they can't be merged @@ -370,10 +371,13 @@ def detect_memory_impact_config( for platforms in component_platforms_map.values(): for p in platforms: platform_counts[p] = platform_counts.get(p, 0) + 1 - # Pick the platform supported by most components, preferring earlier in PLATFORM_PREFERENCE + # Pick the platform supported by most components, preferring earlier in MEMORY_IMPACT_PLATFORM_PREFERENCE platform = max( platform_counts.keys(), - key=lambda p: (platform_counts[p], -PLATFORM_PREFERENCE.index(p)), + key=lambda p: ( + platform_counts[p], + -MEMORY_IMPACT_PLATFORM_PREFERENCE.index(p), + ), ) # Debug output From 1ec9383abe07ea278824ca63e55be8d5751ffe05 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Fri, 17 Oct 2025 16:39:10 -1000 Subject: [PATCH 51/76] preen --- .github/workflows/ci.yml | 34 +++++++++++++++++++++------------- 1 file changed, 21 insertions(+), 13 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 74ba831bc4..f2f3169eae 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -540,13 +540,22 @@ jobs: with: ref: ${{ github.base_ref }} - # Create cache key based on: - # 1. Target branch commit SHA - # 2. Hash of build infrastructure files (scripts and CI workflow) - # 3. Platform being tested - # 4. Component list + # Check if memory impact extraction script exists on target branch + # If not, skip the analysis (this handles older branches that don't have the feature) + - name: Check for memory impact script + id: check-script + run: | + if [ -f "script/ci_memory_impact_extract.py" ]; then + echo "skip=false" >> $GITHUB_OUTPUT + else + echo "skip=true" >> $GITHUB_OUTPUT + echo "::warning::ci_memory_impact_extract.py not found on target branch, skipping memory impact analysis" + fi + + # All remaining steps only run if script exists - name: Generate cache key id: cache-key + if: steps.check-script.outputs.skip != 'true' run: | # Get the commit SHA of the target branch target_sha=$(git rev-parse HEAD) @@ -571,15 +580,16 @@ jobs: echo "cache-key=${cache_key}" >> $GITHUB_OUTPUT echo "Cache key: ${cache_key}" - # Try to restore cached analysis results - name: Restore cached memory analysis id: cache-memory-analysis + if: steps.check-script.outputs.skip != 'true' uses: actions/cache/restore@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 with: path: memory-analysis-target.json key: ${{ steps.cache-key.outputs.cache-key }} - name: Cache status + if: steps.check-script.outputs.skip != 'true' run: | if [ "${{ steps.cache-memory-analysis.outputs.cache-hit }}" == "true" ]; then echo "✓ Cache hit! Using cached memory analysis results." @@ -588,23 +598,22 @@ jobs: echo "✗ Cache miss. Will build and analyze memory usage." fi - # Only restore Python and build if cache miss - name: Restore Python - if: steps.cache-memory-analysis.outputs.cache-hit != 'true' + if: steps.check-script.outputs.skip != 'true' && steps.cache-memory-analysis.outputs.cache-hit != 'true' uses: ./.github/actions/restore-python with: python-version: ${{ env.DEFAULT_PYTHON }} cache-key: ${{ needs.common.outputs.cache-key }} - name: Cache platformio - if: steps.cache-memory-analysis.outputs.cache-hit != 'true' + if: steps.check-script.outputs.skip != 'true' && steps.cache-memory-analysis.outputs.cache-hit != 'true' uses: actions/cache/restore@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 with: path: ~/.platformio key: platformio-memory-${{ fromJSON(needs.determine-jobs.outputs.memory_impact).platform }}-${{ hashFiles('platformio.ini') }} - name: Build, compile, and analyze memory - if: steps.cache-memory-analysis.outputs.cache-hit != 'true' + if: steps.check-script.outputs.skip != 'true' && steps.cache-memory-analysis.outputs.cache-hit != 'true' id: build run: | . venv/bin/activate @@ -631,17 +640,16 @@ jobs: --output-env \ --output-json memory-analysis-target.json - # Save build results to cache for future runs - name: Save memory analysis to cache - if: steps.cache-memory-analysis.outputs.cache-hit != 'true' && steps.build.outcome == 'success' + if: steps.check-script.outputs.skip != 'true' && steps.cache-memory-analysis.outputs.cache-hit != 'true' && steps.build.outcome == 'success' uses: actions/cache/save@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 with: path: memory-analysis-target.json key: ${{ steps.cache-key.outputs.cache-key }} - # Extract outputs from cached or freshly built analysis - name: Extract memory usage for outputs id: extract + if: steps.check-script.outputs.skip != 'true' run: | if [ -f memory-analysis-target.json ]; then ram=$(jq -r '.ram_bytes' memory-analysis-target.json) From 6fe5a0c736c3fa57b69d4d3e5d97931e80fff516 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Fri, 17 Oct 2025 16:44:38 -1000 Subject: [PATCH 52/76] preen --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f2f3169eae..efa9ce0bca 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -667,7 +667,7 @@ jobs: with: name: memory-analysis-target path: memory-analysis-target.json - if-no-files-found: error + if-no-files-found: warn retention-days: 1 memory-impact-pr-branch: From 0475ec55334b57981fdd9dd87cef2196bcb20e4c Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Fri, 17 Oct 2025 17:01:20 -1000 Subject: [PATCH 53/76] preen --- script/ci_memory_impact_comment.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/script/ci_memory_impact_comment.py b/script/ci_memory_impact_comment.py index 60676949e8..0be783ab3d 100755 --- a/script/ci_memory_impact_comment.py +++ b/script/ci_memory_impact_comment.py @@ -73,10 +73,12 @@ def format_change(before: int, after: int) -> str: # Format delta with sign and always show in bytes for precision if delta > 0: delta_str = f"+{delta:,} bytes" - emoji = "📈" + # Use 🚨 for significant increases (>1%), 🔸 for smaller ones + emoji = "🚨" if abs(percentage) > 1.0 else "🔸" elif delta < 0: delta_str = f"{delta:,} bytes" - emoji = "📉" + # Use 🎉 for significant reductions (>1%), ✅ for smaller ones + emoji = "🎉" if abs(percentage) > 1.0 else "✅" else: delta_str = "+0 bytes" emoji = "➡️" From 8fd43f1d96a80311c02d3acc5086e0dfd3211034 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Fri, 17 Oct 2025 17:09:05 -1000 Subject: [PATCH 54/76] tweak --- script/ci_memory_impact_comment.py | 36 ++++++++++++++++++++++-------- 1 file changed, 27 insertions(+), 9 deletions(-) diff --git a/script/ci_memory_impact_comment.py b/script/ci_memory_impact_comment.py index 0be783ab3d..2d36ffa405 100755 --- a/script/ci_memory_impact_comment.py +++ b/script/ci_memory_impact_comment.py @@ -22,6 +22,10 @@ sys.path.insert(0, str(Path(__file__).parent.parent)) # Comment marker to identify our memory impact comments COMMENT_MARKER = "" +# Thresholds for emoji significance indicators (percentage) +OVERALL_CHANGE_THRESHOLD = 1.0 # Overall RAM/Flash changes +COMPONENT_CHANGE_THRESHOLD = 3.0 # Component breakdown changes + def load_analysis_json(json_path: str) -> dict | None: """Load memory analysis results from JSON file. @@ -57,12 +61,16 @@ def format_bytes(bytes_value: int) -> str: return f"{bytes_value:,} bytes" -def format_change(before: int, after: int) -> str: +def format_change( + before: int, after: int, use_trend_icons: bool = False, threshold: float = 1.0 +) -> str: """Format memory change with delta and percentage. Args: before: Memory usage before change (in bytes) after: Memory usage after change (in bytes) + use_trend_icons: If True, use 📈/📉 chart icons; if False, use status emojis + threshold: Percentage threshold for "significant" change (default 1.0%) Returns: Formatted string with delta and percentage @@ -73,12 +81,18 @@ def format_change(before: int, after: int) -> str: # Format delta with sign and always show in bytes for precision if delta > 0: delta_str = f"+{delta:,} bytes" - # Use 🚨 for significant increases (>1%), 🔸 for smaller ones - emoji = "🚨" if abs(percentage) > 1.0 else "🔸" + if use_trend_icons: + emoji = "📈" + else: + # Use 🚨 for significant increases, 🔸 for smaller ones + emoji = "🚨" if abs(percentage) > threshold else "🔸" elif delta < 0: delta_str = f"{delta:,} bytes" - # Use 🎉 for significant reductions (>1%), ✅ for smaller ones - emoji = "🎉" if abs(percentage) > 1.0 else "✅" + if use_trend_icons: + emoji = "📉" + else: + # Use 🎉 for significant reductions, ✅ for smaller ones + emoji = "🎉" if abs(percentage) > threshold else "✅" else: delta_str = "+0 bytes" emoji = "➡️" @@ -173,7 +187,7 @@ def create_symbol_changes_table( for symbol, target_size, pr_size, delta in changed_symbols[:30]: target_str = format_bytes(target_size) pr_str = format_bytes(pr_size) - change_str = format_change(target_size, pr_size) + change_str = format_change(target_size, pr_size, use_trend_icons=True) display_symbol = format_symbol_for_display(symbol) lines.append( f"| {display_symbol} | {target_str} | {pr_str} | {change_str} |" @@ -287,7 +301,9 @@ def create_detailed_breakdown_table( for comp, target_flash, pr_flash, delta in changed_components[:20]: target_str = format_bytes(target_flash) pr_str = format_bytes(pr_flash) - change_str = format_change(target_flash, pr_flash) + change_str = format_change( + target_flash, pr_flash, threshold=COMPONENT_CHANGE_THRESHOLD + ) lines.append(f"| `{comp}` | {target_str} | {pr_str} | {change_str} |") if len(changed_components) > 20: @@ -331,8 +347,10 @@ def create_comment_body( Returns: Formatted comment body """ - ram_change = format_change(target_ram, pr_ram) - flash_change = format_change(target_flash, pr_flash) + ram_change = format_change(target_ram, pr_ram, threshold=OVERALL_CHANGE_THRESHOLD) + flash_change = format_change( + target_flash, pr_flash, threshold=OVERALL_CHANGE_THRESHOLD + ) # Use provided analysis data if available component_breakdown = "" From d98b00f56ddffc2e318e9830952712c1cbfd33f5 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Fri, 17 Oct 2025 17:10:28 -1000 Subject: [PATCH 55/76] tweak --- script/ci_memory_impact_comment.py | 33 ++++++++++++++++-------------- 1 file changed, 18 insertions(+), 15 deletions(-) diff --git a/script/ci_memory_impact_comment.py b/script/ci_memory_impact_comment.py index 2d36ffa405..f381df0ff6 100755 --- a/script/ci_memory_impact_comment.py +++ b/script/ci_memory_impact_comment.py @@ -61,16 +61,15 @@ def format_bytes(bytes_value: int) -> str: return f"{bytes_value:,} bytes" -def format_change( - before: int, after: int, use_trend_icons: bool = False, threshold: float = 1.0 -) -> str: +def format_change(before: int, after: int, threshold: float | None = None) -> str: """Format memory change with delta and percentage. Args: before: Memory usage before change (in bytes) after: Memory usage after change (in bytes) - use_trend_icons: If True, use 📈/📉 chart icons; if False, use status emojis - threshold: Percentage threshold for "significant" change (default 1.0%) + threshold: Optional percentage threshold for "significant" change. + If provided, adds supplemental emoji (🎉/🚨/🔸/✅) to chart icons. + If None, only shows chart icons (📈/📉/➡️). Returns: Formatted string with delta and percentage @@ -78,21 +77,25 @@ def format_change( delta = after - before percentage = 0.0 if before == 0 else (delta / before) * 100 - # Format delta with sign and always show in bytes for precision + # Always use chart icons to show direction if delta > 0: delta_str = f"+{delta:,} bytes" - if use_trend_icons: - emoji = "📈" + trend_icon = "📈" + # Add supplemental emoji based on threshold if provided + if threshold is not None: + significance = "🚨" if abs(percentage) > threshold else "🔸" + emoji = f"{trend_icon} {significance}" else: - # Use 🚨 for significant increases, 🔸 for smaller ones - emoji = "🚨" if abs(percentage) > threshold else "🔸" + emoji = trend_icon elif delta < 0: delta_str = f"{delta:,} bytes" - if use_trend_icons: - emoji = "📉" + trend_icon = "📉" + # Add supplemental emoji based on threshold if provided + if threshold is not None: + significance = "🎉" if abs(percentage) > threshold else "✅" + emoji = f"{trend_icon} {significance}" else: - # Use 🎉 for significant reductions, ✅ for smaller ones - emoji = "🎉" if abs(percentage) > threshold else "✅" + emoji = trend_icon else: delta_str = "+0 bytes" emoji = "➡️" @@ -187,7 +190,7 @@ def create_symbol_changes_table( for symbol, target_size, pr_size, delta in changed_symbols[:30]: target_str = format_bytes(target_size) pr_str = format_bytes(pr_size) - change_str = format_change(target_size, pr_size, use_trend_icons=True) + change_str = format_change(target_size, pr_size) # Chart icons only display_symbol = format_symbol_for_display(symbol) lines.append( f"| {display_symbol} | {target_str} | {pr_str} | {change_str} |" From cd93f7f55a07ea73a21cf78a620f77e03d4bb4c0 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Fri, 17 Oct 2025 17:13:24 -1000 Subject: [PATCH 56/76] tweak --- .coveragerc | 1 + 1 file changed, 1 insertion(+) diff --git a/.coveragerc b/.coveragerc index f23592be24..c15e79a31b 100644 --- a/.coveragerc +++ b/.coveragerc @@ -1,4 +1,5 @@ [run] omit = esphome/components/* + esphome/analyze_memory/* tests/integration/* From 931e3f80f0b1a70adf1264a75e4d568c90af9d1b Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Fri, 17 Oct 2025 17:25:03 -1000 Subject: [PATCH 57/76] no memory when tatget branch does not have --- .github/workflows/ci.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index efa9ce0bca..42f934de9d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -534,6 +534,7 @@ jobs: ram_usage: ${{ steps.extract.outputs.ram_usage }} flash_usage: ${{ steps.extract.outputs.flash_usage }} cache_hit: ${{ steps.cache-memory-analysis.outputs.cache-hit }} + skip: ${{ steps.check-script.outputs.skip }} steps: - name: Check out target branch uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 @@ -735,7 +736,7 @@ jobs: - determine-jobs - memory-impact-target-branch - memory-impact-pr-branch - if: github.event_name == 'pull_request' && fromJSON(needs.determine-jobs.outputs.memory_impact).should_run == 'true' + if: github.event_name == 'pull_request' && fromJSON(needs.determine-jobs.outputs.memory_impact).should_run == 'true' && needs.memory-impact-target-branch.outputs.skip != 'true' permissions: contents: read pull-requests: write From 5080698c3a7ed5b64429ce5d8b0fbfeddb635c9a Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Fri, 17 Oct 2025 17:34:16 -1000 Subject: [PATCH 58/76] no memory when tatget branch does not have --- script/ci_memory_impact_comment.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/script/ci_memory_impact_comment.py b/script/ci_memory_impact_comment.py index f381df0ff6..8b0dbb6f58 100755 --- a/script/ci_memory_impact_comment.py +++ b/script/ci_memory_impact_comment.py @@ -304,9 +304,9 @@ def create_detailed_breakdown_table( for comp, target_flash, pr_flash, delta in changed_components[:20]: target_str = format_bytes(target_flash) pr_str = format_bytes(pr_flash) - change_str = format_change( - target_flash, pr_flash, threshold=COMPONENT_CHANGE_THRESHOLD - ) + # Only apply threshold to ESPHome components, not framework/infrastructure + threshold = COMPONENT_CHANGE_THRESHOLD if comp.startswith("[esphome]") else None + change_str = format_change(target_flash, pr_flash, threshold=threshold) lines.append(f"| `{comp}` | {target_str} | {pr_str} | {change_str} |") if len(changed_components) > 20: From c70937ed01441c97f7c7d8132d05d635ac3a2534 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Fri, 17 Oct 2025 17:55:05 -1000 Subject: [PATCH 59/76] dry --- script/analyze_component_buses.py | 14 +------ script/determine-jobs.py | 64 ++++++++++++++----------------- script/helpers.py | 62 +++++++++++++++++++++++++++--- script/list-components.py | 10 ++--- script/split_components_for_ci.py | 10 ++--- script/test_build_components.py | 9 +++-- 6 files changed, 100 insertions(+), 69 deletions(-) diff --git a/script/analyze_component_buses.py b/script/analyze_component_buses.py index d0882e22e9..78f5ca3344 100755 --- a/script/analyze_component_buses.py +++ b/script/analyze_component_buses.py @@ -34,6 +34,8 @@ from typing import Any # Add esphome to path sys.path.insert(0, str(Path(__file__).parent.parent)) +from helpers import BASE_BUS_COMPONENTS + from esphome import yaml_util from esphome.config_helpers import Extend, Remove @@ -67,18 +69,6 @@ NO_BUSES_SIGNATURE = "no_buses" # Isolated components have unique signatures and cannot be merged with others ISOLATED_SIGNATURE_PREFIX = "isolated_" -# Base bus components - these ARE the bus implementations and should not -# be flagged as needing migration since they are the platform/base components -BASE_BUS_COMPONENTS = { - "i2c", - "spi", - "uart", - "modbus", - "canbus", - "remote_transmitter", - "remote_receiver", -} - # Components that must be tested in isolation (not grouped or batched with others) # These have known build issues that prevent grouping # NOTE: This should be kept in sync with both test_build_components and split_components_for_ci.py diff --git a/script/determine-jobs.py b/script/determine-jobs.py index 8e2c239fe2..5767ced859 100755 --- a/script/determine-jobs.py +++ b/script/determine-jobs.py @@ -38,6 +38,7 @@ Options: from __future__ import annotations import argparse +from collections import Counter from enum import StrEnum from functools import cache import json @@ -48,11 +49,13 @@ import sys from typing import Any from helpers import ( + BASE_BUS_COMPONENTS, CPP_FILE_EXTENSIONS, - ESPHOME_COMPONENTS_PATH, PYTHON_FILE_EXTENSIONS, changed_files, get_all_dependencies, + get_component_from_path, + get_component_test_files, get_components_from_integration_fixtures, parse_test_filename, root_path, @@ -142,12 +145,9 @@ def should_run_integration_tests(branch: str | None = None) -> bool: # Check if any required components changed for file in files: - if file.startswith(ESPHOME_COMPONENTS_PATH): - parts = file.split("/") - if len(parts) >= 3: - component = parts[2] - if component in all_required_components: - return True + component = get_component_from_path(file) + if component and component in all_required_components: + return True return False @@ -261,10 +261,7 @@ def _component_has_tests(component: str) -> bool: Returns: True if the component has test YAML files """ - tests_dir = Path(root_path) / "tests" / "components" / component - if not tests_dir.exists(): - return False - return any(tests_dir.glob("test.*.yaml")) + return bool(get_component_test_files(component)) def detect_memory_impact_config( @@ -291,17 +288,15 @@ def detect_memory_impact_config( files = changed_files(branch) # Find all changed components (excluding core and base bus components) - changed_component_set = set() + changed_component_set: set[str] = set() has_core_changes = False for file in files: - if file.startswith(ESPHOME_COMPONENTS_PATH): - parts = file.split("/") - if len(parts) >= 3: - component = parts[2] - # Skip base bus components as they're used across many builds - if component not in ["i2c", "spi", "uart", "modbus", "canbus"]: - changed_component_set.add(component) + component = get_component_from_path(file) + if component: + # Skip base bus components as they're used across many builds + if component not in BASE_BUS_COMPONENTS: + changed_component_set.add(component) elif file.startswith("esphome/"): # Core ESPHome files changed (not component-specific) has_core_changes = True @@ -321,25 +316,24 @@ def detect_memory_impact_config( return {"should_run": "false"} # Find components that have tests and collect their supported platforms - components_with_tests = [] - component_platforms_map = {} # Track which platforms each component supports + components_with_tests: list[str] = [] + component_platforms_map: dict[ + str, set[Platform] + ] = {} # Track which platforms each component supports for component in sorted(changed_component_set): - tests_dir = Path(root_path) / "tests" / "components" / component - if not tests_dir.exists(): - continue - # Look for test files on preferred platforms - test_files = list(tests_dir.glob("test.*.yaml")) + test_files = get_component_test_files(component) if not test_files: continue # Check if component has tests for any preferred platform - available_platforms = [] - for test_file in test_files: - _, platform = parse_test_filename(test_file) - if platform != "all" and platform in MEMORY_IMPACT_PLATFORM_PREFERENCE: - available_platforms.append(platform) + available_platforms = [ + platform + for test_file in test_files + if (platform := parse_test_filename(test_file)[1]) != "all" + and platform in MEMORY_IMPACT_PLATFORM_PREFERENCE + ] if not available_platforms: continue @@ -367,10 +361,10 @@ def detect_memory_impact_config( else: # No common platform - pick the most commonly supported platform # This allows testing components individually even if they can't be merged - platform_counts = {} - for platforms in component_platforms_map.values(): - for p in platforms: - platform_counts[p] = platform_counts.get(p, 0) + 1 + # Count how many components support each platform + platform_counts = Counter( + p for platforms in component_platforms_map.values() for p in platforms + ) # Pick the platform supported by most components, preferring earlier in MEMORY_IMPACT_PLATFORM_PREFERENCE platform = max( platform_counts.keys(), diff --git a/script/helpers.py b/script/helpers.py index 85e568dcf8..edde3d78af 100644 --- a/script/helpers.py +++ b/script/helpers.py @@ -29,6 +29,18 @@ YAML_FILE_EXTENSIONS = (".yaml", ".yml") # Component path prefix ESPHOME_COMPONENTS_PATH = "esphome/components/" +# Base bus components - these ARE the bus implementations and should not +# be flagged as needing migration since they are the platform/base components +BASE_BUS_COMPONENTS = { + "i2c", + "spi", + "uart", + "modbus", + "canbus", + "remote_transmitter", + "remote_receiver", +} + def parse_list_components_output(output: str) -> list[str]: """Parse the output from list-components.py script. @@ -63,6 +75,48 @@ def parse_test_filename(test_file: Path) -> tuple[str, str]: return parts[0], "all" +def get_component_from_path(file_path: str) -> str | None: + """Extract component name from a file path. + + Args: + file_path: Path to a file (e.g., "esphome/components/wifi/wifi.cpp") + + Returns: + Component name if path is in components directory, None otherwise + """ + if not file_path.startswith(ESPHOME_COMPONENTS_PATH): + return None + parts = file_path.split("/") + if len(parts) >= 3: + return parts[2] + return None + + +def get_component_test_files( + component: str, *, all_variants: bool = False +) -> list[Path]: + """Get test files for a component. + + Args: + component: Component name (e.g., "wifi") + all_variants: If True, returns all test files including variants (test-*.yaml). + If False, returns only base test files (test.*.yaml). + Default is False. + + Returns: + List of test file paths for the component, or empty list if none exist + """ + tests_dir = Path(root_path) / "tests" / "components" / component + if not tests_dir.exists(): + return [] + + if all_variants: + # Match both test.*.yaml and test-*.yaml patterns + return list(tests_dir.glob("test[.-]*.yaml")) + # Match only test.*.yaml (base tests) + return list(tests_dir.glob("test.*.yaml")) + + def styled(color: str | tuple[str, ...], msg: str, reset: bool = True) -> str: prefix = "".join(color) if isinstance(color, tuple) else color suffix = colorama.Style.RESET_ALL if reset else "" @@ -331,11 +385,9 @@ def _filter_changed_ci(files: list[str]) -> list[str]: # because changes in one file can affect other files in the same component. filtered_files = [] for f in files: - if f.startswith(ESPHOME_COMPONENTS_PATH): - # Check if file belongs to any of the changed components - parts = f.split("/") - if len(parts) >= 3 and parts[2] in component_set: - filtered_files.append(f) + component = get_component_from_path(f) + if component and component in component_set: + filtered_files.append(f) return filtered_files diff --git a/script/list-components.py b/script/list-components.py index 9abb2bc345..11533ceb30 100755 --- a/script/list-components.py +++ b/script/list-components.py @@ -4,7 +4,7 @@ from collections.abc import Callable from pathlib import Path import sys -from helpers import changed_files, git_ls_files +from helpers import changed_files, get_component_from_path, git_ls_files from esphome.const import ( KEY_CORE, @@ -30,11 +30,9 @@ def get_all_component_files() -> list[str]: def extract_component_names_array_from_files_array(files): components = [] for file in files: - file_parts = file.split("/") - if len(file_parts) >= 4: - component_name = file_parts[2] - if component_name not in components: - components.append(component_name) + component_name = get_component_from_path(file) + if component_name and component_name not in components: + components.append(component_name) return components diff --git a/script/split_components_for_ci.py b/script/split_components_for_ci.py index dff46d3619..6ba2598eda 100755 --- a/script/split_components_for_ci.py +++ b/script/split_components_for_ci.py @@ -28,6 +28,7 @@ from script.analyze_component_buses import ( create_grouping_signature, merge_compatible_bus_groups, ) +from script.helpers import get_component_test_files # Weighting for batch creation # Isolated components can't be grouped/merged, so they count as 10x @@ -45,17 +46,12 @@ def has_test_files(component_name: str, tests_dir: Path) -> bool: Args: component_name: Name of the component - tests_dir: Path to tests/components directory + tests_dir: Path to tests/components directory (unused, kept for compatibility) Returns: True if the component has test.*.yaml files """ - component_dir = tests_dir / component_name - if not component_dir.exists() or not component_dir.is_dir(): - return False - - # Check for test.*.yaml files - return any(component_dir.glob("test.*.yaml")) + return bool(get_component_test_files(component_name)) def create_intelligent_batches( diff --git a/script/test_build_components.py b/script/test_build_components.py index 07f2680799..77c97a8773 100755 --- a/script/test_build_components.py +++ b/script/test_build_components.py @@ -39,6 +39,7 @@ from script.analyze_component_buses import ( merge_compatible_bus_groups, uses_local_file_references, ) +from script.helpers import get_component_test_files from script.merge_component_configs import merge_component_configs @@ -100,10 +101,10 @@ def find_component_tests( if not comp_dir.is_dir(): continue - # Find test files - either base only (test.*.yaml) or all (test[.-]*.yaml) - pattern = "test.*.yaml" if base_only else "test[.-]*.yaml" - for test_file in comp_dir.glob(pattern): - component_tests[comp_dir.name].append(test_file) + # Get test files using helper function + test_files = get_component_test_files(comp_dir.name, all_variants=not base_only) + if test_files: + component_tests[comp_dir.name] = test_files return dict(component_tests) From b95999aca7cc2d39ff6846627b8a0936f409465d Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Fri, 17 Oct 2025 17:55:37 -1000 Subject: [PATCH 60/76] Update esphome/analyze_memory/__init__.py Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- esphome/analyze_memory/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/esphome/analyze_memory/__init__.py b/esphome/analyze_memory/__init__.py index 5ef9eab526..74299d4e95 100644 --- a/esphome/analyze_memory/__init__.py +++ b/esphome/analyze_memory/__init__.py @@ -295,7 +295,7 @@ class MemoryAnalyzer: cppfilt_cmd = "c++filt" _LOGGER.warning("Demangling %d symbols", len(symbols)) - _LOGGER.warning("objdump_path = %s", self.objdump_path) + _LOGGER.debug("objdump_path = %s", self.objdump_path) # Check if we have a toolchain-specific c++filt if self.objdump_path and self.objdump_path != "objdump": From 9a4288d81a02e7484e393f97a89fab856ae6e4e9 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Fri, 17 Oct 2025 17:56:41 -1000 Subject: [PATCH 61/76] Update script/determine-jobs.py Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- script/determine-jobs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/script/determine-jobs.py b/script/determine-jobs.py index 5767ced859..26e91edbe1 100755 --- a/script/determine-jobs.py +++ b/script/determine-jobs.py @@ -26,7 +26,7 @@ The CI workflow uses this information to: - Skip or run Python linters (ruff, flake8, pylint, pyupgrade) - Determine which components to test individually - Decide how to split component tests (if there are many) -- Run memory impact analysis when exactly one component changes +- Run memory impact analysis whenever there are changed components (merged config), and also for core-only changes Usage: python script/determine-jobs.py [-b BRANCH] From a96cc5e6f20a8b7205a48ea38836fb22ff012239 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Fri, 17 Oct 2025 17:57:33 -1000 Subject: [PATCH 62/76] Update esphome/analyze_memory/__init__.py Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- esphome/analyze_memory/__init__.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/esphome/analyze_memory/__init__.py b/esphome/analyze_memory/__init__.py index 74299d4e95..3e85c4d869 100644 --- a/esphome/analyze_memory/__init__.py +++ b/esphome/analyze_memory/__init__.py @@ -294,24 +294,24 @@ class MemoryAnalyzer: # Try to find the appropriate c++filt for the platform cppfilt_cmd = "c++filt" - _LOGGER.warning("Demangling %d symbols", len(symbols)) + _LOGGER.info("Demangling %d symbols", len(symbols)) _LOGGER.debug("objdump_path = %s", self.objdump_path) # Check if we have a toolchain-specific c++filt if self.objdump_path and self.objdump_path != "objdump": # Replace objdump with c++filt in the path potential_cppfilt = self.objdump_path.replace("objdump", "c++filt") - _LOGGER.warning("Checking for toolchain c++filt at: %s", potential_cppfilt) + _LOGGER.info("Checking for toolchain c++filt at: %s", potential_cppfilt) if Path(potential_cppfilt).exists(): cppfilt_cmd = potential_cppfilt - _LOGGER.warning("✓ Using toolchain c++filt: %s", cppfilt_cmd) + _LOGGER.info("✓ Using toolchain c++filt: %s", cppfilt_cmd) else: - _LOGGER.warning( + _LOGGER.info( "✗ Toolchain c++filt not found at %s, using system c++filt", potential_cppfilt, ) else: - _LOGGER.warning( + _LOGGER.info( "✗ Using system c++filt (objdump_path=%s)", self.objdump_path ) From 0b09e506854decd24b44a6ee77e2831f5a193859 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Fri, 17 Oct 2025 17:57:42 -1000 Subject: [PATCH 63/76] preen --- esphome/analyze_memory/cli.py | 4 ++-- script/determine-jobs.py | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/esphome/analyze_memory/cli.py b/esphome/analyze_memory/cli.py index 7b004353ec..bcf9f45de9 100644 --- a/esphome/analyze_memory/cli.py +++ b/esphome/analyze_memory/cli.py @@ -183,9 +183,9 @@ class MemoryAnalyzerCLI(MemoryAnalyzer): f"{len(symbols):>{self.COL_CORE_COUNT}} | {percentage:>{self.COL_CORE_PERCENT - 1}.1f}%" ) - # Top 10 largest core symbols + # Top 15 largest core symbols lines.append("") - lines.append("Top 10 Largest [esphome]core Symbols:") + lines.append("Top 15 Largest [esphome]core Symbols:") sorted_core_symbols = sorted( self._esphome_core_symbols, key=lambda x: x[2], reverse=True ) diff --git a/script/determine-jobs.py b/script/determine-jobs.py index 5767ced859..bcc357d953 100755 --- a/script/determine-jobs.py +++ b/script/determine-jobs.py @@ -13,9 +13,9 @@ what files have changed. It outputs JSON with the following structure: "component_test_count": 5, "memory_impact": { "should_run": "true/false", - "component": "component_name", - "test_file": "test.esp32-idf.yaml", - "platform": "esp32-idf" + "components": ["component1", "component2", ...], + "platform": "esp32-idf", + "use_merged_config": "true" } } @@ -26,7 +26,7 @@ The CI workflow uses this information to: - Skip or run Python linters (ruff, flake8, pylint, pyupgrade) - Determine which components to test individually - Decide how to split component tests (if there are many) -- Run memory impact analysis when exactly one component changes +- Run memory impact analysis when components change Usage: python script/determine-jobs.py [-b BRANCH] From bbd636a8cc7fecd046ef16cc919a71bf37e3db97 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci-lite[bot]" <117423508+pre-commit-ci-lite[bot]@users.noreply.github.com> Date: Sat, 18 Oct 2025 03:59:23 +0000 Subject: [PATCH 64/76] [pre-commit.ci lite] apply automatic fixes --- esphome/analyze_memory/__init__.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/esphome/analyze_memory/__init__.py b/esphome/analyze_memory/__init__.py index 3e85c4d869..b5d574807e 100644 --- a/esphome/analyze_memory/__init__.py +++ b/esphome/analyze_memory/__init__.py @@ -311,9 +311,7 @@ class MemoryAnalyzer: potential_cppfilt, ) else: - _LOGGER.info( - "✗ Using system c++filt (objdump_path=%s)", self.objdump_path - ) + _LOGGER.info("✗ Using system c++filt (objdump_path=%s)", self.objdump_path) # Strip GCC optimization suffixes and prefixes before demangling # Suffixes like $isra$0, $part$0, $constprop$0 confuse c++filt From 9cf1fd24fd5e1a91bbc5fe49ed941b60dce5eb49 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Fri, 17 Oct 2025 18:06:13 -1000 Subject: [PATCH 65/76] preen --- esphome/analyze_memory/__init__.py | 42 +++---- esphome/analyze_memory/cli.py | 2 +- script/ci_memory_impact_comment.py | 196 +++++++++++++---------------- script/ci_memory_impact_extract.py | 15 +-- 4 files changed, 116 insertions(+), 139 deletions(-) diff --git a/esphome/analyze_memory/__init__.py b/esphome/analyze_memory/__init__.py index 3e85c4d869..942caabe70 100644 --- a/esphome/analyze_memory/__init__.py +++ b/esphome/analyze_memory/__init__.py @@ -77,7 +77,7 @@ class MemoryAnalyzer: readelf_path: str | None = None, external_components: set[str] | None = None, idedata: "IDEData | None" = None, - ): + ) -> None: """Initialize memory analyzer. Args: @@ -311,15 +311,13 @@ class MemoryAnalyzer: potential_cppfilt, ) else: - _LOGGER.info( - "✗ Using system c++filt (objdump_path=%s)", self.objdump_path - ) + _LOGGER.info("✗ Using system c++filt (objdump_path=%s)", self.objdump_path) # Strip GCC optimization suffixes and prefixes before demangling # Suffixes like $isra$0, $part$0, $constprop$0 confuse c++filt # Prefixes like _GLOBAL__sub_I_ need to be removed and tracked - symbols_stripped = [] - symbols_prefixes = [] # Track removed prefixes + symbols_stripped: list[str] = [] + symbols_prefixes: list[str] = [] # Track removed prefixes for symbol in symbols: # Remove GCC optimization markers stripped = re.sub(r"\$(?:isra|part|constprop)\$\d+", "", symbol) @@ -327,12 +325,11 @@ class MemoryAnalyzer: # Handle GCC global constructor/initializer prefixes # _GLOBAL__sub_I_ -> extract for demangling prefix = "" - if stripped.startswith("_GLOBAL__sub_I_"): - prefix = "_GLOBAL__sub_I_" - stripped = stripped[len(prefix) :] - elif stripped.startswith("_GLOBAL__sub_D_"): - prefix = "_GLOBAL__sub_D_" - stripped = stripped[len(prefix) :] + for gcc_prefix in _GCC_PREFIX_ANNOTATIONS: + if stripped.startswith(gcc_prefix): + prefix = gcc_prefix + stripped = stripped[len(prefix) :] + break symbols_stripped.append(stripped) symbols_prefixes.append(prefix) @@ -405,17 +402,18 @@ class MemoryAnalyzer: if stripped == demangled and stripped.startswith("_Z"): failed_count += 1 if failed_count <= 5: # Only log first 5 failures - _LOGGER.warning("Failed to demangle: %s", original[:100]) + _LOGGER.warning("Failed to demangle: %s", original) - if failed_count > 0: - _LOGGER.warning( - "Failed to demangle %d/%d symbols using %s", - failed_count, - len(symbols), - cppfilt_cmd, - ) - else: - _LOGGER.warning("Successfully demangled all %d symbols", len(symbols)) + if failed_count == 0: + _LOGGER.info("Successfully demangled all %d symbols", len(symbols)) + return + + _LOGGER.warning( + "Failed to demangle %d/%d symbols using %s", + failed_count, + len(symbols), + cppfilt_cmd, + ) @staticmethod def _restore_symbol_prefix(prefix: str, stripped: str, demangled: str) -> str: diff --git a/esphome/analyze_memory/cli.py b/esphome/analyze_memory/cli.py index bcf9f45de9..a2366430dd 100644 --- a/esphome/analyze_memory/cli.py +++ b/esphome/analyze_memory/cli.py @@ -83,7 +83,7 @@ class MemoryAnalyzerCLI(MemoryAnalyzer): total_ram = sum(c.ram_total for _, c in components) # Build report - lines = [] + lines: list[str] = [] lines.append("=" * self.TABLE_WIDTH) lines.append("Component Memory Analysis".center(self.TABLE_WIDTH)) diff --git a/script/ci_memory_impact_comment.py b/script/ci_memory_impact_comment.py index 8b0dbb6f58..d177b101a8 100755 --- a/script/ci_memory_impact_comment.py +++ b/script/ci_memory_impact_comment.py @@ -411,137 +411,115 @@ def find_existing_comment(pr_number: str) -> str | None: Returns: Comment numeric ID if found, None otherwise + + Raises: + subprocess.CalledProcessError: If gh command fails """ - try: - print( - f"DEBUG: Looking for existing comment on PR #{pr_number}", file=sys.stderr - ) + print(f"DEBUG: Looking for existing comment on PR #{pr_number}", file=sys.stderr) - # Use gh api to get comments directly - this returns the numeric id field - result = subprocess.run( - [ - "gh", - "api", - f"/repos/{{owner}}/{{repo}}/issues/{pr_number}/comments", - "--jq", - ".[] | {id, body}", - ], - capture_output=True, - text=True, - check=True, - ) + # Use gh api to get comments directly - this returns the numeric id field + result = subprocess.run( + [ + "gh", + "api", + f"/repos/{{owner}}/{{repo}}/issues/{pr_number}/comments", + "--jq", + ".[] | {id, body}", + ], + capture_output=True, + text=True, + check=True, + ) - print( - f"DEBUG: gh api comments output (first 500 chars):\n{result.stdout[:500]}", - file=sys.stderr, - ) + print( + f"DEBUG: gh api comments output (first 500 chars):\n{result.stdout[:500]}", + file=sys.stderr, + ) - # Parse comments and look for our marker - comment_count = 0 - for line in result.stdout.strip().split("\n"): - if not line: - continue + # Parse comments and look for our marker + comment_count = 0 + for line in result.stdout.strip().split("\n"): + if not line: + continue - try: - comment = json.loads(line) - comment_count += 1 - comment_id = comment.get("id") + try: + comment = json.loads(line) + comment_count += 1 + comment_id = comment.get("id") + print( + f"DEBUG: Checking comment {comment_count}: id={comment_id}", + file=sys.stderr, + ) + + body = comment.get("body", "") + if COMMENT_MARKER in body: print( - f"DEBUG: Checking comment {comment_count}: id={comment_id}", + f"DEBUG: Found existing comment with id={comment_id}", file=sys.stderr, ) + # Return the numeric id + return str(comment_id) + print("DEBUG: Comment does not contain marker", file=sys.stderr) + except json.JSONDecodeError as e: + print(f"DEBUG: JSON decode error: {e}", file=sys.stderr) + continue - body = comment.get("body", "") - if COMMENT_MARKER in body: - print( - f"DEBUG: Found existing comment with id={comment_id}", - file=sys.stderr, - ) - # Return the numeric id - return str(comment_id) - print("DEBUG: Comment does not contain marker", file=sys.stderr) - except json.JSONDecodeError as e: - print(f"DEBUG: JSON decode error: {e}", file=sys.stderr) - continue - - print( - f"DEBUG: No existing comment found (checked {comment_count} comments)", - file=sys.stderr, - ) - return None - - except subprocess.CalledProcessError as e: - print(f"Error finding existing comment: {e}", file=sys.stderr) - if e.stderr: - print(f"stderr: {e.stderr.decode()}", file=sys.stderr) - return None + print( + f"DEBUG: No existing comment found (checked {comment_count} comments)", + file=sys.stderr, + ) + return None -def post_or_update_comment(pr_number: str, comment_body: str) -> bool: +def post_or_update_comment(pr_number: str, comment_body: str) -> None: """Post a new comment or update existing one. Args: pr_number: PR number comment_body: Comment body text - Returns: - True if successful, False otherwise + Raises: + subprocess.CalledProcessError: If gh command fails """ # Look for existing comment existing_comment_id = find_existing_comment(pr_number) - try: - if existing_comment_id and existing_comment_id != "None": - # Update existing comment - print( - f"DEBUG: Updating existing comment {existing_comment_id}", - file=sys.stderr, - ) - result = subprocess.run( - [ - "gh", - "api", - f"/repos/{{owner}}/{{repo}}/issues/comments/{existing_comment_id}", - "-X", - "PATCH", - "-f", - f"body={comment_body}", - ], - check=True, - capture_output=True, - text=True, - ) - print(f"DEBUG: Update response: {result.stdout}", file=sys.stderr) - else: - # Post new comment - print( - f"DEBUG: Posting new comment (existing_comment_id={existing_comment_id})", - file=sys.stderr, - ) - result = subprocess.run( - ["gh", "pr", "comment", pr_number, "--body", comment_body], - check=True, - capture_output=True, - text=True, - ) - print(f"DEBUG: Post response: {result.stdout}", file=sys.stderr) + if existing_comment_id and existing_comment_id != "None": + # Update existing comment + print( + f"DEBUG: Updating existing comment {existing_comment_id}", + file=sys.stderr, + ) + result = subprocess.run( + [ + "gh", + "api", + f"/repos/{{owner}}/{{repo}}/issues/comments/{existing_comment_id}", + "-X", + "PATCH", + "-f", + f"body={comment_body}", + ], + check=True, + capture_output=True, + text=True, + ) + print(f"DEBUG: Update response: {result.stdout}", file=sys.stderr) + else: + # Post new comment + print( + f"DEBUG: Posting new comment (existing_comment_id={existing_comment_id})", + file=sys.stderr, + ) + result = subprocess.run( + ["gh", "pr", "comment", pr_number, "--body", comment_body], + check=True, + capture_output=True, + text=True, + ) + print(f"DEBUG: Post response: {result.stdout}", file=sys.stderr) - print("Comment posted/updated successfully", file=sys.stderr) - return True - - except subprocess.CalledProcessError as e: - print(f"Error posting/updating comment: {e}", file=sys.stderr) - if e.stderr: - print( - f"stderr: {e.stderr.decode() if isinstance(e.stderr, bytes) else e.stderr}", - file=sys.stderr, - ) - if e.stdout: - print( - f"stdout: {e.stdout.decode() if isinstance(e.stdout, bytes) else e.stdout}", - file=sys.stderr, - ) - return False + print("Comment posted/updated successfully", file=sys.stderr) def main() -> int: diff --git a/script/ci_memory_impact_extract.py b/script/ci_memory_impact_extract.py index 76632ebc33..5522d522f0 100755 --- a/script/ci_memory_impact_extract.py +++ b/script/ci_memory_impact_extract.py @@ -27,6 +27,11 @@ sys.path.insert(0, str(Path(__file__).parent.parent)) # pylint: disable=wrong-import-position from script.ci_helpers import write_github_output +# Regex patterns for extracting memory usage from PlatformIO output +_RAM_PATTERN = re.compile(r"RAM:\s+\[.*?\]\s+\d+\.\d+%\s+\(used\s+(\d+)\s+bytes") +_FLASH_PATTERN = re.compile(r"Flash:\s+\[.*?\]\s+\d+\.\d+%\s+\(used\s+(\d+)\s+bytes") +_BUILD_PATH_PATTERN = re.compile(r"Build path: (.+)") + def extract_from_compile_output( output_text: str, @@ -42,7 +47,7 @@ def extract_from_compile_output( Flash: [=== ] 34.0% (used 348511 bytes from 1023984 bytes) Also extracts build directory from lines like: - INFO Deleting /path/to/build/.esphome/build/componenttestesp8266ard/.pioenvs + INFO Compiling app... Build path: /path/to/build Args: output_text: Compile output text (may contain multiple builds) @@ -51,12 +56,8 @@ def extract_from_compile_output( Tuple of (total_ram_bytes, total_flash_bytes, build_dir) or (None, None, None) if not found """ # Find all RAM and Flash matches (may be multiple builds) - ram_matches = re.findall( - r"RAM:\s+\[.*?\]\s+\d+\.\d+%\s+\(used\s+(\d+)\s+bytes", output_text - ) - flash_matches = re.findall( - r"Flash:\s+\[.*?\]\s+\d+\.\d+%\s+\(used\s+(\d+)\s+bytes", output_text - ) + ram_matches = _RAM_PATTERN.findall(output_text) + flash_matches = _FLASH_PATTERN.findall(output_text) if not ram_matches or not flash_matches: return None, None, None From 0b077bdfc62c2d2923356ecec37ad27821b610aa Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Fri, 17 Oct 2025 18:08:52 -1000 Subject: [PATCH 66/76] preen --- script/ci_memory_impact_extract.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/script/ci_memory_impact_extract.py b/script/ci_memory_impact_extract.py index 5522d522f0..17ac788ae3 100755 --- a/script/ci_memory_impact_extract.py +++ b/script/ci_memory_impact_extract.py @@ -70,7 +70,7 @@ def extract_from_compile_output( # Look for: INFO Compiling app... Build path: /path/to/build # Note: Multiple builds reuse the same build path (each overwrites the previous) build_dir = None - if match := re.search(r"Build path: (.+)", output_text): + if match := _BUILD_PATH_PATTERN.search(output_text): build_dir = match.group(1).strip() return total_ram, total_flash, build_dir @@ -210,11 +210,7 @@ def main() -> int: return 1 # Count how many builds were found - num_builds = len( - re.findall( - r"RAM:\s+\[.*?\]\s+\d+\.\d+%\s+\(used\s+(\d+)\s+bytes", compile_output - ) - ) + num_builds = len(_RAM_PATTERN.findall(compile_output)) if num_builds > 1: print( From 07ad32968e585919374e9c7c891bdb355501a9f9 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Fri, 17 Oct 2025 18:15:46 -1000 Subject: [PATCH 67/76] template all the things --- script/ci_memory_impact_comment.py | 296 +++++++----------- .../ci_memory_impact_comment_template.j2 | 27 ++ .../ci_memory_impact_component_breakdown.j2 | 15 + script/templates/ci_memory_impact_macros.j2 | 8 + .../ci_memory_impact_symbol_changes.j2 | 51 +++ 5 files changed, 216 insertions(+), 181 deletions(-) create mode 100644 script/templates/ci_memory_impact_comment_template.j2 create mode 100644 script/templates/ci_memory_impact_component_breakdown.j2 create mode 100644 script/templates/ci_memory_impact_macros.j2 create mode 100644 script/templates/ci_memory_impact_symbol_changes.j2 diff --git a/script/ci_memory_impact_comment.py b/script/ci_memory_impact_comment.py index d177b101a8..961c304e40 100755 --- a/script/ci_memory_impact_comment.py +++ b/script/ci_memory_impact_comment.py @@ -14,6 +14,8 @@ from pathlib import Path import subprocess import sys +from jinja2 import Environment, FileSystemLoader + # Add esphome to path for analyze_memory import sys.path.insert(0, str(Path(__file__).parent.parent)) @@ -26,6 +28,22 @@ COMMENT_MARKER = "" OVERALL_CHANGE_THRESHOLD = 1.0 # Overall RAM/Flash changes COMPONENT_CHANGE_THRESHOLD = 3.0 # Component breakdown changes +# Display limits for tables +MAX_COMPONENT_BREAKDOWN_ROWS = 20 # Maximum components to show in breakdown table +MAX_CHANGED_SYMBOLS_ROWS = 30 # Maximum changed symbols to show +MAX_NEW_SYMBOLS_ROWS = 15 # Maximum new symbols to show +MAX_REMOVED_SYMBOLS_ROWS = 15 # Maximum removed symbols to show + +# Symbol display formatting +SYMBOL_DISPLAY_MAX_LENGTH = 100 # Max length before using
tag +SYMBOL_DISPLAY_TRUNCATE_LENGTH = 97 # Length to truncate in summary + +# Component change noise threshold +COMPONENT_CHANGE_NOISE_THRESHOLD = 2 # Ignore component changes ≤ this many bytes + +# Template directory +TEMPLATE_DIR = Path(__file__).parent / "templates" + def load_analysis_json(json_path: str) -> dict | None: """Load memory analysis results from JSON file. @@ -111,35 +129,20 @@ def format_change(before: int, after: int, threshold: float | None = None) -> st return f"{emoji} {delta_str} ({pct_str})" -def format_symbol_for_display(symbol: str) -> str: - """Format a symbol name for display in markdown table. - - Args: - symbol: Symbol name to format - - Returns: - Formatted symbol with backticks or HTML details tag for long names - """ - if len(symbol) <= 100: - return f"`{symbol}`" - # Use HTML details for very long symbols (no backticks inside HTML) - return f"
{symbol[:97]}...{symbol}
" - - -def create_symbol_changes_table( +def prepare_symbol_changes_data( target_symbols: dict | None, pr_symbols: dict | None -) -> str: - """Create a markdown table showing symbols that changed size. +) -> dict | None: + """Prepare symbol changes data for template rendering. Args: target_symbols: Symbol name to size mapping for target branch pr_symbols: Symbol name to size mapping for PR branch Returns: - Formatted markdown table + Dictionary with changed, new, and removed symbols, or None if no changes """ if not target_symbols or not pr_symbols: - return "" + return None # Find all symbols that exist in both branches or only in one all_symbols = set(target_symbols.keys()) | set(pr_symbols.keys()) @@ -165,113 +168,39 @@ def create_symbol_changes_table( changed_symbols.append((symbol, target_size, pr_size, delta)) if not changed_symbols and not new_symbols and not removed_symbols: - return "" + return None - lines = [ - "", - "
", - "🔍 Symbol-Level Changes (click to expand)", - "", - ] + # Sort by size/delta + changed_symbols.sort(key=lambda x: abs(x[3]), reverse=True) + new_symbols.sort(key=lambda x: x[1], reverse=True) + removed_symbols.sort(key=lambda x: x[1], reverse=True) - # Show changed symbols (sorted by absolute delta) - if changed_symbols: - changed_symbols.sort(key=lambda x: abs(x[3]), reverse=True) - lines.extend( - [ - "### Changed Symbols", - "", - "| Symbol | Target Size | PR Size | Change |", - "|--------|-------------|---------|--------|", - ] - ) - - # Show top 30 changes - for symbol, target_size, pr_size, delta in changed_symbols[:30]: - target_str = format_bytes(target_size) - pr_str = format_bytes(pr_size) - change_str = format_change(target_size, pr_size) # Chart icons only - display_symbol = format_symbol_for_display(symbol) - lines.append( - f"| {display_symbol} | {target_str} | {pr_str} | {change_str} |" - ) - - if len(changed_symbols) > 30: - lines.append( - f"| ... | ... | ... | *({len(changed_symbols) - 30} more changed symbols not shown)* |" - ) - lines.append("") - - # Show new symbols - if new_symbols: - new_symbols.sort(key=lambda x: x[1], reverse=True) - lines.extend( - [ - "### New Symbols (top 15)", - "", - "| Symbol | Size |", - "|--------|------|", - ] - ) - - for symbol, size in new_symbols[:15]: - display_symbol = format_symbol_for_display(symbol) - lines.append(f"| {display_symbol} | {format_bytes(size)} |") - - if len(new_symbols) > 15: - total_new_size = sum(s[1] for s in new_symbols) - lines.append( - f"| *{len(new_symbols) - 15} more new symbols...* | *Total: {format_bytes(total_new_size)}* |" - ) - lines.append("") - - # Show removed symbols - if removed_symbols: - removed_symbols.sort(key=lambda x: x[1], reverse=True) - lines.extend( - [ - "### Removed Symbols (top 15)", - "", - "| Symbol | Size |", - "|--------|------|", - ] - ) - - for symbol, size in removed_symbols[:15]: - display_symbol = format_symbol_for_display(symbol) - lines.append(f"| {display_symbol} | {format_bytes(size)} |") - - if len(removed_symbols) > 15: - total_removed_size = sum(s[1] for s in removed_symbols) - lines.append( - f"| *{len(removed_symbols) - 15} more removed symbols...* | *Total: {format_bytes(total_removed_size)}* |" - ) - lines.append("") - - lines.extend(["
", ""]) - - return "\n".join(lines) + return { + "changed_symbols": changed_symbols, + "new_symbols": new_symbols, + "removed_symbols": removed_symbols, + } -def create_detailed_breakdown_table( +def prepare_component_breakdown_data( target_analysis: dict | None, pr_analysis: dict | None -) -> str: - """Create a markdown table showing detailed memory breakdown by component. +) -> list[tuple[str, int, int, int]] | None: + """Prepare component breakdown data for template rendering. Args: target_analysis: Component memory breakdown for target branch pr_analysis: Component memory breakdown for PR branch Returns: - Formatted markdown table + List of tuples (component, target_flash, pr_flash, delta), or None if no changes """ if not target_analysis or not pr_analysis: - return "" + return None # Combine all components from both analyses all_components = set(target_analysis.keys()) | set(pr_analysis.keys()) - # Filter to components that have changed (ignoring noise ≤2 bytes) + # Filter to components that have changed (ignoring noise) changed_components = [] for comp in all_components: target_mem = target_analysis.get(comp, {}) @@ -280,43 +209,18 @@ def create_detailed_breakdown_table( target_flash = target_mem.get("flash_total", 0) pr_flash = pr_mem.get("flash_total", 0) - # Only include if component has meaningful change (>2 bytes) + # Only include if component has meaningful change (above noise threshold) delta = pr_flash - target_flash - if abs(delta) > 2: + if abs(delta) > COMPONENT_CHANGE_NOISE_THRESHOLD: changed_components.append((comp, target_flash, pr_flash, delta)) if not changed_components: - return "" + return None # Sort by absolute delta (largest changes first) changed_components.sort(key=lambda x: abs(x[3]), reverse=True) - # Build table - limit to top 20 changes - lines = [ - "", - "
", - "📊 Component Memory Breakdown", - "", - "| Component | Target Flash | PR Flash | Change |", - "|-----------|--------------|----------|--------|", - ] - - for comp, target_flash, pr_flash, delta in changed_components[:20]: - target_str = format_bytes(target_flash) - pr_str = format_bytes(pr_flash) - # Only apply threshold to ESPHome components, not framework/infrastructure - threshold = COMPONENT_CHANGE_THRESHOLD if comp.startswith("[esphome]") else None - change_str = format_change(target_flash, pr_flash, threshold=threshold) - lines.append(f"| `{comp}` | {target_str} | {pr_str} | {change_str} |") - - if len(changed_components) > 20: - lines.append( - f"| ... | ... | ... | *({len(changed_components) - 20} more components not shown)* |" - ) - - lines.extend(["", "
", ""]) - - return "\n".join(lines) + return changed_components def create_comment_body( @@ -332,7 +236,7 @@ def create_comment_body( pr_symbols: dict | None = None, target_cache_hit: bool = False, ) -> str: - """Create the comment body with memory impact analysis. + """Create the comment body with memory impact analysis using Jinja2 templates. Args: components: List of component names (merged config) @@ -350,57 +254,87 @@ def create_comment_body( Returns: Formatted comment body """ - ram_change = format_change(target_ram, pr_ram, threshold=OVERALL_CHANGE_THRESHOLD) - flash_change = format_change( - target_flash, pr_flash, threshold=OVERALL_CHANGE_THRESHOLD + # Set up Jinja2 environment + env = Environment( + loader=FileSystemLoader(TEMPLATE_DIR), + trim_blocks=True, + lstrip_blocks=True, ) - # Use provided analysis data if available - component_breakdown = "" - symbol_changes = "" + # Register custom filters + env.filters["format_bytes"] = format_bytes + env.filters["format_change"] = format_change - if target_analysis and pr_analysis: - component_breakdown = create_detailed_breakdown_table( - target_analysis, pr_analysis - ) - - if target_symbols and pr_symbols: - symbol_changes = create_symbol_changes_table(target_symbols, pr_symbols) - else: - print("No ELF files provided, skipping detailed analysis", file=sys.stderr) + # Prepare template context + context = { + "comment_marker": COMMENT_MARKER, + "platform": platform, + "target_ram": format_bytes(target_ram), + "pr_ram": format_bytes(pr_ram), + "target_flash": format_bytes(target_flash), + "pr_flash": format_bytes(pr_flash), + "ram_change": format_change( + target_ram, pr_ram, threshold=OVERALL_CHANGE_THRESHOLD + ), + "flash_change": format_change( + target_flash, pr_flash, threshold=OVERALL_CHANGE_THRESHOLD + ), + "target_cache_hit": target_cache_hit, + "component_change_threshold": COMPONENT_CHANGE_THRESHOLD, + } # Format components list if len(components) == 1: - components_str = f"`{components[0]}`" - config_note = "a representative test configuration" + context["components_str"] = f"`{components[0]}`" + context["config_note"] = "a representative test configuration" else: - components_str = ", ".join(f"`{c}`" for c in sorted(components)) - config_note = f"a merged configuration with {len(components)} components" + context["components_str"] = ", ".join(f"`{c}`" for c in sorted(components)) + context["config_note"] = ( + f"a merged configuration with {len(components)} components" + ) - # Add cache info note if target was cached - cache_note = "" - if target_cache_hit: - cache_note = "\n\n> ⚡ Target branch analysis was loaded from cache (build skipped for faster CI)." + # Prepare component breakdown if available + component_breakdown = "" + if target_analysis and pr_analysis: + changed_components = prepare_component_breakdown_data( + target_analysis, pr_analysis + ) + if changed_components: + template = env.get_template("ci_memory_impact_component_breakdown.j2") + component_breakdown = template.render( + changed_components=changed_components, + format_bytes=format_bytes, + format_change=format_change, + component_change_threshold=COMPONENT_CHANGE_THRESHOLD, + max_rows=MAX_COMPONENT_BREAKDOWN_ROWS, + ) - return f"""{COMMENT_MARKER} -## Memory Impact Analysis + # Prepare symbol changes if available + symbol_changes = "" + if target_symbols and pr_symbols: + symbol_data = prepare_symbol_changes_data(target_symbols, pr_symbols) + if symbol_data: + template = env.get_template("ci_memory_impact_symbol_changes.j2") + symbol_changes = template.render( + **symbol_data, + format_bytes=format_bytes, + format_change=format_change, + max_changed_rows=MAX_CHANGED_SYMBOLS_ROWS, + max_new_rows=MAX_NEW_SYMBOLS_ROWS, + max_removed_rows=MAX_REMOVED_SYMBOLS_ROWS, + symbol_max_length=SYMBOL_DISPLAY_MAX_LENGTH, + symbol_truncate_length=SYMBOL_DISPLAY_TRUNCATE_LENGTH, + ) -**Components:** {components_str} -**Platform:** `{platform}` + if not target_analysis or not pr_analysis: + print("No ELF files provided, skipping detailed analysis", file=sys.stderr) -| Metric | Target Branch | This PR | Change | -|--------|--------------|---------|--------| -| **RAM** | {format_bytes(target_ram)} | {format_bytes(pr_ram)} | {ram_change} | -| **Flash** | {format_bytes(target_flash)} | {format_bytes(pr_flash)} | {flash_change} | -{component_breakdown}{symbol_changes}{cache_note} + context["component_breakdown"] = component_breakdown + context["symbol_changes"] = symbol_changes ---- -> **Note:** This analysis measures **static RAM and Flash usage** only (compile-time allocation). -> **Dynamic memory (heap)** cannot be measured automatically. -> **⚠️ You must test this PR on a real device** to measure free heap and ensure no runtime memory issues. - -*This analysis runs automatically when components change. Memory usage is measured from {config_note}.* -""" + # Render main template + template = env.get_template("ci_memory_impact_comment_template.j2") + return template.render(**context) def find_existing_comment(pr_number: str) -> str | None: @@ -605,9 +539,9 @@ def main() -> int: ) # Post or update comment - success = post_or_update_comment(args.pr_number, comment_body) + post_or_update_comment(args.pr_number, comment_body) - return 0 if success else 1 + return 0 if __name__ == "__main__": diff --git a/script/templates/ci_memory_impact_comment_template.j2 b/script/templates/ci_memory_impact_comment_template.j2 new file mode 100644 index 0000000000..4c8d7f4865 --- /dev/null +++ b/script/templates/ci_memory_impact_comment_template.j2 @@ -0,0 +1,27 @@ +{{ comment_marker }} +## Memory Impact Analysis + +**Components:** {{ components_str }} +**Platform:** `{{ platform }}` + +| Metric | Target Branch | This PR | Change | +|--------|--------------|---------|--------| +| **RAM** | {{ target_ram }} | {{ pr_ram }} | {{ ram_change }} | +| **Flash** | {{ target_flash }} | {{ pr_flash }} | {{ flash_change }} | +{% if component_breakdown %} +{{ component_breakdown }} +{%- endif %} +{%- if symbol_changes %} +{{ symbol_changes }} +{%- endif %} +{%- if target_cache_hit %} + +> ⚡ Target branch analysis was loaded from cache (build skipped for faster CI). +{%- endif %} + +--- +> **Note:** This analysis measures **static RAM and Flash usage** only (compile-time allocation). +> **Dynamic memory (heap)** cannot be measured automatically. +> **⚠️ You must test this PR on a real device** to measure free heap and ensure no runtime memory issues. + +*This analysis runs automatically when components change. Memory usage is measured from {{ config_note }}.* diff --git a/script/templates/ci_memory_impact_component_breakdown.j2 b/script/templates/ci_memory_impact_component_breakdown.j2 new file mode 100644 index 0000000000..a781e5c546 --- /dev/null +++ b/script/templates/ci_memory_impact_component_breakdown.j2 @@ -0,0 +1,15 @@ + +
+📊 Component Memory Breakdown + +| Component | Target Flash | PR Flash | Change | +|-----------|--------------|----------|--------| +{% for comp, target_flash, pr_flash, delta in changed_components[:max_rows] -%} +{% set threshold = component_change_threshold if comp.startswith("[esphome]") else none -%} +| `{{ comp }}` | {{ target_flash|format_bytes }} | {{ pr_flash|format_bytes }} | {{ format_change(target_flash, pr_flash, threshold=threshold) }} | +{% endfor -%} +{% if changed_components|length > max_rows -%} +| ... | ... | ... | *({{ changed_components|length - max_rows }} more components not shown)* | +{% endif -%} + +
diff --git a/script/templates/ci_memory_impact_macros.j2 b/script/templates/ci_memory_impact_macros.j2 new file mode 100644 index 0000000000..9fb346a7c5 --- /dev/null +++ b/script/templates/ci_memory_impact_macros.j2 @@ -0,0 +1,8 @@ +{#- Macro for formatting symbol names in tables -#} +{%- macro format_symbol(symbol, max_length, truncate_length) -%} +{%- if symbol|length <= max_length -%} +`{{ symbol }}` +{%- else -%} +
{{ symbol[:truncate_length] }}...{{ symbol }}
+{%- endif -%} +{%- endmacro -%} diff --git a/script/templates/ci_memory_impact_symbol_changes.j2 b/script/templates/ci_memory_impact_symbol_changes.j2 new file mode 100644 index 0000000000..bd540712f8 --- /dev/null +++ b/script/templates/ci_memory_impact_symbol_changes.j2 @@ -0,0 +1,51 @@ +{%- from 'ci_memory_impact_macros.j2' import format_symbol -%} + +
+🔍 Symbol-Level Changes (click to expand) + +{%- if changed_symbols %} + +### Changed Symbols + +| Symbol | Target Size | PR Size | Change | +|--------|-------------|---------|--------| +{% for symbol, target_size, pr_size, delta in changed_symbols[:max_changed_rows] -%} +| {{ format_symbol(symbol, symbol_max_length, symbol_truncate_length) }} | {{ target_size|format_bytes }} | {{ pr_size|format_bytes }} | {{ format_change(target_size, pr_size) }} | +{% endfor -%} +{% if changed_symbols|length > max_changed_rows -%} +| ... | ... | ... | *({{ changed_symbols|length - max_changed_rows }} more changed symbols not shown)* | +{% endif -%} + +{%- endif %} +{%- if new_symbols %} + +### New Symbols (top {{ max_new_rows }}) + +| Symbol | Size | +|--------|------| +{% for symbol, size in new_symbols[:max_new_rows] -%} +| {{ format_symbol(symbol, symbol_max_length, symbol_truncate_length) }} | {{ size|format_bytes }} | +{% endfor -%} +{% if new_symbols|length > max_new_rows -%} +{% set total_new_size = new_symbols|sum(attribute=1) -%} +| *{{ new_symbols|length - max_new_rows }} more new symbols...* | *Total: {{ total_new_size|format_bytes }}* | +{% endif -%} + +{%- endif %} +{%- if removed_symbols %} + +### Removed Symbols (top {{ max_removed_rows }}) + +| Symbol | Size | +|--------|------| +{% for symbol, size in removed_symbols[:max_removed_rows] -%} +| {{ format_symbol(symbol, symbol_max_length, symbol_truncate_length) }} | {{ size|format_bytes }} | +{% endfor -%} +{% if removed_symbols|length > max_removed_rows -%} +{% set total_removed_size = removed_symbols|sum(attribute=1) -%} +| *{{ removed_symbols|length - max_removed_rows }} more removed symbols...* | *Total: {{ total_removed_size|format_bytes }}* | +{% endif -%} + +{%- endif %} + +
From ba18bb6a4fedb7946c0a462957fdbfe960bb1eb3 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Fri, 17 Oct 2025 18:18:15 -1000 Subject: [PATCH 68/76] template all the things --- script/ci_memory_impact_comment.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/script/ci_memory_impact_comment.py b/script/ci_memory_impact_comment.py index 961c304e40..5a399639f5 100755 --- a/script/ci_memory_impact_comment.py +++ b/script/ci_memory_impact_comment.py @@ -148,9 +148,11 @@ def prepare_symbol_changes_data( all_symbols = set(target_symbols.keys()) | set(pr_symbols.keys()) # Track changes - changed_symbols = [] - new_symbols = [] - removed_symbols = [] + changed_symbols: list[ + tuple[str, int, int, int] + ] = [] # (symbol, target_size, pr_size, delta) + new_symbols: list[tuple[str, int]] = [] # (symbol, size) + removed_symbols: list[tuple[str, int]] = [] # (symbol, size) for symbol in all_symbols: target_size = target_symbols.get(symbol, 0) @@ -201,7 +203,9 @@ def prepare_component_breakdown_data( all_components = set(target_analysis.keys()) | set(pr_analysis.keys()) # Filter to components that have changed (ignoring noise) - changed_components = [] + changed_components: list[ + tuple[str, int, int, int] + ] = [] # (comp, target_flash, pr_flash, delta) for comp in all_components: target_mem = target_analysis.get(comp, {}) pr_mem = pr_analysis.get(comp, {}) From a078486a878406a6fd85a8d995e9453bf1d52561 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Fri, 17 Oct 2025 18:21:28 -1000 Subject: [PATCH 69/76] update test --- tests/script/test_determine_jobs.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/script/test_determine_jobs.py b/tests/script/test_determine_jobs.py index f8557ef6b6..24c77b6ae9 100644 --- a/tests/script/test_determine_jobs.py +++ b/tests/script/test_determine_jobs.py @@ -17,6 +17,9 @@ script_dir = os.path.abspath( ) sys.path.insert(0, script_dir) +# Import helpers module for patching +import helpers # noqa: E402 + spec = importlib.util.spec_from_file_location( "determine_jobs", os.path.join(script_dir, "determine-jobs.py") ) @@ -478,9 +481,10 @@ def test_main_filters_components_without_tests( airthings_dir = tests_dir / "airthings_ble" airthings_dir.mkdir(parents=True) - # Mock root_path to use tmp_path + # Mock root_path to use tmp_path (need to patch both determine_jobs and helpers) with ( patch.object(determine_jobs, "root_path", str(tmp_path)), + patch.object(helpers, "root_path", str(tmp_path)), patch("sys.argv", ["determine-jobs.py"]), ): # Clear the cache since we're mocking root_path From 7e54803edea0b24f0892129e4a66d39dd44da5b3 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Fri, 17 Oct 2025 18:25:41 -1000 Subject: [PATCH 70/76] update test --- esphome/analyze_memory/cli.py | 19 +++---- script/ci_memory_impact_comment.py | 82 ++++++++++++++++++------------ script/ci_memory_impact_extract.py | 30 +++++------ 3 files changed, 75 insertions(+), 56 deletions(-) diff --git a/esphome/analyze_memory/cli.py b/esphome/analyze_memory/cli.py index a2366430dd..5713eac94c 100644 --- a/esphome/analyze_memory/cli.py +++ b/esphome/analyze_memory/cli.py @@ -371,15 +371,16 @@ def main(): idedata = None for idedata_path in idedata_candidates: - if idedata_path.exists(): - try: - with open(idedata_path, encoding="utf-8") as f: - raw_data = json.load(f) - idedata = IDEData(raw_data) - print(f"Loaded idedata from: {idedata_path}", file=sys.stderr) - break - except (json.JSONDecodeError, OSError) as e: - print(f"Warning: Failed to load idedata: {e}", file=sys.stderr) + if not idedata_path.exists(): + continue + try: + with open(idedata_path, encoding="utf-8") as f: + raw_data = json.load(f) + idedata = IDEData(raw_data) + print(f"Loaded idedata from: {idedata_path}", file=sys.stderr) + break + except (json.JSONDecodeError, OSError) as e: + print(f"Warning: Failed to load idedata: {e}", file=sys.stderr) if not idedata: print( diff --git a/script/ci_memory_impact_comment.py b/script/ci_memory_impact_comment.py index 5a399639f5..4e3fbb9086 100755 --- a/script/ci_memory_impact_comment.py +++ b/script/ci_memory_impact_comment.py @@ -409,6 +409,54 @@ def find_existing_comment(pr_number: str) -> str | None: return None +def update_existing_comment(comment_id: str, comment_body: str) -> None: + """Update an existing comment. + + Args: + comment_id: Comment ID to update + comment_body: New comment body text + + Raises: + subprocess.CalledProcessError: If gh command fails + """ + print(f"DEBUG: Updating existing comment {comment_id}", file=sys.stderr) + result = subprocess.run( + [ + "gh", + "api", + f"/repos/{{owner}}/{{repo}}/issues/comments/{comment_id}", + "-X", + "PATCH", + "-f", + f"body={comment_body}", + ], + check=True, + capture_output=True, + text=True, + ) + print(f"DEBUG: Update response: {result.stdout}", file=sys.stderr) + + +def create_new_comment(pr_number: str, comment_body: str) -> None: + """Create a new PR comment. + + Args: + pr_number: PR number + comment_body: Comment body text + + Raises: + subprocess.CalledProcessError: If gh command fails + """ + print(f"DEBUG: Posting new comment on PR #{pr_number}", file=sys.stderr) + result = subprocess.run( + ["gh", "pr", "comment", pr_number, "--body", comment_body], + check=True, + capture_output=True, + text=True, + ) + print(f"DEBUG: Post response: {result.stdout}", file=sys.stderr) + + def post_or_update_comment(pr_number: str, comment_body: str) -> None: """Post a new comment or update existing one. @@ -423,39 +471,9 @@ def post_or_update_comment(pr_number: str, comment_body: str) -> None: existing_comment_id = find_existing_comment(pr_number) if existing_comment_id and existing_comment_id != "None": - # Update existing comment - print( - f"DEBUG: Updating existing comment {existing_comment_id}", - file=sys.stderr, - ) - result = subprocess.run( - [ - "gh", - "api", - f"/repos/{{owner}}/{{repo}}/issues/comments/{existing_comment_id}", - "-X", - "PATCH", - "-f", - f"body={comment_body}", - ], - check=True, - capture_output=True, - text=True, - ) - print(f"DEBUG: Update response: {result.stdout}", file=sys.stderr) + update_existing_comment(existing_comment_id, comment_body) else: - # Post new comment - print( - f"DEBUG: Posting new comment (existing_comment_id={existing_comment_id})", - file=sys.stderr, - ) - result = subprocess.run( - ["gh", "pr", "comment", pr_number, "--body", comment_body], - check=True, - capture_output=True, - text=True, - ) - print(f"DEBUG: Post response: {result.stdout}", file=sys.stderr) + create_new_comment(pr_number, comment_body) print("Comment posted/updated successfully", file=sys.stderr) diff --git a/script/ci_memory_impact_extract.py b/script/ci_memory_impact_extract.py index 17ac788ae3..77d59417e3 100755 --- a/script/ci_memory_impact_extract.py +++ b/script/ci_memory_impact_extract.py @@ -25,6 +25,8 @@ import sys sys.path.insert(0, str(Path(__file__).parent.parent)) # pylint: disable=wrong-import-position +from esphome.analyze_memory import MemoryAnalyzer +from esphome.platformio_api import IDEData from script.ci_helpers import write_github_output # Regex patterns for extracting memory usage from PlatformIO output @@ -85,9 +87,6 @@ def run_detailed_analysis(build_dir: str) -> dict | None: Returns: Dictionary with analysis results or None if analysis fails """ - from esphome.analyze_memory import MemoryAnalyzer - from esphome.platformio_api import IDEData - build_path = Path(build_dir) if not build_path.exists(): print(f"Build directory not found: {build_dir}", file=sys.stderr) @@ -120,18 +119,19 @@ def run_detailed_analysis(build_dir: str) -> dict | None: idedata = None for idedata_path in idedata_candidates: - if idedata_path.exists(): - try: - with open(idedata_path, encoding="utf-8") as f: - raw_data = json.load(f) - idedata = IDEData(raw_data) - print(f"Loaded idedata from: {idedata_path}", file=sys.stderr) - break - except (json.JSONDecodeError, OSError) as e: - print( - f"Warning: Failed to load idedata from {idedata_path}: {e}", - file=sys.stderr, - ) + if not idedata_path.exists(): + continue + try: + with open(idedata_path, encoding="utf-8") as f: + raw_data = json.load(f) + idedata = IDEData(raw_data) + print(f"Loaded idedata from: {idedata_path}", file=sys.stderr) + break + except (json.JSONDecodeError, OSError) as e: + print( + f"Warning: Failed to load idedata from {idedata_path}: {e}", + file=sys.stderr, + ) analyzer = MemoryAnalyzer(elf_path, idedata=idedata) components = analyzer.analyze() From 85e0a4fbf9e966a096d61cfcf086640ee88c6be3 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Fri, 17 Oct 2025 18:29:36 -1000 Subject: [PATCH 71/76] update test --- esphome/platformio_api.py | 22 ++++++++++------------ 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/esphome/platformio_api.py b/esphome/platformio_api.py index cc48562b4c..c50bb2acff 100644 --- a/esphome/platformio_api.py +++ b/esphome/platformio_api.py @@ -378,19 +378,17 @@ class IDEData: @property def objdump_path(self) -> str: # replace gcc at end with objdump - - # Windows - if self.cc_path.endswith(".exe"): - return f"{self.cc_path[:-7]}objdump.exe" - - return f"{self.cc_path[:-3]}objdump" + return ( + f"{self.cc_path[:-7]}objdump.exe" + if self.cc_path.endswith(".exe") + else f"{self.cc_path[:-3]}objdump" + ) @property def readelf_path(self) -> str: # replace gcc at end with readelf - - # Windows - if self.cc_path.endswith(".exe"): - return f"{self.cc_path[:-7]}readelf.exe" - - return f"{self.cc_path[:-3]}readelf" + return ( + f"{self.cc_path[:-7]}readelf.exe" + if self.cc_path.endswith(".exe") + else f"{self.cc_path[:-3]}readelf" + ) From 541fb8b27c3cc302923fd41ad6c3f6bdb9b06ea9 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Fri, 17 Oct 2025 18:32:22 -1000 Subject: [PATCH 72/76] update test --- esphome/analyze_memory/__init__.py | 40 +++++++++++++++++++----------- 1 file changed, 25 insertions(+), 15 deletions(-) diff --git a/esphome/analyze_memory/__init__.py b/esphome/analyze_memory/__init__.py index 942caabe70..db16051b8a 100644 --- a/esphome/analyze_memory/__init__.py +++ b/esphome/analyze_memory/__init__.py @@ -33,6 +33,21 @@ _GCC_PREFIX_ANNOTATIONS = { "_GLOBAL__sub_D_": "global destructor for", } +# GCC optimization suffix pattern (e.g., $isra$0, $part$1, $constprop$2) +_GCC_OPTIMIZATION_SUFFIX_PATTERN = re.compile(r"(\$(?:isra|part|constprop)\$\d+)") + +# C++ runtime patterns for categorization +_CPP_RUNTIME_PATTERNS = frozenset(["vtable", "typeinfo", "thunk"]) + +# libc printf/scanf family base names (used to detect variants like _printf_r, vfprintf, etc.) +_LIBC_PRINTF_SCANF_FAMILY = frozenset(["printf", "fprintf", "sprintf", "scanf"]) + +# Regex pattern for parsing readelf section headers +# Format: [ #] name type addr off size +_READELF_SECTION_PATTERN = re.compile( + r"\s*\[\s*\d+\]\s+([\.\w]+)\s+\w+\s+[\da-fA-F]+\s+[\da-fA-F]+\s+([\da-fA-F]+)" +) + @dataclass class MemorySection: @@ -133,12 +148,7 @@ class MemoryAnalyzer: # Parse section headers for line in result.stdout.splitlines(): # Look for section entries - if not ( - match := re.match( - r"\s*\[\s*\d+\]\s+([\.\w]+)\s+\w+\s+[\da-fA-F]+\s+[\da-fA-F]+\s+([\da-fA-F]+)", - line, - ) - ): + if not (match := _READELF_SECTION_PATTERN.match(line)): continue section_name = match.group(1) @@ -273,14 +283,14 @@ class MemoryAnalyzer: # Check if spi_flash vs spi_driver if "spi_" in symbol_name or "SPI" in symbol_name: - if "spi_flash" in symbol_name: - return "spi_flash" - return "spi_driver" + return "spi_flash" if "spi_flash" in symbol_name else "spi_driver" # libc special printf variants - if symbol_name.startswith("_") and symbol_name[1:].replace("_r", "").replace( - "v", "" - ).replace("s", "") in ["printf", "fprintf", "sprintf", "scanf"]: + if ( + symbol_name.startswith("_") + and symbol_name[1:].replace("_r", "").replace("v", "").replace("s", "") + in _LIBC_PRINTF_SCANF_FAMILY + ): return "libc" # Track uncategorized symbols for analysis @@ -320,7 +330,7 @@ class MemoryAnalyzer: symbols_prefixes: list[str] = [] # Track removed prefixes for symbol in symbols: # Remove GCC optimization markers - stripped = re.sub(r"\$(?:isra|part|constprop)\$\d+", "", symbol) + stripped = _GCC_OPTIMIZATION_SUFFIX_PATTERN.sub("", symbol) # Handle GCC global constructor/initializer prefixes # _GLOBAL__sub_I_ -> extract for demangling @@ -450,7 +460,7 @@ class MemoryAnalyzer: Returns: Demangled name with suffix annotation """ - suffix_match = re.search(r"(\$(?:isra|part|constprop)\$\d+)", original) + suffix_match = _GCC_OPTIMIZATION_SUFFIX_PATTERN.search(original) if suffix_match: return f"{demangled} [{suffix_match.group(1)}]" return demangled @@ -462,7 +472,7 @@ class MemoryAnalyzer: def _categorize_esphome_core_symbol(self, demangled: str) -> str: """Categorize ESPHome core symbols into subcategories.""" # Special patterns that need to be checked separately - if any(pattern in demangled for pattern in ["vtable", "typeinfo", "thunk"]): + if any(pattern in demangled for pattern in _CPP_RUNTIME_PATTERNS): return "C++ Runtime (vtables/RTTI)" if demangled.startswith("std::"): From f9807db08ab7218f5f14570814378fc95aba3ff1 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Fri, 17 Oct 2025 18:37:24 -1000 Subject: [PATCH 73/76] preen --- esphome/analyze_memory/__init__.py | 21 +++++++++++++-------- esphome/analyze_memory/cli.py | 22 ++++++++++++++++------ 2 files changed, 29 insertions(+), 14 deletions(-) diff --git a/esphome/analyze_memory/__init__.py b/esphome/analyze_memory/__init__.py index db16051b8a..15cadaf859 100644 --- a/esphome/analyze_memory/__init__.py +++ b/esphome/analyze_memory/__init__.py @@ -48,6 +48,12 @@ _READELF_SECTION_PATTERN = re.compile( r"\s*\[\s*\d+\]\s+([\.\w]+)\s+\w+\s+[\da-fA-F]+\s+[\da-fA-F]+\s+([\da-fA-F]+)" ) +# Component category prefixes +_COMPONENT_PREFIX_ESPHOME = "[esphome]" +_COMPONENT_PREFIX_EXTERNAL = "[external]" +_COMPONENT_CORE = f"{_COMPONENT_PREFIX_ESPHOME}core" +_COMPONENT_API = f"{_COMPONENT_PREFIX_ESPHOME}api" + @dataclass class MemorySection: @@ -222,7 +228,7 @@ class MemoryAnalyzer: self._uncategorized_symbols.append((symbol_name, demangled, size)) # Track ESPHome core symbols for detailed analysis - if component == "[esphome]core" and size > 0: + if component == _COMPONENT_CORE and size > 0: demangled = self._demangle_symbol(symbol_name) self._esphome_core_symbols.append((symbol_name, demangled, size)) @@ -246,7 +252,7 @@ class MemoryAnalyzer: for component_name in get_esphome_components(): patterns = get_component_class_patterns(component_name) if any(pattern in demangled for pattern in patterns): - return f"[esphome]{component_name}" + return f"{_COMPONENT_PREFIX_ESPHOME}{component_name}" # Check for ESPHome component namespaces match = ESPHOME_COMPONENT_PATTERN.search(demangled) @@ -257,17 +263,17 @@ class MemoryAnalyzer: # Check if this is an actual component in the components directory if component_name in get_esphome_components(): - return f"[esphome]{component_name}" + return f"{_COMPONENT_PREFIX_ESPHOME}{component_name}" # Check if this is a known external component from the config if component_name in self.external_components: - return f"[external]{component_name}" + return f"{_COMPONENT_PREFIX_EXTERNAL}{component_name}" # Everything else in esphome:: namespace is core - return "[esphome]core" + return _COMPONENT_CORE # Check for esphome core namespace (no component namespace) if "esphome::" in demangled: # If no component match found, it's core - return "[esphome]core" + return _COMPONENT_CORE # Check against symbol patterns for component, patterns in SYMBOL_PATTERNS.items(): @@ -460,8 +466,7 @@ class MemoryAnalyzer: Returns: Demangled name with suffix annotation """ - suffix_match = _GCC_OPTIMIZATION_SUFFIX_PATTERN.search(original) - if suffix_match: + if suffix_match := _GCC_OPTIMIZATION_SUFFIX_PATTERN.search(original): return f"{demangled} [{suffix_match.group(1)}]" return demangled diff --git a/esphome/analyze_memory/cli.py b/esphome/analyze_memory/cli.py index 5713eac94c..1695a00c19 100644 --- a/esphome/analyze_memory/cli.py +++ b/esphome/analyze_memory/cli.py @@ -3,7 +3,13 @@ from collections import defaultdict import sys -from . import MemoryAnalyzer +from . import ( + _COMPONENT_API, + _COMPONENT_CORE, + _COMPONENT_PREFIX_ESPHOME, + _COMPONENT_PREFIX_EXTERNAL, + MemoryAnalyzer, +) class MemoryAnalyzerCLI(MemoryAnalyzer): @@ -144,7 +150,9 @@ class MemoryAnalyzerCLI(MemoryAnalyzer): if self._esphome_core_symbols: lines.append("") lines.append("=" * self.TABLE_WIDTH) - lines.append("[esphome]core Detailed Analysis".center(self.TABLE_WIDTH)) + lines.append( + f"{_COMPONENT_CORE} Detailed Analysis".center(self.TABLE_WIDTH) + ) lines.append("=" * self.TABLE_WIDTH) lines.append("") @@ -185,7 +193,7 @@ class MemoryAnalyzerCLI(MemoryAnalyzer): # Top 15 largest core symbols lines.append("") - lines.append("Top 15 Largest [esphome]core Symbols:") + lines.append(f"Top 15 Largest {_COMPONENT_CORE} Symbols:") sorted_core_symbols = sorted( self._esphome_core_symbols, key=lambda x: x[2], reverse=True ) @@ -199,10 +207,12 @@ class MemoryAnalyzerCLI(MemoryAnalyzer): esphome_components = [ (name, mem) for name, mem in components - if name.startswith("[esphome]") and name != "[esphome]core" + if name.startswith(_COMPONENT_PREFIX_ESPHOME) and name != _COMPONENT_CORE ] external_components = [ - (name, mem) for name, mem in components if name.startswith("[external]") + (name, mem) + for name, mem in components + if name.startswith(_COMPONENT_PREFIX_EXTERNAL) ] top_esphome_components = sorted( @@ -217,7 +227,7 @@ class MemoryAnalyzerCLI(MemoryAnalyzer): # Check if API component exists and ensure it's included api_component = None for name, mem in components: - if name == "[esphome]api": + if name == _COMPONENT_API: api_component = (name, mem) break From 4f4da1de22acb050c0641a98828f0f6e231c2487 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Fri, 17 Oct 2025 18:41:12 -1000 Subject: [PATCH 74/76] preen --- esphome/analyze_memory/__init__.py | 17 +++++++++++------ esphome/analyze_memory/helpers.py | 13 +++++++++---- esphome/platformio_api.py | 14 ++++++++------ 3 files changed, 28 insertions(+), 16 deletions(-) diff --git a/esphome/analyze_memory/__init__.py b/esphome/analyze_memory/__init__.py index 15cadaf859..71e86e3788 100644 --- a/esphome/analyze_memory/__init__.py +++ b/esphome/analyze_memory/__init__.py @@ -54,15 +54,20 @@ _COMPONENT_PREFIX_EXTERNAL = "[external]" _COMPONENT_CORE = f"{_COMPONENT_PREFIX_ESPHOME}core" _COMPONENT_API = f"{_COMPONENT_PREFIX_ESPHOME}api" +# C++ namespace prefixes +_NAMESPACE_ESPHOME = "esphome::" +_NAMESPACE_STD = "std::" + +# Type alias for symbol information: (symbol_name, size, component) +SymbolInfoType = tuple[str, int, str] + @dataclass class MemorySection: """Represents a memory section with its symbols.""" name: str - symbols: list[tuple[str, int, str]] = field( - default_factory=list - ) # (symbol_name, size, component) + symbols: list[SymbolInfoType] = field(default_factory=list) total_size: int = 0 @@ -246,7 +251,7 @@ class MemoryAnalyzer: # Check for special component classes first (before namespace pattern) # This handles cases like esphome::ESPHomeOTAComponent which should map to ota - if "esphome::" in demangled: + if _NAMESPACE_ESPHOME in demangled: # Check for special component classes that include component name in the class # For example: esphome::ESPHomeOTAComponent -> ota component for component_name in get_esphome_components(): @@ -271,7 +276,7 @@ class MemoryAnalyzer: return _COMPONENT_CORE # Check for esphome core namespace (no component namespace) - if "esphome::" in demangled: + if _NAMESPACE_ESPHOME in demangled: # If no component match found, it's core return _COMPONENT_CORE @@ -480,7 +485,7 @@ class MemoryAnalyzer: if any(pattern in demangled for pattern in _CPP_RUNTIME_PATTERNS): return "C++ Runtime (vtables/RTTI)" - if demangled.startswith("std::"): + if demangled.startswith(_NAMESPACE_STD): return "C++ STL" # Check against patterns from const.py diff --git a/esphome/analyze_memory/helpers.py b/esphome/analyze_memory/helpers.py index 1b5a1c67c2..cb503b37c5 100644 --- a/esphome/analyze_memory/helpers.py +++ b/esphome/analyze_memory/helpers.py @@ -5,6 +5,11 @@ from pathlib import Path from .const import SECTION_MAPPING +# Import namespace constant from parent module +# Note: This would create a circular import if done at module level, +# so we'll define it locally here as well +_NAMESPACE_ESPHOME = "esphome::" + # Get the list of actual ESPHome components by scanning the components directory @cache @@ -40,10 +45,10 @@ def get_component_class_patterns(component_name: str) -> list[str]: component_upper = component_name.upper() component_camel = component_name.replace("_", "").title() return [ - f"esphome::{component_upper}Component", # e.g., esphome::OTAComponent - f"esphome::ESPHome{component_upper}Component", # e.g., esphome::ESPHomeOTAComponent - f"esphome::{component_camel}Component", # e.g., esphome::OtaComponent - f"esphome::ESPHome{component_camel}Component", # e.g., esphome::ESPHomeOtaComponent + f"{_NAMESPACE_ESPHOME}{component_upper}Component", # e.g., esphome::OTAComponent + f"{_NAMESPACE_ESPHOME}ESPHome{component_upper}Component", # e.g., esphome::ESPHomeOTAComponent + f"{_NAMESPACE_ESPHOME}{component_camel}Component", # e.g., esphome::OtaComponent + f"{_NAMESPACE_ESPHOME}ESPHome{component_camel}Component", # e.g., esphome::ESPHomeOtaComponent ] diff --git a/esphome/platformio_api.py b/esphome/platformio_api.py index c50bb2acff..d59523a74a 100644 --- a/esphome/platformio_api.py +++ b/esphome/platformio_api.py @@ -378,17 +378,19 @@ class IDEData: @property def objdump_path(self) -> str: # replace gcc at end with objdump + path = self.cc_path return ( - f"{self.cc_path[:-7]}objdump.exe" - if self.cc_path.endswith(".exe") - else f"{self.cc_path[:-3]}objdump" + f"{path[:-7]}objdump.exe" + if path.endswith(".exe") + else f"{path[:-3]}objdump" ) @property def readelf_path(self) -> str: # replace gcc at end with readelf + path = self.cc_path return ( - f"{self.cc_path[:-7]}readelf.exe" - if self.cc_path.endswith(".exe") - else f"{self.cc_path[:-3]}readelf" + f"{path[:-7]}readelf.exe" + if path.endswith(".exe") + else f"{path[:-3]}readelf" ) From 7f2d8a2c118da393b7758dee2ae215ddd50985fc Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Fri, 17 Oct 2025 18:46:41 -1000 Subject: [PATCH 75/76] whitespace --- .../ci_memory_impact_comment_template.j2 | 6 +- .../ci_memory_impact_symbol_changes.j2 | 12 +- tests/script/test_determine_jobs.py | 182 ++++++++++++++++++ 3 files changed, 191 insertions(+), 9 deletions(-) diff --git a/script/templates/ci_memory_impact_comment_template.j2 b/script/templates/ci_memory_impact_comment_template.j2 index 4c8d7f4865..9fbf78e99f 100644 --- a/script/templates/ci_memory_impact_comment_template.j2 +++ b/script/templates/ci_memory_impact_comment_template.j2 @@ -10,10 +10,10 @@ | **Flash** | {{ target_flash }} | {{ pr_flash }} | {{ flash_change }} | {% if component_breakdown %} {{ component_breakdown }} -{%- endif %} -{%- if symbol_changes %} +{% endif %} +{% if symbol_changes %} {{ symbol_changes }} -{%- endif %} +{% endif %} {%- if target_cache_hit %} > ⚡ Target branch analysis was loaded from cache (build skipped for faster CI). diff --git a/script/templates/ci_memory_impact_symbol_changes.j2 b/script/templates/ci_memory_impact_symbol_changes.j2 index bd540712f8..60f2f50e48 100644 --- a/script/templates/ci_memory_impact_symbol_changes.j2 +++ b/script/templates/ci_memory_impact_symbol_changes.j2 @@ -3,7 +3,7 @@
🔍 Symbol-Level Changes (click to expand) -{%- if changed_symbols %} +{% if changed_symbols %} ### Changed Symbols @@ -16,8 +16,8 @@ | ... | ... | ... | *({{ changed_symbols|length - max_changed_rows }} more changed symbols not shown)* | {% endif -%} -{%- endif %} -{%- if new_symbols %} +{% endif %} +{% if new_symbols %} ### New Symbols (top {{ max_new_rows }}) @@ -31,8 +31,8 @@ | *{{ new_symbols|length - max_new_rows }} more new symbols...* | *Total: {{ total_new_size|format_bytes }}* | {% endif -%} -{%- endif %} -{%- if removed_symbols %} +{% endif %} +{% if removed_symbols %} ### Removed Symbols (top {{ max_removed_rows }}) @@ -46,6 +46,6 @@ | *{{ removed_symbols|length - max_removed_rows }} more removed symbols...* | *Total: {{ total_removed_size|format_bytes }}* | {% endif -%} -{%- endif %} +{% endif %}
diff --git a/tests/script/test_determine_jobs.py b/tests/script/test_determine_jobs.py index 24c77b6ae9..b479fc03c5 100644 --- a/tests/script/test_determine_jobs.py +++ b/tests/script/test_determine_jobs.py @@ -504,3 +504,185 @@ def test_main_filters_components_without_tests( # memory_impact should be present assert "memory_impact" in output assert output["memory_impact"]["should_run"] == "false" + + +# Tests for detect_memory_impact_config function + + +def test_detect_memory_impact_config_with_common_platform(tmp_path: Path) -> None: + """Test memory impact detection when components share a common platform.""" + # Create test directory structure + tests_dir = tmp_path / "tests" / "components" + + # wifi component with esp32-idf test + wifi_dir = tests_dir / "wifi" + wifi_dir.mkdir(parents=True) + (wifi_dir / "test.esp32-idf.yaml").write_text("test: wifi") + + # api component with esp32-idf test + api_dir = tests_dir / "api" + api_dir.mkdir(parents=True) + (api_dir / "test.esp32-idf.yaml").write_text("test: api") + + # Mock changed_files to return wifi and api component changes + with ( + patch.object(determine_jobs, "root_path", str(tmp_path)), + patch.object(helpers, "root_path", str(tmp_path)), + patch.object(determine_jobs, "changed_files") as mock_changed_files, + ): + mock_changed_files.return_value = [ + "esphome/components/wifi/wifi.cpp", + "esphome/components/api/api.cpp", + ] + determine_jobs._component_has_tests.cache_clear() + + result = determine_jobs.detect_memory_impact_config() + + assert result["should_run"] == "true" + assert set(result["components"]) == {"wifi", "api"} + assert result["platform"] == "esp32-idf" # Common platform + assert result["use_merged_config"] == "true" + + +def test_detect_memory_impact_config_core_only_changes(tmp_path: Path) -> None: + """Test memory impact detection with core-only changes (no component changes).""" + # Create test directory structure with fallback component + tests_dir = tmp_path / "tests" / "components" + + # api component (fallback component) with esp32-idf test + api_dir = tests_dir / "api" + api_dir.mkdir(parents=True) + (api_dir / "test.esp32-idf.yaml").write_text("test: api") + + # Mock changed_files to return only core files (no component files) + with ( + patch.object(determine_jobs, "root_path", str(tmp_path)), + patch.object(helpers, "root_path", str(tmp_path)), + patch.object(determine_jobs, "changed_files") as mock_changed_files, + ): + mock_changed_files.return_value = [ + "esphome/core/application.cpp", + "esphome/core/component.h", + ] + determine_jobs._component_has_tests.cache_clear() + + result = determine_jobs.detect_memory_impact_config() + + assert result["should_run"] == "true" + assert result["components"] == ["api"] # Fallback component + assert result["platform"] == "esp32-idf" # Fallback platform + assert result["use_merged_config"] == "true" + + +def test_detect_memory_impact_config_no_common_platform(tmp_path: Path) -> None: + """Test memory impact detection when components have no common platform.""" + # Create test directory structure + tests_dir = tmp_path / "tests" / "components" + + # wifi component only has esp32-idf test + wifi_dir = tests_dir / "wifi" + wifi_dir.mkdir(parents=True) + (wifi_dir / "test.esp32-idf.yaml").write_text("test: wifi") + + # logger component only has esp8266-ard test + logger_dir = tests_dir / "logger" + logger_dir.mkdir(parents=True) + (logger_dir / "test.esp8266-ard.yaml").write_text("test: logger") + + # Mock changed_files to return both components + with ( + patch.object(determine_jobs, "root_path", str(tmp_path)), + patch.object(helpers, "root_path", str(tmp_path)), + patch.object(determine_jobs, "changed_files") as mock_changed_files, + ): + mock_changed_files.return_value = [ + "esphome/components/wifi/wifi.cpp", + "esphome/components/logger/logger.cpp", + ] + determine_jobs._component_has_tests.cache_clear() + + result = determine_jobs.detect_memory_impact_config() + + # Should pick the most frequently supported platform + assert result["should_run"] == "true" + assert set(result["components"]) == {"wifi", "logger"} + # When no common platform, picks most commonly supported + # esp8266-ard is preferred over esp32-idf in the preference list + assert result["platform"] in ["esp32-idf", "esp8266-ard"] + assert result["use_merged_config"] == "true" + + +def test_detect_memory_impact_config_no_changes(tmp_path: Path) -> None: + """Test memory impact detection when no files changed.""" + # Mock changed_files to return empty list + with ( + patch.object(determine_jobs, "root_path", str(tmp_path)), + patch.object(helpers, "root_path", str(tmp_path)), + patch.object(determine_jobs, "changed_files") as mock_changed_files, + ): + mock_changed_files.return_value = [] + determine_jobs._component_has_tests.cache_clear() + + result = determine_jobs.detect_memory_impact_config() + + assert result["should_run"] == "false" + + +def test_detect_memory_impact_config_no_components_with_tests(tmp_path: Path) -> None: + """Test memory impact detection when changed components have no tests.""" + # Create test directory structure + tests_dir = tmp_path / "tests" / "components" + + # Create component directory but no test files + custom_component_dir = tests_dir / "my_custom_component" + custom_component_dir.mkdir(parents=True) + + # Mock changed_files to return component without tests + with ( + patch.object(determine_jobs, "root_path", str(tmp_path)), + patch.object(helpers, "root_path", str(tmp_path)), + patch.object(determine_jobs, "changed_files") as mock_changed_files, + ): + mock_changed_files.return_value = [ + "esphome/components/my_custom_component/component.cpp", + ] + determine_jobs._component_has_tests.cache_clear() + + result = determine_jobs.detect_memory_impact_config() + + assert result["should_run"] == "false" + + +def test_detect_memory_impact_config_skips_base_bus_components(tmp_path: Path) -> None: + """Test that base bus components (i2c, spi, uart) are skipped.""" + # Create test directory structure + tests_dir = tmp_path / "tests" / "components" + + # i2c component (should be skipped as it's a base bus component) + i2c_dir = tests_dir / "i2c" + i2c_dir.mkdir(parents=True) + (i2c_dir / "test.esp32-idf.yaml").write_text("test: i2c") + + # wifi component (should not be skipped) + wifi_dir = tests_dir / "wifi" + wifi_dir.mkdir(parents=True) + (wifi_dir / "test.esp32-idf.yaml").write_text("test: wifi") + + # Mock changed_files to return both i2c and wifi + with ( + patch.object(determine_jobs, "root_path", str(tmp_path)), + patch.object(helpers, "root_path", str(tmp_path)), + patch.object(determine_jobs, "changed_files") as mock_changed_files, + ): + mock_changed_files.return_value = [ + "esphome/components/i2c/i2c.cpp", + "esphome/components/wifi/wifi.cpp", + ] + determine_jobs._component_has_tests.cache_clear() + + result = determine_jobs.detect_memory_impact_config() + + # Should only include wifi, not i2c + assert result["should_run"] == "true" + assert result["components"] == ["wifi"] + assert "i2c" not in result["components"] From e70cb098ae25af4ea59a5b2a8d792d20212c9d50 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Fri, 17 Oct 2025 18:50:07 -1000 Subject: [PATCH 76/76] whitespace --- tests/unit_tests/test_platformio_api.py | 36 +++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/tests/unit_tests/test_platformio_api.py b/tests/unit_tests/test_platformio_api.py index 07948cc6ad..13ef3516e4 100644 --- a/tests/unit_tests/test_platformio_api.py +++ b/tests/unit_tests/test_platformio_api.py @@ -387,6 +387,42 @@ def test_idedata_addr2line_path_unix(setup_core: Path) -> None: assert result == "/usr/bin/addr2line" +def test_idedata_objdump_path_windows(setup_core: Path) -> None: + """Test IDEData.objdump_path on Windows.""" + raw_data = {"prog_path": "/path/to/firmware.elf", "cc_path": "C:\\tools\\gcc.exe"} + idedata = platformio_api.IDEData(raw_data) + + result = idedata.objdump_path + assert result == "C:\\tools\\objdump.exe" + + +def test_idedata_objdump_path_unix(setup_core: Path) -> None: + """Test IDEData.objdump_path on Unix.""" + raw_data = {"prog_path": "/path/to/firmware.elf", "cc_path": "/usr/bin/gcc"} + idedata = platformio_api.IDEData(raw_data) + + result = idedata.objdump_path + assert result == "/usr/bin/objdump" + + +def test_idedata_readelf_path_windows(setup_core: Path) -> None: + """Test IDEData.readelf_path on Windows.""" + raw_data = {"prog_path": "/path/to/firmware.elf", "cc_path": "C:\\tools\\gcc.exe"} + idedata = platformio_api.IDEData(raw_data) + + result = idedata.readelf_path + assert result == "C:\\tools\\readelf.exe" + + +def test_idedata_readelf_path_unix(setup_core: Path) -> None: + """Test IDEData.readelf_path on Unix.""" + raw_data = {"prog_path": "/path/to/firmware.elf", "cc_path": "/usr/bin/gcc"} + idedata = platformio_api.IDEData(raw_data) + + result = idedata.readelf_path + assert result == "/usr/bin/readelf" + + def test_patch_structhash(setup_core: Path) -> None: """Test patch_structhash monkey patches platformio functions.""" # Create simple namespace objects to act as modules