mirror of
https://github.com/esphome/esphome.git
synced 2025-10-21 03:03:50 +01:00
merge
This commit is contained in:
122
.github/workflows/ci.yml
vendored
122
.github/workflows/ci.yml
vendored
@@ -548,45 +548,53 @@ jobs:
|
|||||||
with:
|
with:
|
||||||
path: ~/.platformio
|
path: ~/.platformio
|
||||||
key: platformio-memory-${{ fromJSON(needs.determine-jobs.outputs.memory_impact).platform }}-${{ hashFiles('platformio.ini') }}
|
key: platformio-memory-${{ fromJSON(needs.determine-jobs.outputs.memory_impact).platform }}-${{ hashFiles('platformio.ini') }}
|
||||||
- name: Compile test configuration and extract memory usage
|
- name: Build and compile with test_build_components
|
||||||
id: extract
|
id: extract
|
||||||
run: |
|
run: |
|
||||||
. venv/bin/activate
|
. venv/bin/activate
|
||||||
component="${{ fromJSON(needs.determine-jobs.outputs.memory_impact).component }}"
|
components='${{ toJSON(fromJSON(needs.determine-jobs.outputs.memory_impact).components) }}'
|
||||||
platform="${{ fromJSON(needs.determine-jobs.outputs.memory_impact).platform }}"
|
platform="${{ fromJSON(needs.determine-jobs.outputs.memory_impact).platform }}"
|
||||||
test_file="${{ fromJSON(needs.determine-jobs.outputs.memory_impact).test_file }}"
|
|
||||||
|
|
||||||
echo "Compiling $component for $platform using $test_file"
|
echo "Building with test_build_components.py for $platform with components:"
|
||||||
python script/test_build_components.py -e compile -c "$component" -t "$platform" --no-grouping 2>&1 | \
|
echo "$components" | jq -r '.[]' | sed 's/^/ - /'
|
||||||
|
|
||||||
|
# Use test_build_components.py which handles grouping automatically
|
||||||
|
# Pass components as comma-separated list
|
||||||
|
component_list=$(echo "$components" | jq -r 'join(",")')
|
||||||
|
|
||||||
|
echo "Compiling with test_build_components.py..."
|
||||||
|
python script/test_build_components.py \
|
||||||
|
-e compile \
|
||||||
|
-c "$component_list" \
|
||||||
|
-t "$platform" 2>&1 | \
|
||||||
python script/ci_memory_impact_extract.py --output-env
|
python script/ci_memory_impact_extract.py --output-env
|
||||||
- name: Find and upload ELF file
|
- name: Find and upload final ELF file
|
||||||
run: |
|
run: |
|
||||||
# Find the ELF file - try both common locations
|
# Note: test_build_components.py may run multiple builds, but each overwrites
|
||||||
elf_file=""
|
# the previous firmware.elf. The memory totals (RAM/Flash) are already summed
|
||||||
|
# by ci_memory_impact_extract.py. This ELF is from the last build and is used
|
||||||
|
# for detailed component breakdown (if available).
|
||||||
|
mkdir -p ./elf-artifacts/target
|
||||||
|
|
||||||
# Try .esphome/build first (default location)
|
# Find the most recent firmware.elf
|
||||||
if [ -d ~/.esphome/build ]; then
|
if [ -d ~/.esphome/build ]; then
|
||||||
elf_file=$(find ~/.esphome/build -name "firmware.elf" -o -name "*.elf" | head -1)
|
elf_file=$(find ~/.esphome/build -name "firmware.elf" -type f -printf '%T@ %p\n' | sort -rn | head -1 | cut -d' ' -f2-)
|
||||||
fi
|
|
||||||
|
|
||||||
# Fallback to finding in .platformio if not found
|
if [ -n "$elf_file" ] && [ -f "$elf_file" ]; then
|
||||||
if [ -z "$elf_file" ] && [ -d ~/.platformio ]; then
|
echo "Found final ELF file: $elf_file"
|
||||||
elf_file=$(find ~/.platformio -name "firmware.elf" | head -1)
|
cp "$elf_file" "./elf-artifacts/target/firmware.elf"
|
||||||
fi
|
else
|
||||||
|
echo "Warning: No ELF file found in ~/.esphome/build"
|
||||||
if [ -n "$elf_file" ] && [ -f "$elf_file" ]; then
|
ls -la ~/.esphome/build/ || true
|
||||||
echo "Found ELF file: $elf_file"
|
fi
|
||||||
mkdir -p ./elf-artifacts
|
|
||||||
cp "$elf_file" ./elf-artifacts/target.elf
|
|
||||||
else
|
else
|
||||||
echo "Warning: No ELF file found in ~/.esphome/build or ~/.platformio"
|
echo "Warning: ~/.esphome/build directory not found"
|
||||||
ls -la ~/.esphome/build/ || true
|
|
||||||
fi
|
fi
|
||||||
- name: Upload ELF artifact
|
- name: Upload ELF artifact
|
||||||
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
|
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
|
||||||
with:
|
with:
|
||||||
name: memory-impact-target-elf
|
name: memory-impact-target-elf
|
||||||
path: ./elf-artifacts/target.elf
|
path: ./elf-artifacts/target/firmware.elf
|
||||||
if-no-files-found: warn
|
if-no-files-found: warn
|
||||||
retention-days: 1
|
retention-days: 1
|
||||||
|
|
||||||
@@ -613,45 +621,53 @@ jobs:
|
|||||||
with:
|
with:
|
||||||
path: ~/.platformio
|
path: ~/.platformio
|
||||||
key: platformio-memory-${{ fromJSON(needs.determine-jobs.outputs.memory_impact).platform }}-${{ hashFiles('platformio.ini') }}
|
key: platformio-memory-${{ fromJSON(needs.determine-jobs.outputs.memory_impact).platform }}-${{ hashFiles('platformio.ini') }}
|
||||||
- name: Compile test configuration and extract memory usage
|
- name: Build and compile with test_build_components
|
||||||
id: extract
|
id: extract
|
||||||
run: |
|
run: |
|
||||||
. venv/bin/activate
|
. venv/bin/activate
|
||||||
component="${{ fromJSON(needs.determine-jobs.outputs.memory_impact).component }}"
|
components='${{ toJSON(fromJSON(needs.determine-jobs.outputs.memory_impact).components) }}'
|
||||||
platform="${{ fromJSON(needs.determine-jobs.outputs.memory_impact).platform }}"
|
platform="${{ fromJSON(needs.determine-jobs.outputs.memory_impact).platform }}"
|
||||||
test_file="${{ fromJSON(needs.determine-jobs.outputs.memory_impact).test_file }}"
|
|
||||||
|
|
||||||
echo "Compiling $component for $platform using $test_file"
|
echo "Building with test_build_components.py for $platform with components:"
|
||||||
python script/test_build_components.py -e compile -c "$component" -t "$platform" --no-grouping 2>&1 | \
|
echo "$components" | jq -r '.[]' | sed 's/^/ - /'
|
||||||
|
|
||||||
|
# Use test_build_components.py which handles grouping automatically
|
||||||
|
# Pass components as comma-separated list
|
||||||
|
component_list=$(echo "$components" | jq -r 'join(",")')
|
||||||
|
|
||||||
|
echo "Compiling with test_build_components.py..."
|
||||||
|
python script/test_build_components.py \
|
||||||
|
-e compile \
|
||||||
|
-c "$component_list" \
|
||||||
|
-t "$platform" 2>&1 | \
|
||||||
python script/ci_memory_impact_extract.py --output-env
|
python script/ci_memory_impact_extract.py --output-env
|
||||||
- name: Find and upload ELF file
|
- name: Find and upload final ELF file
|
||||||
run: |
|
run: |
|
||||||
# Find the ELF file - try both common locations
|
# Note: test_build_components.py may run multiple builds, but each overwrites
|
||||||
elf_file=""
|
# the previous firmware.elf. The memory totals (RAM/Flash) are already summed
|
||||||
|
# by ci_memory_impact_extract.py. This ELF is from the last build and is used
|
||||||
|
# for detailed component breakdown (if available).
|
||||||
|
mkdir -p ./elf-artifacts/pr
|
||||||
|
|
||||||
# Try .esphome/build first (default location)
|
# Find the most recent firmware.elf
|
||||||
if [ -d ~/.esphome/build ]; then
|
if [ -d ~/.esphome/build ]; then
|
||||||
elf_file=$(find ~/.esphome/build -name "firmware.elf" -o -name "*.elf" | head -1)
|
elf_file=$(find ~/.esphome/build -name "firmware.elf" -type f -printf '%T@ %p\n' | sort -rn | head -1 | cut -d' ' -f2-)
|
||||||
fi
|
|
||||||
|
|
||||||
# Fallback to finding in .platformio if not found
|
if [ -n "$elf_file" ] && [ -f "$elf_file" ]; then
|
||||||
if [ -z "$elf_file" ] && [ -d ~/.platformio ]; then
|
echo "Found final ELF file: $elf_file"
|
||||||
elf_file=$(find ~/.platformio -name "firmware.elf" | head -1)
|
cp "$elf_file" "./elf-artifacts/pr/firmware.elf"
|
||||||
fi
|
else
|
||||||
|
echo "Warning: No ELF file found in ~/.esphome/build"
|
||||||
if [ -n "$elf_file" ] && [ -f "$elf_file" ]; then
|
ls -la ~/.esphome/build/ || true
|
||||||
echo "Found ELF file: $elf_file"
|
fi
|
||||||
mkdir -p ./elf-artifacts
|
|
||||||
cp "$elf_file" ./elf-artifacts/pr.elf
|
|
||||||
else
|
else
|
||||||
echo "Warning: No ELF file found in ~/.esphome/build or ~/.platformio"
|
echo "Warning: ~/.esphome/build directory not found"
|
||||||
ls -la ~/.esphome/build/ || true
|
|
||||||
fi
|
fi
|
||||||
- name: Upload ELF artifact
|
- name: Upload ELF artifact
|
||||||
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
|
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
|
||||||
with:
|
with:
|
||||||
name: memory-impact-pr-elf
|
name: memory-impact-pr-elf
|
||||||
path: ./elf-artifacts/pr.elf
|
path: ./elf-artifacts/pr/firmware.elf
|
||||||
if-no-files-found: warn
|
if-no-files-found: warn
|
||||||
retention-days: 1
|
retention-days: 1
|
||||||
|
|
||||||
@@ -690,7 +706,7 @@ jobs:
|
|||||||
- name: Post or update PR comment
|
- name: Post or update PR comment
|
||||||
env:
|
env:
|
||||||
GH_TOKEN: ${{ github.token }}
|
GH_TOKEN: ${{ github.token }}
|
||||||
COMPONENT: ${{ fromJSON(needs.determine-jobs.outputs.memory_impact).component }}
|
COMPONENTS: ${{ toJSON(fromJSON(needs.determine-jobs.outputs.memory_impact).components) }}
|
||||||
PLATFORM: ${{ fromJSON(needs.determine-jobs.outputs.memory_impact).platform }}
|
PLATFORM: ${{ fromJSON(needs.determine-jobs.outputs.memory_impact).platform }}
|
||||||
TARGET_RAM: ${{ needs.memory-impact-target-branch.outputs.ram_usage }}
|
TARGET_RAM: ${{ needs.memory-impact-target-branch.outputs.ram_usage }}
|
||||||
TARGET_FLASH: ${{ needs.memory-impact-target-branch.outputs.flash_usage }}
|
TARGET_FLASH: ${{ needs.memory-impact-target-branch.outputs.flash_usage }}
|
||||||
@@ -699,27 +715,27 @@ jobs:
|
|||||||
run: |
|
run: |
|
||||||
. venv/bin/activate
|
. venv/bin/activate
|
||||||
|
|
||||||
# Check if ELF files exist
|
# Check if ELF files exist (from final build)
|
||||||
target_elf_arg=""
|
target_elf_arg=""
|
||||||
pr_elf_arg=""
|
pr_elf_arg=""
|
||||||
|
|
||||||
if [ -f ./elf-artifacts/target/target.elf ]; then
|
if [ -f ./elf-artifacts/target/firmware.elf ]; then
|
||||||
echo "Found target ELF file"
|
echo "Found target ELF file"
|
||||||
target_elf_arg="--target-elf ./elf-artifacts/target/target.elf"
|
target_elf_arg="--target-elf ./elf-artifacts/target/firmware.elf"
|
||||||
else
|
else
|
||||||
echo "No target ELF file found"
|
echo "No target ELF file found"
|
||||||
fi
|
fi
|
||||||
|
|
||||||
if [ -f ./elf-artifacts/pr/pr.elf ]; then
|
if [ -f ./elf-artifacts/pr/firmware.elf ]; then
|
||||||
echo "Found PR ELF file"
|
echo "Found PR ELF file"
|
||||||
pr_elf_arg="--pr-elf ./elf-artifacts/pr/pr.elf"
|
pr_elf_arg="--pr-elf ./elf-artifacts/pr/firmware.elf"
|
||||||
else
|
else
|
||||||
echo "No PR ELF file found"
|
echo "No PR ELF file found"
|
||||||
fi
|
fi
|
||||||
|
|
||||||
python script/ci_memory_impact_comment.py \
|
python script/ci_memory_impact_comment.py \
|
||||||
--pr-number "${{ github.event.pull_request.number }}" \
|
--pr-number "${{ github.event.pull_request.number }}" \
|
||||||
--component "$COMPONENT" \
|
--components "$COMPONENTS" \
|
||||||
--platform "$PLATFORM" \
|
--platform "$PLATFORM" \
|
||||||
--target-ram "$TARGET_RAM" \
|
--target-ram "$TARGET_RAM" \
|
||||||
--target-flash "$TARGET_FLASH" \
|
--target-flash "$TARGET_FLASH" \
|
||||||
|
@@ -316,6 +316,7 @@ def main():
|
|||||||
print(
|
print(
|
||||||
"Usage: python -m esphome.analyze_memory <elf_file> [objdump_path] [readelf_path]"
|
"Usage: python -m esphome.analyze_memory <elf_file> [objdump_path] [readelf_path]"
|
||||||
)
|
)
|
||||||
|
print("\nIf objdump/readelf paths are not provided, you must specify them.")
|
||||||
print("\nExample for ESP8266:")
|
print("\nExample for ESP8266:")
|
||||||
print(" python -m esphome.analyze_memory firmware.elf \\")
|
print(" python -m esphome.analyze_memory firmware.elf \\")
|
||||||
print(
|
print(
|
||||||
@@ -332,6 +333,14 @@ def main():
|
|||||||
print(
|
print(
|
||||||
" ~/.platformio/packages/toolchain-xtensa-esp-elf/bin/xtensa-esp32-elf-readelf"
|
" ~/.platformio/packages/toolchain-xtensa-esp-elf/bin/xtensa-esp32-elf-readelf"
|
||||||
)
|
)
|
||||||
|
print("\nExample for ESP32-C3 (RISC-V):")
|
||||||
|
print(" python -m esphome.analyze_memory firmware.elf \\")
|
||||||
|
print(
|
||||||
|
" ~/.platformio/packages/toolchain-riscv32-esp/bin/riscv32-esp-elf-objdump \\"
|
||||||
|
)
|
||||||
|
print(
|
||||||
|
" ~/.platformio/packages/toolchain-riscv32-esp/bin/riscv32-esp-elf-readelf"
|
||||||
|
)
|
||||||
sys.exit(1)
|
sys.exit(1)
|
||||||
|
|
||||||
elf_file = sys.argv[1]
|
elf_file = sys.argv[1]
|
||||||
|
@@ -145,7 +145,16 @@ def run_compile(config, verbose):
|
|||||||
args = []
|
args = []
|
||||||
if CONF_COMPILE_PROCESS_LIMIT in config[CONF_ESPHOME]:
|
if CONF_COMPILE_PROCESS_LIMIT in config[CONF_ESPHOME]:
|
||||||
args += [f"-j{config[CONF_ESPHOME][CONF_COMPILE_PROCESS_LIMIT]}"]
|
args += [f"-j{config[CONF_ESPHOME][CONF_COMPILE_PROCESS_LIMIT]}"]
|
||||||
return run_platformio_cli_run(config, verbose, *args)
|
result = run_platformio_cli_run(config, verbose, *args)
|
||||||
|
|
||||||
|
# Run memory analysis if enabled
|
||||||
|
if config.get(CONF_ESPHOME, {}).get("analyze_memory", False):
|
||||||
|
try:
|
||||||
|
analyze_memory_usage(config)
|
||||||
|
except Exception as e:
|
||||||
|
_LOGGER.warning("Failed to analyze memory usage: %s", e)
|
||||||
|
|
||||||
|
return result
|
||||||
|
|
||||||
|
|
||||||
def _run_idedata(config):
|
def _run_idedata(config):
|
||||||
@@ -374,3 +383,93 @@ class IDEData:
|
|||||||
return f"{self.cc_path[:-7]}addr2line.exe"
|
return f"{self.cc_path[:-7]}addr2line.exe"
|
||||||
|
|
||||||
return f"{self.cc_path[:-3]}addr2line"
|
return f"{self.cc_path[:-3]}addr2line"
|
||||||
|
|
||||||
|
@property
|
||||||
|
def objdump_path(self) -> str:
|
||||||
|
# replace gcc at end with objdump
|
||||||
|
|
||||||
|
# Windows
|
||||||
|
if self.cc_path.endswith(".exe"):
|
||||||
|
return f"{self.cc_path[:-7]}objdump.exe"
|
||||||
|
|
||||||
|
return f"{self.cc_path[:-3]}objdump"
|
||||||
|
|
||||||
|
@property
|
||||||
|
def readelf_path(self) -> str:
|
||||||
|
# replace gcc at end with readelf
|
||||||
|
|
||||||
|
# Windows
|
||||||
|
if self.cc_path.endswith(".exe"):
|
||||||
|
return f"{self.cc_path[:-7]}readelf.exe"
|
||||||
|
|
||||||
|
return f"{self.cc_path[:-3]}readelf"
|
||||||
|
|
||||||
|
|
||||||
|
def analyze_memory_usage(config: dict[str, Any]) -> None:
|
||||||
|
"""Analyze memory usage by component after compilation."""
|
||||||
|
# Lazy import to avoid overhead when not needed
|
||||||
|
from esphome.analyze_memory import MemoryAnalyzer
|
||||||
|
|
||||||
|
idedata = get_idedata(config)
|
||||||
|
|
||||||
|
# Get paths to tools
|
||||||
|
elf_path = idedata.firmware_elf_path
|
||||||
|
objdump_path = idedata.objdump_path
|
||||||
|
readelf_path = idedata.readelf_path
|
||||||
|
|
||||||
|
# Debug logging
|
||||||
|
_LOGGER.debug("ELF path from idedata: %s", elf_path)
|
||||||
|
|
||||||
|
# Check if file exists
|
||||||
|
if not Path(elf_path).exists():
|
||||||
|
# Try alternate path
|
||||||
|
alt_path = Path(CORE.relative_build_path(".pioenvs", CORE.name, "firmware.elf"))
|
||||||
|
if alt_path.exists():
|
||||||
|
elf_path = str(alt_path)
|
||||||
|
_LOGGER.debug("Using alternate ELF path: %s", elf_path)
|
||||||
|
else:
|
||||||
|
_LOGGER.warning("ELF file not found at %s or %s", elf_path, alt_path)
|
||||||
|
return
|
||||||
|
|
||||||
|
# Extract external components from config
|
||||||
|
external_components = set()
|
||||||
|
|
||||||
|
# Get the list of built-in ESPHome components
|
||||||
|
from esphome.analyze_memory import get_esphome_components
|
||||||
|
|
||||||
|
builtin_components = get_esphome_components()
|
||||||
|
|
||||||
|
# Special non-component keys that appear in configs
|
||||||
|
NON_COMPONENT_KEYS = {
|
||||||
|
CONF_ESPHOME,
|
||||||
|
"substitutions",
|
||||||
|
"packages",
|
||||||
|
"globals",
|
||||||
|
"<<",
|
||||||
|
}
|
||||||
|
|
||||||
|
# Check all top-level keys in config
|
||||||
|
for key in config:
|
||||||
|
if key not in builtin_components and key not in NON_COMPONENT_KEYS:
|
||||||
|
# This is an external component
|
||||||
|
external_components.add(key)
|
||||||
|
|
||||||
|
_LOGGER.debug("Detected external components: %s", external_components)
|
||||||
|
|
||||||
|
# Create analyzer and run analysis
|
||||||
|
analyzer = MemoryAnalyzer(elf_path, objdump_path, readelf_path, external_components)
|
||||||
|
analyzer.analyze()
|
||||||
|
|
||||||
|
# Generate and print report
|
||||||
|
report = analyzer.generate_report()
|
||||||
|
_LOGGER.info("\n%s", report)
|
||||||
|
|
||||||
|
# Optionally save to file
|
||||||
|
if config.get(CONF_ESPHOME, {}).get("memory_report_file"):
|
||||||
|
report_file = Path(config[CONF_ESPHOME]["memory_report_file"])
|
||||||
|
if report_file.suffix == ".json":
|
||||||
|
report_file.write_text(analyzer.to_json())
|
||||||
|
_LOGGER.info("Memory report saved to %s", report_file)
|
||||||
|
else:
|
||||||
|
report_file.write_text(report)
|
||||||
|
_LOGGER.info("Memory report saved to %s", report_file)
|
||||||
|
@@ -24,6 +24,57 @@ from esphome.analyze_memory import MemoryAnalyzer # noqa: E402
|
|||||||
COMMENT_MARKER = "<!-- esphome-memory-impact-analysis -->"
|
COMMENT_MARKER = "<!-- esphome-memory-impact-analysis -->"
|
||||||
|
|
||||||
|
|
||||||
|
def get_platform_toolchain(platform: str) -> tuple[str | None, str | None]:
|
||||||
|
"""Get platform-specific objdump and readelf paths.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
platform: Platform name (e.g., "esp8266-ard", "esp32-idf", "esp32-c3-idf")
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
Tuple of (objdump_path, readelf_path) or (None, None) if not found/supported
|
||||||
|
"""
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
home = Path.home()
|
||||||
|
platformio_packages = home / ".platformio" / "packages"
|
||||||
|
|
||||||
|
# Map platform to toolchain
|
||||||
|
toolchain = None
|
||||||
|
prefix = None
|
||||||
|
|
||||||
|
if "esp8266" in platform:
|
||||||
|
toolchain = "toolchain-xtensa"
|
||||||
|
prefix = "xtensa-lx106-elf"
|
||||||
|
elif "esp32-c" in platform or "esp32-h" in platform or "esp32-p4" in platform:
|
||||||
|
# RISC-V variants (C2, C3, C5, C6, H2, P4)
|
||||||
|
toolchain = "toolchain-riscv32-esp"
|
||||||
|
prefix = "riscv32-esp-elf"
|
||||||
|
elif "esp32" in platform:
|
||||||
|
# Xtensa variants (original, S2, S3)
|
||||||
|
toolchain = "toolchain-xtensa-esp-elf"
|
||||||
|
if "s2" in platform:
|
||||||
|
prefix = "xtensa-esp32s2-elf"
|
||||||
|
elif "s3" in platform:
|
||||||
|
prefix = "xtensa-esp32s3-elf"
|
||||||
|
else:
|
||||||
|
prefix = "xtensa-esp32-elf"
|
||||||
|
else:
|
||||||
|
# Other platforms (RP2040, LibreTiny, etc.) - not supported
|
||||||
|
print(f"Platform {platform} not supported for ELF analysis", file=sys.stderr)
|
||||||
|
return None, None
|
||||||
|
|
||||||
|
toolchain_path = platformio_packages / toolchain / "bin"
|
||||||
|
objdump_path = toolchain_path / f"{prefix}-objdump"
|
||||||
|
readelf_path = toolchain_path / f"{prefix}-readelf"
|
||||||
|
|
||||||
|
if objdump_path.exists() and readelf_path.exists():
|
||||||
|
print(f"Using {platform} toolchain: {prefix}", file=sys.stderr)
|
||||||
|
return str(objdump_path), str(readelf_path)
|
||||||
|
|
||||||
|
print(f"Warning: Toolchain not found at {toolchain_path}", file=sys.stderr)
|
||||||
|
return None, None
|
||||||
|
|
||||||
|
|
||||||
def format_bytes(bytes_value: int) -> str:
|
def format_bytes(bytes_value: int) -> str:
|
||||||
"""Format bytes value with comma separators.
|
"""Format bytes value with comma separators.
|
||||||
|
|
||||||
@@ -314,7 +365,7 @@ def create_detailed_breakdown_table(
|
|||||||
|
|
||||||
|
|
||||||
def create_comment_body(
|
def create_comment_body(
|
||||||
component: str,
|
components: list[str],
|
||||||
platform: str,
|
platform: str,
|
||||||
target_ram: int,
|
target_ram: int,
|
||||||
target_flash: int,
|
target_flash: int,
|
||||||
@@ -328,7 +379,7 @@ def create_comment_body(
|
|||||||
"""Create the comment body with memory impact analysis.
|
"""Create the comment body with memory impact analysis.
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
component: Component name
|
components: List of component names (merged config)
|
||||||
platform: Platform name
|
platform: Platform name
|
||||||
target_ram: RAM usage in target branch
|
target_ram: RAM usage in target branch
|
||||||
target_flash: Flash usage in target branch
|
target_flash: Flash usage in target branch
|
||||||
@@ -374,10 +425,18 @@ def create_comment_body(
|
|||||||
else:
|
else:
|
||||||
print("No ELF files provided, skipping detailed analysis", file=sys.stderr)
|
print("No ELF files provided, skipping detailed analysis", file=sys.stderr)
|
||||||
|
|
||||||
|
# Format components list
|
||||||
|
if len(components) == 1:
|
||||||
|
components_str = f"`{components[0]}`"
|
||||||
|
config_note = "a representative test configuration"
|
||||||
|
else:
|
||||||
|
components_str = ", ".join(f"`{c}`" for c in sorted(components))
|
||||||
|
config_note = f"a merged configuration with {len(components)} components"
|
||||||
|
|
||||||
return f"""{COMMENT_MARKER}
|
return f"""{COMMENT_MARKER}
|
||||||
## Memory Impact Analysis
|
## Memory Impact Analysis
|
||||||
|
|
||||||
**Component:** `{component}`
|
**Components:** {components_str}
|
||||||
**Platform:** `{platform}`
|
**Platform:** `{platform}`
|
||||||
|
|
||||||
| Metric | Target Branch | This PR | Change |
|
| Metric | Target Branch | This PR | Change |
|
||||||
@@ -386,7 +445,7 @@ def create_comment_body(
|
|||||||
| **Flash** | {format_bytes(target_flash)} | {format_bytes(pr_flash)} | {flash_change} |
|
| **Flash** | {format_bytes(target_flash)} | {format_bytes(pr_flash)} | {flash_change} |
|
||||||
{component_breakdown}{symbol_changes}
|
{component_breakdown}{symbol_changes}
|
||||||
---
|
---
|
||||||
*This analysis runs automatically when a single component changes. Memory usage is measured from a representative test configuration.*
|
*This analysis runs automatically when components change. Memory usage is measured from {config_note}.*
|
||||||
"""
|
"""
|
||||||
|
|
||||||
|
|
||||||
@@ -537,7 +596,11 @@ def main() -> int:
|
|||||||
description="Post or update PR comment with memory impact analysis"
|
description="Post or update PR comment with memory impact analysis"
|
||||||
)
|
)
|
||||||
parser.add_argument("--pr-number", required=True, help="PR number")
|
parser.add_argument("--pr-number", required=True, help="PR number")
|
||||||
parser.add_argument("--component", required=True, help="Component name")
|
parser.add_argument(
|
||||||
|
"--components",
|
||||||
|
required=True,
|
||||||
|
help='JSON array of component names (e.g., \'["api", "wifi"]\')',
|
||||||
|
)
|
||||||
parser.add_argument("--platform", required=True, help="Platform name")
|
parser.add_argument("--platform", required=True, help="Platform name")
|
||||||
parser.add_argument(
|
parser.add_argument(
|
||||||
"--target-ram", type=int, required=True, help="Target branch RAM usage"
|
"--target-ram", type=int, required=True, help="Target branch RAM usage"
|
||||||
@@ -560,9 +623,29 @@ def main() -> int:
|
|||||||
|
|
||||||
args = parser.parse_args()
|
args = parser.parse_args()
|
||||||
|
|
||||||
|
# Parse components from JSON
|
||||||
|
try:
|
||||||
|
components = json.loads(args.components)
|
||||||
|
if not isinstance(components, list):
|
||||||
|
print("Error: --components must be a JSON array", file=sys.stderr)
|
||||||
|
sys.exit(1)
|
||||||
|
except json.JSONDecodeError as e:
|
||||||
|
print(f"Error parsing --components JSON: {e}", file=sys.stderr)
|
||||||
|
sys.exit(1)
|
||||||
|
|
||||||
|
# Detect platform-specific toolchain paths
|
||||||
|
objdump_path = args.objdump_path
|
||||||
|
readelf_path = args.readelf_path
|
||||||
|
|
||||||
|
if not objdump_path or not readelf_path:
|
||||||
|
# Auto-detect based on platform
|
||||||
|
objdump_path, readelf_path = get_platform_toolchain(args.platform)
|
||||||
|
|
||||||
# Create comment body
|
# Create comment body
|
||||||
|
# Note: ELF files (if provided) are from the final build when test_build_components
|
||||||
|
# runs multiple builds. Memory totals (RAM/Flash) are already summed across all builds.
|
||||||
comment_body = create_comment_body(
|
comment_body = create_comment_body(
|
||||||
component=args.component,
|
components=components,
|
||||||
platform=args.platform,
|
platform=args.platform,
|
||||||
target_ram=args.target_ram,
|
target_ram=args.target_ram,
|
||||||
target_flash=args.target_flash,
|
target_flash=args.target_flash,
|
||||||
@@ -570,8 +653,8 @@ def main() -> int:
|
|||||||
pr_flash=args.pr_flash,
|
pr_flash=args.pr_flash,
|
||||||
target_elf=args.target_elf,
|
target_elf=args.target_elf,
|
||||||
pr_elf=args.pr_elf,
|
pr_elf=args.pr_elf,
|
||||||
objdump_path=args.objdump_path,
|
objdump_path=objdump_path,
|
||||||
readelf_path=args.readelf_path,
|
readelf_path=readelf_path,
|
||||||
)
|
)
|
||||||
|
|
||||||
# Post or update comment
|
# Post or update comment
|
||||||
|
@@ -28,27 +28,36 @@ from script.ci_helpers import write_github_output
|
|||||||
def extract_from_compile_output(output_text: str) -> tuple[int | None, int | None]:
|
def extract_from_compile_output(output_text: str) -> tuple[int | None, int | None]:
|
||||||
"""Extract memory usage from PlatformIO compile output.
|
"""Extract memory usage from PlatformIO compile output.
|
||||||
|
|
||||||
|
Supports multiple builds (for component groups or isolated components).
|
||||||
|
When test_build_components.py creates multiple builds, this sums the
|
||||||
|
memory usage across all builds.
|
||||||
|
|
||||||
Looks for lines like:
|
Looks for lines like:
|
||||||
RAM: [==== ] 36.1% (used 29548 bytes from 81920 bytes)
|
RAM: [==== ] 36.1% (used 29548 bytes from 81920 bytes)
|
||||||
Flash: [=== ] 34.0% (used 348511 bytes from 1023984 bytes)
|
Flash: [=== ] 34.0% (used 348511 bytes from 1023984 bytes)
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
output_text: Compile output text
|
output_text: Compile output text (may contain multiple builds)
|
||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
Tuple of (ram_bytes, flash_bytes) or (None, None) if not found
|
Tuple of (total_ram_bytes, total_flash_bytes) or (None, None) if not found
|
||||||
"""
|
"""
|
||||||
ram_match = re.search(
|
# Find all RAM and Flash matches (may be multiple builds)
|
||||||
|
ram_matches = re.findall(
|
||||||
r"RAM:\s+\[.*?\]\s+\d+\.\d+%\s+\(used\s+(\d+)\s+bytes", output_text
|
r"RAM:\s+\[.*?\]\s+\d+\.\d+%\s+\(used\s+(\d+)\s+bytes", output_text
|
||||||
)
|
)
|
||||||
flash_match = re.search(
|
flash_matches = re.findall(
|
||||||
r"Flash:\s+\[.*?\]\s+\d+\.\d+%\s+\(used\s+(\d+)\s+bytes", output_text
|
r"Flash:\s+\[.*?\]\s+\d+\.\d+%\s+\(used\s+(\d+)\s+bytes", output_text
|
||||||
)
|
)
|
||||||
|
|
||||||
if ram_match and flash_match:
|
if not ram_matches or not flash_matches:
|
||||||
return int(ram_match.group(1)), int(flash_match.group(1))
|
return None, None
|
||||||
|
|
||||||
return None, None
|
# Sum all builds (handles multiple component groups)
|
||||||
|
total_ram = sum(int(match) for match in ram_matches)
|
||||||
|
total_flash = sum(int(match) for match in flash_matches)
|
||||||
|
|
||||||
|
return total_ram, total_flash
|
||||||
|
|
||||||
|
|
||||||
def main() -> int:
|
def main() -> int:
|
||||||
@@ -83,8 +92,21 @@ def main() -> int:
|
|||||||
)
|
)
|
||||||
return 1
|
return 1
|
||||||
|
|
||||||
print(f"RAM: {ram_bytes} bytes", file=sys.stderr)
|
# Count how many builds were found
|
||||||
print(f"Flash: {flash_bytes} bytes", file=sys.stderr)
|
num_builds = len(
|
||||||
|
re.findall(
|
||||||
|
r"RAM:\s+\[.*?\]\s+\d+\.\d+%\s+\(used\s+(\d+)\s+bytes", compile_output
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
if num_builds > 1:
|
||||||
|
print(
|
||||||
|
f"Found {num_builds} builds - summing memory usage across all builds",
|
||||||
|
file=sys.stderr,
|
||||||
|
)
|
||||||
|
|
||||||
|
print(f"Total RAM: {ram_bytes} bytes", file=sys.stderr)
|
||||||
|
print(f"Total Flash: {flash_bytes} bytes", file=sys.stderr)
|
||||||
|
|
||||||
if args.output_env:
|
if args.output_env:
|
||||||
# Output to GitHub Actions
|
# Output to GitHub Actions
|
||||||
|
@@ -237,14 +237,14 @@ def _component_has_tests(component: str) -> bool:
|
|||||||
return any(tests_dir.glob("test.*.yaml"))
|
return any(tests_dir.glob("test.*.yaml"))
|
||||||
|
|
||||||
|
|
||||||
def detect_single_component_for_memory_impact(
|
def detect_memory_impact_config(
|
||||||
branch: str | None = None,
|
branch: str | None = None,
|
||||||
) -> dict[str, Any]:
|
) -> dict[str, Any]:
|
||||||
"""Detect if exactly one component changed for memory impact analysis.
|
"""Determine memory impact analysis configuration.
|
||||||
|
|
||||||
This analyzes the actual changed files (not dependencies) to determine if
|
Always runs memory impact analysis when there are changed components,
|
||||||
exactly one component has been modified. This is different from the
|
building a merged configuration with all changed components (like
|
||||||
changed_components list which includes all dependencies.
|
test_build_components.py does) to get comprehensive memory analysis.
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
branch: Branch to compare against
|
branch: Branch to compare against
|
||||||
@@ -252,37 +252,25 @@ def detect_single_component_for_memory_impact(
|
|||||||
Returns:
|
Returns:
|
||||||
Dictionary with memory impact analysis parameters:
|
Dictionary with memory impact analysis parameters:
|
||||||
- should_run: "true" or "false"
|
- should_run: "true" or "false"
|
||||||
- component: component name (if should_run is true)
|
- components: list of component names to analyze
|
||||||
- test_file: test file name (if should_run is true)
|
- platform: platform name for the merged build
|
||||||
- platform: platform name (if should_run is true)
|
- use_merged_config: "true" (always use merged config)
|
||||||
"""
|
"""
|
||||||
# Platform preference order for memory impact analysis
|
# Platform preference order for memory impact analysis
|
||||||
# Ordered by production relevance and memory constraint importance
|
# Prefer ESP8266 for memory impact as it's the most constrained platform
|
||||||
PLATFORM_PREFERENCE = [
|
PLATFORM_PREFERENCE = [
|
||||||
|
"esp8266-ard", # ESP8266 Arduino (most memory constrained - best for impact analysis)
|
||||||
"esp32-idf", # Primary ESP32 IDF platform
|
"esp32-idf", # Primary ESP32 IDF platform
|
||||||
"esp32-c3-idf", # ESP32-C3 IDF
|
"esp32-c3-idf", # ESP32-C3 IDF
|
||||||
"esp32-c6-idf", # ESP32-C6 IDF
|
"esp32-c6-idf", # ESP32-C6 IDF
|
||||||
"esp32-s2-idf", # ESP32-S2 IDF
|
"esp32-s2-idf", # ESP32-S2 IDF
|
||||||
"esp32-s3-idf", # ESP32-S3 IDF
|
"esp32-s3-idf", # ESP32-S3 IDF
|
||||||
"esp32-c2-idf", # ESP32-C2 IDF
|
|
||||||
"esp32-c5-idf", # ESP32-C5 IDF
|
|
||||||
"esp32-h2-idf", # ESP32-H2 IDF
|
|
||||||
"esp32-p4-idf", # ESP32-P4 IDF
|
|
||||||
"esp8266-ard", # ESP8266 Arduino (memory constrained)
|
|
||||||
"esp32-ard", # ESP32 Arduino
|
|
||||||
"esp32-c3-ard", # ESP32-C3 Arduino
|
|
||||||
"esp32-s2-ard", # ESP32-S2 Arduino
|
|
||||||
"esp32-s3-ard", # ESP32-S3 Arduino
|
|
||||||
"bk72xx-ard", # BK72xx Arduino
|
|
||||||
"rp2040-ard", # RP2040 Arduino
|
|
||||||
"nrf52-adafruit", # nRF52 Adafruit
|
|
||||||
"host", # Host platform (development/testing)
|
|
||||||
]
|
]
|
||||||
|
|
||||||
# Get actually changed files (not dependencies)
|
# Get actually changed files (not dependencies)
|
||||||
files = changed_files(branch)
|
files = changed_files(branch)
|
||||||
|
|
||||||
# Find all changed components (excluding core)
|
# Find all changed components (excluding core and base bus components)
|
||||||
changed_component_set = set()
|
changed_component_set = set()
|
||||||
|
|
||||||
for file in files:
|
for file in files:
|
||||||
@@ -291,49 +279,53 @@ def detect_single_component_for_memory_impact(
|
|||||||
if len(parts) >= 3:
|
if len(parts) >= 3:
|
||||||
component = parts[2]
|
component = parts[2]
|
||||||
# Skip base bus components as they're used across many builds
|
# Skip base bus components as they're used across many builds
|
||||||
if component not in ["i2c", "spi", "uart", "modbus"]:
|
if component not in ["i2c", "spi", "uart", "modbus", "canbus"]:
|
||||||
changed_component_set.add(component)
|
changed_component_set.add(component)
|
||||||
|
|
||||||
# Only proceed if exactly one component changed
|
# If no components changed, don't run memory impact
|
||||||
if len(changed_component_set) != 1:
|
if not changed_component_set:
|
||||||
return {"should_run": "false"}
|
return {"should_run": "false"}
|
||||||
|
|
||||||
component = list(changed_component_set)[0]
|
# Find components that have tests on the preferred platform
|
||||||
|
components_with_tests = []
|
||||||
|
selected_platform = None
|
||||||
|
|
||||||
# Find a test configuration for this component
|
for component in sorted(changed_component_set):
|
||||||
tests_dir = Path(root_path) / "tests" / "components" / component
|
tests_dir = Path(root_path) / "tests" / "components" / component
|
||||||
|
if not tests_dir.exists():
|
||||||
|
continue
|
||||||
|
|
||||||
if not tests_dir.exists():
|
# Look for test files on preferred platforms
|
||||||
return {"should_run": "false"}
|
test_files = list(tests_dir.glob("test.*.yaml"))
|
||||||
|
if not test_files:
|
||||||
|
continue
|
||||||
|
|
||||||
# Look for test files
|
# Check if component has tests for any preferred platform
|
||||||
test_files = list(tests_dir.glob("test.*.yaml"))
|
|
||||||
if not test_files:
|
|
||||||
return {"should_run": "false"}
|
|
||||||
|
|
||||||
# Try each preferred platform in order
|
|
||||||
for preferred_platform in PLATFORM_PREFERENCE:
|
|
||||||
for test_file in test_files:
|
for test_file in test_files:
|
||||||
parts = test_file.stem.split(".")
|
parts = test_file.stem.split(".")
|
||||||
if len(parts) >= 2:
|
if len(parts) >= 2:
|
||||||
platform = parts[1]
|
platform = parts[1]
|
||||||
if platform == preferred_platform:
|
if platform in PLATFORM_PREFERENCE:
|
||||||
return {
|
components_with_tests.append(component)
|
||||||
"should_run": "true",
|
# Select the most preferred platform across all components
|
||||||
"component": component,
|
if selected_platform is None or PLATFORM_PREFERENCE.index(
|
||||||
"test_file": test_file.name,
|
platform
|
||||||
"platform": platform,
|
) < PLATFORM_PREFERENCE.index(selected_platform):
|
||||||
}
|
selected_platform = platform
|
||||||
|
break
|
||||||
|
|
||||||
|
# If no components have tests, don't run memory impact
|
||||||
|
if not components_with_tests:
|
||||||
|
return {"should_run": "false"}
|
||||||
|
|
||||||
|
# Use the most preferred platform found, or fall back to esp8266-ard
|
||||||
|
platform = selected_platform or "esp8266-ard"
|
||||||
|
|
||||||
# Fall back to first test file
|
|
||||||
test_file = test_files[0]
|
|
||||||
parts = test_file.stem.split(".")
|
|
||||||
platform = parts[1] if len(parts) >= 2 else "esp32-idf"
|
|
||||||
return {
|
return {
|
||||||
"should_run": "true",
|
"should_run": "true",
|
||||||
"component": component,
|
"components": components_with_tests,
|
||||||
"test_file": test_file.name,
|
|
||||||
"platform": platform,
|
"platform": platform,
|
||||||
|
"use_merged_config": "true",
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
@@ -386,8 +378,8 @@ def main() -> None:
|
|||||||
if component not in directly_changed_components
|
if component not in directly_changed_components
|
||||||
]
|
]
|
||||||
|
|
||||||
# Detect single component change for memory impact analysis
|
# Detect components for memory impact analysis (merged config)
|
||||||
memory_impact = detect_single_component_for_memory_impact(args.branch)
|
memory_impact = detect_memory_impact_config(args.branch)
|
||||||
|
|
||||||
# Build output
|
# Build output
|
||||||
output: dict[str, Any] = {
|
output: dict[str, Any] = {
|
||||||
|
Reference in New Issue
Block a user