From 05216db5f0cb397ba3bce637c8ef0d31c1ae0b77 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Mon, 20 Oct 2025 16:26:49 -1000 Subject: [PATCH] ESP8266: Complete testing mode memory patches with DRAM and Flash (#11427) --- esphome/components/esp8266/__init__.py | 16 +- esphome/components/esp8266/iram_fix.py.script | 44 ----- .../components/esp8266/testing_mode.py.script | 166 ++++++++++++++++++ .../build_components_base.esp8266-ard.yaml | 2 +- 4 files changed, 176 insertions(+), 52 deletions(-) delete mode 100644 esphome/components/esp8266/iram_fix.py.script create mode 100644 esphome/components/esp8266/testing_mode.py.script diff --git a/esphome/components/esp8266/__init__.py b/esphome/components/esp8266/__init__.py index 9d8e6b7d1e..a74f9ee8ce 100644 --- a/esphome/components/esp8266/__init__.py +++ b/esphome/components/esp8266/__init__.py @@ -190,7 +190,9 @@ async def to_code(config): cg.add_define("ESPHOME_VARIANT", "ESP8266") cg.add_define(ThreadModel.SINGLE) - cg.add_platformio_option("extra_scripts", ["pre:iram_fix.py", "post:post_build.py"]) + cg.add_platformio_option( + "extra_scripts", ["pre:testing_mode.py", "post:post_build.py"] + ) conf = config[CONF_FRAMEWORK] cg.add_platformio_option("framework", "arduino") @@ -230,9 +232,9 @@ async def to_code(config): # For cases where nullptrs can be handled, use nothrow: `new (std::nothrow) T;` cg.add_build_flag("-DNEW_OOM_ABORT") - # In testing mode, fake a larger IRAM to allow linking grouped component tests - # Real ESP8266 hardware only has 32KB IRAM, but for CI testing we pretend it has 2MB - # This is done via a pre-build script that generates a custom linker script + # In testing mode, fake larger memory to allow linking grouped component tests + # Real ESP8266 hardware only has 32KB IRAM and ~80KB RAM, but for CI testing + # we pretend it has much larger memory to test that components compile together if CORE.testing_mode: cg.add_build_flag("-DESPHOME_TESTING_MODE") @@ -271,8 +273,8 @@ def copy_files(): post_build_file, CORE.relative_build_path("post_build.py"), ) - iram_fix_file = dir / "iram_fix.py.script" + testing_mode_file = dir / "testing_mode.py.script" copy_file_if_changed( - iram_fix_file, - CORE.relative_build_path("iram_fix.py"), + testing_mode_file, + CORE.relative_build_path("testing_mode.py"), ) diff --git a/esphome/components/esp8266/iram_fix.py.script b/esphome/components/esp8266/iram_fix.py.script deleted file mode 100644 index 96bddc2ced..0000000000 --- a/esphome/components/esp8266/iram_fix.py.script +++ /dev/null @@ -1,44 +0,0 @@ -import os -import re - -# pylint: disable=E0602 -Import("env") # noqa - - -def patch_linker_script_after_preprocess(source, target, env): - """Patch the local linker script after PlatformIO preprocesses it.""" - # Check if we're in testing mode by looking for the define - build_flags = env.get("BUILD_FLAGS", []) - testing_mode = any("-DESPHOME_TESTING_MODE" in flag for flag in build_flags) - - if not testing_mode: - return - - # Get the local linker script path - build_dir = env.subst("$BUILD_DIR") - local_ld = os.path.join(build_dir, "ld", "local.eagle.app.v6.common.ld") - - if not os.path.exists(local_ld): - return - - # Read the linker script - with open(local_ld, "r") as f: - content = f.read() - - # Replace IRAM size from 0x8000 (32KB) to 0x200000 (2MB) - # The line looks like: iram1_0_seg : org = 0x40100000, len = 0x8000 - updated = re.sub( - r"(iram1_0_seg\s*:\s*org\s*=\s*0x40100000\s*,\s*len\s*=\s*)0x8000", - r"\g<1>0x200000", - content, - ) - - if updated != content: - with open(local_ld, "w") as f: - f.write(updated) - print("ESPHome: Patched IRAM size to 2MB for testing mode") - - -# Hook into the build process right before linking -# This runs after PlatformIO has already preprocessed the linker scripts -env.AddPreAction("$BUILD_DIR/${PROGNAME}.elf", patch_linker_script_after_preprocess) diff --git a/esphome/components/esp8266/testing_mode.py.script b/esphome/components/esp8266/testing_mode.py.script new file mode 100644 index 0000000000..44d84b765c --- /dev/null +++ b/esphome/components/esp8266/testing_mode.py.script @@ -0,0 +1,166 @@ +import os +import re + +# pylint: disable=E0602 +Import("env") # noqa + + +# Memory sizes for testing mode (allow larger builds for CI component grouping) +TESTING_IRAM_SIZE = "0x200000" # 2MB +TESTING_DRAM_SIZE = "0x200000" # 2MB +TESTING_FLASH_SIZE = "0x2000000" # 32MB + + +def patch_segment_size(content, segment_name, new_size, label): + """Patch a memory segment's length in linker script. + + Args: + content: Linker script content + segment_name: Name of the segment (e.g., 'iram1_0_seg') + new_size: New size as hex string (e.g., '0x200000') + label: Human-readable label for logging (e.g., 'IRAM') + + Returns: + Tuple of (patched_content, was_patched) + """ + # Match: segment_name : org = 0x..., len = 0x... + pattern = rf"({segment_name}\s*:\s*org\s*=\s*0x[0-9a-fA-F]+\s*,\s*len\s*=\s*)0x[0-9a-fA-F]+" + new_content = re.sub(pattern, rf"\g<1>{new_size}", content) + return new_content, new_content != content + + +def apply_memory_patches(content): + """Apply IRAM, DRAM, and Flash patches to linker script content. + + Args: + content: Linker script content as string + + Returns: + Patched content as string + """ + patches_applied = [] + + # Patch IRAM (for larger code in IRAM) + content, patched = patch_segment_size(content, "iram1_0_seg", TESTING_IRAM_SIZE, "IRAM") + if patched: + patches_applied.append("IRAM") + + # Patch DRAM (for larger BSS/data sections) + content, patched = patch_segment_size(content, "dram0_0_seg", TESTING_DRAM_SIZE, "DRAM") + if patched: + patches_applied.append("DRAM") + + # Patch Flash (for larger code sections) + content, patched = patch_segment_size(content, "irom0_0_seg", TESTING_FLASH_SIZE, "Flash") + if patched: + patches_applied.append("Flash") + + if patches_applied: + iram_mb = int(TESTING_IRAM_SIZE, 16) // (1024 * 1024) + dram_mb = int(TESTING_DRAM_SIZE, 16) // (1024 * 1024) + flash_mb = int(TESTING_FLASH_SIZE, 16) // (1024 * 1024) + print(f" Patched memory segments: {', '.join(patches_applied)} (IRAM/DRAM: {iram_mb}MB, Flash: {flash_mb}MB)") + + return content + + +def patch_linker_script_file(filepath, description): + """Patch a linker script file in the build directory with enlarged memory segments. + + This function modifies linker scripts in the build directory only (never SDK files). + It patches IRAM, DRAM, and Flash segments to allow larger builds in testing mode. + + Args: + filepath: Path to the linker script file in the build directory + description: Human-readable description for logging + + Returns: + True if the file was patched, False if already patched or not found + """ + if not os.path.exists(filepath): + print(f"ESPHome: {description} not found at {filepath}") + return False + + print(f"ESPHome: Patching {description}...") + with open(filepath, "r") as f: + content = f.read() + + patched_content = apply_memory_patches(content) + + if patched_content != content: + with open(filepath, "w") as f: + f.write(patched_content) + print(f"ESPHome: Successfully patched {description}") + return True + else: + print(f"ESPHome: {description} already patched or no changes needed") + return False + + +def patch_local_linker_script(source, target, env): + """Patch the local.eagle.app.v6.common.ld in build directory. + + This patches the preprocessed linker script that PlatformIO creates in the build + directory, enlarging IRAM, DRAM, and Flash segments for testing mode. + + Args: + source: SCons source nodes + target: SCons target nodes + env: SCons environment + """ + # Check if we're in testing mode + build_flags = env.get("BUILD_FLAGS", []) + testing_mode = any("-DESPHOME_TESTING_MODE" in flag for flag in build_flags) + + if not testing_mode: + return + + # Patch the local linker script if it exists + build_dir = env.subst("$BUILD_DIR") + ld_dir = os.path.join(build_dir, "ld") + if os.path.exists(ld_dir): + local_ld = os.path.join(ld_dir, "local.eagle.app.v6.common.ld") + if os.path.exists(local_ld): + patch_linker_script_file(local_ld, "local.eagle.app.v6.common.ld") + + +# Check if we're in testing mode +build_flags = env.get("BUILD_FLAGS", []) +testing_mode = any("-DESPHOME_TESTING_MODE" in flag for flag in build_flags) + +if testing_mode: + # Create a custom linker script in the build directory with patched memory limits + # This allows larger IRAM/DRAM/Flash for CI component grouping tests + build_dir = env.subst("$BUILD_DIR") + ldscript = env.GetProjectOption("board_build.ldscript", "") + assert ldscript, "No linker script configured in board_build.ldscript" + + framework_dir = env.PioPlatform().get_package_dir("framework-arduinoespressif8266") + assert framework_dir is not None, "Could not find framework-arduinoespressif8266 package" + + # Read the original SDK linker script (read-only, SDK is never modified) + sdk_ld = os.path.join(framework_dir, "tools", "sdk", "ld", ldscript) + # Create a custom version in the build directory (isolated, temporary) + custom_ld = os.path.join(build_dir, f"testing_{ldscript}") + + if os.path.exists(sdk_ld) and not os.path.exists(custom_ld): + # Read the SDK linker script + with open(sdk_ld, "r") as f: + content = f.read() + + # Apply memory patches (IRAM: 2MB, DRAM: 2MB, Flash: 32MB) + patched_content = apply_memory_patches(content) + + # Write the patched linker script to the build directory + with open(custom_ld, "w") as f: + f.write(patched_content) + + print(f"ESPHome: Created custom linker script: {custom_ld}") + + # Tell the linker to use our custom script from the build directory + assert os.path.exists(custom_ld), f"Custom linker script not found: {custom_ld}" + env.Replace(LDSCRIPT_PATH=custom_ld) + print(f"ESPHome: Using custom linker script with patched memory limits") + + # Also patch local.eagle.app.v6.common.ld after PlatformIO creates it + env.AddPreAction("$BUILD_DIR/${PROGNAME}.elf", patch_local_linker_script) diff --git a/tests/test_build_components/build_components_base.esp8266-ard.yaml b/tests/test_build_components/build_components_base.esp8266-ard.yaml index e4d6607c86..1e2d614392 100644 --- a/tests/test_build_components/build_components_base.esp8266-ard.yaml +++ b/tests/test_build_components/build_components_base.esp8266-ard.yaml @@ -3,7 +3,7 @@ esphome: friendly_name: $component_name esp8266: - board: d1_mini + board: d1_mini_pro logger: level: VERY_VERBOSE