From a59b1494d80955a73cef001fceca889f94b654b1 Mon Sep 17 00:00:00 2001 From: Javier Peletier Date: Mon, 20 Oct 2025 03:17:16 +0200 Subject: [PATCH] [substitutions] Recursive substitutions and better jinja error handling and debug help (#10806) --- esphome/components/substitutions/__init__.py | 17 +-- esphome/components/substitutions/jinja.py | 109 +++++++++++++++--- .../02-expressions.approved.yaml | 2 + .../substitutions/02-expressions.input.yaml | 2 + 4 files changed, 100 insertions(+), 30 deletions(-) diff --git a/esphome/components/substitutions/__init__.py b/esphome/components/substitutions/__init__.py index e6bcdc063a..098d56bfad 100644 --- a/esphome/components/substitutions/__init__.py +++ b/esphome/components/substitutions/__init__.py @@ -6,7 +6,7 @@ import esphome.config_validation as cv from esphome.const import CONF_SUBSTITUTIONS, VALID_SUBSTITUTIONS_CHARACTERS from esphome.yaml_util import ESPHomeDataBase, ESPLiteralValue, make_data_base -from .jinja import Jinja, JinjaStr, TemplateError, TemplateRuntimeError, has_jinja +from .jinja import Jinja, JinjaError, JinjaStr, has_jinja CODEOWNERS = ["@esphome/core"] _LOGGER = logging.getLogger(__name__) @@ -57,17 +57,12 @@ def _expand_jinja(value, orig_value, path, jinja, ignore_missing): "->".join(str(x) for x in path), err.message, ) - except ( - TemplateError, - TemplateRuntimeError, - RuntimeError, - ArithmeticError, - AttributeError, - TypeError, - ) as err: + except JinjaError as err: raise cv.Invalid( - f"{type(err).__name__} Error evaluating jinja expression '{value}': {str(err)}." - f" See {'->'.join(str(x) for x in path)}", + f"{err.error_name()} Error evaluating jinja expression '{value}': {str(err.parent())}." + f"\nEvaluation stack: (most recent evaluation last)\n{err.stack_trace_str()}" + f"\nRelevant context:\n{err.context_trace_str()}" + f"\nSee {'->'.join(str(x) for x in path)}", path, ) return value diff --git a/esphome/components/substitutions/jinja.py b/esphome/components/substitutions/jinja.py index e7164d8fff..dde0162993 100644 --- a/esphome/components/substitutions/jinja.py +++ b/esphome/components/substitutions/jinja.py @@ -6,6 +6,8 @@ import re import jinja2 as jinja from jinja2.sandbox import SandboxedEnvironment +from esphome.yaml_util import ESPLiteralValue + TemplateError = jinja.TemplateError TemplateSyntaxError = jinja.TemplateSyntaxError TemplateRuntimeError = jinja.TemplateRuntimeError @@ -26,18 +28,20 @@ def has_jinja(st): return detect_jinja_re.search(st) is not None -# SAFE_GLOBAL_FUNCTIONS defines a allowlist of built-in functions that are considered safe to expose +# SAFE_GLOBALS defines a allowlist of built-in functions or modules that are considered safe to expose # in Jinja templates or other sandboxed evaluation contexts. Only functions that do not allow # arbitrary code execution, file access, or other security risks are included. # # The following functions are considered safe: +# - math: The entire math module is injected, allowing access to mathematical functions like sin, cos, sqrt, etc. # - ord: Converts a character to its Unicode code point integer. # - chr: Converts an integer to its corresponding Unicode character. # - len: Returns the length of a sequence or collection. # # These functions were chosen because they are pure, have no side effects, and do not provide access # to the file system, environment, or other potentially sensitive resources. -SAFE_GLOBAL_FUNCTIONS = { +SAFE_GLOBALS = { + "math": math, # Inject entire math module "ord": ord, "chr": chr, "len": len, @@ -56,22 +60,62 @@ class JinjaStr(str): later in the main substitutions pass. """ + Undefined = object() + def __new__(cls, value: str, upvalues=None): - obj = super().__new__(cls, value) - obj.upvalues = upvalues or {} + if isinstance(value, JinjaStr): + base = str(value) + merged = {**value.upvalues, **(upvalues or {})} + else: + base = value + merged = dict(upvalues or {}) + obj = super().__new__(cls, base) + obj.upvalues = merged + obj.result = JinjaStr.Undefined return obj - def __init__(self, value: str, upvalues=None): - self.upvalues = upvalues or {} + +class JinjaError(Exception): + def __init__(self, context_trace: dict, expr: str): + self.context_trace = context_trace + self.eval_stack = [expr] + + def parent(self): + return self.__context__ + + def error_name(self): + return type(self.parent()).__name__ + + def context_trace_str(self): + return "\n".join( + f" {k} = {repr(v)} ({type(v).__name__})" + for k, v in self.context_trace.items() + ) + + def stack_trace_str(self): + return "\n".join( + f" {len(self.eval_stack) - i}: {expr}{i == 0 and ' <-- ' + self.error_name() or ''}" + for i, expr in enumerate(self.eval_stack) + ) -class Jinja: +class TrackerContext(jinja.runtime.Context): + def resolve_or_missing(self, key): + val = super().resolve_or_missing(key) + if isinstance(val, JinjaStr): + self.environment.context_trace[key] = val + val, _ = self.environment.expand(val) + self.environment.context_trace[key] = val + return val + + +class Jinja(SandboxedEnvironment): """ Wraps a Jinja environment """ def __init__(self, context_vars): - self.env = SandboxedEnvironment( + super().__init__( trim_blocks=True, lstrip_blocks=True, block_start_string="<%", @@ -82,13 +126,20 @@ class Jinja: variable_end_string="}", undefined=jinja.StrictUndefined, ) - self.env.add_extension("jinja2.ext.do") - self.env.globals["math"] = math # Inject entire math module + self.context_class = TrackerContext + self.add_extension("jinja2.ext.do") + self.context_trace = {} self.context_vars = {**context_vars} - self.env.globals = { - **self.env.globals, + for k, v in self.context_vars.items(): + if isinstance(v, ESPLiteralValue): + continue + if isinstance(v, str) and not isinstance(v, JinjaStr) and has_jinja(v): + self.context_vars[k] = JinjaStr(v, self.context_vars) + + self.globals = { + **self.globals, **self.context_vars, - **SAFE_GLOBAL_FUNCTIONS, + **SAFE_GLOBALS, } def safe_eval(self, expr): @@ -110,23 +161,43 @@ class Jinja: result = None override_vars = {} if isinstance(content_str, JinjaStr): + if content_str.result is not JinjaStr.Undefined: + return content_str.result, None # If `value` is already a JinjaStr, it means we are trying to evaluate it again # in a parent pass. # Hopefully, all required variables are visible now. override_vars = content_str.upvalues + + old_trace = self.context_trace + self.context_trace = {} try: - template = self.env.from_string(content_str) + template = self.from_string(content_str) result = self.safe_eval(template.render(override_vars)) if isinstance(result, Undefined): - # This happens when the expression is simply an undefined variable. Jinja does not - # raise an exception, instead we get "Undefined". - # Trigger an UndefinedError exception so we skip to below. - print("" + result) + print("" + result) # force a UndefinedError exception except (TemplateSyntaxError, UndefinedError) as err: # `content_str` contains a Jinja expression that refers to a variable that is undefined # in this scope. Perhaps it refers to a root substitution that is not visible yet. - # Therefore, return the original `content_str` as a JinjaStr, which contains the variables + # Therefore, return `content_str` as a JinjaStr, which contains the variables # that are actually visible to it at this point to postpone evaluation. return JinjaStr(content_str, {**self.context_vars, **override_vars}), err + except JinjaError as err: + err.context_trace = {**self.context_trace, **err.context_trace} + err.eval_stack.append(content_str) + raise err + except ( + TemplateError, + TemplateRuntimeError, + RuntimeError, + ArithmeticError, + AttributeError, + TypeError, + ) as err: + raise JinjaError(self.context_trace, content_str) from err + finally: + self.context_trace = old_trace + + if isinstance(content_str, JinjaStr): + content_str.result = result return result, None diff --git a/tests/unit_tests/fixtures/substitutions/02-expressions.approved.yaml b/tests/unit_tests/fixtures/substitutions/02-expressions.approved.yaml index 443cba144e..1a51fc44cf 100644 --- a/tests/unit_tests/fixtures/substitutions/02-expressions.approved.yaml +++ b/tests/unit_tests/fixtures/substitutions/02-expressions.approved.yaml @@ -8,6 +8,7 @@ substitutions: area: 25 numberOne: 1 var1: 79 + double_width: 14 test_list: - The area is 56 - 56 @@ -25,3 +26,4 @@ test_list: - ord("a") = 97 - chr(97) = a - len([1,2,3]) = 3 + - width = 7, double_width = 14 diff --git a/tests/unit_tests/fixtures/substitutions/02-expressions.input.yaml b/tests/unit_tests/fixtures/substitutions/02-expressions.input.yaml index 07ad992f1f..4612f581b5 100644 --- a/tests/unit_tests/fixtures/substitutions/02-expressions.input.yaml +++ b/tests/unit_tests/fixtures/substitutions/02-expressions.input.yaml @@ -8,6 +8,7 @@ substitutions: area: 25 numberOne: 1 var1: 79 + double_width: ${width * 2} test_list: - "The area is ${width * height}" @@ -23,3 +24,4 @@ test_list: - ord("a") = ${ ord("a") } - chr(97) = ${ chr(97) } - len([1,2,3]) = ${ len([1,2,3]) } + - width = ${width}, double_width = ${double_width}