sascha/esphome
1
0
Fork 0
mirror of https://github.com/esphome/esphome.git synced 2025-10-29 22:24:26 +00:00
Code Issues Releases Wiki Activity

Jinja expressions in configs (Take #3) (#8955)

Browse Source
This commit is contained in:
Javier Peletier
2025-07-01 04:57:00 +02:00
committed by GitHub
parent 27c745d5a1
commit 8c34b72b62
21 changed files with 486 additions and 24 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
esphome/components/packages/__init__.py
View File

@@ -74,7 +74,7 @@ BASE_SCHEMA = cv.All(
{
cv.Required(CONF_PATH): validate_yaml_filename,
cv.Optional(CONF_VARS, default={}): cv.Schema(
{cv.string: cv.string}
{cv.string: object}
),
}
),
@@ -148,7 +148,6 @@ def _process_base_package(config: dict) -> dict:
raise cv.Invalid(
f"Current ESPHome Version is too old to use this package: {ESPHOME_VERSION} < {min_version}"
)
vars = {k: str(v) for k, v in vars.items()}
new_yaml = yaml_util.substitute_vars(new_yaml, vars)
packages[f"{filename}{idx}"] = new_yaml
except EsphomeError as e:

95
esphome/components/substitutions/__init__.py
View File

@@ -5,6 +5,13 @@ from esphome.config_helpers import Extend, Remove, merge_config
import esphome.config_validation as cv
from esphome.const import CONF_SUBSTITUTIONS, VALID_SUBSTITUTIONS_CHARACTERS
from esphome.yaml_util import ESPHomeDataBase, make_data_base
from .jinja import (
Jinja,
JinjaStr,
has_jinja,
TemplateError,
TemplateRuntimeError,
)
CODEOWNERS = ["@esphome/core"]
_LOGGER = logging.getLogger(__name__)
@@ -28,7 +35,7 @@ def validate_substitution_key(value):
CONFIG_SCHEMA = cv.Schema(
{
validate_substitution_key: cv.string_strict,
validate_substitution_key: object,
}
)
@@ -37,7 +44,42 @@ async def to_code(config):
pass
def _expand_substitutions(substitutions, value, path, ignore_missing):
def _expand_jinja(value, orig_value, path, jinja, ignore_missing):
if has_jinja(value):
# If the original value passed in to this function is a JinjaStr, it means it contains an unresolved
# Jinja expression from a previous pass.
if isinstance(orig_value, JinjaStr):
# Rebuild the JinjaStr in case it was lost while replacing substitutions.
value = JinjaStr(value, orig_value.upvalues)
try:
# Invoke the jinja engine to evaluate the expression.
value, err = jinja.expand(value)
if err is not None:
if not ignore_missing and "password" not in path:
_LOGGER.warning(
"Found '%s' (see %s) which looks like an expression,"
" but could not resolve all the variables: %s",
value,
"->".join(str(x) for x in path),
err.message,
)
except (
TemplateError,
TemplateRuntimeError,
RuntimeError,
ArithmeticError,
AttributeError,
TypeError,
) as err:
raise cv.Invalid(
f"{type(err).__name__} Error evaluating jinja expression '{value}': {str(err)}."
f" See {'->'.join(str(x) for x in path)}",
path,
)
return value
def _expand_substitutions(substitutions, value, path, jinja, ignore_missing):
if "$" not in value:
return value
@@ -47,7 +89,8 @@ def _expand_substitutions(substitutions, value, path, ignore_missing):
while True:
m = cv.VARIABLE_PROG.search(value, i)
if not m:
# Nothing more to match. Done
# No more variable substitutions found. See if the remainder looks like a jinja template
value = _expand_jinja(value, orig_value, path, jinja, ignore_missing)
break
i, j = m.span(0)
@@ -67,8 +110,15 @@ def _expand_substitutions(substitutions, value, path, ignore_missing):
continue
sub = substitutions[name]
if i == 0 and j == len(value):
# The variable spans the whole expression, e.g., "${varName}". Return its resolved value directly
# to conserve its type.
value = sub
break
tail = value[j:]
value = value[:i] + sub
value = value[:i] + str(sub)
i = len(value)
value += tail
@@ -77,36 +127,40 @@ def _expand_substitutions(substitutions, value, path, ignore_missing):
if isinstance(orig_value, ESPHomeDataBase):
# even though string can get larger or smaller, the range should point
# to original document marks
return make_data_base(value, orig_value)
value = make_data_base(value, orig_value)
return value
def _substitute_item(substitutions, item, path, ignore_missing):
def _substitute_item(substitutions, item, path, jinja, ignore_missing):
if isinstance(item, list):
for i, it in enumerate(item):
sub = _substitute_item(substitutions, it, path + [i], ignore_missing)
sub = _substitute_item(substitutions, it, path + [i], jinja, ignore_missing)
if sub is not None:
item[i] = sub
elif isinstance(item, dict):
replace_keys = []
for k, v in item.items():
if path or k != CONF_SUBSTITUTIONS:
sub = _substitute_item(substitutions, k, path + [k], ignore_missing)
sub = _substitute_item(
substitutions, k, path + [k], jinja, ignore_missing
)
if sub is not None:
replace_keys.append((k, sub))
sub = _substitute_item(substitutions, v, path + [k], ignore_missing)
sub = _substitute_item(substitutions, v, path + [k], jinja, ignore_missing)
if sub is not None:
item[k] = sub
for old, new in replace_keys:
item[new] = merge_config(item.get(old), item.get(new))
del item[old]
elif isinstance(item, str):
sub = _expand_substitutions(substitutions, item, path, ignore_missing)
if sub != item:
sub = _expand_substitutions(substitutions, item, path, jinja, ignore_missing)
if isinstance(sub, JinjaStr) or sub != item:
return sub
elif isinstance(item, (core.Lambda, Extend, Remove)):
sub = _expand_substitutions(substitutions, item.value, path, ignore_missing)
sub = _expand_substitutions(
substitutions, item.value, path, jinja, ignore_missing
)
if sub != item:
item.value = sub
return None
@@ -116,11 +170,11 @@ def do_substitution_pass(config, command_line_substitutions, ignore_missing=Fals
if CONF_SUBSTITUTIONS not in config and not command_line_substitutions:
return
substitutions = config.get(CONF_SUBSTITUTIONS)
if substitutions is None:
substitutions = command_line_substitutions
elif command_line_substitutions:
substitutions = {**substitutions, **command_line_substitutions}
# Merge substitutions in config, overriding with substitutions coming from command line:
substitutions = {
**config.get(CONF_SUBSTITUTIONS, {}),
**(command_line_substitutions or {}),
}
with cv.prepend_path("substitutions"):
if not isinstance(substitutions, dict):
raise cv.Invalid(
@@ -133,7 +187,7 @@ def do_substitution_pass(config, command_line_substitutions, ignore_missing=Fals
sub = validate_substitution_key(key)
if sub != key:
replace_keys.append((key, sub))
substitutions[key] = cv.string_strict(value)
substitutions[key] = value
for old, new in replace_keys:
substitutions[new] = substitutions[old]
del substitutions[old]
@@ -141,4 +195,7 @@ def do_substitution_pass(config, command_line_substitutions, ignore_missing=Fals
config[CONF_SUBSTITUTIONS] = substitutions
# Move substitutions to the first place to replace substitutions in them correctly
config.move_to_end(CONF_SUBSTITUTIONS, False)
_substitute_item(substitutions, config, [], ignore_missing)
# Create a Jinja environment that will consider substitutions in scope:
jinja = Jinja(substitutions)
_substitute_item(substitutions, config, [], jinja, ignore_missing)

99
esphome/components/substitutions/jinja.py Normal file
View File

@@ -0,0 +1,99 @@
import logging
import math
import re
import jinja2 as jinja
from jinja2.nativetypes import NativeEnvironment
TemplateError = jinja.TemplateError
TemplateSyntaxError = jinja.TemplateSyntaxError
TemplateRuntimeError = jinja.TemplateRuntimeError
UndefinedError = jinja.UndefinedError
Undefined = jinja.Undefined
_LOGGER = logging.getLogger(__name__)
DETECT_JINJA = r"(\$\{)"
detect_jinja_re = re.compile(
r"<%.+?%>" # Block form expression: <% ... %>
r"|\$\{[^}]+\}", # Braced form expression: ${ ... }
flags=re.MULTILINE,
)
def has_jinja(st):
return detect_jinja_re.search(st) is not None
class JinjaStr(str):
"""
Wraps a string containing an unresolved Jinja expression,
storing the variables visible to it when it failed to resolve.
For example, an expression inside a package, `${ A * B }` may fail
to resolve at package parsing time if `A` is a local package var
but `B` is a substitution defined in the root yaml.
Therefore, we store the value of `A` as an upvalue bound
to the original string so we may be able to resolve `${ A * B }`
later in the main substitutions pass.
"""
def __new__(cls, value: str, upvalues=None):
obj = super().__new__(cls, value)
obj.upvalues = upvalues or {}
return obj
def __init__(self, value: str, upvalues=None):
self.upvalues = upvalues or {}
class Jinja:
"""
Wraps a Jinja environment
"""
def __init__(self, context_vars):
self.env = NativeEnvironment(
trim_blocks=True,
lstrip_blocks=True,
block_start_string="<%",
block_end_string="%>",
line_statement_prefix="#",
line_comment_prefix="##",
variable_start_string="${",
variable_end_string="}",
undefined=jinja.StrictUndefined,
)
self.env.add_extension("jinja2.ext.do")
self.env.globals["math"] = math # Inject entire math module
self.context_vars = {**context_vars}
self.env.globals = {**self.env.globals, **self.context_vars}
def expand(self, content_str):
"""
Renders a string that may contain Jinja expressions or statements
Returns the resulting processed string if all values could be resolved.
Otherwise, it returns a tagged (JinjaStr) string that captures variables
in scope (upvalues), like a closure for later evaluation.
"""
result = None
override_vars = {}
if isinstance(content_str, JinjaStr):
# If `value` is already a JinjaStr, it means we are trying to evaluate it again
# in a parent pass.
# Hopefully, all required variables are visible now.
override_vars = content_str.upvalues
try:
template = self.env.from_string(content_str)
result = template.render(override_vars)
if isinstance(result, Undefined):
# This happens when the expression is simply an undefined variable. Jinja does not
# raise an exception, instead we get "Undefined".
# Trigger an UndefinedError exception so we skip to below.
print("" + result)
except (TemplateSyntaxError, UndefinedError) as err:
# `content_str` contains a Jinja expression that refers to a variable that is undefined
# in this scope. Perhaps it refers to a root substitution that is not visible yet.
# Therefore, return the original `content_str` as a JinjaStr, which contains the variables
# that are actually visible to it at this point to postpone evaluation.
return JinjaStr(content_str, {**self.context_vars, **override_vars}), err
return result, None

1
esphome/config.py
View File

@@ -789,7 +789,6 @@ def validate_config(
result.add_output_path([CONF_SUBSTITUTIONS], CONF_SUBSTITUTIONS)
try:
substitutions.do_substitution_pass(config, command_line_substitutions)
substitutions.do_substitution_pass(config, command_line_substitutions)
except vol.Invalid as err:
result.add_error(err)
return result

2
esphome/yaml_util.py
View File

@@ -292,8 +292,6 @@ class ESPHomeLoaderMixin:
if file is None:
raise yaml.MarkedYAMLError("Must include 'file'", node.start_mark)
vars = fields.get(CONF_VARS)
if vars:
vars = {k: str(v) for k, v in vars.items()}
return file, vars
if isinstance(node, yaml.nodes.MappingNode):