Jinja expressions in configs (Take #3) (#8955)

tests/unit_tests/fixtures/substitutions/.gitignore

@@ -0,0 +1 @@
+*.received.yaml

tests/unit_tests/fixtures/substitutions/00-simple_var.approved.yaml

@@ -0,0 +1,19 @@
+substitutions:
+  var1: '1'
+  var2: '2'
+  var21: '79'
+esphome:
+  name: test
+test_list:
+  - '1'
+  - '1'
+  - '1'
+  - '1'
+  - 'Values: 1 2'
+  - 'Value: 79'
+  - 1 + 2
+  - 1 * 2
+  - 'Undefined var: ${undefined_var}'
+  - ${undefined_var}
+  - $undefined_var
+  - ${ undefined_var }

tests/unit_tests/fixtures/substitutions/00-simple_var.input.yaml

@@ -0,0 +1,21 @@
+esphome:
+  name: test
+
+substitutions:
+  var1: "1"
+  var2: "2"
+  var21: "79"
+
+test_list:
+  - "$var1"
+  - "${var1}"
+  - $var1
+  - ${var1}
+  - "Values: $var1 ${var2}"
+  - "Value: ${var2${var1}}"
+  - "$var1 + $var2"
+  - "${ var1 } * ${ var2 }"
+  - "Undefined var: ${undefined_var}"
+  - ${undefined_var}
+  - $undefined_var
+  - ${ undefined_var }

tests/unit_tests/fixtures/substitutions/01-include.approved.yaml

@@ -0,0 +1,15 @@
+substitutions:
+  var1: '1'
+  var2: '2'
+  a: alpha
+test_list:
+  - values:
+      - var1: '1'
+      - a: A
+      - b: B-default
+      - c: The value of C is C
+  - values:
+      - var1: '1'
+      - a: alpha
+      - b: beta
+      - c: The value of C is $c

tests/unit_tests/fixtures/substitutions/01-include.input.yaml

@@ -0,0 +1,15 @@
+substitutions:
+  var1: "1"
+  var2: "2"
+  a: "alpha"
+
+test_list:
+  - !include
+    file: inc1.yaml
+    vars:
+      a: "A"
+      c: "C"
+  - !include
+    file: inc1.yaml
+    vars:
+      b: "beta"

tests/unit_tests/fixtures/substitutions/02-expressions.approved.yaml

@@ -0,0 +1,24 @@
+substitutions:
+  width: 7
+  height: 8
+  enabled: true
+  pin: &id001
+    number: 18
+    inverted: true
+  area: 25
+  numberOne: 1
+  var1: 79
+test_list:
+  - The area is 56
+  - 56
+  - 56 + 1
+  - ENABLED
+  - list:
+      - 7
+      - 8
+  - width: 7
+    height: 8
+  - *id001
+  - The pin number is 18
+  - The square root is: 5.0
+  - The number is 80

tests/unit_tests/fixtures/substitutions/02-expressions.input.yaml

@@ -0,0 +1,22 @@
+substitutions:
+  width: 7
+  height: 8
+  enabled: true
+  pin:
+    number: 18
+    inverted: true
+  area: 25
+  numberOne: 1
+  var1: 79
+
+test_list:
+  - "The area is ${width * height}"
+  - ${width * height}
+  - ${width * height} + 1
+  - ${enabled and "ENABLED" or "DISABLED"}
+  - list: ${ [width, height] }
+  - "${ {'width': width, 'height': height} }"
+  - ${pin}
+  - The pin number is ${pin.number}
+  - The square root is: ${math.sqrt(area)}
+  - The number is ${var${numberOne} + 1}

tests/unit_tests/fixtures/substitutions/03-closures.approved.yaml

@@ -0,0 +1,17 @@
+substitutions:
+  B: 5
+  var7: 79
+package_result:
+  - The value of A*B is 35, where A is a package var and B is a substitution in the
+    root file
+  - Double substitution also works; the value of var7 is 79, where A is a package
+    var
+local_results:
+  - The value of B is 5
+  - 'You will see, however, that
+
+    ${A} is not substituted here, since
+
+    it is out of scope.
+
+    '

tests/unit_tests/fixtures/substitutions/03-closures.input.yaml

@@ -0,0 +1,16 @@
+substitutions:
+  B: 5
+  var7: 79
+
+packages:
+  closures_package: !include
+    file: closures_package.yaml
+    vars:
+      A: 7
+
+local_results:
+  - The value of B is ${B}
+  - |
+    You will see, however, that
+    ${A} is not substituted here, since
+    it is out of scope.

tests/unit_tests/fixtures/substitutions/04-display_example.approved.yaml

@@ -0,0 +1,5 @@
+display:
+  - platform: ili9xxx
+    dimensions:
+      width: 960
+      height: 544

tests/unit_tests/fixtures/substitutions/04-display_example.input.yaml

@@ -0,0 +1,7 @@
+# main.yaml
+packages:
+  my_display: !include
+    file: display.yaml
+    vars:
+      high_dpi: true
+      native_height: 272

tests/unit_tests/fixtures/substitutions/closures_package.yaml

@@ -0,0 +1,3 @@
+package_result:
+  - The value of A*B is ${A * B}, where A is a package var and B is a substitution in the root file
+  - Double substitution also works; the value of var7 is ${var$A}, where A is a package var

tests/unit_tests/fixtures/substitutions/display.yaml

@@ -0,0 +1,11 @@
+# display.yaml
+
+defaults:
+  native_width: 480
+  native_height: 480
+
+display:
+  - platform: ili9xxx
+    dimensions:
+      width: ${high_dpi and native_width * 2 or native_width}
+      height: ${high_dpi and native_height * 2 or native_height}

tests/unit_tests/fixtures/substitutions/inc1.yaml

@@ -0,0 +1,8 @@
+defaults:
+  b: "B-default"
+
+values:
+  - var1: $var1
+  - a: $a
+  - b: ${b}
+  - c: The value of C is $c

tests/unit_tests/test_substitutions.py

@@ -0,0 +1,125 @@
+import glob
+import logging
+import os
+
+from esphome import yaml_util
+from esphome.components import substitutions
+from esphome.const import CONF_PACKAGES
+
+_LOGGER = logging.getLogger(__name__)
+
+# Set to True for dev mode behavior
+# This will generate the expected version of the test files.
+
+DEV_MODE = False
+
+
+def sort_dicts(obj):
+    """Recursively sort dictionaries for order-insensitive comparison."""
+    if isinstance(obj, dict):
+        return {k: sort_dicts(obj[k]) for k in sorted(obj)}
+    elif isinstance(obj, list):
+        # Lists are not sorted; we preserve order
+        return [sort_dicts(i) for i in obj]
+    else:
+        return obj
+
+
+def dict_diff(a, b, path=""):
+    """Recursively find differences between two dict/list structures."""
+    diffs = []
+    if isinstance(a, dict) and isinstance(b, dict):
+        a_keys = set(a)
+        b_keys = set(b)
+        for key in a_keys - b_keys:
+            diffs.append(f"{path}/{key} only in actual")
+        for key in b_keys - a_keys:
+            diffs.append(f"{path}/{key} only in expected")
+        for key in a_keys & b_keys:
+            diffs.extend(dict_diff(a[key], b[key], f"{path}/{key}"))
+    elif isinstance(a, list) and isinstance(b, list):
+        min_len = min(len(a), len(b))
+        for i in range(min_len):
+            diffs.extend(dict_diff(a[i], b[i], f"{path}[{i}]"))
+        if len(a) > len(b):
+            for i in range(min_len, len(a)):
+                diffs.append(f"{path}[{i}] only in actual: {a[i]!r}")
+        elif len(b) > len(a):
+            for i in range(min_len, len(b)):
+                diffs.append(f"{path}[{i}] only in expected: {b[i]!r}")
+    else:
+        if a != b:
+            diffs.append(f"\t{path}: actual={a!r} expected={b!r}")
+    return diffs
+
+
+def write_yaml(path, data):
+    with open(path, "w", encoding="utf-8") as f:
+        f.write(yaml_util.dump(data))
+
+
+def test_substitutions_fixtures(fixture_path):
+    base_dir = fixture_path / "substitutions"
+    sources = sorted(glob.glob(str(base_dir / "*.input.yaml")))
+    assert sources, f"No input YAML files found in {base_dir}"
+
+    failures = []
+    for source_path in sources:
+        try:
+            expected_path = source_path.replace(".input.yaml", ".approved.yaml")
+            test_case = os.path.splitext(os.path.basename(source_path))[0].replace(
+                ".input", ""
+            )
+
+            # Load using ESPHome's YAML loader
+            config = yaml_util.load_yaml(source_path)
+
+            if CONF_PACKAGES in config:
+                from esphome.components.packages import do_packages_pass
+
+                config = do_packages_pass(config)
+
+            substitutions.do_substitution_pass(config, None)
+
+            # Also load expected using ESPHome's loader, or use {} if missing and DEV_MODE
+            if os.path.isfile(expected_path):
+                expected = yaml_util.load_yaml(expected_path)
+            elif DEV_MODE:
+                expected = {}
+            else:
+                assert os.path.isfile(expected_path), (
+                    f"Expected file missing: {expected_path}"
+                )
+
+            # Sort dicts only (not lists) for comparison
+            got_sorted = sort_dicts(config)
+            expected_sorted = sort_dicts(expected)
+
+            if got_sorted != expected_sorted:
+                diff = "\n".join(dict_diff(got_sorted, expected_sorted))
+                msg = (
+                    f"Substitution result mismatch for {os.path.basename(source_path)}\n"
+                    f"Diff:\n{diff}\n\n"
+                    f"Got:      {got_sorted}\n"
+                    f"Expected: {expected_sorted}"
+                )
+                # Write out the received file when test fails
+                if DEV_MODE:
+                    received_path = os.path.join(
+                        os.path.dirname(source_path), f"{test_case}.received.yaml"
+                    )
+                    write_yaml(received_path, config)
+                    print(msg)
+                    failures.append(msg)
+                else:
+                    raise AssertionError(msg)
+        except Exception as err:
+            _LOGGER.error("Error in test file %s", source_path)
+            raise err
+
+    if DEV_MODE and failures:
+        print(f"\n{len(failures)} substitution test case(s) failed.")
+
+    if DEV_MODE:
+        _LOGGER.error("Tests passed, but Dev mode is enabled.")
+    assert not DEV_MODE  # make sure DEV_MODE is disabled after you are finished.