Proposal: Test yaml for each component (#5398)

* Test for each component.

* When possible use commandline substitution.

* Add wildcard support.

* end file with new line.

* Move component tests into subfolder.

* Add component test to pipeline.

* Remove trailing whitespace.

* add restore python step.

* Add `. venv/bin/activate` to pipeline.

* step `changed-components` needs `common` step.

* start `list-components-changed.py` different.

* iterate on pipeline stage `list-components`.

* Update `checkout` action.

* Rename test folder from `tests` to `_test`.

* validate file exists.

* Move component test folder.

* extend list-components to include child components.

* File does not end with a newline

* Handle empty list-components matrix.

* list-components also check for changes in tests folder.

* Improve `list-components.py`.

* `*` is a forbidden character for filenames on windows.

---------

Co-authored-by: Your Name <you@example.com>
Co-authored-by: Keith Burzinski <kbx81x@gmail.com>

script/fulltest

@@ -12,3 +12,4 @@ script/lint-cpp
 script/unit_test
 script/component_test
 script/test
+script/test_build_components

script/list-components.py

@@ -0,0 +1,153 @@
+#!/usr/bin/env python3
+from pathlib import Path
+import sys
+import argparse
+
+from helpers import git_ls_files, changed_files
+from esphome.loader import get_component, get_platform
+from esphome.core import CORE
+from esphome.const import (
+    KEY_CORE,
+    KEY_TARGET_FRAMEWORK,
+    KEY_TARGET_PLATFORM,
+    PLATFORM_ESP32,
+    PLATFORM_ESP8266,
+)
+
+
+def filter_component_files(str):
+    return str.startswith("esphome/components/") | str.startswith("tests/components/")
+
+
+def extract_component_names_array_from_files_array(files):
+    components = []
+    for file in files:
+        file_parts = file.split("/")
+        if len(file_parts) >= 4:
+            component_name = file_parts[2]
+            if component_name not in components:
+                components.append(component_name)
+    return components
+
+
+def add_item_to_components_graph(components_graph, parent, child):
+    if not parent.startswith("__") and parent != child:
+        if parent not in components_graph:
+            components_graph[parent] = []
+        if child not in components_graph[parent]:
+            components_graph[parent].append(child)
+
+
+def create_components_graph():
+    # The root directory of the repo
+    root = Path(__file__).parent.parent
+    components_dir = root / "esphome" / "components"
+    # Fake some directory so that get_component works
+    CORE.config_path = str(root)
+    # Various configuration to capture different outcomes used by `AUTO_LOAD` function.
+    TARGET_CONFIGURATIONS = [
+        {KEY_TARGET_FRAMEWORK: None, KEY_TARGET_PLATFORM: None},
+        {KEY_TARGET_FRAMEWORK: "arduino", KEY_TARGET_PLATFORM: None},
+        {KEY_TARGET_FRAMEWORK: "esp-idf", KEY_TARGET_PLATFORM: None},
+        {KEY_TARGET_FRAMEWORK: None, KEY_TARGET_PLATFORM: PLATFORM_ESP32},
+    ]
+    CORE.data[KEY_CORE] = TARGET_CONFIGURATIONS[0]
+
+    components_graph = {}
+
+    for path in components_dir.iterdir():
+        if not path.is_dir():
+            continue
+        if not (path / "__init__.py").is_file():
+            continue
+        name = path.name
+        comp = get_component(name)
+        if comp is None:
+            print(
+                f"Cannot find component {name}. Make sure current path is pip installed ESPHome"
+            )
+            sys.exit(1)
+
+        for dependency in comp.dependencies:
+            add_item_to_components_graph(components_graph, dependency, name)
+
+        for target_config in TARGET_CONFIGURATIONS:
+            CORE.data[KEY_CORE] = target_config
+            for auto_load in comp.auto_load:
+                add_item_to_components_graph(components_graph, auto_load, name)
+        # restore config
+        CORE.data[KEY_CORE] = TARGET_CONFIGURATIONS[0]
+
+        for platform_path in path.iterdir():
+            platform_name = platform_path.stem
+            platform = get_platform(platform_name, name)
+            if platform is None:
+                continue
+
+            add_item_to_components_graph(components_graph, platform_name, name)
+
+            for dependency in platform.dependencies:
+                add_item_to_components_graph(components_graph, dependency, name)
+
+            for target_config in TARGET_CONFIGURATIONS:
+                CORE.data[KEY_CORE] = target_config
+                for auto_load in platform.auto_load:
+                    add_item_to_components_graph(components_graph, auto_load, name)
+            # restore config
+            CORE.data[KEY_CORE] = TARGET_CONFIGURATIONS[0]
+
+    return components_graph
+
+
+def find_children_of_component(components_graph, component_name, depth=0):
+    if component_name not in components_graph:
+        return []
+
+    children = []
+
+    for child in components_graph[component_name]:
+        children.append(child)
+        if depth < 10:
+            children.extend(
+                find_children_of_component(components_graph, child, depth + 1)
+            )
+    # Remove duplicate values
+    return list(set(children))
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "-c", "--changed", action="store_true", help="Only run on changed files"
+    )
+    args = parser.parse_args()
+
+    files = git_ls_files()
+    files = filter(filter_component_files, files)
+
+    if args.changed:
+        changed = changed_files()
+        files = [f for f in files if f in changed]
+
+    components = extract_component_names_array_from_files_array(files)
+
+    if args.changed:
+        components_graph = create_components_graph()
+
+        all_changed_components = components.copy()
+        for c in components:
+            all_changed_components.extend(
+                find_children_of_component(components_graph, c)
+            )
+        # Remove duplicate values
+        all_changed_components = list(set(all_changed_components))
+
+        for c in sorted(all_changed_components):
+            print(c)
+    else:
+        for c in sorted(components):
+            print(c)
+
+
+if __name__ == "__main__":
+    main()

script/test_build_components

@@ -0,0 +1,85 @@
+#!/usr/bin/env bash
+
+set -e
+
+# Parse parameter:
+# - `e` - Parameter for `esphome` command. Default `compile`. Common alternative is `config`.
+# - `c` - Component folder name to test. Default `*`.
+esphome_command="compile"
+target_component="*"
+while getopts e:c: flag
+do
+    case $flag in
+        e) esphome_command=${OPTARG};;
+        c) target_component=${OPTARG};;
+        \?) echo "Usage: $0 [-e <config|compile|clean>] [-c <string>]" 1>&2; exit 1;;
+    esac
+done
+
+cd "$(dirname "$0")/.."
+
+if ! [ -d "./tests/test_build_components/build" ]; then
+  mkdir ./tests/test_build_components/build
+fi
+
+start_esphome() {
+  # create dynamic yaml file in `build` folder.
+  # `./tests/test_build_components/build/[target_component].[test_name].[target_platform].yaml`
+  component_test_file="./tests/test_build_components/build/$target_component.$test_name.$target_platform.yaml"
+
+  cp $target_platform_file $component_test_file
+  sed -i "s!\$component_test_file!../../.$f!g" $component_test_file
+
+  # Start esphome process
+  echo "> [$target_component] [$test_name] [$target_platform]"
+  echo "esphome -s component_name $target_component -s test_name $test_name -s target_platform $target_platform $esphome_command $component_test_file"
+  # TODO: Validate escape of Command line substitution value
+  esphome -s component_name $target_component -s test_name $test_name -s target_platform $target_platform $esphome_command $component_test_file
+}
+
+# Find all test yaml files.
+# - `./tests/components/[target_component]/[test_name].[target_platform].yaml`
+# - `./tests/components/[target_component]/[test_name].all.yaml`
+for f in ./tests/components/$target_component/*.*.yaml; do
+  [ -f "$f" ] || continue
+  IFS='/' read -r -a folder_name <<< "$f"
+  target_component="${folder_name[3]}"
+
+  IFS='.' read -r -a file_name <<< "${folder_name[4]}"
+  test_name="${file_name[0]}"
+  target_platform="${file_name[1]}"
+  file_name_parts=${#file_name[@]}
+
+  if [ "$target_platform" = "all" ] || [ $file_name_parts = 2 ]; then
+    # Test has *not* defined a specific target platform. Need to run tests for all possible target platforms.
+    
+    for target_platform_file in ./tests/test_build_components/build_components_base.*.yaml; do
+      IFS='/' read -r -a folder_name <<< "$target_platform_file"
+      IFS='.' read -r -a file_name <<< "${folder_name[3]}"
+      target_platform="${file_name[1]}"
+
+      start_esphome
+    done
+
+  else
+    # Test has defined a specific target platform.
+  
+    # Validate we have a base test yaml for selected platform.
+    # The target_platform is sourced from the following location.
+    # 1. `./tests/test_build_components/build_components_base.[target_platform].yaml`
+    # 2. `./tests/test_build_components/build_components_base.[target_platform]-ard.yaml`
+    target_platform_file="./tests/test_build_components/build_components_base.$target_platform.yaml"
+    if ! [ -f "$target_platform_file" ]; then
+      # Try find arduino test framework as platform.
+      target_platform_ard="$target_platform-ard"
+      target_platform_file="./tests/test_build_components/build_components_base.$target_platform_ard.yaml"
+      if ! [ -f "$target_platform_file" ]; then
+        echo "No base test file [./tests/test_build_components/build_components_base.$target_platform.yaml, ./tests/build_components_base.$target_platform_ard.yaml] for component test [$f] found."
+        exit 1
+      fi
+      target_platform=$target_platform_ard
+    fi
+
+    start_esphome
+  fi
+done