Merge components in packages (#3555)

Co-authored-by: Paul Monigatti <pm@paul.pm>
Co-authored-by: Jesse Hills <3060199+jesserockz@users.noreply.github.com>

esphome/components/packages/__init__.py

@@ -165,7 +165,7 @@ def do_packages_pass(config: dict):
                 f"Packages must be a key to value mapping, got {type(packages)} instead"
             )
 
-        for package_name, package_config in packages.items():
+        for package_name, package_config in reversed(packages.items()):
             with cv.prepend_path(package_name):
                 recursive_package = package_config
                 if CONF_URL in package_config:

esphome/config.py

@@ -14,6 +14,7 @@ from esphome import core, yaml_util, loader
 import esphome.core.config as core_config
 from esphome.const import (
     CONF_ESPHOME,
+    CONF_ID,
     CONF_PLATFORM,
     CONF_PACKAGES,
     CONF_SUBSTITUTIONS,
@@ -24,6 +25,7 @@ from esphome.core import CORE, EsphomeError
 from esphome.helpers import indent
 from esphome.util import safe_print, OrderedDict
 
+from esphome.config_helpers import Extend
 from esphome.loader import get_component, get_platform, ComponentManifest
 from esphome.yaml_util import is_secret, ESPHomeDataBase, ESPForceValue
 from esphome.voluptuous_schema import ExtraKeysInvalid
@@ -334,6 +336,13 @@ class LoadValidationStep(ConfigValidationStep):
                 continue
             p_name = p_config.get("platform")
             if p_name is None:
+                p_id = p_config.get(CONF_ID)
+                if isinstance(p_id, Extend):
+                    result.add_str_error(
+                        f"Source for extension of ID '{p_id.value}' was not found.",
+                        path + [CONF_ID],
+                    )
+                    continue
                 result.add_str_error("No platform specified! See 'platform' key.", path)
                 continue
             # Remove temp output path and construct new one

esphome/config_helpers.py

@@ -1,10 +1,27 @@
 import json
 import os
 
+from esphome.const import CONF_ID
 from esphome.core import CORE
 from esphome.helpers import read_file
 
 
+class Extend:
+    def __init__(self, value):
+        self.value = value
+
+    def __str__(self):
+        return f"!extend {self.value}"
+
+    def __eq__(self, b):
+        """
+        Check if two Extend objects contain the same ID.
+
+        Only used in unit tests.
+        """
+        return isinstance(b, Extend) and self.value == b.value
+
+
 def read_config_file(path: str) -> str:
     if CORE.vscode and (
         not CORE.ace or os.path.abspath(path) == os.path.abspath(CORE.config_path)
@@ -36,7 +53,25 @@ def merge_config(full_old, full_new):
         if isinstance(new, list):
             if not isinstance(old, list):
                 return new
-            return old + new
+            res = old.copy()
+            ids = {
+                v[CONF_ID]: i
+                for i, v in enumerate(res)
+                if CONF_ID in v and isinstance(v[CONF_ID], str)
+            }
+            for v in new:
+                if CONF_ID in v:
+                    new_id = v[CONF_ID]
+                    if isinstance(new_id, Extend):
+                        new_id = new_id.value
+                        if new_id in ids:
+                            v[CONF_ID] = new_id
+                            res[ids[new_id]] = merge(res[ids[new_id]], v)
+                            continue
+                    else:
+                        ids[new_id] = len(res)
+                res.append(v)
+            return res
         if new is None:
             return old
 

esphome/config_validation.py

@@ -13,6 +13,7 @@ import voluptuous as vol
 
 from esphome import core
 import esphome.codegen as cg
+from esphome.config_helpers import Extend
 from esphome.const import (
     ALLOWED_NAME_CHARS,
     CONF_AVAILABILITY,
@@ -490,6 +491,8 @@ def declare_id(type):
         if value is None:
             return core.ID(None, is_declaration=True, type=type)
 
+        if isinstance(value, Extend):
+            raise Invalid(f"Source for extension of ID '{value.value}' was not found.")
         return core.ID(validate_id_name(value), is_declaration=True, type=type)
 
     return validator

esphome/yaml_util.py

@@ -10,7 +10,7 @@ import yaml
 import yaml.constructor
 
 from esphome import core
-from esphome.config_helpers import read_config_file
+from esphome.config_helpers import read_config_file, Extend
 from esphome.core import (
     EsphomeError,
     IPAddress,
@@ -338,6 +338,10 @@ class ESPHomeLoader(yaml.SafeLoader):
         obj = self.construct_scalar(node)
         return add_class_to_obj(obj, ESPForceValue)
 
+    @_add_data_ref
+    def construct_extend(self, node):
+        return Extend(str(node.value))
+
 
 ESPHomeLoader.add_constructor("tag:yaml.org,2002:int", ESPHomeLoader.construct_yaml_int)
 ESPHomeLoader.add_constructor(
@@ -369,6 +373,7 @@ ESPHomeLoader.add_constructor(
 )
 ESPHomeLoader.add_constructor("!lambda", ESPHomeLoader.construct_lambda)
 ESPHomeLoader.add_constructor("!force", ESPHomeLoader.construct_force)
+ESPHomeLoader.add_constructor("!extend", ESPHomeLoader.construct_extend)
 
 
 def load_yaml(fname, clear_secrets=True):