diff --git a/esphome/components/esp_adf/__init__.py b/esphome/components/esp_adf/__init__.py
index d51ff8afda..a82d160722 100644
--- a/esphome/components/esp_adf/__init__.py
+++ b/esphome/components/esp_adf/__init__.py
@@ -2,16 +2,17 @@ import os
 
 import esphome.config_validation as cv
 import esphome.codegen as cg
+import esphome.final_validate as fv
 
 from esphome.components import esp32
 
 from esphome.const import CONF_ID, CONF_BOARD
 
-CONFLICTS_WITH = ["i2s_audio"]
 CODEOWNERS = ["@jesserockz"]
 DEPENDENCIES = ["esp32"]
 
 CONF_ESP_ADF_ID = "esp_adf_id"
+CONF_ESP_ADF = "esp_adf"
 
 esp_adf_ns = cg.esphome_ns.namespace("esp_adf")
 ESPADF = esp_adf_ns.class_("ESPADF", cg.Component)
@@ -23,13 +24,28 @@ SUPPORTED_BOARDS = {
 }
 
 
-def _validate_board(config):
-    board = config.get(CONF_BOARD, esp32.get_board())
-    if board not in SUPPORTED_BOARDS:
-        raise cv.Invalid(f"Board {board} is not supported by esp-adf")
+def _default_board(config):
+    config = config.copy()
+    if board := config.get(CONF_BOARD) is None:
+        board = esp32.get_board()
+        if board in SUPPORTED_BOARDS:
+            config[CONF_BOARD] = board
     return config
 
 
+def final_validate_usable_board(platform: str):
+    def _validate(adf_config):
+        board = adf_config.get(CONF_BOARD)
+        if board not in SUPPORTED_BOARDS:
+            raise cv.Invalid(f"Board {board} is not supported by esp-adf {platform}")
+        return adf_config
+
+    return cv.Schema(
+        {cv.Required(CONF_ESP_ADF_ID): fv.id_declaration_match_schema(_validate)},
+        extra=cv.ALLOW_EXTRA,
+    )
+
+
 CONFIG_SCHEMA = cv.All(
     cv.Schema(
         {
@@ -37,7 +53,7 @@ CONFIG_SCHEMA = cv.All(
             cv.Optional(CONF_BOARD): cv.string_strict,
         }
     ),
-    _validate_board,
+    _default_board,
     cv.only_with_esp_idf,
 )
 
@@ -62,15 +78,18 @@ async def to_code(config):
     cg.add_platformio_option(
         "board_build.embed_txtfiles", "components/dueros_service/duer_profile"
     )
-    board = config.get(CONF_BOARD, esp32.get_board())
-    esp32.add_idf_sdkconfig_option(SUPPORTED_BOARDS[board], True)
 
-    esp32.add_extra_script(
-        "pre",
-        "apply_adf_patches.py",
-        os.path.join(os.path.dirname(__file__), "apply_adf_patches.py.script"),
-    )
-    esp32.add_extra_build_file(
-        "esp_adf_patches/idf_v4.4_freertos.patch",
-        "https://github.com/espressif/esp-adf/raw/v2.5/idf_patches/idf_v4.4_freertos.patch",
-    )
+    if board := config.get(CONF_BOARD):
+        cg.add_define("USE_ESP_ADF_BOARD")
+
+        esp32.add_idf_sdkconfig_option(SUPPORTED_BOARDS[board], True)
+
+        esp32.add_extra_script(
+            "pre",
+            "apply_adf_patches.py",
+            os.path.join(os.path.dirname(__file__), "apply_adf_patches.py.script"),
+        )
+        esp32.add_extra_build_file(
+            "esp_adf_patches/idf_v4.4_freertos.patch",
+            "https://github.com/espressif/esp-adf/raw/v2.5/idf_patches/idf_v4.4_freertos.patch",
+        )
diff --git a/esphome/components/esp_adf/esp_adf.cpp b/esphome/components/esp_adf/esp_adf.cpp
index 2de4f80599..25e1193ee4 100644
--- a/esphome/components/esp_adf/esp_adf.cpp
+++ b/esphome/components/esp_adf/esp_adf.cpp
@@ -12,10 +12,11 @@ namespace esp_adf {
 static const char *const TAG = "esp_adf";
 
 void ESPADF::setup() {
+#ifdef USE_ESP_ADF_BOARD
   ESP_LOGI(TAG, "Start codec chip");
-
   audio_board_handle_t board_handle = audio_board_init();
   audio_hal_ctrl_codec(board_handle->audio_hal, AUDIO_HAL_CODEC_MODE_BOTH, AUDIO_HAL_CTRL_START);
+#endif
 }
 
 float ESPADF::get_setup_priority() const { return setup_priority::HARDWARE; }
diff --git a/esphome/components/esp_adf/microphone/__init__.py b/esphome/components/esp_adf/microphone/__init__.py
index ccfc280d50..245e7cc99d 100644
--- a/esphome/components/esp_adf/microphone/__init__.py
+++ b/esphome/components/esp_adf/microphone/__init__.py
@@ -3,9 +3,16 @@ import esphome.config_validation as cv
 from esphome.components import microphone
 from esphome.const import CONF_ID
 
-from .. import CONF_ESP_ADF_ID, ESPADF, ESPADFPipeline, esp_adf_ns
+from .. import (
+    CONF_ESP_ADF_ID,
+    ESPADF,
+    ESPADFPipeline,
+    esp_adf_ns,
+    final_validate_usable_board,
+)
 
 AUTO_LOAD = ["esp_adf"]
+CONFLICTS_WITH = ["i2s_audio"]
 DEPENDENCIES = ["esp32"]
 
 ESPADFMicrophone = esp_adf_ns.class_(
@@ -23,6 +30,8 @@ CONFIG_SCHEMA = cv.All(
     cv.only_with_esp_idf,
 )
 
+FINAL_VALIDATE_SCHEMA = final_validate_usable_board("microphone")
+
 
 async def to_code(config):
     var = cg.new_Pvariable(config[CONF_ID])
diff --git a/esphome/components/esp_adf/speaker/__init__.py b/esphome/components/esp_adf/speaker/__init__.py
index 2fb30ff849..1b30989fa2 100644
--- a/esphome/components/esp_adf/speaker/__init__.py
+++ b/esphome/components/esp_adf/speaker/__init__.py
@@ -3,9 +3,16 @@ import esphome.config_validation as cv
 from esphome.components import speaker
 from esphome.const import CONF_ID
 
-from .. import CONF_ESP_ADF_ID, ESPADF, ESPADFPipeline, esp_adf_ns
+from .. import (
+    CONF_ESP_ADF_ID,
+    ESPADF,
+    ESPADFPipeline,
+    esp_adf_ns,
+    final_validate_usable_board,
+)
 
 AUTO_LOAD = ["esp_adf"]
+CONFLICTS_WITH = ["i2s_audio"]
 DEPENDENCIES = ["esp32"]
 
 ESPADFSpeaker = esp_adf_ns.class_(
@@ -23,6 +30,8 @@ CONFIG_SCHEMA = cv.All(
     cv.only_with_esp_idf,
 )
 
+FINAL_VALIDATE_SCHEMA = final_validate_usable_board("speaker")
+
 
 async def to_code(config):
     var = cg.new_Pvariable(config[CONF_ID])