diff --git a/esphome/components/esp32/__init__.py b/esphome/components/esp32/__init__.py
index ca10f36c1b..e2359a8470 100644
--- a/esphome/components/esp32/__init__.py
+++ b/esphome/components/esp32/__init__.py
@@ -841,6 +841,9 @@ async def to_code(config):
     if conf[CONF_ADVANCED][CONF_IGNORE_EFUSE_CUSTOM_MAC]:
         cg.add_define("USE_ESP32_IGNORE_EFUSE_CUSTOM_MAC")
 
+    for clean_var in ("IDF_PATH", "IDF_TOOLS_PATH"):
+        os.environ.pop(clean_var, None)
+
     add_extra_script(
         "post",
         "post_build.py",