diff --git a/esphome/components/esp32/__init__.py b/esphome/components/esp32/__init__.py
index cb6354cc74..48d11f46fa 100644
--- a/esphome/components/esp32/__init__.py
+++ b/esphome/components/esp32/__init__.py
@@ -877,6 +877,11 @@ async def to_code(config):
     for clean_var in ("IDF_PATH", "IDF_TOOLS_PATH"):
         os.environ.pop(clean_var, None)
 
+    # Set the location of the IDF component manager cache
+    os.environ["IDF_COMPONENT_CACHE_PATH"] = str(
+        CORE.relative_internal_path(".espressif")
+    )
+
     add_extra_script(
         "post",
         "post_build.py",