diff --git a/esphome/components/esp32/__init__.py b/esphome/components/esp32/__init__.py
index 69fc8d3262..ba11aeef21 100644
--- a/esphome/components/esp32/__init__.py
+++ b/esphome/components/esp32/__init__.py
@@ -523,11 +523,11 @@ def copy_files():
             __version__,
         )
 
+        import shutil
+
+        shutil.rmtree(CORE.relative_build_path("components"), ignore_errors=True)
+
         if CORE.data[KEY_ESP32][KEY_COMPONENTS]:
-            import shutil
-
-            shutil.rmtree(CORE.relative_build_path("components"), ignore_errors=True)
-
             components: dict = CORE.data[KEY_ESP32][KEY_COMPONENTS]
 
             for name, component in components.items():
diff --git a/esphome/git.py b/esphome/git.py
index 130cd4f5e1..a607325b73 100644
--- a/esphome/git.py
+++ b/esphome/git.py
@@ -44,7 +44,7 @@ def clone_or_update(
     *,
     url: str,
     ref: str = None,
-    refresh: TimePeriodSeconds,
+    refresh: Optional[TimePeriodSeconds],
     domain: str,
     username: str = None,
     password: str = None,
@@ -81,7 +81,7 @@ def clone_or_update(
         if not file_timestamp.exists():
             file_timestamp = Path(repo_dir / ".git" / "HEAD")
         age = datetime.now() - datetime.fromtimestamp(file_timestamp.stat().st_mtime)
-        if age.total_seconds() > refresh.total_seconds:
+        if refresh is None or age.total_seconds() > refresh.total_seconds:
             old_sha = run_git_command(["git", "rev-parse", "HEAD"], str(repo_dir))
             _LOGGER.info("Updating %s", key)
             _LOGGER.debug("Location: %s", repo_dir)