Optimize subprocess performance with close_fds=False (#10145)

docker/build.py

@@ -90,7 +90,7 @@ def main():
     def run_command(*cmd, ignore_error: bool = False):
         print(f"$ {shlex.join(list(cmd))}")
         if not args.dry_run:
-            rc = subprocess.call(list(cmd))
+            rc = subprocess.call(list(cmd), close_fds=False)
             if rc != 0 and not ignore_error:
                 print("Command failed")
                 sys.exit(1)

esphome/components/sdl/display.py

@@ -36,7 +36,9 @@ def get_sdl_options(value):
     if value != "":
         return value
     try:
-        return subprocess.check_output(["sdl2-config", "--cflags", "--libs"]).decode()
+        return subprocess.check_output(
+            ["sdl2-config", "--cflags", "--libs"], close_fds=False
+        ).decode()
     except Exception as e:
         raise cv.Invalid("Unable to run sdl2-config - have you installed sdl2?") from e
 

esphome/dashboard/web_server.py

@@ -229,6 +229,7 @@ class EsphomeCommandWebSocket(tornado.websocket.WebSocketHandler):
                 stdin=subprocess.PIPE,
                 stdout=subprocess.PIPE,
                 stderr=subprocess.STDOUT,
+                close_fds=False,
             )
             stdout_thread = threading.Thread(target=self._stdout_thread)
             stdout_thread.daemon = True

esphome/git.py

@@ -17,7 +17,9 @@ _LOGGER = logging.getLogger(__name__)
 def run_git_command(cmd, cwd=None) -> str:
     _LOGGER.debug("Running git command: %s", " ".join(cmd))
     try:
-        ret = subprocess.run(cmd, cwd=cwd, capture_output=True, check=False)
+        ret = subprocess.run(
+            cmd, cwd=cwd, capture_output=True, check=False, close_fds=False
+        )
     except FileNotFoundError as err:
         raise cv.Invalid(
             "git is not installed but required for external_components.\n"

esphome/helpers.py

@@ -114,7 +114,9 @@ def cpp_string_escape(string, encoding="utf-8"):
 def run_system_command(*args):
     import subprocess
 
-    with subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE) as p:
+    with subprocess.Popen(
+        args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, close_fds=False
+    ) as p:
         stdout, stderr = p.communicate()
         rc = p.returncode
         return rc, stdout, stderr

esphome/platformio_api.py

@@ -211,7 +211,7 @@ def _decode_pc(config, addr):
         return
     command = [idedata.addr2line_path, "-pfiaC", "-e", idedata.firmware_elf_path, addr]
     try:
-        translation = subprocess.check_output(command).decode().strip()
+        translation = subprocess.check_output(command, close_fds=False).decode().strip()
     except Exception:  # pylint: disable=broad-except
         _LOGGER.debug("Caught exception for command %s", command, exc_info=1)
         return

esphome/util.py

@@ -239,7 +239,12 @@ def run_external_process(*cmd: str, **kwargs: Any) -> int | str:
 
     try:
         proc = subprocess.run(
-            cmd, stdout=sub_stdout, stderr=sub_stderr, encoding="utf-8", check=False
+            cmd,
+            stdout=sub_stdout,
+            stderr=sub_stderr,
+            encoding="utf-8",
+            check=False,
+            close_fds=False,
         )
         return proc.stdout if capture_stdout else proc.returncode
     except KeyboardInterrupt:  # pylint: disable=try-except-raise

script/clang-format

@@ -31,7 +31,11 @@ def run_format(executable, args, queue, lock, failed_files):
         invocation.append(path)
 
         proc = subprocess.run(
-            invocation, capture_output=True, encoding="utf-8", check=False
+            invocation,
+            capture_output=True,
+            encoding="utf-8",
+            check=False,
+            close_fds=False,
         )
         if proc.returncode != 0:
             with lock:

script/clang-tidy

@@ -158,7 +158,11 @@ def run_tidy(executable, args, options, tmpdir, path_queue, lock, failed_files):
         invocation.extend(options)
 
         proc = subprocess.run(
-            invocation, capture_output=True, encoding="utf-8", check=False
+            invocation,
+            capture_output=True,
+            encoding="utf-8",
+            check=False,
+            close_fds=False,
         )
         if proc.returncode != 0:
             with lock:
@@ -320,9 +324,11 @@ def main():
         print("Applying fixes ...")
         try:
             try:
-                subprocess.call(["clang-apply-replacements-18", tmpdir])
+                subprocess.call(
+                    ["clang-apply-replacements-18", tmpdir], close_fds=False
+                )
             except FileNotFoundError:
-                subprocess.call(["clang-apply-replacements", tmpdir])
+                subprocess.call(["clang-apply-replacements", tmpdir], close_fds=False)
         except FileNotFoundError:
             print(
                 "Error please install clang-apply-replacements-18 or clang-apply-replacements.\n",

script/platformio_install_deps.py

@@ -55,4 +55,6 @@ for section in config.sections():
             tools.append("-t")
             tools.append(tool)
 
-subprocess.check_call(["platformio", "pkg", "install", "-g", *libs, *platforms, *tools])
+subprocess.check_call(
+    ["platformio", "pkg", "install", "-g", *libs, *platforms, *tools], close_fds=False
+)

script/run-in-env.py

@@ -13,7 +13,7 @@ def find_and_activate_virtualenv():
     try:
         # Get the top-level directory of the git repository
         my_path = subprocess.check_output(
-            ["git", "rev-parse", "--show-toplevel"], text=True
+            ["git", "rev-parse", "--show-toplevel"], text=True, close_fds=False
         ).strip()
     except subprocess.CalledProcessError:
         print(
@@ -44,7 +44,7 @@ def find_and_activate_virtualenv():
 def run_command():
     # Execute the remaining arguments in the new environment
     if len(sys.argv) > 1:
-        subprocess.run(sys.argv[1:], check=False)
+        subprocess.run(sys.argv[1:], check=False, close_fds=False)
     else:
         print(
             "No command provided to run in the virtual environment.",

tests/integration/conftest.py

@@ -105,6 +105,7 @@ logger:
                     check=True,
                     cwd=init_dir,
                     env=env,
+                    close_fds=False,
                 )
 
         # Lock is held until here, ensuring cache is fully populated before any test proceeds
@@ -245,6 +246,7 @@ async def compile_esphome(
                 # Start in a new process group to isolate signal handling
                 start_new_session=True,
                 env=env,
+                close_fds=False,
             )
             await proc.wait()
 
@@ -477,6 +479,7 @@ async def run_binary_and_wait_for_port(
         # Start in a new process group to isolate signal handling
         start_new_session=True,
         pass_fds=(device_fd,),
+        close_fds=False,
     )
 
     # Close the device end in the parent process