Refactor clang-tidy script to use actual compiler flags and includes (#2133)

Co-authored-by: Otto winter <otto@otto-winter.com>

script/build_compile_commands.py

@@ -1,16 +0,0 @@
-#!/usr/bin/env python3
-import sys
-import os.path
-
-sys.path.append(os.path.dirname(__file__))
-from helpers import build_all_include, build_compile_commands
-
-
-def main():
-    build_all_include()
-    build_compile_commands()
-    print("Done.")
-
-
-if __name__ == "__main__":
-    main()

script/clang-tidy

@@ -1,6 +1,7 @@
 #!/usr/bin/env python3
 
 import argparse
+import json
 import multiprocessing
 import os
 import queue
@@ -15,27 +16,57 @@ import click
 import pexpect
 
 sys.path.append(os.path.dirname(__file__))
-from helpers import basepath, shlex_quote, get_output, build_compile_commands, \
-    build_all_include, temp_header_file, git_ls_files, filter_changed
+from helpers import shlex_quote, get_output, \
+    build_all_include, temp_header_file, git_ls_files, filter_changed, load_idedata
 
 
-def run_tidy(args, tmpdir, queue, lock, failed_files):
+def clang_options(idedata):
+    cmd = [
+        # target 32-bit arch (this prevents size mismatch errors on a 64-bit host)
+        '-m32',
+        # disable built-in include directories from the host
+        '-nostdinc',
+        '-nostdinc++',
+        # allow to condition code on the presence of clang-tidy
+        '-DCLANG_TIDY'
+    ]
+
+    # copy compiler flags, except those clang doesn't understand.
+    cmd.extend(flag for flag in idedata['cxx_flags'].split(' ')
+               if flag not in ('-free', '-fipa-pta', '-mlongcalls', '-mtext-section-literals'))
+
+    # defines
+    cmd.extend(f'-D{define}' for define in idedata['defines'])
+
+    # add include directories, using -isystem for dependencies to suppress their errors
+    for directory in idedata['includes']['toolchain']:
+        cmd.extend(['-isystem', directory])
+    for directory in sorted(set(idedata['includes']['build'])):
+        dependency = "framework-arduino" in directory or "/libdeps/" in directory
+        cmd.extend(['-isystem' if dependency else '-I', directory])
+
+    return cmd
+
+
+def run_tidy(args, options, tmpdir, queue, lock, failed_files):
     while True:
         path = queue.get()
-        invocation = ['clang-tidy-11', '-header-filter=^{}/.*'.format(re.escape(basepath))]
+        invocation = ['clang-tidy-11']
+
         if tmpdir is not None:
-            invocation.append('-export-fixes')
+            invocation.append('--export-fixes')
             # Get a temporary file. We immediately close the handle so clang-tidy can
             # overwrite it.
             (handle, name) = tempfile.mkstemp(suffix='.yaml', dir=tmpdir)
             os.close(handle)
             invocation.append(name)
-        invocation.append('-p=.')
+
         if args.quiet:
             invocation.append('-quiet')
-        for arg in ['-Wfor-loop-analysis', '-Wshadow-field', '-Wshadow-field-in-constructor']:
-            invocation.append('-extra-arg={}'.format(arg))
+
         invocation.append(os.path.abspath(path))
+        invocation.append('--')
+        invocation.extend(options)
         invocation_s = ' '.join(shlex_quote(x) for x in invocation)
 
         # Use pexpect for a pseudy-TTY with colored output
@@ -95,8 +126,8 @@ def main():
         """)
         return 1
 
-    build_all_include()
-    build_compile_commands()
+    idedata = load_idedata("esp8266-tidy")
+    options = clang_options(idedata)
 
     files = []
     for path in git_ls_files(['*.cpp']):
@@ -116,6 +147,7 @@ def main():
         files = split_list(files, args.split_num)[args.split_at - 1]
 
     if args.all_headers and args.split_at in (None, 1):
+        build_all_include()
         files.insert(0, temp_header_file)
 
     tmpdir = None
@@ -128,7 +160,7 @@ def main():
         lock = threading.Lock()
         for _ in range(args.jobs):
             t = threading.Thread(target=run_tidy,
-                                 args=(args, tmpdir, task_queue, lock, failed_files))
+                                 args=(args, options, tmpdir, task_queue, lock, failed_files))
             t.daemon = True
             t.start()
 

script/helpers.py

@@ -1,13 +1,14 @@
 import codecs
-import json
 import os.path
 import re
 import subprocess
-import sys
+import json
+from pathlib import Path
 
 root_path = os.path.abspath(os.path.normpath(os.path.join(__file__, "..", "..")))
 basepath = os.path.join(root_path, "esphome")
-temp_header_file = os.path.join(root_path, ".temp-clang-tidy.cpp")
+temp_folder = os.path.join(root_path, ".temp")
+temp_header_file = os.path.join(temp_folder, "all-include.cpp")
 
 
 def shlex_quote(s):
@@ -33,63 +34,9 @@ def build_all_include():
     headers.sort()
     headers.append("")
     content = "\n".join(headers)
-    with codecs.open(temp_header_file, "w", encoding="utf-8") as f:
-        f.write(content)
-
-
-def build_compile_commands():
-    gcc_flags_json = os.path.join(root_path, ".gcc-flags.json")
-    if not os.path.isfile(gcc_flags_json):
-        print("Could not find {} file which is required for clang-tidy.".format(gcc_flags_json))
-        print(
-            'Please run "pio init --ide atom" in the root esphome folder to generate that file.'
-        )
-        sys.exit(1)
-    with codecs.open(gcc_flags_json, "r", encoding="utf-8") as f:
-        gcc_flags = json.load(f)
-    exec_path = gcc_flags["execPath"]
-    include_paths = gcc_flags["gccIncludePaths"].split(",")
-    includes = [f"-I{p}" for p in include_paths]
-    cpp_flags = gcc_flags["gccDefaultCppFlags"].split(" ")
-    defines = [flag for flag in cpp_flags if flag.startswith("-D")]
-    command = [exec_path]
-    command.extend(includes)
-    command.extend(defines)
-    command.append("-std=gnu++11")
-    command.append("-Wall")
-    command.append("-Wno-delete-non-virtual-dtor")
-    command.append("-Wno-unused-variable")
-    command.append("-Wunreachable-code")
-
-    source_files = []
-    for path in walk_files(basepath):
-        filetypes = (".cpp",)
-        ext = os.path.splitext(path)[1]
-        if ext in filetypes:
-            source_files.append(os.path.abspath(path))
-    source_files.append(temp_header_file)
-    source_files.sort()
-    compile_commands = [
-        {
-            "directory": root_path,
-            "command": " ".join(
-                shlex_quote(x) for x in (command + ["-o", p + ".o", "-c", p])
-            ),
-            "file": p,
-        }
-        for p in source_files
-    ]
-    compile_commands_json = os.path.join(root_path, "compile_commands.json")
-    if os.path.isfile(compile_commands_json):
-        with codecs.open(compile_commands_json, "r", encoding="utf-8") as f:
-            try:
-                if json.load(f) == compile_commands:
-                    return
-            # pylint: disable=bare-except
-            except:
-                pass
-    with codecs.open(compile_commands_json, "w", encoding="utf-8") as f:
-        json.dump(compile_commands, f, indent=2)
+    p = Path(temp_header_file)
+    p.parent.mkdir(exist_ok=True)
+    p.write_text(content)
 
 
 def walk_files(path):
@@ -153,3 +100,28 @@ def git_ls_files(patterns=None):
     output, err = proc.communicate()
     lines = [x.split() for x in output.decode("utf-8").splitlines()]
     return {s[3].strip(): int(s[0]) for s in lines}
+
+
+def load_idedata(environment):
+    platformio_ini = Path(root_path) / "platformio.ini"
+    temp_idedata = Path(temp_folder) / f"idedata-{environment}.json"
+    if not platformio_ini.is_file() or not temp_idedata.is_file():
+        changed = True
+    elif platformio_ini.stat().st_mtime >= temp_idedata.stat().st_mtime:
+        changed = True
+    else:
+        changed = False
+
+    if not changed:
+        text = temp_idedata.read_text()
+    else:
+        stdout = subprocess.check_output(
+            ["pio", "run", "-t", "idedata", "-e", environment]
+        )
+        match = re.search(r'{\s*".*}', stdout.decode("utf-8"))
+        text = match.group()
+
+        temp_idedata.parent.mkdir(exist_ok=True)
+        temp_idedata.write_text(text)
+
+    return json.loads(text)