GH Actions Update (#1134)

script/ci-custom.py

@@ -101,10 +101,12 @@ def lint_re_check(regex, **kwargs):
                 if 'NOLINT' in match.group(0):
                     continue
                 lineno = content.count("\n", 0, match.start()) + 1
+                substr = content[:match.start()]
+                col = len(substr) - substr.rfind('\n')
                 err = func(fname, match)
                 if err is None:
                     continue
-                errors.append(f"{err} See line {lineno}.")
+                errors.append((lineno, col+1, err))
             return errors
         return decor(new_func)
     return decorator
@@ -121,8 +123,7 @@ def lint_content_find_check(find, **kwargs):
             errors = []
             for line, col in find_all(content, find_):
                 err = func(fname)
-                errors.append("{err} See line {line}:{col}."
-                              "".format(err=err, line=line+1, col=col+1))
+                errors.append((line+1, col+1, err))
             return errors
         return decor(new_func)
     return decorator
@@ -215,9 +216,10 @@ def lint_const_ordered(fname, content):
                 continue
             target = next(i for i, l in ordered if l == ml)
             target_text = next(l for i, l in matching if target == i)
-            errors.append("Constant {} is not ordered, please make sure all constants are ordered. "
-                          "See line {} (should go to line {}, {})"
-                          "".format(highlight(ml), mi, target, target_text))
+            errors.append((ml, None,
+                           "Constant {} is not ordered, please make sure all constants are ordered. "
+                           "See line {} (should go to line {}, {})"
+                           "".format(highlight(ml), mi, target, target_text)))
     return errors
 
 
@@ -354,13 +356,22 @@ errors = collections.defaultdict(list)
 def add_errors(fname, errs):
     if not isinstance(errs, list):
         errs = [errs]
-    errs = [x for x in errs if x is not None]
     for err in errs:
+        if err is None:
+            continue
+        try:
+            lineno, col, msg = err
+        except ValueError:
+            lineno = 1
+            col = 1
+            msg = err
         if not isinstance(err, str):
             raise ValueError("Error is not instance of string!")
-    if not errs:
-        return
-    errors[fname].extend(errs)
+        if not isinstance(lineno, int):
+            raise ValueError("Line number is not an int!")
+        if not isinstance(col, int):
+            raise ValueError("Column number is not an int!")
+        errors[fname].append((lineno, col, msg))
 
 
 for fname in files:
@@ -380,8 +391,8 @@ run_checks(LINT_POST_CHECKS, 'POST')
 
 for f, errs in sorted(errors.items()):
     print(f"\033[0;32m************* File \033[1;32m{f}\033[0m")
-    for err in errs:
-        print(err)
+    for lineno, col, msg in errs:
+        print(f"ERROR {f}:{lineno}:{col} - {msg}")
     print()
 
 sys.exit(len(errors))

script/clang-tidy

@@ -2,20 +2,19 @@
 
 from __future__ import print_function
 
+import argparse
 import multiprocessing
 import os
 import re
-
-import pexpect
 import shutil
 import subprocess
 import sys
 import tempfile
-
-import argparse
-import click
 import threading
 
+import click
+import pexpect
+
 sys.path.append(os.path.dirname(__file__))
 from helpers import basepath, shlex_quote, get_output, build_compile_commands, \
     build_all_include, temp_header_file, git_ls_files, filter_changed
@@ -49,7 +48,7 @@ def run_tidy(args, tmpdir, queue, lock, failed_files):
 
         # Use pexpect for a pseudy-TTY with colored output
         output, rc = pexpect.run(invocation_s, withexitstatus=True, encoding='utf-8',
-                                 timeout=15*60)
+                                 timeout=15 * 60)
         with lock:
             if rc != 0:
                 print()
@@ -65,6 +64,11 @@ def progress_bar_show(value):
         return ''
 
 
+def split_list(a, n):
+    k, m = divmod(len(a), n)
+    return [a[i * k + min(i, m):(i + 1) * k + min(i + 1, m)] for i in range(n)]
+
+
 def main():
     parser = argparse.ArgumentParser()
     parser.add_argument('-j', '--jobs', type=int,
@@ -77,6 +81,10 @@ def main():
                         help='Run clang-tidy in quiet mode')
     parser.add_argument('-c', '--changed', action='store_true',
                         help='Only run on changed files')
+    parser.add_argument('--split-num', type=int, help='Split the files into X jobs.',
+                        default=None)
+    parser.add_argument('--split-at', type=int, help='Which split is this? Starts at 1',
+                        default=None)
     parser.add_argument('--all-headers', action='store_true',
                         help='Create a dummy file that checks all headers')
     args = parser.parse_args()
@@ -114,7 +122,10 @@ def main():
 
     files.sort()
 
-    if args.all_headers:
+    if args.split_num:
+        files = split_list(files, args.split_num)[args.split_at - 1]
+
+    if args.all_headers and args.split_at in (None, 1):
         files.insert(0, temp_header_file)
 
     tmpdir = None
@@ -157,8 +168,8 @@ def main():
             print('Error applying fixes.\n', file=sys.stderr)
             raise
 
-    sys.exit(return_code)
+    return return_code
 
 
 if __name__ == '__main__':
-    main()
+    sys.exit(main())

script/helpers.py

@@ -101,8 +101,10 @@ def splitlines_no_ends(string):
 
 
 def changed_files():
-    for remote in ('upstream', 'origin'):
-        command = ['git', 'merge-base', f'{remote}/dev', 'HEAD']
+    check_remotes = ['upstream', 'origin']
+    check_remotes.extend(splitlines_no_ends(get_output('git', 'remote')))
+    for remote in check_remotes:
+        command = ['git', 'merge-base', f'refs/remotes/{remote}/dev', 'HEAD']
         try:
             merge_base = splitlines_no_ends(get_output(*command))[0]
             break