sascha/esphome
1
0
Fork 0
mirror of https://github.com/esphome/esphome.git synced 2025-06-19 06:45:49 +01:00
Code Issues Releases Wiki Activity

Lint the script folder files ()

Browse Source
This commit is contained in:
Jesse Hills
2023-12-22 20:03:47 +13:00
committed by GitHub
parent 676ae6b26e
commit d2d0058386
6 changed files with 398 additions and 377 deletions

233
script/ci-custom.py

@ -1,10 +1,8 @@
#!/usr/bin/env python3
from helpers import styled, print_error_for_file, git_ls_files, filter_changed
import argparse
import codecs
import collections
import colorama
import fnmatch
import functools
import os.path
@ -12,6 +10,9 @@ import re
import sys
import time
import colorama
from helpers import filter_changed, git_ls_files, print_error_for_file, styled
sys.path.append(os.path.dirname(__file__))
@ -30,31 +31,6 @@ def find_all(a_str, sub):
column += len(sub)
colorama.init()
parser = argparse.ArgumentParser()
parser.add_argument(
"files", nargs="*", default=[], help="files to be processed (regex on path)"
)
parser.add_argument(
"-c", "--changed", action="store_true", help="Only run on changed files"
)
parser.add_argument(
"--print-slowest", action="store_true", help="Print the slowest checks"
)
args = parser.parse_args()
EXECUTABLE_BIT = git_ls_files()
files = list(EXECUTABLE_BIT.keys())
# Match against re
file_name_re = re.compile("|".join(args.files))
files = [p for p in files if file_name_re.search(p)]
if args.changed:
files = filter_changed(files)
files.sort()
file_types = (
".h",
".c",
@ -86,6 +62,30 @@ ignore_types = (".ico", ".png", ".woff", ".woff2", "")
LINT_FILE_CHECKS = []
LINT_CONTENT_CHECKS = []
LINT_POST_CHECKS = []
EXECUTABLE_BIT = {}
errors = collections.defaultdict(list)
def add_errors(fname, errs):
if not isinstance(errs, list):
errs = [errs]
for err in errs:
if err is None:
continue
try:
lineno, col, msg = err
except ValueError:
lineno = 1
col = 1
msg = err
if not isinstance(msg, str):
raise ValueError("Error is not instance of string!")
if not isinstance(lineno, int):
raise ValueError("Line number is not an int!")
if not isinstance(col, int):
raise ValueError("Column number is not an int!")
errors[fname].append((lineno, col, msg))
def run_check(lint_obj, fname, *args):
@ -155,7 +155,7 @@ def lint_re_check(regex, **kwargs):
def decorator(func):
@functools.wraps(func)
def new_func(fname, content):
errors = []
errs = []
for match in prog.finditer(content):
if "NOLINT" in match.group(0):
continue
@ -165,8 +165,8 @@ def lint_re_check(regex, **kwargs):
err = func(fname, match)
if err is None:
continue
errors.append((lineno, col + 1, err))
return errors
errs.append((lineno, col + 1, err))
return errs
return decor(new_func)
@ -182,13 +182,13 @@ def lint_content_find_check(find, only_first=False, **kwargs):
find_ = find
if callable(find):
find_ = find(fname, content)
errors = []
errs = []
for line, col in find_all(content, find_):
err = func(fname)
errors.append((line + 1, col + 1, err))
errs.append((line + 1, col + 1, err))
if only_first:
break
return errors
return errs
return decor(new_func)
@ -235,8 +235,8 @@ def lint_executable_bit(fname):
ex = EXECUTABLE_BIT[fname]
if ex != 100644:
return (
"File has invalid executable bit {}. If running from a windows machine please "
"see disabling executable bit in git.".format(ex)
f"File has invalid executable bit {ex}. If running from a windows machine please "
"see disabling executable bit in git."
)
return None
@ -285,8 +285,8 @@ def lint_no_defines(fname, match):
s = highlight(f"static const uint8_t {match.group(1)} = {match.group(2)};")
return (
"#define macros for integer constants are not allowed, please use "
"{} style instead (replace uint8_t with the appropriate "
"datatype). See also Google style guide.".format(s)
f"{s} style instead (replace uint8_t with the appropriate "
"datatype). See also Google style guide."
)
@ -296,11 +296,11 @@ def lint_no_long_delays(fname, match):
if duration_ms < 50:
return None
return (
"{} - long calls to delay() are not allowed in ESPHome because everything executes "
"in one thread. Calling delay() will block the main thread and slow down ESPHome.\n"
f"{highlight(match.group(0).strip())} - long calls to delay() are not allowed "
"in ESPHome because everything executes in one thread. Calling delay() will "
"block the main thread and slow down ESPHome.\n"
"If there's no way to work around the delay() and it doesn't execute often, please add "
"a '// NOLINT' comment to the line."
"".format(highlight(match.group(0).strip()))
)
@ -311,28 +311,28 @@ def lint_const_ordered(fname, content):
Reason: Otherwise people add it to the end, and then that results in merge conflicts.
"""
lines = content.splitlines()
errors = []
errs = []
for start in ["CONF_", "ICON_", "UNIT_"]:
matching = [
(i + 1, line) for i, line in enumerate(lines) if line.startswith(start)
]
ordered = list(sorted(matching, key=lambda x: x[1].replace("_", " ")))
ordered = [(mi, ol) for (mi, _), (_, ol) in zip(matching, ordered)]
for (mi, ml), (oi, ol) in zip(matching, ordered):
if ml == ol:
for (mi, mline), (_, ol) in zip(matching, ordered):
if mline == ol:
continue
target = next(i for i, l in ordered if l == ml)
target_text = next(l for i, l in matching if target == i)
errors.append(
target = next(i for i, line in ordered if line == mline)
target_text = next(line for i, line in matching if target == i)
errs.append(
(
mi,
1,
f"Constant {highlight(ml)} is not ordered, please make sure all "
f"Constant {highlight(mline)} is not ordered, please make sure all "
f"constants are ordered. See line {mi} (should go to line {target}, "
f"{target_text})",
)
)
return errors
return errs
@lint_re_check(r'^\s*CONF_([A-Z_0-9a-z]+)\s+=\s+[\'"](.*?)[\'"]\s*?$', include=["*.py"])
@ -344,15 +344,14 @@ def lint_conf_matches(fname, match):
if const_norm == value_norm:
return None
return (
"Constant {} does not match value {}! Please make sure the constant's name matches its "
"value!"
"".format(highlight("CONF_" + const), highlight(value))
f"Constant {highlight('CONF_' + const)} does not match value {highlight(value)}! "
"Please make sure the constant's name matches its value!"
)
CONF_RE = r'^(CONF_[a-zA-Z0-9_]+)\s*=\s*[\'"].*?[\'"]\s*?$'
with codecs.open("esphome/const.py", "r", encoding="utf-8") as f_handle:
constants_content = f_handle.read()
with codecs.open("esphome/const.py", "r", encoding="utf-8") as const_f_handle:
constants_content = const_f_handle.read()
CONSTANTS = [m.group(1) for m in re.finditer(CONF_RE, constants_content, re.MULTILINE)]
CONSTANTS_USES = collections.defaultdict(list)
@ -365,8 +364,8 @@ def lint_conf_from_const_py(fname, match):
CONSTANTS_USES[name].append(fname)
return None
return (
"Constant {} has already been defined in const.py - please import the constant from "
"const.py directly.".format(highlight(name))
f"Constant {highlight(name)} has already been defined in const.py - "
"please import the constant from const.py directly."
)
@ -473,16 +472,15 @@ def lint_no_byte_datatype(fname, match):
@lint_post_check
def lint_constants_usage():
errors = []
errs = []
for constant, uses in CONSTANTS_USES.items():
if len(uses) < 4:
continue
errors.append(
"Constant {} is defined in {} files. Please move all definitions of the "
"constant to const.py (Uses: {})"
"".format(highlight(constant), len(uses), ", ".join(uses))
errs.append(
f"Constant {highlight(constant)} is defined in {len(uses)} files. Please move all definitions of the "
f"constant to const.py (Uses: {', '.join(uses)})"
)
return errors
return errs
def relative_cpp_search_text(fname, content):
@ -553,7 +551,7 @@ def lint_namespace(fname, content):
return (
"Invalid namespace found in C++ file. All integration C++ files should put all "
"functions in a separate namespace that matches the integration's name. "
"Please make sure the file contains {}".format(highlight(search))
f"Please make sure the file contains {highlight(search)}"
)
@ -639,66 +637,73 @@ def lint_log_in_header(fname):
)
errors = collections.defaultdict(list)
def main():
colorama.init()
parser = argparse.ArgumentParser()
parser.add_argument(
"files", nargs="*", default=[], help="files to be processed (regex on path)"
)
parser.add_argument(
"-c", "--changed", action="store_true", help="Only run on changed files"
)
parser.add_argument(
"--print-slowest", action="store_true", help="Print the slowest checks"
)
args = parser.parse_args()
def add_errors(fname, errs):
if not isinstance(errs, list):
errs = [errs]
for err in errs:
if err is None:
global EXECUTABLE_BIT
EXECUTABLE_BIT = git_ls_files()
files = list(EXECUTABLE_BIT.keys())
# Match against re
file_name_re = re.compile("|".join(args.files))
files = [p for p in files if file_name_re.search(p)]
if args.changed:
files = filter_changed(files)
files.sort()
for fname in files:
_, ext = os.path.splitext(fname)
run_checks(LINT_FILE_CHECKS, fname, fname)
if ext in ignore_types:
continue
try:
lineno, col, msg = err
except ValueError:
lineno = 1
col = 1
msg = err
if not isinstance(msg, str):
raise ValueError("Error is not instance of string!")
if not isinstance(lineno, int):
raise ValueError("Line number is not an int!")
if not isinstance(col, int):
raise ValueError("Column number is not an int!")
errors[fname].append((lineno, col, msg))
with codecs.open(fname, "r", encoding="utf-8") as f_handle:
content = f_handle.read()
except UnicodeDecodeError:
add_errors(
fname,
"File is not readable as UTF-8. Please set your editor to UTF-8 mode.",
)
continue
run_checks(LINT_CONTENT_CHECKS, fname, fname, content)
run_checks(LINT_POST_CHECKS, "POST")
for fname in files:
_, ext = os.path.splitext(fname)
run_checks(LINT_FILE_CHECKS, fname, fname)
if ext in ignore_types:
continue
try:
with codecs.open(fname, "r", encoding="utf-8") as f_handle:
content = f_handle.read()
except UnicodeDecodeError:
add_errors(
fname,
"File is not readable as UTF-8. Please set your editor to UTF-8 mode.",
for f, errs in sorted(errors.items()):
bold = functools.partial(styled, colorama.Style.BRIGHT)
bold_red = functools.partial(styled, (colorama.Style.BRIGHT, colorama.Fore.RED))
err_str = (
f"{bold(f'{f}:{lineno}:{col}:')} {bold_red('lint:')} {msg}\n"
for lineno, col, msg in errs
)
continue
run_checks(LINT_CONTENT_CHECKS, fname, fname, content)
print_error_for_file(f, "\n".join(err_str))
run_checks(LINT_POST_CHECKS, "POST")
if args.print_slowest:
lint_times = []
for lint in LINT_FILE_CHECKS + LINT_CONTENT_CHECKS + LINT_POST_CHECKS:
durations = lint.get("durations", [])
lint_times.append((sum(durations), len(durations), lint["func"].__name__))
lint_times.sort(key=lambda x: -x[0])
for i in range(min(len(lint_times), 10)):
dur, invocations, name = lint_times[i]
print(f" - '{name}' took {dur:.2f}s total (ran on {invocations} files)")
print(f"Total time measured: {sum(x[0] for x in lint_times):.2f}s")
for f, errs in sorted(errors.items()):
bold = functools.partial(styled, colorama.Style.BRIGHT)
bold_red = functools.partial(styled, (colorama.Style.BRIGHT, colorama.Fore.RED))
err_str = (
f"{bold(f'{f}:{lineno}:{col}:')} {bold_red('lint:')} {msg}\n"
for lineno, col, msg in errs
)
print_error_for_file(f, "\n".join(err_str))
return len(errors)
if args.print_slowest:
lint_times = []
for lint in LINT_FILE_CHECKS + LINT_CONTENT_CHECKS + LINT_POST_CHECKS:
durations = lint.get("durations", [])
lint_times.append((sum(durations), len(durations), lint["func"].__name__))
lint_times.sort(key=lambda x: -x[0])
for i in range(min(len(lint_times), 10)):
dur, invocations, name = lint_times[i]
print(f" - '{name}' took {dur:.2f}s total (ran on {invocations} files)")
print(f"Total time measured: {sum(x[0] for x in lint_times):.2f}s")
sys.exit(len(errors))
if __name__ == "__main__":
sys.exit(main())