From dc8dfede3a54ccf4b6f6a2fd2f70673c7a3f8a6d Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Sun, 19 Oct 2025 15:36:41 -1000 Subject: [PATCH] tweak --- .github/workflows/ci.yml | 205 ++++++++++++++++++++++------ script/determine-jobs.py | 13 ++ tests/script/test_determine_jobs.py | 3 + 3 files changed, 183 insertions(+), 38 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 807f1af878..afb48495b0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -170,6 +170,7 @@ jobs: outputs: integration-tests: ${{ steps.determine.outputs.integration-tests }} clang-tidy: ${{ steps.determine.outputs.clang-tidy }} + clang-tidy-mode: ${{ steps.determine.outputs.clang-tidy-mode }} python-linters: ${{ steps.determine.outputs.python-linters }} changed-components: ${{ steps.determine.outputs.changed-components }} changed-components-with-tests: ${{ steps.determine.outputs.changed-components-with-tests }} @@ -201,6 +202,7 @@ jobs: # Extract individual fields echo "integration-tests=$(echo "$output" | jq -r '.integration_tests')" >> $GITHUB_OUTPUT echo "clang-tidy=$(echo "$output" | jq -r '.clang_tidy')" >> $GITHUB_OUTPUT + echo "clang-tidy-mode=$(echo "$output" | jq -r '.clang_tidy_mode')" >> $GITHUB_OUTPUT echo "python-linters=$(echo "$output" | jq -r '.python_linters')" >> $GITHUB_OUTPUT echo "changed-components=$(echo "$output" | jq -c '.changed_components')" >> $GITHUB_OUTPUT echo "changed-components-with-tests=$(echo "$output" | jq -c '.changed_components_with_tests')" >> $GITHUB_OUTPUT @@ -245,21 +247,13 @@ jobs: . venv/bin/activate pytest -vv --no-cov --tb=native -n auto tests/integration/ - clang-tidy: + clang-tidy-single: name: ${{ matrix.name }} runs-on: ubuntu-24.04 needs: - common - determine-jobs - # Skip split jobs for small PRs (< 30 files) - run single job instead for faster completion - # Skip unsplit job for large PRs (>= 30 files) - run split jobs instead for parallelization - # Jobs without split attribute (ESP8266, IDF, ZEPHYR) always run when clang-tidy is enabled - if: | - needs.determine-jobs.outputs.clang-tidy == 'true' && - (matrix.split == null || ( - !(matrix.split == true && fromJSON(needs.determine-jobs.outputs.changed-cpp-file-count) < 30) && - !(matrix.split == false && fromJSON(needs.determine-jobs.outputs.changed-cpp-file-count) >= 30) - )) + if: needs.determine-jobs.outputs.clang-tidy == 'true' env: GH_TOKEN: ${{ github.token }} strategy: @@ -271,33 +265,6 @@ jobs: name: Run script/clang-tidy for ESP8266 options: --environment esp8266-arduino-tidy --grep USE_ESP8266 pio_cache_key: tidyesp8266 - # For small PRs (< 30 files), run ESP32 Arduino as a single job for faster completion - # For larger PRs, split into 4 parallel jobs to distribute the load - - id: clang-tidy - name: Run script/clang-tidy for ESP32 Arduino - options: --environment esp32-arduino-tidy - pio_cache_key: tidyesp32 - split: false - - id: clang-tidy - name: Run script/clang-tidy for ESP32 Arduino 1/4 - options: --environment esp32-arduino-tidy --split-num 4 --split-at 1 - pio_cache_key: tidyesp32 - split: true - - id: clang-tidy - name: Run script/clang-tidy for ESP32 Arduino 2/4 - options: --environment esp32-arduino-tidy --split-num 4 --split-at 2 - pio_cache_key: tidyesp32 - split: true - - id: clang-tidy - name: Run script/clang-tidy for ESP32 Arduino 3/4 - options: --environment esp32-arduino-tidy --split-num 4 --split-at 3 - pio_cache_key: tidyesp32 - split: true - - id: clang-tidy - name: Run script/clang-tidy for ESP32 Arduino 4/4 - options: --environment esp32-arduino-tidy --split-num 4 --split-at 4 - pio_cache_key: tidyesp32 - split: true - id: clang-tidy name: Run script/clang-tidy for ESP32 IDF options: --environment esp32-idf-tidy --grep USE_ESP_IDF @@ -378,6 +345,166 @@ jobs: # yamllint disable-line rule:line-length if: always() + clang-tidy-nosplit: + name: Run script/clang-tidy for ESP32 Arduino + runs-on: ubuntu-24.04 + needs: + - common + - determine-jobs + if: needs.determine-jobs.outputs.clang-tidy-mode == 'nosplit' + env: + GH_TOKEN: ${{ github.token }} + steps: + - name: Check out code from GitHub + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + with: + # Need history for HEAD~1 to work for checking changed files + fetch-depth: 2 + + - name: Restore Python + uses: ./.github/actions/restore-python + with: + python-version: ${{ env.DEFAULT_PYTHON }} + cache-key: ${{ needs.common.outputs.cache-key }} + + - name: Cache platformio + if: github.ref == 'refs/heads/dev' + uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 + with: + path: ~/.platformio + key: platformio-tidyesp32-${{ hashFiles('platformio.ini') }} + + - name: Cache platformio + if: github.ref != 'refs/heads/dev' + uses: actions/cache/restore@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 + with: + path: ~/.platformio + key: platformio-tidyesp32-${{ hashFiles('platformio.ini') }} + + - name: Register problem matchers + run: | + echo "::add-matcher::.github/workflows/matchers/gcc.json" + echo "::add-matcher::.github/workflows/matchers/clang-tidy.json" + + - name: Check if full clang-tidy scan needed + id: check_full_scan + run: | + . venv/bin/activate + if python script/clang_tidy_hash.py --check; then + echo "full_scan=true" >> $GITHUB_OUTPUT + echo "reason=hash_changed" >> $GITHUB_OUTPUT + else + echo "full_scan=false" >> $GITHUB_OUTPUT + echo "reason=normal" >> $GITHUB_OUTPUT + fi + + - name: Run clang-tidy + run: | + . venv/bin/activate + if [ "${{ steps.check_full_scan.outputs.full_scan }}" = "true" ]; then + echo "Running FULL clang-tidy scan (hash changed)" + script/clang-tidy --all-headers --fix --environment esp32-arduino-tidy + else + echo "Running clang-tidy on changed files only" + script/clang-tidy --all-headers --fix --changed --environment esp32-arduino-tidy + fi + env: + # Also cache libdeps, store them in a ~/.platformio subfolder + PLATFORMIO_LIBDEPS_DIR: ~/.platformio/libdeps + + - name: Suggested changes + run: script/ci-suggest-changes + if: always() + + clang-tidy-split: + name: ${{ matrix.name }} + runs-on: ubuntu-24.04 + needs: + - common + - determine-jobs + if: needs.determine-jobs.outputs.clang-tidy-mode == 'split' + env: + GH_TOKEN: ${{ github.token }} + strategy: + fail-fast: false + max-parallel: 2 + matrix: + include: + - id: clang-tidy + name: Run script/clang-tidy for ESP32 Arduino 1/4 + options: --environment esp32-arduino-tidy --split-num 4 --split-at 1 + - id: clang-tidy + name: Run script/clang-tidy for ESP32 Arduino 2/4 + options: --environment esp32-arduino-tidy --split-num 4 --split-at 2 + - id: clang-tidy + name: Run script/clang-tidy for ESP32 Arduino 3/4 + options: --environment esp32-arduino-tidy --split-num 4 --split-at 3 + - id: clang-tidy + name: Run script/clang-tidy for ESP32 Arduino 4/4 + options: --environment esp32-arduino-tidy --split-num 4 --split-at 4 + + steps: + - name: Check out code from GitHub + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + with: + # Need history for HEAD~1 to work for checking changed files + fetch-depth: 2 + + - name: Restore Python + uses: ./.github/actions/restore-python + with: + python-version: ${{ env.DEFAULT_PYTHON }} + cache-key: ${{ needs.common.outputs.cache-key }} + + - name: Cache platformio + if: github.ref == 'refs/heads/dev' + uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 + with: + path: ~/.platformio + key: platformio-tidyesp32-${{ hashFiles('platformio.ini') }} + + - name: Cache platformio + if: github.ref != 'refs/heads/dev' + uses: actions/cache/restore@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 + with: + path: ~/.platformio + key: platformio-tidyesp32-${{ hashFiles('platformio.ini') }} + + - name: Register problem matchers + run: | + echo "::add-matcher::.github/workflows/matchers/gcc.json" + echo "::add-matcher::.github/workflows/matchers/clang-tidy.json" + + - name: Check if full clang-tidy scan needed + id: check_full_scan + run: | + . venv/bin/activate + if python script/clang_tidy_hash.py --check; then + echo "full_scan=true" >> $GITHUB_OUTPUT + echo "reason=hash_changed" >> $GITHUB_OUTPUT + else + echo "full_scan=false" >> $GITHUB_OUTPUT + echo "reason=normal" >> $GITHUB_OUTPUT + fi + + - name: Run clang-tidy + run: | + . venv/bin/activate + if [ "${{ steps.check_full_scan.outputs.full_scan }}" = "true" ]; then + echo "Running FULL clang-tidy scan (hash changed)" + script/clang-tidy --all-headers --fix ${{ matrix.options }} + else + echo "Running clang-tidy on changed files only" + script/clang-tidy --all-headers --fix --changed ${{ matrix.options }} + fi + env: + # Also cache libdeps, store them in a ~/.platformio subfolder + PLATFORMIO_LIBDEPS_DIR: ~/.platformio/libdeps + + - name: Suggested changes + run: script/ci-suggest-changes + if: always() + test-build-components-splitter: name: Split components for intelligent grouping (40 weighted per batch) runs-on: ubuntu-24.04 @@ -818,7 +945,9 @@ jobs: - pylint - pytest - integration-tests - - clang-tidy + - clang-tidy-single + - clang-tidy-nosplit + - clang-tidy-split - determine-jobs - test-build-components-splitter - test-build-components-split diff --git a/script/determine-jobs.py b/script/determine-jobs.py index ec2817c12b..41e5763621 100755 --- a/script/determine-jobs.py +++ b/script/determine-jobs.py @@ -462,10 +462,23 @@ def main() -> None: # Detect components for memory impact analysis (merged config) memory_impact = detect_memory_impact_config(args.branch) + # Determine clang-tidy split mode based on file count + # For small PRs (< 30 files), use nosplit for faster CI + # For large PRs (>= 30 files), use split for better parallelization + CLANG_TIDY_SPLIT_THRESHOLD = 30 + if run_clang_tidy: + if changed_cpp_file_count < CLANG_TIDY_SPLIT_THRESHOLD: + clang_tidy_mode = "nosplit" + else: + clang_tidy_mode = "split" + else: + clang_tidy_mode = "disabled" + # Build output output: dict[str, Any] = { "integration_tests": run_integration, "clang_tidy": run_clang_tidy, + "clang_tidy_mode": clang_tidy_mode, "clang_format": run_clang_format, "python_linters": run_python_linters, "changed_components": changed_components, diff --git a/tests/script/test_determine_jobs.py b/tests/script/test_determine_jobs.py index 360efe6e99..54693861e6 100644 --- a/tests/script/test_determine_jobs.py +++ b/tests/script/test_determine_jobs.py @@ -107,6 +107,7 @@ def test_main_all_tests_should_run( assert output["integration_tests"] is True assert output["clang_tidy"] is True + assert output["clang_tidy_mode"] in ["nosplit", "split"] assert output["clang_format"] is True assert output["python_linters"] is True assert output["changed_components"] == ["wifi", "api", "sensor"] @@ -159,6 +160,7 @@ def test_main_no_tests_should_run( assert output["integration_tests"] is False assert output["clang_tidy"] is False + assert output["clang_tidy_mode"] == "disabled" assert output["clang_format"] is False assert output["python_linters"] is False assert output["changed_components"] == [] @@ -244,6 +246,7 @@ def test_main_with_branch_argument( assert output["integration_tests"] is False assert output["clang_tidy"] is True + assert output["clang_tidy_mode"] in ["nosplit", "split"] assert output["clang_format"] is False assert output["python_linters"] is True assert output["changed_components"] == ["mqtt"]