Redo docker build system with buildkit+multi-stage and cache pio packages (#2338)

.github/workflows/ci.yml

@@ -9,61 +9,7 @@ on:
   pull_request:
 
 jobs:
-  ci-with-container:
-    name: ${{ matrix.name }}
-    runs-on: ubuntu-latest
-    strategy:
-      fail-fast: false
-      matrix:
-        include:
-          - id: clang-format
-            name: Run script/clang-format
-          - id: clang-tidy
-            name: Run script/clang-tidy for ESP8266
-            options: --environment esp8266-tidy --grep ARDUINO_ARCH_ESP8266
-          - id: clang-tidy
-            name: Run script/clang-tidy for ESP32 1/4
-            options: --environment esp32-tidy --split-num 4 --split-at 1
-          - id: clang-tidy
-            name: Run script/clang-tidy for ESP32 2/4
-            options: --environment esp32-tidy --split-num 4 --split-at 2
-          - id: clang-tidy
-            name: Run script/clang-tidy for ESP32 3/4
-            options: --environment esp32-tidy --split-num 4 --split-at 3
-          - id: clang-tidy
-            name: Run script/clang-tidy for ESP32 4/4
-            options: --environment esp32-tidy --split-num 4 --split-at 4
-
-    # cpp lint job runs with esphome-lint docker image so that clang-format-*
-    # doesn't have to be installed
-    container: ghcr.io/esphome/esphome-lint:1.2
-    steps:
-      - uses: actions/checkout@v2
-
-      - name: Register problem matchers
-        run: |
-          echo "::add-matcher::.github/workflows/matchers/clang-tidy.json"
-          echo "::add-matcher::.github/workflows/matchers/gcc.json"
-
-      # Also run git-diff-index so that the step is marked as failed on formatting errors,
-      # since clang-format doesn't do anything but change files if -i is passed.
-      - name: Run clang-format
-        run: |
-          script/clang-format -i
-          git diff-index --quiet HEAD --
-        if: ${{ matrix.id == 'clang-format' }}
-
-      - name: Run clang-tidy
-        run: script/clang-tidy --all-headers --fix ${{ matrix.options }}
-        if: ${{ matrix.id == 'clang-tidy' }}
-
-      - name: Suggested changes
-        run: script/ci-suggest-changes
-        if: always()
-
   ci:
-    # Don't use the esphome-lint docker image because it may contain outdated requirements.
-    # This way, all dependencies are cached via the cache action.
     name: ${{ matrix.name }}
     runs-on: ubuntu-latest
     strategy:
@@ -77,48 +23,85 @@ jobs:
           - id: test
             file: tests/test1.yaml
             name: Test tests/test1.yaml
+            pio_cache_key: test1
           - id: test
             file: tests/test2.yaml
             name: Test tests/test2.yaml
+            pio_cache_key: test2
           - id: test
             file: tests/test3.yaml
             name: Test tests/test3.yaml
+            pio_cache_key: test1
           - id: test
             file: tests/test4.yaml
             name: Test tests/test4.yaml
+            pio_cache_key: test4
           - id: test
             file: tests/test5.yaml
             name: Test tests/test5.yaml
+            pio_cache_key: test5
           - id: pytest
             name: Run pytest
+          - id: clang-format
+            name: Run script/clang-format
+          - id: clang-tidy
+            name: Run script/clang-tidy for ESP8266
+            options: --environment esp8266-tidy --grep ARDUINO_ARCH_ESP8266
+            pio_cache_key: tidyesp8266
+          - id: clang-tidy
+            name: Run script/clang-tidy for ESP32 1/4
+            options: --environment esp32-tidy --split-num 4 --split-at 1
+            pio_cache_key: tidyesp32
+          - id: clang-tidy
+            name: Run script/clang-tidy for ESP32 2/4
+            options: --environment esp32-tidy --split-num 4 --split-at 2
+            pio_cache_key: tidyesp32
+          - id: clang-tidy
+            name: Run script/clang-tidy for ESP32 3/4
+            options: --environment esp32-tidy --split-num 4 --split-at 3
+            pio_cache_key: tidyesp32
+          - id: clang-tidy
+            name: Run script/clang-tidy for ESP32 4/4
+            options: --environment esp32-tidy --split-num 4 --split-at 4
+            pio_cache_key: tidyesp32
 
     steps:
       - uses: actions/checkout@v2
       - name: Set up Python
         uses: actions/setup-python@v2
+        id: python
         with:
           python-version: '3.7'
 
       - name: Cache pip modules
-        uses: actions/cache@v1
+        uses: actions/cache@v2
         with:
           path: ~/.cache/pip
-          key: esphome-pip-3.7-${{ hashFiles('setup.py') }}
+          key: pip-${{ steps.python.outputs.python-version }}-${{ hashFiles('requirements*.txt') }}
           restore-keys: |
-            esphome-pip-3.7-
-
-      # Use per test platformio cache because tests have different platform versions
-      - name: Cache ~/.platformio
-        uses: actions/cache@v1
-        with:
-          path: ~/.platformio
-          key: test-home-platformio-${{ matrix.file }}-${{ hashFiles('esphome/core/config.py') }}
-          restore-keys: |
-            test-home-platformio-${{ matrix.file }}-
-        if: ${{ matrix.id == 'test' }}
+            pip-${{ steps.python.outputs.python-version }}-
 
       - name: Set up python environment
-        run: script/setup
+        run: |
+          pip3 install -r requirements.txt -r requirements_optional.txt -r requirements_test.txt
+          pip3 install -e .
+
+      # Use per check platformio cache because checks use different parts
+      - name: Cache platformio
+        uses: actions/cache@v2
+        with:
+          path: ~/.platformio
+          key: platformio-${{ matrix.pio_cache_key }}-${{ hashFiles('platformio.ini') }}
+          restore-keys: |
+            platformio-${{ matrix.pio_cache_key }}-
+        if: matrix.id == 'test' || matrix.id == 'clang-tidy'
+
+      - name: Install clang tools
+        run: |
+          sudo apt-get install \
+              clang-format-11 \
+              clang-tidy-11
+        if: matrix.id == 'clang-tidy' || matrix.id == 'clang-format'
 
       - name: Register problem matchers
         run: |
@@ -127,20 +110,45 @@ jobs:
           echo "::add-matcher::.github/workflows/matchers/python.json"
           echo "::add-matcher::.github/workflows/matchers/pytest.json"
           echo "::add-matcher::.github/workflows/matchers/gcc.json"
+          echo "::add-matcher::.github/workflows/matchers/clang-tidy.json"
 
       - name: Lint Custom
         run: |
           script/ci-custom.py
           script/build_codeowners.py --check
-        if: ${{ matrix.id == 'ci-custom' }}
+        if: matrix.id == 'ci-custom'
+
       - name: Lint Python
         run: script/lint-python
-        if: ${{ matrix.id == 'lint-python' }}
+        if: matrix.id == 'lint-python'
 
       - run: esphome compile ${{ matrix.file }}
-        if: ${{ matrix.id == 'test' }}
+        if: matrix.id == 'test'
+        env:
+          # Also cache libdeps, store them in a ~/.platformio subfolder
+          PLATFORMIO_LIBDEPS_DIR: ~/.platformio/libdeps
 
       - name: Run pytest
         run: |
           pytest -vv --tb=native tests
-        if: ${{ matrix.id == 'pytest' }}
+        if: matrix.id == 'pytest'
+
+      # Also run git-diff-index so that the step is marked as failed on formatting errors,
+      # since clang-format doesn't do anything but change files if -i is passed.
+      - name: Run clang-format
+        run: |
+          script/clang-format -i
+          git diff-index --quiet HEAD --
+        if: matrix.id == 'clang-format'
+
+      - name: Run clang-tidy
+        run: |
+          script/clang-tidy --all-headers --fix ${{ matrix.options }}
+        if: matrix.id == 'clang-tidy'
+        env:
+          # Also cache libdeps, store them in a ~/.platformio subfolder
+          PLATFORMIO_LIBDEPS_DIR: ~/.platformio/libdeps
+
+      - name: Suggested changes
+        run: script/ci-suggest-changes
+        if: always() && (matrix.id == 'clang-tidy' || matrix.id == 'clang-format')