Redo docker build system with buildkit+multi-stage and cache pio packages (#2338)

.github/workflows/ci-docker.yml

@@ -27,6 +27,11 @@ jobs:
       uses: actions/setup-python@v2
       with:
         python-version: '3.9'
+    - name: Set up Docker Buildx
+      uses: docker/setup-buildx-action@v1
+    - name: Set up QEMU
+      uses: docker/setup-qemu-action@v1
+
     - name: Set TAG
       run: |
         echo "TAG=check" >> $GITHUB_ENV

.github/workflows/ci.yml

@@ -9,61 +9,7 @@ on:
   pull_request:
 
 jobs:
-  ci-with-container:
-    name: ${{ matrix.name }}
-    runs-on: ubuntu-latest
-    strategy:
-      fail-fast: false
-      matrix:
-        include:
-          - id: clang-format
-            name: Run script/clang-format
-          - id: clang-tidy
-            name: Run script/clang-tidy for ESP8266
-            options: --environment esp8266-tidy --grep ARDUINO_ARCH_ESP8266
-          - id: clang-tidy
-            name: Run script/clang-tidy for ESP32 1/4
-            options: --environment esp32-tidy --split-num 4 --split-at 1
-          - id: clang-tidy
-            name: Run script/clang-tidy for ESP32 2/4
-            options: --environment esp32-tidy --split-num 4 --split-at 2
-          - id: clang-tidy
-            name: Run script/clang-tidy for ESP32 3/4
-            options: --environment esp32-tidy --split-num 4 --split-at 3
-          - id: clang-tidy
-            name: Run script/clang-tidy for ESP32 4/4
-            options: --environment esp32-tidy --split-num 4 --split-at 4
-
-    # cpp lint job runs with esphome-lint docker image so that clang-format-*
-    # doesn't have to be installed
-    container: ghcr.io/esphome/esphome-lint:1.2
-    steps:
-      - uses: actions/checkout@v2
-
-      - name: Register problem matchers
-        run: |
-          echo "::add-matcher::.github/workflows/matchers/clang-tidy.json"
-          echo "::add-matcher::.github/workflows/matchers/gcc.json"
-
-      # Also run git-diff-index so that the step is marked as failed on formatting errors,
-      # since clang-format doesn't do anything but change files if -i is passed.
-      - name: Run clang-format
-        run: |
-          script/clang-format -i
-          git diff-index --quiet HEAD --
-        if: ${{ matrix.id == 'clang-format' }}
-
-      - name: Run clang-tidy
-        run: script/clang-tidy --all-headers --fix ${{ matrix.options }}
-        if: ${{ matrix.id == 'clang-tidy' }}
-
-      - name: Suggested changes
-        run: script/ci-suggest-changes
-        if: always()
-
   ci:
-    # Don't use the esphome-lint docker image because it may contain outdated requirements.
-    # This way, all dependencies are cached via the cache action.
     name: ${{ matrix.name }}
     runs-on: ubuntu-latest
     strategy:
@@ -77,48 +23,85 @@ jobs:
           - id: test
             file: tests/test1.yaml
             name: Test tests/test1.yaml
+            pio_cache_key: test1
           - id: test
             file: tests/test2.yaml
             name: Test tests/test2.yaml
+            pio_cache_key: test2
           - id: test
             file: tests/test3.yaml
             name: Test tests/test3.yaml
+            pio_cache_key: test1
           - id: test
             file: tests/test4.yaml
             name: Test tests/test4.yaml
+            pio_cache_key: test4
           - id: test
             file: tests/test5.yaml
             name: Test tests/test5.yaml
+            pio_cache_key: test5
           - id: pytest
             name: Run pytest
+          - id: clang-format
+            name: Run script/clang-format
+          - id: clang-tidy
+            name: Run script/clang-tidy for ESP8266
+            options: --environment esp8266-tidy --grep ARDUINO_ARCH_ESP8266
+            pio_cache_key: tidyesp8266
+          - id: clang-tidy
+            name: Run script/clang-tidy for ESP32 1/4
+            options: --environment esp32-tidy --split-num 4 --split-at 1
+            pio_cache_key: tidyesp32
+          - id: clang-tidy
+            name: Run script/clang-tidy for ESP32 2/4
+            options: --environment esp32-tidy --split-num 4 --split-at 2
+            pio_cache_key: tidyesp32
+          - id: clang-tidy
+            name: Run script/clang-tidy for ESP32 3/4
+            options: --environment esp32-tidy --split-num 4 --split-at 3
+            pio_cache_key: tidyesp32
+          - id: clang-tidy
+            name: Run script/clang-tidy for ESP32 4/4
+            options: --environment esp32-tidy --split-num 4 --split-at 4
+            pio_cache_key: tidyesp32
 
     steps:
       - uses: actions/checkout@v2
       - name: Set up Python
         uses: actions/setup-python@v2
+        id: python
         with:
           python-version: '3.7'
 
       - name: Cache pip modules
-        uses: actions/cache@v1
+        uses: actions/cache@v2
         with:
           path: ~/.cache/pip
-          key: esphome-pip-3.7-${{ hashFiles('setup.py') }}
+          key: pip-${{ steps.python.outputs.python-version }}-${{ hashFiles('requirements*.txt') }}
           restore-keys: |
-            esphome-pip-3.7-
-
-      # Use per test platformio cache because tests have different platform versions
-      - name: Cache ~/.platformio
-        uses: actions/cache@v1
-        with:
-          path: ~/.platformio
-          key: test-home-platformio-${{ matrix.file }}-${{ hashFiles('esphome/core/config.py') }}
-          restore-keys: |
-            test-home-platformio-${{ matrix.file }}-
-        if: ${{ matrix.id == 'test' }}
+            pip-${{ steps.python.outputs.python-version }}-
 
       - name: Set up python environment
-        run: script/setup
+        run: |
+          pip3 install -r requirements.txt -r requirements_optional.txt -r requirements_test.txt
+          pip3 install -e .
+
+      # Use per check platformio cache because checks use different parts
+      - name: Cache platformio
+        uses: actions/cache@v2
+        with:
+          path: ~/.platformio
+          key: platformio-${{ matrix.pio_cache_key }}-${{ hashFiles('platformio.ini') }}
+          restore-keys: |
+            platformio-${{ matrix.pio_cache_key }}-
+        if: matrix.id == 'test' || matrix.id == 'clang-tidy'
+
+      - name: Install clang tools
+        run: |
+          sudo apt-get install \
+              clang-format-11 \
+              clang-tidy-11
+        if: matrix.id == 'clang-tidy' || matrix.id == 'clang-format'
 
       - name: Register problem matchers
         run: |
@@ -127,20 +110,45 @@ jobs:
           echo "::add-matcher::.github/workflows/matchers/python.json"
           echo "::add-matcher::.github/workflows/matchers/pytest.json"
           echo "::add-matcher::.github/workflows/matchers/gcc.json"
+          echo "::add-matcher::.github/workflows/matchers/clang-tidy.json"
 
       - name: Lint Custom
         run: |
           script/ci-custom.py
           script/build_codeowners.py --check
-        if: ${{ matrix.id == 'ci-custom' }}
+        if: matrix.id == 'ci-custom'
+
       - name: Lint Python
         run: script/lint-python
-        if: ${{ matrix.id == 'lint-python' }}
+        if: matrix.id == 'lint-python'
 
       - run: esphome compile ${{ matrix.file }}
-        if: ${{ matrix.id == 'test' }}
+        if: matrix.id == 'test'
+        env:
+          # Also cache libdeps, store them in a ~/.platformio subfolder
+          PLATFORMIO_LIBDEPS_DIR: ~/.platformio/libdeps
 
       - name: Run pytest
         run: |
           pytest -vv --tb=native tests
-        if: ${{ matrix.id == 'pytest' }}
+        if: matrix.id == 'pytest'
+
+      # Also run git-diff-index so that the step is marked as failed on formatting errors,
+      # since clang-format doesn't do anything but change files if -i is passed.
+      - name: Run clang-format
+        run: |
+          script/clang-format -i
+          git diff-index --quiet HEAD --
+        if: matrix.id == 'clang-format'
+
+      - name: Run clang-tidy
+        run: |
+          script/clang-tidy --all-headers --fix ${{ matrix.options }}
+        if: matrix.id == 'clang-tidy'
+        env:
+          # Also cache libdeps, store them in a ~/.platformio subfolder
+          PLATFORMIO_LIBDEPS_DIR: ~/.platformio/libdeps
+
+      - name: Suggested changes
+        run: script/ci-suggest-changes
+        if: always() && (matrix.id == 'clang-tidy' || matrix.id == 'clang-format')

.github/workflows/docker-lint-build.yml

@@ -1,100 +0,0 @@
-name: Build and publish lint docker image
-
-# Only run when docker paths change
-on:
-  push:
-    branches: [dev]
-    paths:
-      - 'docker/Dockerfile.lint'
-      - 'requirements.txt'
-      - 'requirements_optional.txt'
-      - 'requirements_test.txt'
-      - 'platformio.ini'
-      - '.github/workflows/docker-lint-build.yml'
-
-jobs:
-  deploy-docker:
-    name: Build and publish docker containers
-    if: github.repository == 'esphome/esphome'
-    runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        arch: [amd64, armv7, aarch64]
-        build_type: ["lint"]
-    steps:
-    - uses: actions/checkout@v2
-    - name: Set up Python
-      uses: actions/setup-python@v2
-      with:
-        python-version: '3.9'
-    - name: Set TAG
-      run: |
-        echo "TAG=1.2" >> $GITHUB_ENV
-
-    - name: Run build
-      run: |
-        docker/build.py \
-          --tag "${TAG}" \
-          --arch "${{ matrix.arch }}" \
-          --build-type "${{ matrix.build_type }}" \
-          build
-
-    - name: Log in to docker hub
-      uses: docker/login-action@v1
-      with:
-        username: ${{ secrets.DOCKER_USER }}
-        password: ${{ secrets.DOCKER_PASSWORD }}
-    - name: Log in to the GitHub container registry
-      uses: docker/login-action@v1
-      with:
-          registry: ghcr.io
-          username: ${{ github.actor }}
-          password: ${{ secrets.GITHUB_TOKEN }}
-
-    - name: Run push
-      run: |
-        docker/build.py \
-          --tag "${TAG}" \
-          --arch "${{ matrix.arch }}" \
-          --build-type "${{ matrix.build_type }}" \
-          push
-
-  deploy-docker-manifest:
-    if: github.repository == 'esphome/esphome'
-    runs-on: ubuntu-latest
-    needs: [deploy-docker]
-    strategy:
-      matrix:
-        build_type: ["lint"]
-    steps:
-    - uses: actions/checkout@v2
-    - name: Set up Python
-      uses: actions/setup-python@v2
-      with:
-        python-version: '3.9'
-    - name: Set TAG
-      run: |
-        echo "TAG=1.2" >> $GITHUB_ENV
-    - name: Enable experimental manifest support
-      run: |
-        mkdir -p ~/.docker
-        echo "{\"experimental\": \"enabled\"}" > ~/.docker/config.json
-
-    - name: Log in to docker hub
-      uses: docker/login-action@v1
-      with:
-        username: ${{ secrets.DOCKER_USER }}
-        password: ${{ secrets.DOCKER_PASSWORD }}
-    - name: Log in to the GitHub container registry
-      uses: docker/login-action@v1
-      with:
-          registry: ghcr.io
-          username: ${{ github.actor }}
-          password: ${{ secrets.GITHUB_TOKEN }}
-
-    - name: Run manifest
-      run: |
-        docker/build.py \
-          --tag "${TAG}" \
-          --build-type "${{ matrix.build_type }}" \
-          manifest

.github/workflows/release.yml

@@ -57,7 +57,7 @@ jobs:
     strategy:
       matrix:
         arch: [amd64, armv7, aarch64]
-        build_type: ["ha-addon", "docker"]
+        build_type: ["ha-addon", "docker", "lint"]
     steps:
     - uses: actions/checkout@v2
     - name: Set up Python
@@ -65,13 +65,10 @@ jobs:
       with:
         python-version: '3.9'
 
-    - name: Run build
-      run: |
-        docker/build.py \
-          --tag "${{ needs.init.outputs.tag }}" \
-          --arch "${{ matrix.arch }}" \
-          --build-type "${{ matrix.build_type }}" \
-          build
+    - name: Set up Docker Buildx
+      uses: docker/setup-buildx-action@v1
+    - name: Set up QEMU
+      uses: docker/setup-qemu-action@v1
 
     - name: Log in to docker hub
       uses: docker/login-action@v1
@@ -85,13 +82,14 @@ jobs:
           username: ${{ github.actor }}
           password: ${{ secrets.GITHUB_TOKEN }}
 
-    - name: Run push
+    - name: Build and push
       run: |
         docker/build.py \
           --tag "${{ needs.init.outputs.tag }}" \
           --arch "${{ matrix.arch }}" \
           --build-type "${{ matrix.build_type }}" \
-          push
+          build \
+          --push
 
   deploy-docker-manifest:
     if: github.repository == 'esphome/esphome'
@@ -99,7 +97,7 @@ jobs:
     needs: [init, deploy-docker]
     strategy:
       matrix:
-        build_type: ["ha-addon", "docker"]
+        build_type: ["ha-addon", "docker", "lint"]
     steps:
     - uses: actions/checkout@v2
     - name: Set up Python