Refactor docker build system and workflows (#2023)

.github/workflows/ci-docker.yml

@@ -18,38 +18,23 @@ jobs:
     name: Build docker containers
     runs-on: ubuntu-latest
     strategy:
-      fail-fast: false
       matrix:
         arch: [amd64, armv7, aarch64]
-        build_type: ["hassio", "docker"]
+        build_type: ["ha-addon", "docker", "lint"]
     steps:
-      - uses: actions/checkout@v2
-      - name: Set up env variables
-        run: |
-          base_version="3.4.0"
+    - uses: actions/checkout@v2
+    - name: Set up Python
+      uses: actions/setup-python@v2
+      with:
+        python-version: '3.9'
+    - name: Set TAG
+      run: |
+        echo "TAG=check" >> $GITHUB_ENV
 
-          if [[ "${{ matrix.build_type }}" == "hassio" ]]; then
-            build_from="esphome/esphome-hassio-base-${{ matrix.arch }}:${base_version}"
-            build_to="esphome/esphome-hassio-${{ matrix.arch }}"
-            dockerfile="docker/Dockerfile.hassio"
-          else
-            build_from="esphome/esphome-base-${{ matrix.arch }}:${base_version}"
-            build_to="esphome/esphome-${{ matrix.arch }}"
-            dockerfile="docker/Dockerfile"
-          fi
-
-          echo "BUILD_FROM=${build_from}" >> $GITHUB_ENV
-          echo "BUILD_TO=${build_to}" >> $GITHUB_ENV
-          echo "DOCKERFILE=${dockerfile}" >> $GITHUB_ENV
-      - name: Pull for cache
-        run: |
-          docker pull "${BUILD_TO}:dev" || true
-      - name: Register QEMU binfmt
-        run: docker run --rm --privileged multiarch/qemu-user-static:5.2.0-2 --reset -p yes
-      - run: |
-          docker build \
-            --build-arg "BUILD_FROM=${BUILD_FROM}" \
-            --build-arg "BUILD_VERSION=ci" \
-            --cache-from "${BUILD_TO}:dev" \
-            --file "${DOCKERFILE}" \
-            .
+    - name: Run build
+      run: |
+        docker/build.py \
+          --tag "${TAG}" \
+          --arch "${{ matrix.arch }}" \
+          --build-type "${{ matrix.build_type }}" \
+          build

.github/workflows/ci.yml

@@ -4,40 +4,36 @@ name: CI
 
 on:
   push:
-    # On dev branch release-dev already performs CI checks
-    # On other branches the `pull_request` trigger will be used
-    branches: [beta, release]
+    branches: [dev, beta, release]
 
   pull_request:
 
 jobs:
-  lint-clang-format:
+  ci-with-container:
+    name: ${{ matrix.name }}
     runs-on: ubuntu-latest
-    # cpp lint job runs with esphome-lint docker image so that clang-format-*
-    # doesn't have to be installed
-    container: esphome/esphome-lint:1.1
-    steps:
-      - uses: actions/checkout@v2
-      # Set up the pio project so that the cpp checks know how files are compiled
-      # (build flags, libraries etc)
-      - name: Set up platformio environment
-        run: pio init --ide atom
-
-      - name: Run clang-format
-        run: script/clang-format -i
-      - name: Suggest changes
-        run: script/ci-suggest-changes
-
-  lint-clang-tidy:
-    runs-on: ubuntu-latest
-    # cpp lint job runs with esphome-lint docker image so that clang-format-*
-    # doesn't have to be installed
-    container: esphome/esphome-lint:1.1
-    # Split clang-tidy check into 4 jobs. Each one will check 1/4th of the .cpp files
     strategy:
       fail-fast: false
       matrix:
-        split: [1, 2, 3, 4]
+        include:
+          - id: clang-format
+            name: Run script/clang-format
+          - id: clang-tidy
+            name: Run script/clang-tidy 1/4
+            split: 1
+          - id: clang-tidy
+            name: Run script/clang-tidy 2/4
+            split: 2
+          - id: clang-tidy
+            name: Run script/clang-tidy 3/4
+            split: 3
+          - id: clang-tidy
+            name: Run script/clang-tidy 4/4
+            split: 4
+
+    # cpp lint job runs with esphome-lint docker image so that clang-format-*
+    # doesn't have to be installed
+    container: esphome/esphome-lint:1.1
     steps:
       - uses: actions/checkout@v2
       # Set up the pio project so that the cpp checks know how files are compiled
@@ -45,26 +41,57 @@ jobs:
       - name: Set up platformio environment
         run: pio init --ide atom
 
-
       - name: Register problem matchers
         run: |
           echo "::add-matcher::.github/workflows/matchers/clang-tidy.json"
           echo "::add-matcher::.github/workflows/matchers/gcc.json"
+
+      - name: Run clang-format
+        run: script/clang-format -i
+        if: ${{ matrix.id == 'clang-format' }}
+
       - name: Run clang-tidy
         run: script/clang-tidy --all-headers --fix --split-num 4 --split-at ${{ matrix.split }}
+        if: ${{ matrix.id == 'clang-tidy' }}
+
       - name: Suggest changes
         run: script/ci-suggest-changes
 
-  lint-python:
+  ci:
     # Don't use the esphome-lint docker image because it may contain outdated requirements.
     # This way, all dependencies are cached via the cache action.
+    name: ${{ matrix.name }}
     runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - id: ci-custom
+            name: Run script/ci-custom
+          - id: lint-python
+            name: Run script/lint-python
+          - id: test
+            file: tests/test1.yaml
+            name: Test tests/test1.yaml
+          - id: test
+            file: tests/test2.yaml
+            name: Test tests/test2.yaml
+          - id: test
+            file: tests/test3.yaml
+            name: Test tests/test3.yaml
+          - id: test
+            file: tests/test4.yaml
+            name: Test tests/test4.yaml
+          - id: pytest
+            name: Run pytest
+
     steps:
       - uses: actions/checkout@v2
       - name: Set up Python
         uses: actions/setup-python@v2
         with:
           python-version: '3.7'
+
       - name: Cache pip modules
         uses: actions/cache@v1
         with:
@@ -72,6 +99,17 @@ jobs:
           key: esphome-pip-3.7-${{ hashFiles('setup.py') }}
           restore-keys: |
             esphome-pip-3.7-
+
+      # Use per test platformio cache because tests have different platform versions
+      - name: Cache ~/.platformio
+        uses: actions/cache@v1
+        with:
+          path: ~/.platformio
+          key: test-home-platformio-${{ matrix.file }}-${{ hashFiles('esphome/core/config.py') }}
+          restore-keys: |
+            test-home-platformio-${{ matrix.file }}-
+        if: ${{ matrix.id == 'test' }}
+
       - name: Set up python environment
         run: script/setup
 
@@ -80,82 +118,22 @@ jobs:
           echo "::add-matcher::.github/workflows/matchers/ci-custom.json"
           echo "::add-matcher::.github/workflows/matchers/lint-python.json"
           echo "::add-matcher::.github/workflows/matchers/python.json"
+          echo "::add-matcher::.github/workflows/matchers/pytest.json"
+          echo "::add-matcher::.github/workflows/matchers/gcc.json"
+
       - name: Lint Custom
-        run: script/ci-custom.py
+        run: |
+          script/ci-custom.py
+          script/build_codeowners.py --check
+        if: ${{ matrix.id == 'ci-custom' }}
       - name: Lint Python
         run: script/lint-python
-      - name: Lint CODEOWNERS
-        run: script/build_codeowners.py --check
+        if: ${{ matrix.id == 'lint-python' }}
 
-  test:
-    runs-on: ubuntu-latest
-    strategy:
-      fail-fast: false
-      matrix:
-          test:
-          - test1
-          - test2
-          - test3
-          - test4
-          - test5
-    steps:
-      - uses: actions/checkout@v2
-      - name: Set up Python
-        uses: actions/setup-python@v2
-        with:
-          python-version: '3.7'
-      - name: Cache pip modules
-        uses: actions/cache@v1
-        with:
-          path: ~/.cache/pip
-          key: esphome-pip-3.7-${{ hashFiles('setup.py') }}
-          restore-keys: |
-            esphome-pip-3.7-
-      # Use per test platformio cache because tests have different platform versions
-      - name: Cache ~/.platformio
-        uses: actions/cache@v1
-        with:
-          path: ~/.platformio
-          key: test-home-platformio-${{ matrix.test }}-${{ hashFiles('esphome/core/config.py') }}
-          restore-keys: |
-            test-home-platformio-${{ matrix.test }}-
-      - name: Set up environment
-        run: script/setup
+      - run: esphome compile ${{ matrix.file }}
+        if: ${{ matrix.id == 'test' }}
 
-
-      - name: Register problem matchers
-        run: |
-          echo "::add-matcher::.github/workflows/matchers/gcc.json"
-          echo "::add-matcher::.github/workflows/matchers/python.json"
-      - run: esphome compile tests/${{ matrix.test }}.yaml
-
-  pytest:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-      - name: Set up Python
-        uses: actions/setup-python@v2
-        with:
-          python-version: '3.7'
-      - name: Cache pip modules
-        uses: actions/cache@v1
-        with:
-          path: ~/.cache/pip
-          key: esphome-pip-3.7-${{ hashFiles('setup.py') }}
-          restore-keys: |
-            esphome-pip-3.7-
-      - name: Set up environment
-        run: script/setup
-      - name: Install Github Actions annotator
-        run: pip install pytest-github-actions-annotate-failures
-
-      - name: Register problem matchers
-        run: |
-          echo "::add-matcher::.github/workflows/matchers/python.json"
       - name: Run pytest
         run: |
-          pytest \
-            -qq \
-            --durations=10 \
-            -o console_output_style=count \
-            tests
+          pytest -vv --tb=native tests
+        if: ${{ matrix.id == 'pytest' }}

.github/workflows/docker-lint-build.yml

@@ -13,30 +13,88 @@ on:
       - '.github/workflows/docker-lint-build.yml'
 
 jobs:
-  publish-docker-lint-iage:
-    name: Build docker containers
+  deploy-docker:
+    name: Build and publish docker containers
+    if: github.repository == 'esphome/esphome'
     runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        arch: [amd64, armv7, aarch64]
+        build_type: ["lint"]
     steps:
-      - uses: actions/checkout@v2
-      - name: Set TAG
-        run: |
-          echo "TAG=1.1" >> $GITHUB_ENV
-      - name: Pull for cache
-        run: |
-          docker pull "esphome/esphome-lint:latest" || true
-      - name: Build
-        run: |
-          docker build \
-            --cache-from "esphome/esphome-lint:latest" \
-            --file "docker/Dockerfile.lint" \
-            --tag "esphome/esphome-lint:latest" \
-            --tag "esphome/esphome-lint:${TAG}" \
-            .
-      - name: Log in to docker hub
-        env:
-          DOCKER_USER: ${{ secrets.DOCKER_USER }}
-          DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}
-        run: docker login -u "${DOCKER_USER}" -p "${DOCKER_PASSWORD}"
-      - run: |
-          docker push "esphome/esphome-lint:${TAG}"
-          docker push "esphome/esphome-lint:latest"
+    - uses: actions/checkout@v2
+    - name: Set up Python
+      uses: actions/setup-python@v2
+      with:
+        python-version: '3.9'
+    - name: Set TAG
+      run: |
+        echo "TAG=1.1" >> $GITHUB_ENV
+
+    - name: Run build
+      run: |
+        docker/build.py \
+          --tag "${TAG}" \
+          --arch "${{ matrix.arch }}" \
+          --build-type "${{ matrix.build_type }}" \
+          build
+
+    - name: Log in to docker hub
+      uses: docker/login-action@v1
+      with:
+        username: ${{ secrets.DOCKER_USER }}
+        password: ${{ secrets.DOCKER_PASSWORD }}
+    - name: Log in to the GitHub container registry
+      uses: docker/login-action@v1
+      with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+    - name: Run push
+      run: |
+        docker/build.py \
+          --tag "${TAG}" \
+          --arch "${{ matrix.arch }}" \
+          --build-type "${{ matrix.build_type }}" \
+          push
+
+  deploy-docker-manifest:
+    if: github.repository == 'esphome/esphome'
+    runs-on: ubuntu-latest
+    needs: [deploy-docker]
+    strategy:
+      matrix:
+        build_type: ["lint"]
+    steps:
+    - uses: actions/checkout@v2
+    - name: Set up Python
+      uses: actions/setup-python@v2
+      with:
+        python-version: '3.9'
+    - name: Set TAG
+      run: |
+        echo "TAG=1.1" >> $GITHUB_ENV
+    - name: Enable experimental manifest support
+      run: |
+        mkdir -p ~/.docker
+        echo "{\"experimental\": \"enabled\"}" > ~/.docker/config.json
+
+    - name: Log in to docker hub
+      uses: docker/login-action@v1
+      with:
+        username: ${{ secrets.DOCKER_USER }}
+        password: ${{ secrets.DOCKER_PASSWORD }}
+    - name: Log in to the GitHub container registry
+      uses: docker/login-action@v1
+      with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+    - name: Run manifest
+      run: |
+        docker/build.py \
+          --tag "${TAG}" \
+          --build-type "${{ matrix.build_type }}" \
+          manifest

.github/workflows/matchers/pytest.json

@@ -0,0 +1,19 @@
+{
+  "problemMatcher": [
+    {
+      "owner": "pytest",
+      "fileLocation": "absolute",
+      "pattern": [
+        {
+          "regexp": "^\\s+File \"(.*)\", line (\\d+), in (.*)$",
+          "file": 1,
+          "line": 2
+        },
+        {
+          "regexp": "^\\s+(.*)$",
+          "message": 1
+        }
+      ]
+    }
+  ]
+}

.github/workflows/release-dev.yml

@@ -1,247 +0,0 @@
-name: Publish dev releases to docker hub
-
-on:
-  push:
-    branches:
-    - dev
-
-jobs:
-  # THE LINT/TEST JOBS ARE COPIED FROM ci.yaml
-
-  lint-clang-format:
-    runs-on: ubuntu-latest
-    # cpp lint job runs with esphome-lint docker image so that clang-format-*
-    # doesn't have to be installed
-    container: esphome/esphome-lint:1.1
-    steps:
-      - uses: actions/checkout@v2
-      # Set up the pio project so that the cpp checks know how files are compiled
-      # (build flags, libraries etc)
-      - name: Set up platformio environment
-        run: pio init --ide atom
-
-      - name: Run clang-format
-        run: script/clang-format -i
-      - name: Suggest changes
-        run: script/ci-suggest-changes
-
-  lint-clang-tidy:
-    runs-on: ubuntu-latest
-    # cpp lint job runs with esphome-lint docker image so that clang-format-*
-    # doesn't have to be installed
-    container: esphome/esphome-lint:1.1
-    # Split clang-tidy check into 4 jobs. Each one will check 1/4th of the .cpp files
-    strategy:
-      fail-fast: false
-      matrix:
-        split: [1, 2, 3, 4]
-    steps:
-      - uses: actions/checkout@v2
-      # Set up the pio project so that the cpp checks know how files are compiled
-      # (build flags, libraries etc)
-      - name: Set up platformio environment
-        run: pio init --ide atom
-
-
-      - name: Register problem matchers
-        run: |
-          echo "::add-matcher::.github/workflows/matchers/clang-tidy.json"
-          echo "::add-matcher::.github/workflows/matchers/gcc.json"
-      - name: Run clang-tidy
-        run: script/clang-tidy --all-headers --fix --split-num 4 --split-at ${{ matrix.split }}
-      - name: Suggest changes
-        run: script/ci-suggest-changes
-
-  lint-python:
-    # Don't use the esphome-lint docker image because it may contain outdated requirements.
-    # This way, all dependencies are cached via the cache action.
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-      - name: Set up Python
-        uses: actions/setup-python@v2
-        with:
-          python-version: '3.7'
-      - name: Cache pip modules
-        uses: actions/cache@v1
-        with:
-          path: ~/.cache/pip
-          key: esphome-pip-3.7-${{ hashFiles('setup.py') }}
-          restore-keys: |
-            esphome-pip-3.7-
-      - name: Set up python environment
-        run: script/setup
-
-      - name: Register problem matchers
-        run: |
-          echo "::add-matcher::.github/workflows/matchers/ci-custom.json"
-          echo "::add-matcher::.github/workflows/matchers/lint-python.json"
-          echo "::add-matcher::.github/workflows/matchers/python.json"
-      - name: Lint Custom
-        run: script/ci-custom.py
-      - name: Lint Python
-        run: script/lint-python
-      - name: Lint CODEOWNERS
-        run: script/build_codeowners.py --check
-
-  test:
-    runs-on: ubuntu-latest
-    strategy:
-      fail-fast: false
-      matrix:
-          test:
-          - test1
-          - test2
-          - test3
-          - test4
-          - test5
-    steps:
-      - uses: actions/checkout@v2
-      - name: Set up Python
-        uses: actions/setup-python@v2
-        with:
-          python-version: '3.7'
-      - name: Cache pip modules
-        uses: actions/cache@v1
-        with:
-          path: ~/.cache/pip
-          key: esphome-pip-3.7-${{ hashFiles('setup.py') }}
-          restore-keys: |
-            esphome-pip-3.7-
-      # Use per test platformio cache because tests have different platform versions
-      - name: Cache ~/.platformio
-        uses: actions/cache@v1
-        with:
-          path: ~/.platformio
-          key: test-home-platformio-${{ matrix.test }}-${{ hashFiles('esphome/core/config.py') }}
-          restore-keys: |
-            test-home-platformio-${{ matrix.test }}-
-      - name: Set up environment
-        run: script/setup
-
-
-      - name: Register problem matchers
-        run: |
-          echo "::add-matcher::.github/workflows/matchers/gcc.json"
-          echo "::add-matcher::.github/workflows/matchers/python.json"
-      - run: esphome compile tests/${{ matrix.test }}.yaml
-
-  pytest:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-      - name: Set up Python
-        uses: actions/setup-python@v2
-        with:
-          python-version: '3.7'
-      - name: Cache pip modules
-        uses: actions/cache@v1
-        with:
-          path: ~/.cache/pip
-          key: esphome-pip-3.7-${{ hashFiles('setup.py') }}
-          restore-keys: |
-            esphome-pip-3.7-
-      - name: Set up environment
-        run: script/setup
-      - name: Install Github Actions annotator
-        run: pip install pytest-github-actions-annotate-failures
-
-      - name: Register problem matchers
-        run: |
-          echo "::add-matcher::.github/workflows/matchers/python.json"
-      - name: Run pytest
-        run: |
-          pytest \
-            -qq \
-            --durations=10 \
-            -o console_output_style=count \
-            tests
-
-  deploy-docker:
-    name: Build and publish docker containers
-    if: github.repository == 'esphome/esphome'
-    runs-on: ubuntu-latest
-    needs: [lint-clang-format, lint-clang-tidy, lint-python, test, pytest]
-    strategy:
-      matrix:
-        arch: [amd64, armv7, aarch64]
-        # Hassio dev image doesn't use esphome/esphome-hassio-$arch and uses base directly
-        build_type: ["docker"]
-    steps:
-      - uses: actions/checkout@v2
-      - name: Set TAG
-        run: |
-          TAG="${GITHUB_SHA:0:7}"
-          echo "TAG=${TAG}" >> $GITHUB_ENV
-      - name: Set up env variables
-        run: |
-          base_version="3.4.0"
-
-          if [[ "${{ matrix.build_type }}" == "hassio" ]]; then
-            build_from="esphome/esphome-hassio-base-${{ matrix.arch }}:${base_version}"
-            build_to="esphome/esphome-hassio-${{ matrix.arch }}"
-            dockerfile="docker/Dockerfile.hassio"
-          else
-            build_from="esphome/esphome-base-${{ matrix.arch }}:${base_version}"
-            build_to="esphome/esphome-${{ matrix.arch }}"
-            dockerfile="docker/Dockerfile"
-          fi
-
-          echo "BUILD_FROM=${build_from}" >> $GITHUB_ENV
-          echo "BUILD_TO=${build_to}" >> $GITHUB_ENV
-          echo "DOCKERFILE=${dockerfile}" >> $GITHUB_ENV
-      - name: Pull for cache
-        run: |
-          docker pull "${BUILD_TO}:dev" || true
-      - name: Register QEMU binfmt
-        run: docker run --rm --privileged multiarch/qemu-user-static:5.2.0-2 --reset -p yes
-      - run: |
-          docker build \
-            --build-arg "BUILD_FROM=${BUILD_FROM}" \
-            --build-arg "BUILD_VERSION=${TAG}" \
-            --tag "${BUILD_TO}:${TAG}" \
-            --tag "${BUILD_TO}:dev" \
-            --cache-from "${BUILD_TO}:dev" \
-            --file "${DOCKERFILE}" \
-            .
-      - name: Log in to docker hub
-        env:
-          DOCKER_USER: ${{ secrets.DOCKER_USER }}
-          DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}
-        run: docker login -u "${DOCKER_USER}" -p "${DOCKER_PASSWORD}"
-      - run: |
-          docker push "${BUILD_TO}:${TAG}"
-          docker push "${BUILD_TO}:dev"
-
-
-  deploy-docker-manifest:
-    if: github.repository == 'esphome/esphome'
-    runs-on: ubuntu-latest
-    needs: [deploy-docker]
-    steps:
-    - name: Enable experimental manifest support
-      run: |
-        mkdir -p ~/.docker
-        echo "{\"experimental\": \"enabled\"}" > ~/.docker/config.json
-    - name: Set TAG
-      run: |
-        TAG="${GITHUB_SHA:0:7}"
-        echo "TAG=${TAG}" >> $GITHUB_ENV
-    - name: Log in to docker hub
-      env:
-        DOCKER_USER: ${{ secrets.DOCKER_USER }}
-        DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}
-      run: docker login -u "${DOCKER_USER}" -p "${DOCKER_PASSWORD}"
-    - name: "Create the manifest"
-      run: |
-        docker manifest create esphome/esphome:${TAG} \
-          esphome/esphome-aarch64:${TAG} \
-          esphome/esphome-amd64:${TAG} \
-          esphome/esphome-armv7:${TAG}
-        docker manifest push esphome/esphome:${TAG}
-
-        docker manifest create esphome/esphome:dev \
-          esphome/esphome-aarch64:${TAG} \
-          esphome/esphome-amd64:${TAG} \
-          esphome/esphome-armv7:${TAG}
-        docker manifest push esphome/esphome:dev

.github/workflows/release.yml

@@ -1,164 +1,35 @@
 name: Publish Release
 
 on:
+  workflow_dispatch:
   release:
     types: [published]
+  schedule:
+    - cron: "0 2 * * *"
 
 jobs:
-  # THE LINT/TEST JOBS ARE COPIED FROM ci.yaml
-
-  lint-clang-format:
+  init:
+    name: Initialize build
     runs-on: ubuntu-latest
-    # cpp lint job runs with esphome-lint docker image so that clang-format-*
-    # doesn't have to be installed
-    container: esphome/esphome-lint:1.1
+    outputs:
+      tag: ${{ steps.tag.outputs.tag }}
     steps:
       - uses: actions/checkout@v2
-      # Set up the pio project so that the cpp checks know how files are compiled
-      # (build flags, libraries etc)
-      - name: Set up platformio environment
-        run: pio init --ide atom
-
-      - name: Run clang-format
-        run: script/clang-format -i
-      - name: Suggest changes
-        run: script/ci-suggest-changes
-
-  lint-clang-tidy:
-    runs-on: ubuntu-latest
-    # cpp lint job runs with esphome-lint docker image so that clang-format-*
-    # doesn't have to be installed
-    container: esphome/esphome-lint:1.1
-    # Split clang-tidy check into 4 jobs. Each one will check 1/4th of the .cpp files
-    strategy:
-      fail-fast: false
-      matrix:
-        split: [1, 2, 3, 4]
-    steps:
-      - uses: actions/checkout@v2
-      # Set up the pio project so that the cpp checks know how files are compiled
-      # (build flags, libraries etc)
-      - name: Set up platformio environment
-        run: pio init --ide atom
-
-
-      - name: Register problem matchers
+      - name: Get tag
+        id: tag
         run: |
-          echo "::add-matcher::.github/workflows/matchers/clang-tidy.json"
-          echo "::add-matcher::.github/workflows/matchers/gcc.json"
-      - name: Run clang-tidy
-        run: script/clang-tidy --all-headers --fix --split-num 4 --split-at ${{ matrix.split }}
-      - name: Suggest changes
-        run: script/ci-suggest-changes
-
-  lint-python:
-    # Don't use the esphome-lint docker image because it may contain outdated requirements.
-    # This way, all dependencies are cached via the cache action.
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-      - name: Set up Python
-        uses: actions/setup-python@v2
-        with:
-          python-version: '3.7'
-      - name: Cache pip modules
-        uses: actions/cache@v1
-        with:
-          path: ~/.cache/pip
-          key: esphome-pip-3.7-${{ hashFiles('setup.py') }}
-          restore-keys: |
-            esphome-pip-3.7-
-      - name: Set up python environment
-        run: script/setup
-
-      - name: Register problem matchers
-        run: |
-          echo "::add-matcher::.github/workflows/matchers/ci-custom.json"
-          echo "::add-matcher::.github/workflows/matchers/lint-python.json"
-          echo "::add-matcher::.github/workflows/matchers/python.json"
-      - name: Lint Custom
-        run: script/ci-custom.py
-      - name: Lint Python
-        run: script/lint-python
-      - name: Lint CODEOWNERS
-        run: script/build_codeowners.py --check
-
-  test:
-    runs-on: ubuntu-latest
-    strategy:
-      fail-fast: false
-      matrix:
-          test:
-          - test1
-          - test2
-          - test3
-          - test4
-          - test5
-    steps:
-      - uses: actions/checkout@v2
-      - name: Set up Python
-        uses: actions/setup-python@v2
-        with:
-          python-version: '3.7'
-      - name: Cache pip modules
-        uses: actions/cache@v1
-        with:
-          path: ~/.cache/pip
-          key: esphome-pip-3.7-${{ hashFiles('setup.py') }}
-          restore-keys: |
-            esphome-pip-3.7-
-      # Use per test platformio cache because tests have different platform versions
-      - name: Cache ~/.platformio
-        uses: actions/cache@v1
-        with:
-          path: ~/.platformio
-          key: test-home-platformio-${{ matrix.test }}-${{ hashFiles('esphome/core/config.py') }}
-          restore-keys: |
-            test-home-platformio-${{ matrix.test }}-
-      - name: Set up environment
-        run: script/setup
-
-      - name: Register problem matchers
-        run: |
-          echo "::add-matcher::.github/workflows/matchers/gcc.json"
-          echo "::add-matcher::.github/workflows/matchers/python.json"
-      - run: esphome compile tests/${{ matrix.test }}.yaml
-
-  pytest:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-      - name: Set up Python
-        uses: actions/setup-python@v2
-        with:
-          python-version: '3.7'
-      - name: Cache pip modules
-        uses: actions/cache@v1
-        with:
-          path: ~/.cache/pip
-          key: esphome-pip-3.7-${{ hashFiles('setup.py') }}
-          restore-keys: |
-            esphome-pip-3.7-
-      - name: Set up environment
-        run: script/setup
-      - name: Install Github Actions annotator
-        run: pip install pytest-github-actions-annotate-failures
-
-      - name: Register problem matchers
-        run: |
-          echo "::add-matcher::.github/workflows/matchers/python.json"
-      - name: Run pytest
-        run: |
-          pytest \
-            -qq \
-            --durations=10 \
-            -o console_output_style=count \
-            tests
+          if [[ "$GITHUB_EVENT_NAME" = "release" ]]; then
+            TAG="${GITHUB_REF#refs/tags/v}"
+          else
+            TAG=$(cat esphome/const.py | sed -n -E "s/^__version__\s+=\s+\"(.+)\"$/\1/p")
+            today="$(date --utc '+%Y%m%d')"
+            TAG="${TAG}${today}"
+          fi
+          echo "::set-output name=tag::${TAG}"
 
   deploy-pypi:
     name: Build and publish to PyPi
-    if: github.repository == 'esphome/esphome'
-    needs: [lint-clang-format, lint-clang-tidy, lint-python, test, pytest]
+    if: github.repository == 'esphome/esphome' && github.event_name == 'release'
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
@@ -182,119 +53,85 @@ jobs:
     name: Build and publish docker containers
     if: github.repository == 'esphome/esphome'
     runs-on: ubuntu-latest
-    needs: [lint-clang-format, lint-clang-tidy, lint-python, test, pytest]
+    needs: [init]
     strategy:
       matrix:
         arch: [amd64, armv7, aarch64]
-        build_type: ["hassio", "docker"]
+        build_type: ["ha-addon", "docker"]
     steps:
-      - uses: actions/checkout@v2
-      - name: Set TAG
-        run: |
-          TAG="${GITHUB_REF#refs/tags/v}"
-          echo "TAG=${TAG}" >> $GITHUB_ENV
-      - name: Set up env variables
-        run: |
-          base_version="3.4.0"
+    - uses: actions/checkout@v2
+    - name: Set up Python
+      uses: actions/setup-python@v2
+      with:
+        python-version: '3.9'
 
-          if [[ "${{ matrix.build_type }}" == "hassio" ]]; then
-            build_from="esphome/esphome-hassio-base-${{ matrix.arch }}:${base_version}"
-            build_to="esphome/esphome-hassio-${{ matrix.arch }}"
-            dockerfile="docker/Dockerfile.hassio"
-          else
-            build_from="esphome/esphome-base-${{ matrix.arch }}:${base_version}"
-            build_to="esphome/esphome-${{ matrix.arch }}"
-            dockerfile="docker/Dockerfile"
-          fi
+    - name: Run build
+      run: |
+        docker/build.py \
+          --tag "${{ needs.init.outputs.tag }}" \
+          --arch "${{ matrix.arch }}" \
+          --build-type "${{ matrix.build_type }}" \
+          build
 
-          if [[ "${{ github.event.release.prerelease }}" == "true" ]]; then
-            cache_tag="beta"
-          else
-            cache_tag="latest"
-          fi
+    - name: Log in to docker hub
+      uses: docker/login-action@v1
+      with:
+        username: ${{ secrets.DOCKER_USER }}
+        password: ${{ secrets.DOCKER_PASSWORD }}
+    - name: Log in to the GitHub container registry
+      uses: docker/login-action@v1
+      with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
 
-          # Set env variables so these values don't need to be calculated again
-          echo "BUILD_FROM=${build_from}" >> $GITHUB_ENV
-          echo "BUILD_TO=${build_to}" >> $GITHUB_ENV
-          echo "DOCKERFILE=${dockerfile}" >> $GITHUB_ENV
-          echo "CACHE_TAG=${cache_tag}" >> $GITHUB_ENV
-      - name: Pull for cache
-        run: |
-          docker pull "${BUILD_TO}:${CACHE_TAG}" || true
-      - name: Register QEMU binfmt
-        run: docker run --rm --privileged multiarch/qemu-user-static:5.2.0-2 --reset -p yes
-      - run: |
-          docker build \
-            --build-arg "BUILD_FROM=${BUILD_FROM}" \
-            --build-arg "BUILD_VERSION=${TAG}" \
-            --tag "${BUILD_TO}:${TAG}" \
-            --cache-from "${BUILD_TO}:${CACHE_TAG}" \
-            --file "${DOCKERFILE}" \
-            .
-      - name: Log in to docker hub
-        env:
-          DOCKER_USER: ${{ secrets.DOCKER_USER }}
-          DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}
-        run: docker login -u "${DOCKER_USER}" -p "${DOCKER_PASSWORD}"
-      - run: docker push "${BUILD_TO}:${TAG}"
-
-      # Always publish to beta tag (also full releases)
-      - name: Publish docker beta tag
-        run: |
-          docker tag "${BUILD_TO}:${TAG}" "${BUILD_TO}:beta"
-          docker push "${BUILD_TO}:beta"
-
-      - if: ${{ !github.event.release.prerelease }}
-        name: Publish docker latest tag
-        run: |
-          docker tag "${BUILD_TO}:${TAG}" "${BUILD_TO}:latest"
-          docker push "${BUILD_TO}:latest"
+    - name: Run push
+      run: |
+        docker/build.py \
+          --tag "${{ needs.init.outputs.tag }}" \
+          --arch "${{ matrix.arch }}" \
+          --build-type "${{ matrix.build_type }}" \
+          push
 
   deploy-docker-manifest:
     if: github.repository == 'esphome/esphome'
     runs-on: ubuntu-latest
-    needs: [deploy-docker]
+    needs: [init, deploy-docker]
+    strategy:
+      matrix:
+        build_type: ["ha-addon", "docker"]
     steps:
+    - uses: actions/checkout@v2
+    - name: Set up Python
+      uses: actions/setup-python@v2
+      with:
+        python-version: '3.9'
     - name: Enable experimental manifest support
       run: |
         mkdir -p ~/.docker
         echo "{\"experimental\": \"enabled\"}" > ~/.docker/config.json
-    - name: Set TAG
-      run: |
-        TAG="${GITHUB_REF#refs/tags/v}"
-        echo "TAG=${TAG}" >> $GITHUB_ENV
+
     - name: Log in to docker hub
-      env:
-        DOCKER_USER: ${{ secrets.DOCKER_USER }}
-        DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}
-      run: docker login -u "${DOCKER_USER}" -p "${DOCKER_PASSWORD}"
-    - name: "Create the manifest"
-      run: |
-        docker manifest create esphome/esphome:${TAG} \
-          esphome/esphome-aarch64:${TAG} \
-          esphome/esphome-amd64:${TAG} \
-          esphome/esphome-armv7:${TAG}
-        docker manifest push esphome/esphome:${TAG}
+      uses: docker/login-action@v1
+      with:
+        username: ${{ secrets.DOCKER_USER }}
+        password: ${{ secrets.DOCKER_PASSWORD }}
+    - name: Log in to the GitHub container registry
+      uses: docker/login-action@v1
+      with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
 
-    - name: Publish docker beta tag
+    - name: Run manifest
       run: |
-        docker manifest create esphome/esphome:beta \
-          esphome/esphome-aarch64:${TAG} \
-          esphome/esphome-amd64:${TAG} \
-          esphome/esphome-armv7:${TAG}
-        docker manifest push esphome/esphome:beta
-
-    - name: Publish docker latest tag
-      if: ${{ !github.event.release.prerelease }}
-      run: |
-        docker manifest create esphome/esphome:latest \
-          esphome/esphome-aarch64:${TAG} \
-          esphome/esphome-amd64:${TAG} \
-          esphome/esphome-armv7:${TAG}
-        docker manifest push esphome/esphome:latest
+        docker/build.py \
+          --tag "${{ needs.init.outputs.tag }}" \
+          --build-type "${{ matrix.build_type }}" \
+          manifest
 
   deploy-hassio-repo:
-    if: github.repository == 'esphome/esphome'
+    if: github.repository == 'esphome/esphome' && github.event_name == 'release'
     runs-on: ubuntu-latest
     needs: [deploy-docker]
     steps: