GH Actions Update (#1134)

.github/workflows/ci.yml

@@ -0,0 +1,196 @@
+# THESE JOBS ARE COPIED IN release.yml and release-dev.yml
+# PLEASE ALSO UPDATE THOSE FILES WHEN CHANGING LINES HERE
+name: CI
+
+on:
+  push:
+    # On dev branch release-dev already performs CI checks
+    # On other branches the `pull_request` trigger will be used
+    branches: [beta, master]
+
+  pull_request:
+    # Only trigger on certain events (not when comments are added)
+    types: [opened, reopened, synchronize]
+    # Only run when PR is against dev branch (all PRs should be against dev branch)
+    # Helps prevent accidentally merging PRs against master branch
+    branches: [dev]
+
+jobs:
+  # A fast overview job that checks only changed files
+  overview:
+    runs-on: ubuntu-latest
+    container: esphome/esphome-lint:dev
+    steps:
+      # Also fetch history and dev branch so that we can check which files changed
+      - uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
+      - name: Fetch dev branch
+        run: git fetch origin dev
+      
+      # Cache the .pio directory with (primarily) library dependencies
+      - name: Cache .pio lib_deps
+        uses: actions/cache@v1
+        with:
+          path: .pio
+          key: lint-cpp-pio-${{ hashFiles('platformio.ini') }}
+          restore-keys: |
+            lint-cpp-pio-
+      - name: Set up python environment
+        run: script/setup
+      # Set up the pio project so that the cpp checks know how files are compiled
+      # (build flags, libraries etc)
+      - name: Set up platformio environment
+        run: pio init --ide atom
+
+      - name: Register problem matchers
+        run: |
+          echo "::add-matcher::.github/workflows/matchers/ci-custom.json"
+          echo "::add-matcher::.github/workflows/matchers/clang-tidy.json"
+          echo "::add-matcher::.github/workflows/matchers/gcc.json"
+          echo "::add-matcher::.github/workflows/matchers/lint-python.json"
+          echo "::add-matcher::.github/workflows/matchers/python.json"
+      - name: Run a quick lint over all changed files
+        run: script/quicklint
+      - name: Suggest changes
+        run: script/ci-suggest-changes
+
+  lint-clang-format:
+    runs-on: ubuntu-latest
+    # cpp lint job runs with esphome-lint docker image so that clang-format-*
+    # doesn't have to be installed
+    container: esphome/esphome-lint:dev
+    steps:
+      - uses: actions/checkout@v2
+      # Cache platformio intermediary files (like libraries etc)
+      # Note: platformio platform versions should be cached via the esphome-lint image
+      - name: Cache Platformio
+        uses: actions/cache@v1
+        with:
+          path: .pio
+          key: lint-cpp-pio-${{ hashFiles('platformio.ini') }}
+          restore-keys: |
+            lint-cpp-pio-
+      # Set up the pio project so that the cpp checks know how files are compiled
+      # (build flags, libraries etc)
+      - name: Set up platformio environment
+        run: pio init --ide atom
+
+      - name: Run clang-format
+        run: script/clang-format -i
+      - name: Suggest changes
+        run: script/ci-suggest-changes
+
+  lint-clang-tidy:
+    runs-on: ubuntu-latest
+    # cpp lint job runs with esphome-lint docker image so that clang-format-*
+    # doesn't have to be installed
+    container: esphome/esphome-lint:dev
+    # Split clang-tidy check into 4 jobs. Each one will check 1/4th of the .cpp files
+    strategy:
+      matrix:
+        split: [1, 2, 3, 4]
+    steps:
+      - uses: actions/checkout@v2
+      # Cache platformio intermediary files (like libraries etc)
+      # Note: platformio platform versions should be cached via the esphome-lint image
+      - name: Cache Platformio
+        uses: actions/cache@v1
+        with:
+          path: .pio
+          key: lint-cpp-pio-${{ hashFiles('platformio.ini') }}
+          restore-keys: |
+            lint-cpp-pio-
+      # Set up the pio project so that the cpp checks know how files are compiled
+      # (build flags, libraries etc)
+      - name: Set up platformio environment
+        run: pio init --ide atom
+
+      
+      - name: Register problem matchers
+        run: |
+          echo "::add-matcher::.github/workflows/matchers/clang-tidy.json"
+          echo "::add-matcher::.github/workflows/matchers/gcc.json"
+      - name: Run clang-tidy
+        run: script/clang-tidy --all-headers --fix --split-num 4 --split-at ${{ matrix.split }}
+      - name: Suggest changes
+        run: script/ci-suggest-changes
+
+  lint-python:
+    # Don't use the esphome-lint docker image because it may contain outdated requirements.
+    # This way, all dependencies are cached via the cache action.
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up Python
+        uses: actions/setup-python@v2
+        with:
+          python-version: '3.7'
+      - name: Cache pip modules
+        uses: actions/cache@v1
+        with:
+          path: ~/.cache/pip
+          key: esphome-pip-3.7-${{ hashFiles('setup.py') }}
+          restore-keys: |
+            esphome-pip-3.7-
+      - name: Set up python environment
+        run: script/setup
+
+      - name: Register problem matchers
+        run: |
+          echo "::add-matcher::.github/workflows/matchers/ci-custom.json"
+          echo "::add-matcher::.github/workflows/matchers/lint-python.json"
+          echo "::add-matcher::.github/workflows/matchers/python.json"
+      - name: Lint Custom
+        run: script/ci-custom.py
+      - name: Lint Python
+        run: script/lint-python
+
+  test:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+          test:
+          - test1
+          - test2
+          - test3
+          - test4
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up Python
+        uses: actions/setup-python@v2
+        with:
+          python-version: '3.7'
+      - name: Cache pip modules
+        uses: actions/cache@v1
+        with:
+          path: ~/.cache/pip
+          key: esphome-pip-3.7-${{ hashFiles('setup.py') }}
+          restore-keys: |
+            esphome-pip-3.7-
+      # Use per test platformio cache because tests have different platform versions
+      - name: Cache ~/.platformio
+        uses: actions/cache@v1
+        with:
+          path: ~/.platformio
+          key: test-home-platformio-${{ matrix.test }}-${{ hashFiles('esphome/core_config.py') }}
+          restore-keys: |
+            test-home-platformio-${{ matrix.test }}-
+      # Cache the intermediary build files
+      - name: Cache Test Build
+        uses: actions/cache@v1
+        with:
+          path: tests/build/${{ matrix.test }}
+          key: test-pio-${{ matrix.test }}-${{ hashFiles('esphome/core_config.py') }}-${{ hashFiles('esphome/**') }}
+          restore-keys: |
+            test-pio-${{ matrix.test }}-${{ hashFiles('esphome/core_config.py') }}-
+            test-pio-${{ matrix.test }}-
+      - name: Set up environment
+        run: script/setup
+
+      
+      - name: Register problem matchers
+        run: |
+          echo "::add-matcher::.github/workflows/matchers/gcc.json"
+          echo "::add-matcher::.github/workflows/matchers/python.json"
+      - run: esphome tests/${{ matrix.test }}.yaml compile

.github/workflows/matchers/ci-custom.json

@@ -0,0 +1,16 @@
+{
+    "problemMatcher": [
+        {
+            "owner": "ci-custom",
+            "pattern": [
+                {
+                    "regexp": "^ERROR (.*):(\\d+):(\\d+) - (.*)$",
+                    "file": 1,
+                    "line": 2,
+                    "column": 3,
+                    "message": 4
+                }
+            ]
+        }
+    ]
+}

.github/workflows/matchers/clang-tidy.json

@@ -0,0 +1,17 @@
+{
+    "problemMatcher": [
+        {
+            "owner": "clang-tidy",
+            "pattern": [
+                {
+                    "regexp": "^(.*):(\\d+):(\\d+):\\s+(error):\\s+(.*) \\[([a-z0-9,\\-]+)\\]\\s*$",
+                    "file": 1,
+                    "line": 2,
+                    "column": 3,
+                    "severity": 4,
+                    "message": 5
+                }
+            ]
+        }
+    ]
+}

.github/workflows/matchers/gcc.json

@@ -0,0 +1,18 @@
+{
+  "problemMatcher": [
+    {
+      "owner": "gcc",
+      "severity": "error",
+      "pattern": [
+        {
+          "regexp": "^(.*):(\\d+):(\\d+):\\s+(?:fatal\\s+)?(warning|error):\\s+(.*)$",
+          "file": 1,
+          "line": 2,
+          "column": 3,
+          "severity": 4,
+          "message": 5
+        }
+      ]
+    }
+  ]
+}

.github/workflows/matchers/lint-python.json

@@ -0,0 +1,28 @@
+{
+  "problemMatcher": [
+    {
+      "owner": "flake8",
+      "severity": "error",
+      "pattern": [
+          {
+          "regexp": "^(.*):(\\d+) - ([EFCDNW]\\d{3}.*)$",
+          "file": 1,
+          "line": 2,
+          "message": 3
+          }
+      ]
+    },
+    {
+      "owner": "pylint",
+      "severity": "error",
+      "pattern": [
+        {
+          "regexp": "^(.*):(\\d+) - (\\[[EFCRW]\\d{4}\\(.*\\),.*\\].*)$",
+          "file": 1,
+          "line": 2,
+          "message": 3
+        }
+      ]
+    }
+  ]
+}

.github/workflows/matchers/python.json

@@ -0,0 +1,18 @@
+{
+    "problemMatcher": [
+        {
+            "owner": "python",
+            "pattern": [
+                {
+                    "regexp": "^\\s*File\\s\\\"(.*)\\\",\\sline\\s(\\d+),\\sin\\s(.*)$",
+                    "file": 1,
+                    "line": 2
+                },
+                {
+                    "regexp": "^\\s*raise\\s(.*)\\(\\'(.*)\\'\\)$",
+                    "message": 2
+                }
+            ]
+        }
+    ]
+}

.github/workflows/pull.yaml

@@ -1,59 +0,0 @@
-name: PR testing
-
-on: [pull_request]
-
-jobs:
-  lint-custom:
-    runs-on: ubuntu-latest
-    container: jesserockz/esphome-lint
-    name: Lint Custom
-    steps:
-    - uses: actions/checkout@v2
-    - run: script/setup
-    - run: script/ci-custom.py
-  lint-python:
-    runs-on: ubuntu-latest
-    container: jesserockz/esphome-lint
-    name: Lint Python
-    steps:
-    - uses: actions/checkout@v2
-    - run: script/setup
-    - run: script/lint-python
-  lint-tidy:
-    runs-on: ubuntu-latest
-    container: jesserockz/esphome-lint
-    name: Lint Tidy
-    steps:
-    - uses: actions/checkout@v2
-      with:
-        fetch-depth: 0
-    - run: script/setup
-    - run: pio init --ide atom
-    - run: script/clang-tidy --all-headers --fix
-    - run: script/ci-suggest-changes
-  lint-format:
-    runs-on: ubuntu-latest
-    container: jesserockz/esphome-lint
-    name: Lint Format
-    steps:
-    - uses: actions/checkout@v2
-      with:
-        fetch-depth: 0
-    - run: script/setup
-    - run: script/clang-format -i
-    - run: script/ci-suggest-changes
-
-  test:
-    runs-on: ubuntu-latest
-    container: jesserockz/esphome-lint
-    strategy:
-      matrix:
-        test:
-        - test1
-        - test2
-        - test3
-        - test4
-    steps:
-    - uses: actions/checkout@v2
-    - run: script/setup
-    - run: esphome tests/${{ matrix.test }}.yaml compile

.github/workflows/release-dev.yaml

@@ -1,124 +0,0 @@
-name: Release dev
-
-on:
-  push:
-    branches:
-    - dev
-
-jobs:
-  lint-custom:
-    runs-on: ubuntu-latest
-    container: jesserockz/esphome-lint
-    name: Lint Custom
-    steps:
-    - uses: actions/checkout@v2
-    - run: script/setup
-    - run: script/ci-custom.py
-  lint-python:
-    runs-on: ubuntu-latest
-    container: jesserockz/esphome-lint
-    name: Lint Python
-    steps:
-    - uses: actions/checkout@v2
-    - run: script/setup
-    - run: script/lint-python
-  lint-tidy:
-    runs-on: ubuntu-latest
-    container: jesserockz/esphome-lint
-    name: Lint Tidy
-    steps:
-    - uses: actions/checkout@v2
-    - run: script/setup
-    - run: pio init --ide atom
-    - run: script/clang-tidy --all-headers --fix
-    - run: script/ci-suggest-changes
-  lint-format:
-    runs-on: ubuntu-latest
-    container: jesserockz/esphome-lint
-    name: Lint Format
-    steps:
-    - uses: actions/checkout@v2
-    - run: script/setup
-    - run: script/clang-format -i
-    - run: script/ci-suggest-changes
-
-  test:
-    runs-on: ubuntu-latest
-    container: jesserockz/esphome-lint
-    strategy:
-      matrix:
-        test:
-        - test1
-        - test2
-        - test3
-        - test4
-    steps:
-    - uses: actions/checkout@v2
-    - run: script/setup
-    - run: esphome tests/${{ matrix.test }}.yaml compile
-
-  deploy-docker:
-    runs-on: ubuntu-latest
-    needs: [lint-custom, lint-python, lint-tidy, lint-format, test]
-    strategy:
-      matrix:
-        arch: [aarch64, amd64, armv7, i386]
-        build-type: [hassio, docker]
-    steps:
-    - uses: actions/checkout@v2
-    - run: docker info
-    - run: docker login -u "${DOCKER_USER}" -p "${DOCKER_PASSWORD}"
-      env:
-        DOCKER_USER: ${{ secrets.DOCKER_USER }}
-        DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}
-    - run: docker run --rm --privileged multiarch/qemu-user-static:5.0.0-2 --reset -p yes
-    - env:
-        BASE_VERSION: 2.1.1
-      run: |
-        if [[ "${{ matrix.build-type }}" == "hassio" ]]; then
-          BUILD_FROM=esphome/esphome-hassio-base-${{ matrix.arch }}:${BASE_VERSION}
-          BUILD_TO=${{ github.repository }}-hassio-${{ matrix.arch }}
-          DOCKERFILE=docker/Dockerfile.hassio
-        else
-          BUILD_FROM=esphome/esphome-base-${{ matrix.arch }}:${BASE_VERSION}
-          BUILD_TO=${{ github.repository }}-${{ matrix.arch }}
-          DOCKERFILE=docker/Dockerfile
-        fi
-
-        TAG=${{ github.sha }}
-        TAG=${TAG:0:7}
-
-        echo "Building tag: ${TAG}"
-
-        docker build \
-          --build-arg BUILD_FROM=${BUILD_FROM} \
-          --build-arg BUILD_VERSION=${TAG} \
-          --tag ${BUILD_TO}:dev \
-          --file ${DOCKERFILE} \
-          .
-
-        echo "Pushing to ${BUILD_TO}:dev"
-        docker push ${BUILD_TO}:dev
-
-  deploy-docker-manifest-version:
-    runs-on: ubuntu-latest
-    needs: [deploy-docker]
-    steps:
-    - run: mkdir -p ~/.docker
-    - run: |
-        echo "{\"experimental\": \"enabled\"}" > ~/.docker/config.json
-    - run: docker login -u "${DOCKER_USER}" -p "${DOCKER_PASSWORD}"
-      env:
-        DOCKER_USER: ${{ secrets.DOCKER_USER }}
-        DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}
-    - run: |
-        REPO=${{ github.repository }}
-
-        docker manifest create ${REPO}:dev \
-          ${REPO}-aarch64:dev \
-          ${REPO}-amd64:dev \
-          ${REPO}-armv7:dev \
-          ${REPO}-i386:dev
-
-        echo "Pushing to ${REPO}:dev"
-        docker manifest push ${REPO}:dev

.github/workflows/release-dev.yml

@@ -0,0 +1,226 @@
+name: Publish dev releases to docker hub
+
+on:
+  push:
+    branches:
+    - dev
+
+jobs:
+  # THE LINT/TEST JOBS ARE COPIED FROM ci.yaml
+ 
+  lint-clang-format:
+    runs-on: ubuntu-latest
+    # cpp lint job runs with esphome-lint docker image so that clang-format-*
+    # doesn't have to be installed
+    container: esphome/esphome-lint:dev
+    steps:
+      - uses: actions/checkout@v2
+      # Cache platformio intermediary files (like libraries etc)
+      # Note: platformio platform versions should be cached via the esphome-lint image
+      - name: Cache Platformio
+        uses: actions/cache@v1
+        with:
+          path: .pio
+          key: lint-cpp-pio-${{ hashFiles('platformio.ini') }}
+          restore-keys: |
+            lint-cpp-pio-
+      # Set up the pio project so that the cpp checks know how files are compiled
+      # (build flags, libraries etc)
+      - name: Set up platformio environment
+        run: pio init --ide atom
+
+      - name: Run clang-format
+        run: script/clang-format -i
+      - name: Suggest changes
+        run: script/ci-suggest-changes
+
+  lint-clang-tidy:
+    runs-on: ubuntu-latest
+    # cpp lint job runs with esphome-lint docker image so that clang-format-*
+    # doesn't have to be installed
+    container: esphome/esphome-lint:dev
+    # Split clang-tidy check into 4 jobs. Each one will check 1/4th of the .cpp files
+    strategy:
+      matrix:
+        split: [1, 2, 3, 4]
+    steps:
+      - uses: actions/checkout@v2
+      # Cache platformio intermediary files (like libraries etc)
+      # Note: platformio platform versions should be cached via the esphome-lint image
+      - name: Cache Platformio
+        uses: actions/cache@v1
+        with:
+          path: .pio
+          key: lint-cpp-pio-${{ hashFiles('platformio.ini') }}
+          restore-keys: |
+            lint-cpp-pio-
+      # Set up the pio project so that the cpp checks know how files are compiled
+      # (build flags, libraries etc)
+      - name: Set up platformio environment
+        run: pio init --ide atom
+
+      
+      - name: Register problem matchers
+        run: |
+          echo "::add-matcher::.github/workflows/matchers/clang-tidy.json"
+          echo "::add-matcher::.github/workflows/matchers/gcc.json"
+      - name: Run clang-tidy
+        run: script/clang-tidy --all-headers --fix --split-num 4 --split-at ${{ matrix.split }}
+      - name: Suggest changes
+        run: script/ci-suggest-changes
+
+  lint-python:
+    # Don't use the esphome-lint docker image because it may contain outdated requirements.
+    # This way, all dependencies are cached via the cache action.
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up Python
+        uses: actions/setup-python@v2
+        with:
+          python-version: '3.7'
+      - name: Cache pip modules
+        uses: actions/cache@v1
+        with:
+          path: ~/.cache/pip
+          key: esphome-pip-3.7-${{ hashFiles('setup.py') }}
+          restore-keys: |
+            esphome-pip-3.7-
+      - name: Set up python environment
+        run: script/setup
+
+      - name: Register problem matchers
+        run: |
+          echo "::add-matcher::.github/workflows/matchers/ci-custom.json"
+          echo "::add-matcher::.github/workflows/matchers/lint-python.json"
+          echo "::add-matcher::.github/workflows/matchers/python.json"
+      - name: Lint Custom
+        run: script/ci-custom.py
+      - name: Lint Python
+        run: script/lint-python
+
+  test:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+          test:
+          - test1
+          - test2
+          - test3
+          - test4
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up Python
+        uses: actions/setup-python@v2
+        with:
+          python-version: '3.7'
+      - name: Cache pip modules
+        uses: actions/cache@v1
+        with:
+          path: ~/.cache/pip
+          key: esphome-pip-3.7-${{ hashFiles('setup.py') }}
+          restore-keys: |
+            esphome-pip-3.7-
+      # Use per test platformio cache because tests have different platform versions
+      - name: Cache ~/.platformio
+        uses: actions/cache@v1
+        with:
+          path: ~/.platformio
+          key: test-home-platformio-${{ matrix.test }}-${{ hashFiles('esphome/core_config.py') }}
+          restore-keys: |
+            test-home-platformio-${{ matrix.test }}-
+      # Cache the intermediary build files
+      - name: Cache Test Build
+        uses: actions/cache@v1
+        with:
+          path: tests/build/${{ matrix.test }}
+          key: test-pio-${{ matrix.test }}-${{ hashFiles('esphome/core_config.py') }}-${{ hashFiles('esphome/**') }}
+          restore-keys: |
+            test-pio-${{ matrix.test }}-${{ hashFiles('esphome/core_config.py') }}-
+            test-pio-${{ matrix.test }}-
+      - name: Set up environment
+        run: script/setup
+
+      
+      - name: Register problem matchers
+        run: |
+          echo "::add-matcher::.github/workflows/matchers/gcc.json"
+          echo "::add-matcher::.github/workflows/matchers/python.json"
+      - run: esphome tests/${{ matrix.test }}.yaml compile
+
+  deploy-docker:
+    name: Build and publish docker containers
+    runs-on: ubuntu-latest
+    needs: [lint-clang-format, lint-clang-tidy, lint-python, test]
+    strategy:
+      matrix:
+        arch: [amd64, i386, armv7, aarch64]
+        build_type: ["hassio", "docker"]
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up env variables
+        run: |
+          tag="dev"
+          base_version="2.1.2"
+
+          if [[ "${{ matrix.build_type }}" == "hassio" ]]; then
+            build_from="esphome/esphome-hassio-base-${{ matrix.arch }}:${base_version}"
+            build_to="esphome/esphome-hassio-${{ matrix.arch }}"
+            dockerfile="docker/Dockerfile.hassio"
+          else
+            build_from="esphome/esphome-base-${{ matrix.arch }}:${base_version}"
+            build_to="esphome/esphome-${{ matrix.arch }}"
+            dockerfile="docker/Dockerfile"
+            if [[ "${{ matrix.arch }}" == "amd64" ]]; then
+              build_to="esphome/esphome"
+            fi
+          fi
+
+          # Set env variables so these values don't need to be calculated again
+          echo "::set-env name=TAG::${tag}"
+          echo "::set-env name=BUILD_FROM::${build_from}"
+          echo "::set-env name=BUILD_TO::${build_to}"
+          echo "::set-env name=IMAGE::${build_to}:${tag}"
+          echo "::set-env name=DOCKERFILE::${dockerfile}"
+      - name: Register QEMU binfmt
+        run: docker run --rm --privileged multiarch/qemu-user-static:5.0.0-2 --reset -p yes
+      - run: |
+          docker build \
+            --build-arg "BUILD_FROM=${BUILD_FROM}" \
+            --build-arg "BUILD_VERSION=${TAG}" \
+            --tag "${IMAGE}" \
+            --file "${DOCKERFILE}" \
+            .
+      - name: Log in to docker hub
+        env:
+          DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }}
+          DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}
+        run: docker login -u "$DOCKER_USER" -p "$DOCKER_PASSWORD"
+      - run: docker push "${IMAGE}"
+
+  deploy-docker-manifest:
+    runs-on: ubuntu-latest
+    needs: [deploy-docker]
+    steps:
+    - name: Enable experimental manifest support
+      run: |
+        mkdir -p ~/.docker
+        echo "{\"experimental\": \"enabled\"}" > ~/.docker/config.json
+    - name: Log in to docker hub
+      env:
+        DOCKER_USER: ${{ secrets.DOCKER_USER }}
+        DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}
+      run: docker login -u "${DOCKER_USER}" -p "${DOCKER_PASSWORD}"
+    - name: "Create the manifest"
+      run: |
+        REPO=esphome/esphome
+        TAG="dev"
+
+        docker manifest create ${REPO}:${TAG} \
+          ${REPO}-aarch64:${TAG} \
+          ${REPO}-amd64:${TAG} \
+          ${REPO}-armv7:${TAG} \
+          ${REPO}-i386:${TAG}
+        echo "::set-env name=TAG::${TAG}"
+        echo "::set-env name=REPO::${REPO}"
+    - run: docker push ${REPO}:${TAG}

.github/workflows/release-version.yaml

@@ -1,180 +0,0 @@
-name: Release a version
-
-on:
-  push:
-    tags:
-    - "*"
-
-jobs:
-  lint-custom:
-    runs-on: ubuntu-latest
-    container: jesserockz/esphome-lint
-    name: Lint Custom
-    steps:
-    - uses: actions/checkout@v2
-    - run: script/setup
-    - run: script/ci-custom.py
-  lint-python:
-    runs-on: ubuntu-latest
-    container: jesserockz/esphome-lint
-    name: Lint Python
-    steps:
-    - uses: actions/checkout@v2
-    - run: script/setup
-    - run: script/lint-python
-  lint-tidy:
-    runs-on: ubuntu-latest
-    container: jesserockz/esphome-lint
-    name: Lint Tidy
-    steps:
-    - uses: actions/checkout@v2
-    - run: script/setup
-    - run: pio init --ide atom
-    - run: script/clang-tidy --all-headers --fix
-    - run: script/ci-suggest-changes
-  lint-format:
-    runs-on: ubuntu-latest
-    container: jesserockz/esphome-lint
-    name: Lint Format
-    steps:
-    - uses: actions/checkout@v2
-    - run: script/setup
-    - run: script/clang-format -i
-    - run: script/ci-suggest-changes
-
-  test:
-    runs-on: ubuntu-latest
-    container: jesserockz/esphome-lint
-    strategy:
-      matrix:
-        test:
-        - test1
-        - test2
-        - test3
-        - test4
-    steps:
-    - uses: actions/checkout@v2
-    - run: script/setup
-    - run: esphome tests/${{ matrix.test }}.yaml compile
-
-  deploy-pypi:
-    runs-on: ubuntu-latest
-    container: jesserockz/esphome-lint
-    needs: [lint-custom, lint-python, lint-tidy, lint-format, test]
-    steps:
-    - run: pip install twine wheel
-    - run: python setup.py sdist bdist_wheel
-    - run: twine upload dist/*
-      env:
-        TWINE_USERNAME: ${{ secrets.TWINE_USERNAME }}
-        TWINE_PASSWORD: ${{ secrets.TWINE_PASSWORD }}
-
-  deploy-docker:
-    runs-on: ubuntu-latest
-    needs: [lint-custom, lint-python, lint-tidy, lint-format, test]
-    strategy:
-      matrix:
-        arch: [aarch64, amd64, armv7, i386]
-        build-type: [hassio, docker]
-    steps:
-    - uses: actions/checkout@v2
-    - run: docker info
-    - run: docker login -u "${DOCKER_USER}" -p "${DOCKER_PASSWORD}"
-      env:
-        DOCKER_USER: ${{ secrets.DOCKER_USER }}
-        DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}
-    - run: docker run --rm --privileged multiarch/qemu-user-static:5.0.0-2 --reset -p yes
-    - env:
-        BASE_VERSION: 2.1.1
-      run: |
-        if [[ "${{ matrix.build-type }}" == "hassio" ]]; then
-          BUILD_FROM=esphome/esphome-hassio-base-${{ matrix.arch }}:${BASE_VERSION}
-          BUILD_TO=${{ github.repository }}-hassio-${{ matrix.arch }}
-          DOCKERFILE=docker/Dockerfile.hassio
-        else
-          BUILD_FROM=esphome/esphome-base-${{ matrix.arch }}:${BASE_VERSION}
-          BUILD_TO=${{ github.repository }}-${{ matrix.arch }}
-          DOCKERFILE=docker/Dockerfile
-        fi
-
-        TAG=${{ github.ref }}
-        TAG=${TAG#refs/tags/v}
-
-        echo "Building tag: ${TAG}"
-
-        docker build \
-          --build-arg BUILD_FROM=${BUILD_FROM} \
-          --build-arg BUILD_VERSION=${TAG} \
-          --tag ${BUILD_TO}:${TAG} \
-          --file ${DOCKERFILE} \
-          .
-
-        echo "Pushing to ${BUILD_TO}:${TAG}"
-        docker push ${BUILD_TO}:${TAG}
-
-        beta_tag="^v\d+\.\d+\.\d+b\d+$"
-        if [[ "${TAG}" ~= "${beta_tag}" ]]; then
-          echo "Pushing to ${BUILD_TO}:beta"
-          docker tag ${BUILD_TO}:${TAG} ${BUILD_TO}:beta
-          docker push ${BUILD_TO}:beta
-        else
-          echo "Pushing to ${BUILD_TO}:latest"
-          docker tag ${BUILD_TO}:${TAG} ${BUILD_TO}:latest
-          docker push ${BUILD_TO}:latest
-        fi
-
-  deploy-docker-manifest-version:
-    runs-on: ubuntu-latest
-    steps:
-    - run: mkdir -p ~/.docker
-    - run: |
-        echo "{\"experimental\": \"enabled\"}" > ~/.docker/config.json
-    - run: docker login -u "${DOCKER_USER}" -p "${DOCKER_PASSWORD}"
-      env:
-        DOCKER_USER: ${{ secrets.DOCKER_USER }}
-        DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}
-    - run: |
-        REPO=${{ github.repository }}
-
-        TAG=${{ github.ref }}
-        TAG=${TAG#refs/tags/v}
-
-        docker manifest create ${REPO}:${TAG} \
-          ${REPO}-aarch64:${TAG} \
-          ${REPO}-amd64:${TAG} \
-          ${REPO}-armv7:${TAG} \
-          ${REPO}-i386:${TAG}
-
-        echo "Pushing to ${REPO}:${TAG}"
-        docker push ${REPO}:${TAG}
-
-  deploy-docker-manifest:
-    runs-on: ubuntu-latest
-    steps:
-    - run: mkdir -p ~/.docker
-    - run: |
-        echo "{\"experimental\": \"enabled\"}" > ~/.docker/config.json
-    - run: docker login -u "${DOCKER_USER}" -p "${DOCKER_PASSWORD}"
-      env:
-        DOCKER_USER: ${{ secrets.DOCKER_USER }}
-        DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}
-    - run: |
-        REPO=${{ github.repository }}
-
-        TAG=${{ github.ref }}
-        TAG=${TAG#refs/tags/v}
-
-        beta_tag="^v\d+\.\d+\.\d+b\d+$"
-        if [[ "${TAG}" ~= "${beta_tag}" ]]; then
-          TAG=beta
-        else
-          TAG=latest
-        fi
-        docker manifest create ${REPO}:${TAG} \
-          ${REPO}-aarch64:${TAG} \
-          ${REPO}-amd64:${TAG} \
-          ${REPO}-armv7:${TAG} \
-          ${REPO}-i386:${TAG}
-
-        echo "Pushing to ${REPO}:${TAG}"
-        docker push ${REPO}:${TAG}

.github/workflows/release.yml

@@ -0,0 +1,276 @@
+name: Publish Release
+
+on:
+  release:
+    types: [published]
+
+jobs:
+  # THE LINT/TEST JOBS ARE COPIED FROM ci.yaml
+ 
+  lint-clang-format:
+    runs-on: ubuntu-latest
+    # cpp lint job runs with esphome-lint docker image so that clang-format-*
+    # doesn't have to be installed
+    container: esphome/esphome-lint:dev
+    steps:
+      - uses: actions/checkout@v2
+      # Cache platformio intermediary files (like libraries etc)
+      # Note: platformio platform versions should be cached via the esphome-lint image
+      - name: Cache Platformio
+        uses: actions/cache@v1
+        with:
+          path: .pio
+          key: lint-cpp-pio-${{ hashFiles('platformio.ini') }}
+          restore-keys: |
+            lint-cpp-pio-
+      # Set up the pio project so that the cpp checks know how files are compiled
+      # (build flags, libraries etc)
+      - name: Set up platformio environment
+        run: pio init --ide atom
+
+      - name: Run clang-format
+        run: script/clang-format -i
+      - name: Suggest changes
+        run: script/ci-suggest-changes
+
+  lint-clang-tidy:
+    runs-on: ubuntu-latest
+    # cpp lint job runs with esphome-lint docker image so that clang-format-*
+    # doesn't have to be installed
+    container: esphome/esphome-lint:dev
+    # Split clang-tidy check into 4 jobs. Each one will check 1/4th of the .cpp files
+    strategy:
+      matrix:
+        split: [1, 2, 3, 4]
+    steps:
+      - uses: actions/checkout@v2
+      # Cache platformio intermediary files (like libraries etc)
+      # Note: platformio platform versions should be cached via the esphome-lint image
+      - name: Cache Platformio
+        uses: actions/cache@v1
+        with:
+          path: .pio
+          key: lint-cpp-pio-${{ hashFiles('platformio.ini') }}
+          restore-keys: |
+            lint-cpp-pio-
+      # Set up the pio project so that the cpp checks know how files are compiled
+      # (build flags, libraries etc)
+      - name: Set up platformio environment
+        run: pio init --ide atom
+
+      
+      - name: Register problem matchers
+        run: |
+          echo "::add-matcher::.github/workflows/matchers/clang-tidy.json"
+          echo "::add-matcher::.github/workflows/matchers/gcc.json"
+      - name: Run clang-tidy
+        run: script/clang-tidy --all-headers --fix --split-num 4 --split-at ${{ matrix.split }}
+      - name: Suggest changes
+        run: script/ci-suggest-changes
+
+  lint-python:
+    # Don't use the esphome-lint docker image because it may contain outdated requirements.
+    # This way, all dependencies are cached via the cache action.
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up Python
+        uses: actions/setup-python@v2
+        with:
+          python-version: '3.7'
+      - name: Cache pip modules
+        uses: actions/cache@v1
+        with:
+          path: ~/.cache/pip
+          key: esphome-pip-3.7-${{ hashFiles('setup.py') }}
+          restore-keys: |
+            esphome-pip-3.7-
+      - name: Set up python environment
+        run: script/setup
+
+      - name: Register problem matchers
+        run: |
+          echo "::add-matcher::.github/workflows/matchers/ci-custom.json"
+          echo "::add-matcher::.github/workflows/matchers/lint-python.json"
+          echo "::add-matcher::.github/workflows/matchers/python.json"
+      - name: Lint Custom
+        run: script/ci-custom.py
+      - name: Lint Python
+        run: script/lint-python
+
+  test:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+          test:
+          - test1
+          - test2
+          - test3
+          - test4
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up Python
+        uses: actions/setup-python@v2
+        with:
+          python-version: '3.7'
+      - name: Cache pip modules
+        uses: actions/cache@v1
+        with:
+          path: ~/.cache/pip
+          key: esphome-pip-3.7-${{ hashFiles('setup.py') }}
+          restore-keys: |
+            esphome-pip-3.7-
+      # Use per test platformio cache because tests have different platform versions
+      - name: Cache ~/.platformio
+        uses: actions/cache@v1
+        with:
+          path: ~/.platformio
+          key: test-home-platformio-${{ matrix.test }}-${{ hashFiles('esphome/core_config.py') }}
+          restore-keys: |
+            test-home-platformio-${{ matrix.test }}-
+      # Cache the intermediary build files
+      - name: Cache Test Build
+        uses: actions/cache@v1
+        with:
+          path: tests/build/${{ matrix.test }}
+          key: test-pio-${{ matrix.test }}-${{ hashFiles('esphome/core_config.py') }}-${{ hashFiles('esphome/**') }}
+          restore-keys: |
+            test-pio-${{ matrix.test }}-${{ hashFiles('esphome/core_config.py') }}-
+            test-pio-${{ matrix.test }}-
+      - name: Set up environment
+        run: script/setup
+
+      
+      - name: Register problem matchers
+        run: |
+          echo "::add-matcher::.github/workflows/matchers/gcc.json"
+          echo "::add-matcher::.github/workflows/matchers/python.json"
+      - run: esphome tests/${{ matrix.test }}.yaml compile
+
+  deploy-pypi:
+    name: Build and publish to PyPi
+    needs: [lint-clang-format, lint-clang-tidy, lint-python, test]
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up Python
+        uses: actions/setup-python@v1
+        with:
+          python-version: '3.x'
+      - name: Set up python environment
+        run: |
+          script/setup
+          pip install setuptools wheel twine
+      - name: Build
+        run: python setup.py sdist bdist_wheel
+      - name: Upload
+        env:
+          TWINE_USERNAME: __token__
+          TWINE_PASSWORD: ${{ secrets.PYPI_TOKEN }}
+        run: twine upload dist/*
+
+  deploy-docker:
+    name: Build and publish docker containers
+    runs-on: ubuntu-latest
+    needs: [lint-clang-format, lint-clang-tidy, lint-python, test]
+    strategy:
+      matrix:
+        arch: [amd64, i386, armv7, aarch64]
+        build_type: ["hassio", "docker"]
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up env variables
+        run: |
+          tag="${GITHUB_REF#v}"
+          base_version="2.1.2"
+
+          if [[ "${{ matrix.build_type }}" == "hassio" ]]; then
+            build_from="esphome/esphome-hassio-base-${{ matrix.arch }}:${base_version}"
+            build_to="esphome/esphome-hassio-${{ matrix.arch }}"
+            dockerfile="docker/Dockerfile.hassio"
+          else
+            build_from="esphome/esphome-base-${{ matrix.arch }}:${base_version}"
+            build_to="esphome/esphome-${{ matrix.arch }}"
+            dockerfile="docker/Dockerfile"
+            if [[ "${{ matrix.arch }}" == "amd64" ]]; then
+              build_to="esphome/esphome"
+            fi
+          fi
+
+          # Set env variables so these values don't need to be calculated again
+          echo "::set-env name=TAG::${tag}"
+          echo "::set-env name=BUILD_FROM::${build_from}"
+          echo "::set-env name=BUILD_TO::${build_to}"
+          echo "::set-env name=IMAGE::${build_to}:${tag}"
+          echo "::set-env name=DOCKERFILE::${dockerfile}"
+      - name: Register QEMU binfmt
+        run: docker run --rm --privileged multiarch/qemu-user-static:5.0.0-2 --reset -p yes
+      - run: |
+          docker build \
+            --build-arg "BUILD_FROM=${BUILD_FROM}" \
+            --build-arg "BUILD_VERSION=${TAG}" \
+            --tag "${IMAGE}" \
+            --file "${DOCKERFILE}" \
+            .
+      - name: Log in to docker hub
+        env:
+          DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }}
+          DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}
+        run: docker login -u "$DOCKER_USER" -p "$DOCKER_PASSWORD"
+      - run: docker push "${IMAGE}"
+      - if: ${{ github.event.release.prerelease) }}
+        name: Publish docker beta tag
+        run: |
+          docker tag "${IMAGE}" ${BUILD_TO}:beta
+          docker push "${BUILD_TO}:beta"
+      - if: ${{ !github.event.release.prerelease) }}
+        name: Publish docker latest tag
+        run: |
+          docker tag "${IMAGE}" ${BUILD_TO}:latest
+          docker push "${BUILD_TO}:latest"
+
+  deploy-docker-manifest:
+    runs-on: ubuntu-latest
+    needs: [deploy-docker]
+    steps:
+    - name: Enable experimental manifest support
+      run: |
+        mkdir -p ~/.docker
+        echo "{\"experimental\": \"enabled\"}" > ~/.docker/config.json
+    - name: Log in to docker hub
+      env:
+        DOCKER_USER: ${{ secrets.DOCKER_USER }}
+        DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}
+      run: docker login -u "${DOCKER_USER}" -p "${DOCKER_PASSWORD}"
+    - name: "Create the manifest"
+      run: |
+        REPO=esphome/esphome
+        TAG="${GITHUB_REF#v}"
+
+        docker manifest create ${REPO}:${TAG} \
+          ${REPO}-aarch64:${TAG} \
+          ${REPO}-amd64:${TAG} \
+          ${REPO}-armv7:${TAG} \
+          ${REPO}-i386:${TAG}
+        echo "::set-env name=TAG::${TAG}"
+        echo "::set-env name=REPO::${REPO}"
+    - run: docker push ${REPO}:${TAG}
+
+    - name: Publish docker beta tag
+      if: ${{ github.event.release.prerelease) }}
+      run: |
+        docker manifest create ${REPO}:beta \
+          ${REPO}-aarch64:beta \
+          ${REPO}-amd64:beta \
+          ${REPO}-armv7:beta \
+          ${REPO}-i386:beta
+        docker push ${REPO}:beta
+    - name: Publish docker latest tag
+      if: ${{ !github.event.release.prerelease) }}
+      run: |
+        docker manifest create ${REPO}:latest \
+          ${REPO}-aarch64:latest \
+          ${REPO}-amd64:latest \
+          ${REPO}-armv7:latest \
+          ${REPO}-i386:latest
+        docker push ${REPO}:latest

docker/Dockerfile.lint

@@ -1,4 +1,4 @@
-FROM esphome/esphome-base-amd64:2.1.1
+FROM esphome/esphome-base-amd64:2.1.2
 
 RUN \
     apt-get update \
@@ -7,6 +7,8 @@ RUN \
         clang-tidy-7 \
         patch \
         software-properties-common \
+    # Update to latest git version because of github actions
+    # https://github.com/actions/checkout/issues/126
     && apt-add-repository ppa:git-core/ppa \
     && apt-get install -y --no-install-recommends \
         git \
@@ -16,7 +18,7 @@ RUN \
         /var/lib/apt/lists/*
 
 COPY requirements_test.txt /requirements_test.txt
-RUN pip3 install --no-cache-dir wheel && pip3 install --no-cache-dir -r /requirements_test.txt
+RUN pip3 install --no-cache-dir -r /requirements_test.txt
 
 VOLUME ["/esphome"]
 WORKDIR /esphome

script/ci-custom.py

@@ -101,10 +101,12 @@ def lint_re_check(regex, **kwargs):
                 if 'NOLINT' in match.group(0):
                     continue
                 lineno = content.count("\n", 0, match.start()) + 1
+                substr = content[:match.start()]
+                col = len(substr) - substr.rfind('\n')
                 err = func(fname, match)
                 if err is None:
                     continue
-                errors.append(f"{err} See line {lineno}.")
+                errors.append((lineno, col+1, err))
             return errors
         return decor(new_func)
     return decorator
@@ -121,8 +123,7 @@ def lint_content_find_check(find, **kwargs):
             errors = []
             for line, col in find_all(content, find_):
                 err = func(fname)
-                errors.append("{err} See line {line}:{col}."
+                errors.append((line+1, col+1, err))
-                              "".format(err=err, line=line+1, col=col+1))
             return errors
         return decor(new_func)
     return decorator
@@ -215,9 +216,10 @@ def lint_const_ordered(fname, content):
                 continue
             target = next(i for i, l in ordered if l == ml)
             target_text = next(l for i, l in matching if target == i)
-            errors.append("Constant {} is not ordered, please make sure all constants are ordered. "
+            errors.append((ml, None,
-                          "See line {} (should go to line {}, {})"
+                           "Constant {} is not ordered, please make sure all constants are ordered. "
-                          "".format(highlight(ml), mi, target, target_text))
+                           "See line {} (should go to line {}, {})"
+                           "".format(highlight(ml), mi, target, target_text)))
     return errors
 
 
@@ -354,13 +356,22 @@ errors = collections.defaultdict(list)
 def add_errors(fname, errs):
     if not isinstance(errs, list):
         errs = [errs]
-    errs = [x for x in errs if x is not None]
     for err in errs:
+        if err is None:
+            continue
+        try:
+            lineno, col, msg = err
+        except ValueError:
+            lineno = 1
+            col = 1
+            msg = err
         if not isinstance(err, str):
             raise ValueError("Error is not instance of string!")
-    if not errs:
+        if not isinstance(lineno, int):
-        return
+            raise ValueError("Line number is not an int!")
-    errors[fname].extend(errs)
+        if not isinstance(col, int):
+            raise ValueError("Column number is not an int!")
+        errors[fname].append((lineno, col, msg))
 
 
 for fname in files:
@@ -380,8 +391,8 @@ run_checks(LINT_POST_CHECKS, 'POST')
 
 for f, errs in sorted(errors.items()):
     print(f"\033[0;32m************* File \033[1;32m{f}\033[0m")
-    for err in errs:
+    for lineno, col, msg in errs:
-        print(err)
+        print(f"ERROR {f}:{lineno}:{col} - {msg}")
     print()
 
 sys.exit(len(errors))

script/clang-tidy

@@ -2,20 +2,19 @@
 
 from __future__ import print_function
 
+import argparse
 import multiprocessing
 import os
 import re
-
-import pexpect
 import shutil
 import subprocess
 import sys
 import tempfile
-
-import argparse
-import click
 import threading
 
+import click
+import pexpect
+
 sys.path.append(os.path.dirname(__file__))
 from helpers import basepath, shlex_quote, get_output, build_compile_commands, \
     build_all_include, temp_header_file, git_ls_files, filter_changed
@@ -49,7 +48,7 @@ def run_tidy(args, tmpdir, queue, lock, failed_files):
 
         # Use pexpect for a pseudy-TTY with colored output
         output, rc = pexpect.run(invocation_s, withexitstatus=True, encoding='utf-8',
-                                 timeout=15*60)
+                                 timeout=15 * 60)
         with lock:
             if rc != 0:
                 print()
@@ -65,6 +64,11 @@ def progress_bar_show(value):
         return ''
 
 
+def split_list(a, n):
+    k, m = divmod(len(a), n)
+    return [a[i * k + min(i, m):(i + 1) * k + min(i + 1, m)] for i in range(n)]
+
+
 def main():
     parser = argparse.ArgumentParser()
     parser.add_argument('-j', '--jobs', type=int,
@@ -77,6 +81,10 @@ def main():
                         help='Run clang-tidy in quiet mode')
     parser.add_argument('-c', '--changed', action='store_true',
                         help='Only run on changed files')
+    parser.add_argument('--split-num', type=int, help='Split the files into X jobs.',
+                        default=None)
+    parser.add_argument('--split-at', type=int, help='Which split is this? Starts at 1',
+                        default=None)
     parser.add_argument('--all-headers', action='store_true',
                         help='Create a dummy file that checks all headers')
     args = parser.parse_args()
@@ -114,7 +122,10 @@ def main():
 
     files.sort()
 
-    if args.all_headers:
+    if args.split_num:
+        files = split_list(files, args.split_num)[args.split_at - 1]
+
+    if args.all_headers and args.split_at in (None, 1):
         files.insert(0, temp_header_file)
 
     tmpdir = None
@@ -157,8 +168,8 @@ def main():
             print('Error applying fixes.\n', file=sys.stderr)
             raise
 
-    sys.exit(return_code)
+    return return_code
 
 
 if __name__ == '__main__':
-    main()
+    sys.exit(main())

script/helpers.py

@@ -101,8 +101,10 @@ def splitlines_no_ends(string):
 
 
 def changed_files():
-    for remote in ('upstream', 'origin'):
+    check_remotes = ['upstream', 'origin']
-        command = ['git', 'merge-base', f'{remote}/dev', 'HEAD']
+    check_remotes.extend(splitlines_no_ends(get_output('git', 'remote')))
+    for remote in check_remotes:
+        command = ['git', 'merge-base', f'refs/remotes/{remote}/dev', 'HEAD']
         try:
             merge_base = splitlines_no_ends(get_output(*command))[0]
             break