sascha/esphome
1
0
Fork 0
mirror of https://github.com/esphome/esphome.git synced 2025-10-31 07:03:55 +00:00
Code Issues Releases Wiki Activity

Refactor docker build system and workflows (#2023)

Browse Source
This commit is contained in:
Otto Winter
2021-07-15 21:30:04 +02:00
committed by Jesse Hills
parent 3715ba030b
commit 4f9a56c884
13 changed files with 456 additions and 708 deletions
Download Patch File Download Diff File Expand all files Collapse all files

41
.github/workflows/ci-docker.yml vendored
View File

@@ -18,38 +18,23 @@ jobs:
name: Build docker containers
runs-on: ubuntu-latest
strategy:
fail-fast: false
matrix:
arch: [amd64, armv7, aarch64]
build_type: ["hassio", "docker"]
build_type: ["ha-addon", "docker", "lint"]
steps:
- uses: actions/checkout@v2
- name: Set up env variables
- name: Set up Python
uses: actions/setup-python@v2
with:
python-version: '3.9'
- name: Set TAG
run: |
base_version="3.4.0"
echo "TAG=check" >> $GITHUB_ENV
if [[ "${{ matrix.build_type }}" == "hassio" ]]; then
build_from="esphome/esphome-hassio-base-${{ matrix.arch }}:${base_version}"
build_to="esphome/esphome-hassio-${{ matrix.arch }}"
dockerfile="docker/Dockerfile.hassio"
else
build_from="esphome/esphome-base-${{ matrix.arch }}:${base_version}"
build_to="esphome/esphome-${{ matrix.arch }}"
dockerfile="docker/Dockerfile"
fi
echo "BUILD_FROM=${build_from}" >> $GITHUB_ENV
echo "BUILD_TO=${build_to}" >> $GITHUB_ENV
echo "DOCKERFILE=${dockerfile}" >> $GITHUB_ENV
- name: Pull for cache
- name: Run build
run: |
docker pull "${BUILD_TO}:dev" || true
- name: Register QEMU binfmt
run: docker run --rm --privileged multiarch/qemu-user-static:5.2.0-2 --reset -p yes
- run: |
docker build \
--build-arg "BUILD_FROM=${BUILD_FROM}" \
--build-arg "BUILD_VERSION=ci" \
--cache-from "${BUILD_TO}:dev" \
--file "${DOCKERFILE}" \
.
docker/build.py \
--tag "${TAG}" \
--arch "${{ matrix.arch }}" \
--build-type "${{ matrix.build_type }}" \
build

178
.github/workflows/ci.yml vendored
View File

@@ -4,40 +4,36 @@ name: CI
on:
push:
# On dev branch release-dev already performs CI checks
# On other branches the `pull_request` trigger will be used
branches: [beta, release]
branches: [dev, beta, release]
pull_request:
jobs:
lint-clang-format:
ci-with-container:
name: ${{ matrix.name }}
runs-on: ubuntu-latest
# cpp lint job runs with esphome-lint docker image so that clang-format-*
# doesn't have to be installed
container: esphome/esphome-lint:1.1
steps:
- uses: actions/checkout@v2
# Set up the pio project so that the cpp checks know how files are compiled
# (build flags, libraries etc)
- name: Set up platformio environment
run: pio init --ide atom
- name: Run clang-format
run: script/clang-format -i
- name: Suggest changes
run: script/ci-suggest-changes
lint-clang-tidy:
runs-on: ubuntu-latest
# cpp lint job runs with esphome-lint docker image so that clang-format-*
# doesn't have to be installed
container: esphome/esphome-lint:1.1
# Split clang-tidy check into 4 jobs. Each one will check 1/4th of the .cpp files
strategy:
fail-fast: false
matrix:
split: [1, 2, 3, 4]
include:
- id: clang-format
name: Run script/clang-format
- id: clang-tidy
name: Run script/clang-tidy 1/4
split: 1
- id: clang-tidy
name: Run script/clang-tidy 2/4
split: 2
- id: clang-tidy
name: Run script/clang-tidy 3/4
split: 3
- id: clang-tidy
name: Run script/clang-tidy 4/4
split: 4
# cpp lint job runs with esphome-lint docker image so that clang-format-*
# doesn't have to be installed
container: esphome/esphome-lint:1.1
steps:
- uses: actions/checkout@v2
# Set up the pio project so that the cpp checks know how files are compiled
@@ -45,26 +41,57 @@ jobs:
- name: Set up platformio environment
run: pio init --ide atom
- name: Register problem matchers
run: |
echo "::add-matcher::.github/workflows/matchers/clang-tidy.json"
echo "::add-matcher::.github/workflows/matchers/gcc.json"
- name: Run clang-format
run: script/clang-format -i
if: ${{ matrix.id == 'clang-format' }}
- name: Run clang-tidy
run: script/clang-tidy --all-headers --fix --split-num 4 --split-at ${{ matrix.split }}
if: ${{ matrix.id == 'clang-tidy' }}
- name: Suggest changes
run: script/ci-suggest-changes
lint-python:
ci:
# Don't use the esphome-lint docker image because it may contain outdated requirements.
# This way, all dependencies are cached via the cache action.
name: ${{ matrix.name }}
runs-on: ubuntu-latest
strategy:
fail-fast: false
matrix:
include:
- id: ci-custom
name: Run script/ci-custom
- id: lint-python
name: Run script/lint-python
- id: test
file: tests/test1.yaml
name: Test tests/test1.yaml
- id: test
file: tests/test2.yaml
name: Test tests/test2.yaml
- id: test
file: tests/test3.yaml
name: Test tests/test3.yaml
- id: test
file: tests/test4.yaml
name: Test tests/test4.yaml
- id: pytest
name: Run pytest
steps:
- uses: actions/checkout@v2
- name: Set up Python
uses: actions/setup-python@v2
with:
python-version: '3.7'
- name: Cache pip modules
uses: actions/cache@v1
with:
@@ -72,6 +99,17 @@ jobs:
key: esphome-pip-3.7-${{ hashFiles('setup.py') }}
restore-keys: |
esphome-pip-3.7-
# Use per test platformio cache because tests have different platform versions
- name: Cache ~/.platformio
uses: actions/cache@v1
with:
path: ~/.platformio
key: test-home-platformio-${{ matrix.file }}-${{ hashFiles('esphome/core/config.py') }}
restore-keys: |
test-home-platformio-${{ matrix.file }}-
if: ${{ matrix.id == 'test' }}
- name: Set up python environment
run: script/setup
@@ -80,82 +118,22 @@ jobs:
echo "::add-matcher::.github/workflows/matchers/ci-custom.json"
echo "::add-matcher::.github/workflows/matchers/lint-python.json"
echo "::add-matcher::.github/workflows/matchers/python.json"
echo "::add-matcher::.github/workflows/matchers/pytest.json"
echo "::add-matcher::.github/workflows/matchers/gcc.json"
- name: Lint Custom
run: script/ci-custom.py
run: |
script/ci-custom.py
script/build_codeowners.py --check
if: ${{ matrix.id == 'ci-custom' }}
- name: Lint Python
run: script/lint-python
- name: Lint CODEOWNERS
run: script/build_codeowners.py --check
if: ${{ matrix.id == 'lint-python' }}
test:
runs-on: ubuntu-latest
strategy:
fail-fast: false
matrix:
test:
- test1
- test2
- test3
- test4
- test5
steps:
- uses: actions/checkout@v2
- name: Set up Python
uses: actions/setup-python@v2
with:
python-version: '3.7'
- name: Cache pip modules
uses: actions/cache@v1
with:
path: ~/.cache/pip
key: esphome-pip-3.7-${{ hashFiles('setup.py') }}
restore-keys: |
esphome-pip-3.7-
# Use per test platformio cache because tests have different platform versions
- name: Cache ~/.platformio
uses: actions/cache@v1
with:
path: ~/.platformio
key: test-home-platformio-${{ matrix.test }}-${{ hashFiles('esphome/core/config.py') }}
restore-keys: |
test-home-platformio-${{ matrix.test }}-
- name: Set up environment
run: script/setup
- run: esphome compile ${{ matrix.file }}
if: ${{ matrix.id == 'test' }}
- name: Register problem matchers
run: |
echo "::add-matcher::.github/workflows/matchers/gcc.json"
echo "::add-matcher::.github/workflows/matchers/python.json"
- run: esphome compile tests/${{ matrix.test }}.yaml
pytest:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- name: Set up Python
uses: actions/setup-python@v2
with:
python-version: '3.7'
- name: Cache pip modules
uses: actions/cache@v1
with:
path: ~/.cache/pip
key: esphome-pip-3.7-${{ hashFiles('setup.py') }}
restore-keys: |
esphome-pip-3.7-
- name: Set up environment
run: script/setup
- name: Install Github Actions annotator
run: pip install pytest-github-actions-annotate-failures
- name: Register problem matchers
run: |
echo "::add-matcher::.github/workflows/matchers/python.json"
- name: Run pytest
run: |
pytest \
-qq \
--durations=10 \
-o console_output_style=count \
tests
pytest -vv --tb=native tests
if: ${{ matrix.id == 'pytest' }}

96
.github/workflows/docker-lint-build.yml vendored
View File

@@ -13,30 +13,88 @@ on:
- '.github/workflows/docker-lint-build.yml'
jobs:
publish-docker-lint-iage:
name: Build docker containers
deploy-docker:
name: Build and publish docker containers
if: github.repository == 'esphome/esphome'
runs-on: ubuntu-latest
strategy:
matrix:
arch: [amd64, armv7, aarch64]
build_type: ["lint"]
steps:
- uses: actions/checkout@v2
- name: Set up Python
uses: actions/setup-python@v2
with:
python-version: '3.9'
- name: Set TAG
run: |
echo "TAG=1.1" >> $GITHUB_ENV
- name: Pull for cache
- name: Run build
run: |
docker pull "esphome/esphome-lint:latest" || true
- name: Build
run: |
docker build \
--cache-from "esphome/esphome-lint:latest" \
--file "docker/Dockerfile.lint" \
--tag "esphome/esphome-lint:latest" \
--tag "esphome/esphome-lint:${TAG}" \
.
docker/build.py \
--tag "${TAG}" \
--arch "${{ matrix.arch }}" \
--build-type "${{ matrix.build_type }}" \
build
- name: Log in to docker hub
env:
DOCKER_USER: ${{ secrets.DOCKER_USER }}
DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}
run: docker login -u "${DOCKER_USER}" -p "${DOCKER_PASSWORD}"
- run: |
docker push "esphome/esphome-lint:${TAG}"
docker push "esphome/esphome-lint:latest"
uses: docker/login-action@v1
with:
username: ${{ secrets.DOCKER_USER }}
password: ${{ secrets.DOCKER_PASSWORD }}
- name: Log in to the GitHub container registry
uses: docker/login-action@v1
with:
registry: ghcr.io
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Run push
run: |
docker/build.py \
--tag "${TAG}" \
--arch "${{ matrix.arch }}" \
--build-type "${{ matrix.build_type }}" \
push
deploy-docker-manifest:
if: github.repository == 'esphome/esphome'
runs-on: ubuntu-latest
needs: [deploy-docker]
strategy:
matrix:
build_type: ["lint"]
steps:
- uses: actions/checkout@v2
- name: Set up Python
uses: actions/setup-python@v2
with:
python-version: '3.9'
- name: Set TAG
run: |
echo "TAG=1.1" >> $GITHUB_ENV
- name: Enable experimental manifest support
run: |
mkdir -p ~/.docker
echo "{\"experimental\": \"enabled\"}" > ~/.docker/config.json
- name: Log in to docker hub
uses: docker/login-action@v1
with:
username: ${{ secrets.DOCKER_USER }}
password: ${{ secrets.DOCKER_PASSWORD }}
- name: Log in to the GitHub container registry
uses: docker/login-action@v1
with:
registry: ghcr.io
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Run manifest
run: |
docker/build.py \
--tag "${TAG}" \
--build-type "${{ matrix.build_type }}" \
manifest

19
.github/workflows/matchers/pytest.json vendored Normal file
View File

@@ -0,0 +1,19 @@
{
"problemMatcher": [
{
"owner": "pytest",
"fileLocation": "absolute",
"pattern": [
{
"regexp": "^\\s+File \"(.*)\", line (\\d+), in (.*)$",
"file": 1,
"line": 2
},
{
"regexp": "^\\s+(.*)$",
"message": 1
}
]
}
]
}

247
.github/workflows/release-dev.yml vendored
View File

@@ -1,247 +0,0 @@
name: Publish dev releases to docker hub
on:
push:
branches:
- dev
jobs:
# THE LINT/TEST JOBS ARE COPIED FROM ci.yaml
lint-clang-format:
runs-on: ubuntu-latest
# cpp lint job runs with esphome-lint docker image so that clang-format-*
# doesn't have to be installed
container: esphome/esphome-lint:1.1
steps:
- uses: actions/checkout@v2
# Set up the pio project so that the cpp checks know how files are compiled
# (build flags, libraries etc)
- name: Set up platformio environment
run: pio init --ide atom
- name: Run clang-format
run: script/clang-format -i
- name: Suggest changes
run: script/ci-suggest-changes
lint-clang-tidy:
runs-on: ubuntu-latest
# cpp lint job runs with esphome-lint docker image so that clang-format-*
# doesn't have to be installed
container: esphome/esphome-lint:1.1
# Split clang-tidy check into 4 jobs. Each one will check 1/4th of the .cpp files
strategy:
fail-fast: false
matrix:
split: [1, 2, 3, 4]
steps:
- uses: actions/checkout@v2
# Set up the pio project so that the cpp checks know how files are compiled
# (build flags, libraries etc)
- name: Set up platformio environment
run: pio init --ide atom
- name: Register problem matchers
run: |
echo "::add-matcher::.github/workflows/matchers/clang-tidy.json"
echo "::add-matcher::.github/workflows/matchers/gcc.json"
- name: Run clang-tidy
run: script/clang-tidy --all-headers --fix --split-num 4 --split-at ${{ matrix.split }}
- name: Suggest changes
run: script/ci-suggest-changes
lint-python:
# Don't use the esphome-lint docker image because it may contain outdated requirements.
# This way, all dependencies are cached via the cache action.
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- name: Set up Python
uses: actions/setup-python@v2
with:
python-version: '3.7'
- name: Cache pip modules
uses: actions/cache@v1
with:
path: ~/.cache/pip
key: esphome-pip-3.7-${{ hashFiles('setup.py') }}
restore-keys: |
esphome-pip-3.7-
- name: Set up python environment
run: script/setup
- name: Register problem matchers
run: |
echo "::add-matcher::.github/workflows/matchers/ci-custom.json"
echo "::add-matcher::.github/workflows/matchers/lint-python.json"
echo "::add-matcher::.github/workflows/matchers/python.json"
- name: Lint Custom
run: script/ci-custom.py
- name: Lint Python
run: script/lint-python
- name: Lint CODEOWNERS
run: script/build_codeowners.py --check
test:
runs-on: ubuntu-latest
strategy:
fail-fast: false
matrix:
test:
- test1
- test2
- test3
- test4
- test5
steps:
- uses: actions/checkout@v2
- name: Set up Python
uses: actions/setup-python@v2
with:
python-version: '3.7'
- name: Cache pip modules
uses: actions/cache@v1
with:
path: ~/.cache/pip
key: esphome-pip-3.7-${{ hashFiles('setup.py') }}
restore-keys: |
esphome-pip-3.7-
# Use per test platformio cache because tests have different platform versions
- name: Cache ~/.platformio
uses: actions/cache@v1
with:
path: ~/.platformio
key: test-home-platformio-${{ matrix.test }}-${{ hashFiles('esphome/core/config.py') }}
restore-keys: |
test-home-platformio-${{ matrix.test }}-
- name: Set up environment
run: script/setup
- name: Register problem matchers
run: |
echo "::add-matcher::.github/workflows/matchers/gcc.json"
echo "::add-matcher::.github/workflows/matchers/python.json"
- run: esphome compile tests/${{ matrix.test }}.yaml
pytest:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- name: Set up Python
uses: actions/setup-python@v2
with:
python-version: '3.7'
- name: Cache pip modules
uses: actions/cache@v1
with:
path: ~/.cache/pip
key: esphome-pip-3.7-${{ hashFiles('setup.py') }}
restore-keys: |
esphome-pip-3.7-
- name: Set up environment
run: script/setup
- name: Install Github Actions annotator
run: pip install pytest-github-actions-annotate-failures
- name: Register problem matchers
run: |
echo "::add-matcher::.github/workflows/matchers/python.json"
- name: Run pytest
run: |
pytest \
-qq \
--durations=10 \
-o console_output_style=count \
tests
deploy-docker:
name: Build and publish docker containers
if: github.repository == 'esphome/esphome'
runs-on: ubuntu-latest
needs: [lint-clang-format, lint-clang-tidy, lint-python, test, pytest]
strategy:
matrix:
arch: [amd64, armv7, aarch64]
# Hassio dev image doesn't use esphome/esphome-hassio-$arch and uses base directly
build_type: ["docker"]
steps:
- uses: actions/checkout@v2
- name: Set TAG
run: |
TAG="${GITHUB_SHA:0:7}"
echo "TAG=${TAG}" >> $GITHUB_ENV
- name: Set up env variables
run: |
base_version="3.4.0"
if [[ "${{ matrix.build_type }}" == "hassio" ]]; then
build_from="esphome/esphome-hassio-base-${{ matrix.arch }}:${base_version}"
build_to="esphome/esphome-hassio-${{ matrix.arch }}"
dockerfile="docker/Dockerfile.hassio"
else
build_from="esphome/esphome-base-${{ matrix.arch }}:${base_version}"
build_to="esphome/esphome-${{ matrix.arch }}"
dockerfile="docker/Dockerfile"
fi
echo "BUILD_FROM=${build_from}" >> $GITHUB_ENV
echo "BUILD_TO=${build_to}" >> $GITHUB_ENV
echo "DOCKERFILE=${dockerfile}" >> $GITHUB_ENV
- name: Pull for cache
run: |
docker pull "${BUILD_TO}:dev" || true
- name: Register QEMU binfmt
run: docker run --rm --privileged multiarch/qemu-user-static:5.2.0-2 --reset -p yes
- run: |
docker build \
--build-arg "BUILD_FROM=${BUILD_FROM}" \
--build-arg "BUILD_VERSION=${TAG}" \
--tag "${BUILD_TO}:${TAG}" \
--tag "${BUILD_TO}:dev" \
--cache-from "${BUILD_TO}:dev" \
--file "${DOCKERFILE}" \
.
- name: Log in to docker hub
env:
DOCKER_USER: ${{ secrets.DOCKER_USER }}
DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}
run: docker login -u "${DOCKER_USER}" -p "${DOCKER_PASSWORD}"
- run: |
docker push "${BUILD_TO}:${TAG}"
docker push "${BUILD_TO}:dev"
deploy-docker-manifest:
if: github.repository == 'esphome/esphome'
runs-on: ubuntu-latest
needs: [deploy-docker]
steps:
- name: Enable experimental manifest support
run: |
mkdir -p ~/.docker
echo "{\"experimental\": \"enabled\"}" > ~/.docker/config.json
- name: Set TAG
run: |
TAG="${GITHUB_SHA:0:7}"
echo "TAG=${TAG}" >> $GITHUB_ENV
- name: Log in to docker hub
env:
DOCKER_USER: ${{ secrets.DOCKER_USER }}
DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}
run: docker login -u "${DOCKER_USER}" -p "${DOCKER_PASSWORD}"
- name: "Create the manifest"
run: |
docker manifest create esphome/esphome:${TAG} \
esphome/esphome-aarch64:${TAG} \
esphome/esphome-amd64:${TAG} \
esphome/esphome-armv7:${TAG}
docker manifest push esphome/esphome:${TAG}
docker manifest create esphome/esphome:dev \
esphome/esphome-aarch64:${TAG} \
esphome/esphome-amd64:${TAG} \
esphome/esphome-armv7:${TAG}
docker manifest push esphome/esphome:dev

309
.github/workflows/release.yml vendored
View File

@@ -1,164 +1,35 @@
name: Publish Release
on:
workflow_dispatch:
release:
types: [published]
schedule:
- cron: "0 2 * * *"
jobs:
# THE LINT/TEST JOBS ARE COPIED FROM ci.yaml
lint-clang-format:
init:
name: Initialize build
runs-on: ubuntu-latest
# cpp lint job runs with esphome-lint docker image so that clang-format-*
# doesn't have to be installed
container: esphome/esphome-lint:1.1
outputs:
tag: ${{ steps.tag.outputs.tag }}
steps:
- uses: actions/checkout@v2
# Set up the pio project so that the cpp checks know how files are compiled
# (build flags, libraries etc)
- name: Set up platformio environment
run: pio init --ide atom
- name: Run clang-format
run: script/clang-format -i
- name: Suggest changes
run: script/ci-suggest-changes
lint-clang-tidy:
runs-on: ubuntu-latest
# cpp lint job runs with esphome-lint docker image so that clang-format-*
# doesn't have to be installed
container: esphome/esphome-lint:1.1
# Split clang-tidy check into 4 jobs. Each one will check 1/4th of the .cpp files
strategy:
fail-fast: false
matrix:
split: [1, 2, 3, 4]
steps:
- uses: actions/checkout@v2
# Set up the pio project so that the cpp checks know how files are compiled
# (build flags, libraries etc)
- name: Set up platformio environment
run: pio init --ide atom
- name: Register problem matchers
- name: Get tag
id: tag
run: |
echo "::add-matcher::.github/workflows/matchers/clang-tidy.json"
echo "::add-matcher::.github/workflows/matchers/gcc.json"
- name: Run clang-tidy
run: script/clang-tidy --all-headers --fix --split-num 4 --split-at ${{ matrix.split }}
- name: Suggest changes
run: script/ci-suggest-changes
lint-python:
# Don't use the esphome-lint docker image because it may contain outdated requirements.
# This way, all dependencies are cached via the cache action.
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- name: Set up Python
uses: actions/setup-python@v2
with:
python-version: '3.7'
- name: Cache pip modules
uses: actions/cache@v1
with:
path: ~/.cache/pip
key: esphome-pip-3.7-${{ hashFiles('setup.py') }}
restore-keys: |
esphome-pip-3.7-
- name: Set up python environment
run: script/setup
- name: Register problem matchers
run: |
echo "::add-matcher::.github/workflows/matchers/ci-custom.json"
echo "::add-matcher::.github/workflows/matchers/lint-python.json"
echo "::add-matcher::.github/workflows/matchers/python.json"
- name: Lint Custom
run: script/ci-custom.py
- name: Lint Python
run: script/lint-python
- name: Lint CODEOWNERS
run: script/build_codeowners.py --check
test:
runs-on: ubuntu-latest
strategy:
fail-fast: false
matrix:
test:
- test1
- test2
- test3
- test4
- test5
steps:
- uses: actions/checkout@v2
- name: Set up Python
uses: actions/setup-python@v2
with:
python-version: '3.7'
- name: Cache pip modules
uses: actions/cache@v1
with:
path: ~/.cache/pip
key: esphome-pip-3.7-${{ hashFiles('setup.py') }}
restore-keys: |
esphome-pip-3.7-
# Use per test platformio cache because tests have different platform versions
- name: Cache ~/.platformio
uses: actions/cache@v1
with:
path: ~/.platformio
key: test-home-platformio-${{ matrix.test }}-${{ hashFiles('esphome/core/config.py') }}
restore-keys: |
test-home-platformio-${{ matrix.test }}-
- name: Set up environment
run: script/setup
- name: Register problem matchers
run: |
echo "::add-matcher::.github/workflows/matchers/gcc.json"
echo "::add-matcher::.github/workflows/matchers/python.json"
- run: esphome compile tests/${{ matrix.test }}.yaml
pytest:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- name: Set up Python
uses: actions/setup-python@v2
with:
python-version: '3.7'
- name: Cache pip modules
uses: actions/cache@v1
with:
path: ~/.cache/pip
key: esphome-pip-3.7-${{ hashFiles('setup.py') }}
restore-keys: |
esphome-pip-3.7-
- name: Set up environment
run: script/setup
- name: Install Github Actions annotator
run: pip install pytest-github-actions-annotate-failures
- name: Register problem matchers
run: |
echo "::add-matcher::.github/workflows/matchers/python.json"
- name: Run pytest
run: |
pytest \
-qq \
--durations=10 \
-o console_output_style=count \
tests
if [[ "$GITHUB_EVENT_NAME" = "release" ]]; then
TAG="${GITHUB_REF#refs/tags/v}"
else
TAG=$(cat esphome/const.py | sed -n -E "s/^__version__\s+=\s+\"(.+)\"$/\1/p")
today="$(date --utc '+%Y%m%d')"
TAG="${TAG}${today}"
fi
echo "::set-output name=tag::${TAG}"
deploy-pypi:
name: Build and publish to PyPi
if: github.repository == 'esphome/esphome'
needs: [lint-clang-format, lint-clang-tidy, lint-python, test, pytest]
if: github.repository == 'esphome/esphome' && github.event_name == 'release'
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
@@ -182,119 +53,85 @@ jobs:
name: Build and publish docker containers
if: github.repository == 'esphome/esphome'
runs-on: ubuntu-latest
needs: [lint-clang-format, lint-clang-tidy, lint-python, test, pytest]
needs: [init]
strategy:
matrix:
arch: [amd64, armv7, aarch64]
build_type: ["hassio", "docker"]
build_type: ["ha-addon", "docker"]
steps:
- uses: actions/checkout@v2
- name: Set TAG
run: |
TAG="${GITHUB_REF#refs/tags/v}"
echo "TAG=${TAG}" >> $GITHUB_ENV
- name: Set up env variables
run: |
base_version="3.4.0"
- name: Set up Python
uses: actions/setup-python@v2
with:
python-version: '3.9'
if [[ "${{ matrix.build_type }}" == "hassio" ]]; then
build_from="esphome/esphome-hassio-base-${{ matrix.arch }}:${base_version}"
build_to="esphome/esphome-hassio-${{ matrix.arch }}"
dockerfile="docker/Dockerfile.hassio"
else
build_from="esphome/esphome-base-${{ matrix.arch }}:${base_version}"
build_to="esphome/esphome-${{ matrix.arch }}"
dockerfile="docker/Dockerfile"
fi
if [[ "${{ github.event.release.prerelease }}" == "true" ]]; then
cache_tag="beta"
else
cache_tag="latest"
fi
# Set env variables so these values don't need to be calculated again
echo "BUILD_FROM=${build_from}" >> $GITHUB_ENV
echo "BUILD_TO=${build_to}" >> $GITHUB_ENV
echo "DOCKERFILE=${dockerfile}" >> $GITHUB_ENV
echo "CACHE_TAG=${cache_tag}" >> $GITHUB_ENV
- name: Pull for cache
- name: Run build
run: |
docker pull "${BUILD_TO}:${CACHE_TAG}" || true
- name: Register QEMU binfmt
run: docker run --rm --privileged multiarch/qemu-user-static:5.2.0-2 --reset -p yes
- run: |
docker build \
--build-arg "BUILD_FROM=${BUILD_FROM}" \
--build-arg "BUILD_VERSION=${TAG}" \
--tag "${BUILD_TO}:${TAG}" \
--cache-from "${BUILD_TO}:${CACHE_TAG}" \
--file "${DOCKERFILE}" \
.
docker/build.py \
--tag "${{ needs.init.outputs.tag }}" \
--arch "${{ matrix.arch }}" \
--build-type "${{ matrix.build_type }}" \
build
- name: Log in to docker hub
env:
DOCKER_USER: ${{ secrets.DOCKER_USER }}
DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}
run: docker login -u "${DOCKER_USER}" -p "${DOCKER_PASSWORD}"
- run: docker push "${BUILD_TO}:${TAG}"
uses: docker/login-action@v1
with:
username: ${{ secrets.DOCKER_USER }}
password: ${{ secrets.DOCKER_PASSWORD }}
- name: Log in to the GitHub container registry
uses: docker/login-action@v1
with:
registry: ghcr.io
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
# Always publish to beta tag (also full releases)
- name: Publish docker beta tag
- name: Run push
run: |
docker tag "${BUILD_TO}:${TAG}" "${BUILD_TO}:beta"
docker push "${BUILD_TO}:beta"
- if: ${{ !github.event.release.prerelease }}
name: Publish docker latest tag
run: |
docker tag "${BUILD_TO}:${TAG}" "${BUILD_TO}:latest"
docker push "${BUILD_TO}:latest"
docker/build.py \
--tag "${{ needs.init.outputs.tag }}" \
--arch "${{ matrix.arch }}" \
--build-type "${{ matrix.build_type }}" \
push
deploy-docker-manifest:
if: github.repository == 'esphome/esphome'
runs-on: ubuntu-latest
needs: [deploy-docker]
needs: [init, deploy-docker]
strategy:
matrix:
build_type: ["ha-addon", "docker"]
steps:
- uses: actions/checkout@v2
- name: Set up Python
uses: actions/setup-python@v2
with:
python-version: '3.9'
- name: Enable experimental manifest support
run: |
mkdir -p ~/.docker
echo "{\"experimental\": \"enabled\"}" > ~/.docker/config.json
- name: Set TAG
run: |
TAG="${GITHUB_REF#refs/tags/v}"
echo "TAG=${TAG}" >> $GITHUB_ENV
- name: Log in to docker hub
env:
DOCKER_USER: ${{ secrets.DOCKER_USER }}
DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}
run: docker login -u "${DOCKER_USER}" -p "${DOCKER_PASSWORD}"
- name: "Create the manifest"
run: |
docker manifest create esphome/esphome:${TAG} \
esphome/esphome-aarch64:${TAG} \
esphome/esphome-amd64:${TAG} \
esphome/esphome-armv7:${TAG}
docker manifest push esphome/esphome:${TAG}
uses: docker/login-action@v1
with:
username: ${{ secrets.DOCKER_USER }}
password: ${{ secrets.DOCKER_PASSWORD }}
- name: Log in to the GitHub container registry
uses: docker/login-action@v1
with:
registry: ghcr.io
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Publish docker beta tag
- name: Run manifest
run: |
docker manifest create esphome/esphome:beta \
esphome/esphome-aarch64:${TAG} \
esphome/esphome-amd64:${TAG} \
esphome/esphome-armv7:${TAG}
docker manifest push esphome/esphome:beta
- name: Publish docker latest tag
if: ${{ !github.event.release.prerelease }}
run: |
docker manifest create esphome/esphome:latest \
esphome/esphome-aarch64:${TAG} \
esphome/esphome-amd64:${TAG} \
esphome/esphome-armv7:${TAG}
docker manifest push esphome/esphome:latest
docker/build.py \
--tag "${{ needs.init.outputs.tag }}" \
--build-type "${{ matrix.build_type }}" \
manifest
deploy-hassio-repo:
if: github.repository == 'esphome/esphome'
if: github.repository == 'esphome/esphome' && github.event_name == 'release'
runs-on: ubuntu-latest
needs: [deploy-docker]
steps:

2
docker/Dockerfile
View File

@@ -1,4 +1,4 @@
ARG BUILD_FROM=esphome/esphome-base-amd64:3.4.0
ARG BUILD_FROM=esphome/esphome-base:latest
FROM ${BUILD_FROM}
# First install requirements to leverage caching when requirements don't change

2
docker/Dockerfile.hassio
View File

@@ -1,4 +1,4 @@
ARG BUILD_FROM
ARG BUILD_FROM=esphome/esphome-hassio-base:latest
FROM ${BUILD_FROM}
# First install requirements to leverage caching when requirements don't change

3
docker/Dockerfile.lint
View File

@@ -1,4 +1,5 @@
FROM esphome/esphome-lint-base:3.4.0
ARG BUILD_FROM=esphome/esphome-lint-base:latest
FROM ${BUILD_FROM}
COPY requirements.txt requirements_optional.txt requirements_test.txt docker/platformio_install_deps.py platformio.ini /
RUN \

177
docker/build.py Executable file
View File

@@ -0,0 +1,177 @@
#!/usr/bin/env python3
from dataclasses import dataclass
import subprocess
import argparse
import platform
import shlex
import re
import sys
CHANNEL_DEV = 'dev'
CHANNEL_BETA = 'beta'
CHANNEL_RELEASE = 'release'
CHANNELS = [CHANNEL_DEV, CHANNEL_BETA, CHANNEL_RELEASE]
ARCH_AMD64 = 'amd64'
ARCH_ARMV7 = 'armv7'
ARCH_AARCH64 = 'aarch64'
ARCHS = [ARCH_AMD64, ARCH_ARMV7, ARCH_AARCH64]
TYPE_DOCKER = 'docker'
TYPE_HA_ADDON = 'ha-addon'
TYPE_LINT = 'lint'
TYPES = [TYPE_DOCKER, TYPE_HA_ADDON, TYPE_LINT]
BASE_VERSION = "3.6.0"
parser = argparse.ArgumentParser()
parser.add_argument("--tag", type=str, required=True, help="The main docker tag to push to. If a version number also adds latest and/or beta tag")
parser.add_argument("--arch", choices=ARCHS, required=False, help="The architecture to build for")
parser.add_argument("--build-type", choices=TYPES, required=True, help="The type of build to run")
parser.add_argument("--dry-run", action="store_true", help="Don't run any commands, just print them")
subparsers = parser.add_subparsers(help="Action to perform", dest="command", required=True)
build_parser = subparsers.add_parser("build", help="Build the image")
push_parser = subparsers.add_parser("push", help="Tag the already built image and push it to docker hub")
manifest_parser = subparsers.add_parser("manifest", help="Create a manifest from already pushed images")
# only lists some possibilities, doesn't have to be perfect
# https://stackoverflow.com/a/45125525
UNAME_TO_ARCH = {
"x86_64": ARCH_AMD64,
"aarch64": ARCH_AARCH64,
"aarch64_be": ARCH_AARCH64,
"arm": ARCH_ARMV7,
}
@dataclass(frozen=True)
class DockerParams:
build_from: str
build_to: str
manifest_to: str
dockerfile: str
@classmethod
def for_type_arch(cls, build_type, arch):
prefix = {
TYPE_DOCKER: "esphome/esphome",
TYPE_HA_ADDON: "esphome/esphome-hassio",
TYPE_LINT: "esphome/esphome-lint"
}[build_type]
build_from = f"ghcr.io/{prefix}-base-{arch}:{BASE_VERSION}"
build_to = f"{prefix}-{arch}"
dockerfile = {
TYPE_DOCKER: "docker/Dockerfile",
TYPE_HA_ADDON: "docker/Dockerfile.hassio",
TYPE_LINT: "docker/Dockerfile.lint",
}[build_type]
return cls(
build_from=build_from,
build_to=build_to,
manifest_to=prefix,
dockerfile=dockerfile
)
def main():
args = parser.parse_args()
def run_command(*cmd, ignore_error: bool = False):
print(f"$ {shlex.join(list(cmd))}")
if not args.dry_run:
rc = subprocess.call(list(cmd))
if rc != 0 and not ignore_error:
print("Command failed")
sys.exit(1)
# detect channel from tag
match = re.match(r'\d+\.\d+(?:\.\d+)?(b\d+)?', args.tag)
if match is None:
channel = CHANNEL_DEV
elif match.group(1) is None:
channel = CHANNEL_RELEASE
else:
channel = CHANNEL_BETA
tags_to_push = [args.tag]
if channel == CHANNEL_DEV:
tags_to_push.append("dev")
elif channel == CHANNEL_BETA:
tags_to_push.append("beta")
elif channel == CHANNEL_RELEASE:
# Additionally push to beta
tags_to_push.append("beta")
tags_to_push.append("latest")
if args.command == "build":
# 1. pull cache image
params = DockerParams.for_type_arch(args.build_type, args.arch)
cache_tag = {
CHANNEL_DEV: "dev",
CHANNEL_BETA: "beta",
CHANNEL_RELEASE: "latest",
}[channel]
cache_img = f"ghcr.io/{params.build_to}:{cache_tag}"
run_command("docker", "pull", cache_img, ignore_error=True)
# 2. register QEMU binfmt (if not host arch)
is_native = UNAME_TO_ARCH.get(platform.machine()) == args.arch
if not is_native:
run_command(
"docker", "run", "--rm", "--privileged", "multiarch/qemu-user-static:5.2.0-2",
"--reset", "-p", "yes"
)
# 3. build
run_command(
"docker", "build",
"--build-arg", f"BUILD_FROM={params.build_from}",
"--build-arg", f"BUILD_VERSION={args.tag}",
"--tag", f"{params.build_to}:{args.tag}",
"--cache-from", cache_img,
"--file", params.dockerfile,
"."
)
elif args.command == "push":
params = DockerParams.for_type_arch(args.build_type, args.arch)
imgs = [f"{params.build_to}:{tag}" for tag in tags_to_push]
imgs += [f"ghcr.io/{params.build_to}:{tag}" for tag in tags_to_push]
src = imgs[0]
# 1. tag images
for img in imgs[1:]:
run_command(
"docker", "tag", src, img
)
# 2. push images
for img in imgs:
run_command(
"docker", "push", img
)
elif args.command == "manifest":
manifest = DockerParams.for_type_arch(args.build_type, ARCH_AMD64).manifest_to
targets = [f"{manifest}:{tag}" for tag in tags_to_push]
targets += [f"ghcr.io/{manifest}:{tag}" for tag in tags_to_push]
# 1. Create manifests
for target in targets:
cmd = ["docker", "manifest", "create", target]
for arch in ARCHS:
src = f"{DockerParams.for_type_arch(args.build_type, arch).build_to}:{args.tag}"
if target.startswith("ghcr.io"):
src = f"ghcr.io/{src}"
cmd.append(src)
run_command(*cmd)
# 2. Push manifests
for target in targets:
run_command(
"docker", "manifest", "push", target
)
if __name__ == "__main__":
main()

6
esphome/const.py
View File

@@ -1,10 +1,6 @@
"""Constants used by esphome."""
MAJOR_VERSION = 1
MINOR_VERSION = 20
PATCH_VERSION = "0b2"
__short_version__ = f"{MAJOR_VERSION}.{MINOR_VERSION}"
__version__ = f"{__short_version__}.{PATCH_VERSION}"
__version__ = "1.20.0b2"
ESP_PLATFORM_ESP32 = "ESP32"
ESP_PLATFORM_ESP8266 = "ESP8266"

50
script/bump-docker-base-version.py
View File

@@ -1,50 +0,0 @@
#!/usr/bin/env python3
import argparse
import re
import sys
def sub(path, pattern, repl, expected_count=1):
with open(path) as fh:
content = fh.read()
content, count = re.subn(pattern, repl, content, flags=re.MULTILINE)
if expected_count is not None:
assert count == expected_count, f"Pattern {pattern} replacement failed!"
with open(path, "wt") as fh:
fh.write(content)
def write_version(version: str):
for p in [
".github/workflows/ci-docker.yml",
".github/workflows/release-dev.yml",
".github/workflows/release.yml",
]:
sub(p, r'base_version=".*"', f'base_version="{version}"')
sub(
"docker/Dockerfile",
r"ARG BUILD_FROM=esphome/esphome-base-amd64:.*",
f"ARG BUILD_FROM=esphome/esphome-base-amd64:{version}",
)
sub(
"docker/Dockerfile.lint",
r"FROM esphome/esphome-lint-base:.*",
f"FROM esphome/esphome-lint-base:{version}",
)
def main():
parser = argparse.ArgumentParser()
parser.add_argument("new_version", type=str)
args = parser.parse_args()
version = args.new_version
print(f"Bumping to {version}")
write_version(version)
return 0
if __name__ == "__main__":
sys.exit(main() or 0)

10
script/bump-version.py
View File

@@ -50,16 +50,10 @@ def sub(path, pattern, repl, expected_count=1):
def write_version(version: Version):
sub(
"esphome/const.py", r"^MAJOR_VERSION = \d+$", f"MAJOR_VERSION = {version.major}"
)
sub(
"esphome/const.py", r"^MINOR_VERSION = \d+$", f"MINOR_VERSION = {version.minor}"
)
sub(
"esphome/const.py",
r"^PATCH_VERSION = .*$",
f'PATCH_VERSION = "{version.full_patch}"',
r"^__version__ = .*$",
f'__version__ = "{version}"',
)