sascha/esphome
1
0
Fork 0
mirror of https://github.com/esphome/esphome.git synced 2025-09-03 20:02:22 +01:00
Code Issues Releases Wiki Activity

🏗 Merge C++ into python codebase (#504)

Browse Source 
## Description:

Move esphome-core codebase into esphome (and a bunch of other refactors). See https://github.com/esphome/feature-requests/issues/97

Yes this is a shit ton of work and no there's no way to automate it :( But it will be worth it 👍

Progress:
- Core support (file copy etc): 80%
- Base Abstractions (light, switch): ~50%
- Integrations: ~10%
- Working? Yes, (but only with ported components).

Other refactors:
- Moves all codegen related stuff into a single class: `esphome.codegen` (imported as `cg`)
- Rework coroutine syntax
- Move from `component/platform.py` to `domain/component.py` structure as with HA
- Move all defaults out of C++ and into config validation.
- Remove `make_...` helpers from Application class. Reason: Merge conflicts with every single new integration.
- Pointer Variables are stored globally instead of locally in setup(). Reason: stack size limit.

Future work:
- Rework const.py - Move all `CONF_...` into a conf class (usage `conf.UPDATE_INTERVAL` vs `CONF_UPDATE_INTERVAL`). Reason: Less convoluted import block
- Enable loading from `custom_components` folder.

**Related issue (if applicable):** https://github.com/esphome/feature-requests/issues/97

**Pull request in [esphome-docs](https://github.com/esphome/esphome-docs) with documentation (if applicable):** esphome/esphome-docs#<esphome-docs PR number goes here>

## Checklist:
  - [ ] The code change is tested and works locally.
  - [ ] Tests have been added to verify that the new code works (under `tests/` folder).

If user exposed functionality or configuration variables are added/changed:
  - [ ] Documentation added/updated in [esphomedocs](https://github.com/OttoWinter/esphomedocs).
This commit is contained in:
Otto Winter
2019-04-17 12:06:00 +02:00
committed by GitHub
parent 049807e3ab
commit 6682c43dfa
817 changed files with 54156 additions and 10830 deletions
Download Patch File Download Diff File Expand all files Collapse all files

360
esphome/writer.py
View File

@@ -1,22 +1,15 @@
from __future__ import print_function
import codecs
import logging
import os
import re
from esphome.config import iter_components
from esphome.const import ARDUINO_VERSION_ESP32_1_0_0, ARDUINO_VERSION_ESP8266_2_5_0, \
ARDUINO_VERSION_ESP8266_DEV, CONF_BOARD_FLASH_MODE, CONF_BRANCH, CONF_COMMIT, CONF_ESPHOME, \
CONF_LOCAL, CONF_PLATFORMIO_OPTIONS, CONF_REPOSITORY, CONF_TAG, CONF_USE_CUSTOM_CODE, \
ARDUINO_VERSION_ESP8266_2_3_0
from esphome.const import CONF_BOARD_FLASH_MODE, CONF_ESPHOME, CONF_PLATFORMIO_OPTIONS, \
HEADER_FILE_EXTENSIONS, SOURCE_FILE_EXTENSIONS
from esphome.core import CORE, EsphomeError
from esphome.core_config import GITHUB_ARCHIVE_ZIP, LIBRARY_URI_REPO, VERSION_REGEX
from esphome.helpers import mkdir_p, run_system_command
from esphome.pins import ESP8266_FLASH_SIZES, ESP8266_LD_SCRIPTS
from esphome.py_compat import IS_PY3, string_types
from esphome.helpers import mkdir_p, read_file, write_file_if_changed
from esphome.storage_json import StorageJSON, storage_path
from esphome.util import safe_print
_LOGGER = logging.getLogger(__name__)
@@ -60,20 +53,11 @@ UPLOAD_SPEED_OVERRIDE = {
}
def get_build_flags(key):
build_flags = set()
def get_flags(key):
flags = set()
for _, component, conf in iter_components(CORE.config):
if not hasattr(component, key):
continue
flags = getattr(component, key)
if callable(flags):
flags = flags(conf)
if flags is None:
continue
if isinstance(flags, string_types):
flags = [flags]
build_flags |= set(flags)
return build_flags
flags |= getattr(component, key)(conf)
return flags
def get_include_text():
@@ -95,37 +79,6 @@ def get_include_text():
return include_text
def update_esphome_core_repo():
if CONF_REPOSITORY not in CORE.esphome_core_version:
return
if CONF_BRANCH not in CORE.esphome_core_version:
# Git commit hash or tag cannot be updated
return
esphome_core_path = CORE.relative_piolibdeps_path('esphome-core')
rc, _, _ = run_system_command('git', '-C', esphome_core_path, '--help')
if rc != 0:
# git not installed or repo not downloaded yet
return
rc, _, _ = run_system_command('git', '-C', esphome_core_path, 'diff-index', '--quiet', 'HEAD',
'--')
if rc != 0:
# local changes, cannot update
_LOGGER.warning("Local changes in esphome-core copy from git. Will not auto-update.")
return
_LOGGER.info("Updating esphome-core copy from git (%s)", esphome_core_path)
rc, stdout, _ = run_system_command('git', '-c', 'color.ui=always', '-C', esphome_core_path,
'pull', '--stat')
if rc != 0:
_LOGGER.warning("Couldn't auto-update local git copy of esphome-core.")
return
if IS_PY3:
stdout = stdout.decode('utf-8', 'backslashreplace')
safe_print(stdout.strip())
def replace_file_content(text, pattern, repl):
content_new, count = re.subn(pattern, repl, text, flags=re.M)
return content_new, count
@@ -136,8 +89,7 @@ def migrate_src_version_0_to_1():
if not os.path.isfile(main_cpp):
return
with codecs.open(main_cpp, 'r', encoding='utf-8') as f_handle:
content = orig_content = f_handle.read()
content = read_file(main_cpp)
if CPP_INCLUDE_BEGIN in content:
return
@@ -160,10 +112,7 @@ def migrate_src_version_0_to_1():
"auto-generated again.", main_cpp, main_cpp)
_LOGGER.info("Migration: Added include section to %s", main_cpp)
if orig_content == content:
return
with codecs.open(main_cpp, 'w', encoding='utf-8') as f_handle:
f_handle.write(content)
write_file_if_changed(content, main_cpp)
def migrate_src_version(old, new):
@@ -181,8 +130,6 @@ def storage_should_clean(old, new): # type: (StorageJSON, StorageJSON) -> bool
if old is None:
return True
if old.esphome_core_version != new.esphome_core_version:
return True
if old.esphome_version != new.esphome_version:
return True
if old.src_version != new.src_version:
@@ -213,29 +160,6 @@ def update_storage_json():
new.save(path)
def symlink_esphome_core_version(esphome_core_version):
from esphome.symlink_ops import symlink, islink, readlink, unlink
lib_path = CORE.relative_build_path('lib')
dst_path = CORE.relative_build_path('lib', 'esphome-core')
if CORE.is_local_esphome_core_copy:
src_path = CORE.relative_path(esphome_core_version[CONF_LOCAL])
do_write = True
if islink(dst_path):
old_path = os.path.join(readlink(dst_path), lib_path)
if old_path != lib_path:
unlink(dst_path)
else:
do_write = False
if do_write:
mkdir_p(lib_path)
symlink(src_path, dst_path)
else:
# Remove symlink when changing back from local version
if islink(dst_path):
unlink(dst_path)
def format_ini(data):
content = u''
for key, value in sorted(data.items()):
@@ -249,119 +173,13 @@ def format_ini(data):
def gather_lib_deps():
import json
lib_deps = set()
if CONF_REPOSITORY in CORE.esphome_core_version:
repo = CORE.esphome_core_version[CONF_REPOSITORY]
ref = next((CORE.esphome_core_version[x] for x in (CONF_COMMIT, CONF_BRANCH, CONF_TAG)
if x in CORE.esphome_core_version), None)
if CONF_TAG in CORE.esphome_core_version and repo == LIBRARY_URI_REPO:
this_version = GITHUB_ARCHIVE_ZIP.format(ref)
elif ref is not None:
this_version = repo + '#' + ref
lib_deps.add(this_version)
elif CORE.is_local_esphome_core_copy:
src_path = CORE.relative_path(CORE.esphome_core_version[CONF_LOCAL])
# Manually add lib_deps because platformio seems to ignore them inside libs/
library_json_path = os.path.join(src_path, 'library.json')
with codecs.open(library_json_path, 'r', encoding='utf-8') as f_handle:
library_json_text = f_handle.read()
library_json = json.loads(library_json_text)
for dep in library_json.get('dependencies', []):
if 'version' in dep and VERSION_REGEX.match(dep['version']) is not None:
lib_deps.add(dep['name'] + '@' + dep['version'])
else:
lib_deps.add(dep['version'])
else:
lib_deps.add(CORE.esphome_core_version)
lib_deps |= get_build_flags('LIB_DEPS')
lib_deps |= get_build_flags('lib_deps')
if CORE.is_esp32:
lib_deps |= {
'Preferences', # Preferences helper
'AsyncTCP@1.0.3', # Pin AsyncTCP version
}
# Manual fix for AsyncTCP
if CORE.arduino_version == ARDUINO_VERSION_ESP32_1_0_0:
lib_deps.discard('AsyncTCP@1.0.3')
lib_deps.add('AsyncTCP@1.0.1')
lib_deps.add('ESPmDNS')
elif CORE.is_esp8266:
lib_deps.add('ESPAsyncTCP@1.2.0')
lib_deps.add('ESP8266mDNS')
# avoid changing build flags order
lib_deps_l = list(lib_deps)
lib_deps_l = [x.as_lib_dep for x in CORE.libraries]
lib_deps_l.sort()
# Move AsyncTCP to front, see https://github.com/platformio/platformio-core/issues/2115
if 'AsyncTCP@1.0.3' in lib_deps_l:
lib_deps_l.insert(0, lib_deps_l.pop(lib_deps_l.index('AsyncTCP@1.0.3')))
if 'AsyncTCP@1.0.1' in lib_deps_l:
lib_deps_l.insert(0, lib_deps_l.pop(lib_deps_l.index('AsyncTCP@1.0.1')))
return lib_deps_l
def gather_build_flags():
build_flags = set()
if not CORE.config[CONF_ESPHOME][CONF_USE_CUSTOM_CODE]:
build_flags |= get_build_flags('build_flags')
build_flags |= get_build_flags('BUILD_FLAGS')
build_flags.add('-DESPHOME_USE')
build_flags.add("-Wno-unused-variable")
build_flags.add("-Wno-unused-but-set-variable")
build_flags.add("-Wno-sign-compare")
build_flags |= get_build_flags('required_build_flags')
build_flags |= get_build_flags('REQUIRED_BUILD_FLAGS')
if not CORE.config[CONF_ESPHOME][CONF_USE_CUSTOM_CODE]:
# For new users, include common components out of the box.
# So that first impression is improved and user doesn't need to wait
# an eternity.
# It's not a perfect solution but shouldn't cause any issues I think
# Common components determined through Google Analytics page views
# and only components that are lightweight (e.g. not lights because they
# take up memory)
build_flags |= {
'-DUSE_ADC_SENSOR',
'-DUSE_BINARY_SENSOR',
'-DUSE_DALLAS_SENSOR',
'-DUSE_DHT_SENSOR',
'-DUSE_GPIO_BINARY_SENSOR',
'-DUSE_GPIO_SWITCH',
'-DUSE_SENSOR',
'-DUSE_STATUS_BINARY_SENSOR',
'-DUSE_STATUS_LED',
'-DUSE_SWITCH',
'-DUSE_TEMPLATE_BINARY_SENSOR',
'-DUSE_TEMPLATE_SENSOR',
'-DUSE_TEMPLATE_SWITCH',
'-DUSE_WIFI_SIGNAL_SENSOR',
}
if CORE.is_esp8266 and CORE.board in ESP8266_FLASH_SIZES and \
CORE.arduino_version != ARDUINO_VERSION_ESP8266_2_3_0:
flash_size = ESP8266_FLASH_SIZES[CORE.board]
ld_scripts = ESP8266_LD_SCRIPTS[flash_size]
ld_script = None
if CORE.arduino_version in ('espressif8266@1.8.0', 'espressif8266@1.7.3',
'espressif8266@1.6.0'):
ld_script = ld_scripts[0]
elif CORE.arduino_version in (ARDUINO_VERSION_ESP8266_DEV, ARDUINO_VERSION_ESP8266_2_5_0):
ld_script = ld_scripts[1]
if ld_script is not None:
build_flags.add('-Wl,-T{}'.format(ld_script))
if CORE.is_esp8266 and CORE.arduino_version in (ARDUINO_VERSION_ESP8266_DEV,
ARDUINO_VERSION_ESP8266_2_5_0):
build_flags.add('-fno-exceptions')
build_flags = CORE.build_flags
# avoid changing build flags order
return list(sorted(list(build_flags)))
@@ -396,31 +214,9 @@ def get_ini_content():
flash_mode = CORE.config[CONF_ESPHOME][CONF_BOARD_FLASH_MODE]
data['board_build.flash_mode'] = flash_mode
if not CORE.config[CONF_ESPHOME][CONF_USE_CUSTOM_CODE]:
# Ignore libraries that are not explicitly used, but may
# be added by LDF
data['lib_ldf_mode'] = 'chain'
REMOVABLE_LIBRARIES = [
'ArduinoOTA',
'Update',
'Wire',
'FastLED',
'NeoPixelBus',
'ESP Async WebServer',
'AsyncMqttClient',
'AsyncTCP',
'ESPAsyncTCP',
]
ignore = []
for x in REMOVABLE_LIBRARIES:
for o in lib_deps:
if o.startswith(x):
break
else:
ignore.append(x)
if ignore:
data['lib_ignore'] = ignore
# Ignore libraries that are not explicitly used, but may
# be added by LDF
# data['lib_ldf_mode'] = 'chain'
data.update(CORE.config[CONF_ESPHOME].get(CONF_PLATFORMIO_OPTIONS, {}))
content = u'[env:{}]\n'.format(CORE.name)
@@ -448,65 +244,131 @@ def find_begin_end(text, begin_s, end_s):
return text[:begin_index], text[(end_index + len(end_s)):]
def write_platformio_ini(content, path):
symlink_esphome_core_version(CORE.esphome_core_version)
update_esphome_core_repo()
def write_platformio_ini(content):
update_storage_json()
path = CORE.relative_build_path('platformio.ini')
if os.path.isfile(path):
try:
with codecs.open(path, 'r', encoding='utf-8') as f_handle:
text = f_handle.read()
except OSError:
raise EsphomeError(u"Could not read ini file at {}".format(path))
prev_file = text
text = read_file(path)
content_format = find_begin_end(text, INI_AUTO_GENERATE_BEGIN, INI_AUTO_GENERATE_END)
else:
prev_file = None
content_format = INI_BASE_FORMAT
full_file = content_format[0] + INI_AUTO_GENERATE_BEGIN + '\n' + content
full_file += INI_AUTO_GENERATE_END + content_format[1]
if prev_file == full_file:
return
with codecs.open(path, mode='w+', encoding='utf-8') as f_handle:
f_handle.write(full_file)
write_file_if_changed(full_file, path)
def write_platformio_project():
mkdir_p(CORE.build_path)
platformio_ini = CORE.relative_build_path('platformio.ini')
content = get_ini_content()
write_gitignore()
write_platformio_ini(content, platformio_ini)
write_platformio_ini(content)
DEFINES_H_FORMAT = u"""\
#pragma once
{}
"""
DEFINES_H_TARGET = 'esphome/core/defines.h'
ESPHOME_README_TXT = u"""
THIS DIRECTORY IS AUTO-GENERATED, DO NOT MODIFY
ESPHome automatically populates the esphome/ directory, and any
changes to this directory will be removed the next time esphome is
run.
For modifying esphome's core files, please use a development esphome install
or use the custom_components folder.
"""
def walk_files(path):
for root, _, files in os.walk(path):
for name in files:
yield os.path.join(root, name)
def copy_src_tree():
import filecmp
import shutil
source_files = {}
for _, component, _ in iter_components(CORE.config):
source_files.update(component.source_files)
# Convert to list and sort
source_files_l = [it for it in source_files.items()]
source_files_l.sort()
# Build #include list for main.cpp
include_l = []
for target, path in source_files_l:
if os.path.splitext(path)[1] in HEADER_FILE_EXTENSIONS:
include_l.append(u'#include "{}"'.format(target))
include_l.append(u'')
include_s = u'\n'.join(include_l)
source_files_copy = source_files.copy()
source_files_copy.pop(DEFINES_H_TARGET)
for path in walk_files(CORE.relative_src_path('esphome')):
if os.path.splitext(path)[1] not in SOURCE_FILE_EXTENSIONS:
# Not a source file, ignore
continue
# Transform path to target path name
target = os.path.relpath(path, CORE.relative_src_path()).replace(os.path.sep, '/')
if target == DEFINES_H_TARGET:
# Ignore defines.h, will be dealt with later
continue
if target not in source_files_copy:
# Source file removed, delete target
os.remove(path)
else:
src_path = source_files_copy.pop(target)
if not filecmp.cmp(path, src_path):
# Files are not same, copy
shutil.copy(src_path, path)
# Now copy new files
for target, src_path in source_files_copy.items():
dst_path = CORE.relative_src_path(*target.split('/'))
mkdir_p(os.path.dirname(dst_path))
shutil.copy(src_path, dst_path)
# Finally copy defines
write_file_if_changed(generate_defines_h(),
CORE.relative_src_path('esphome', 'core', 'defines.h'))
write_file_if_changed(ESPHOME_README_TXT,
CORE.relative_src_path('esphome', 'README.txt'))
return include_s
def generate_defines_h():
define_content_l = [x.as_macro for x in CORE.defines]
define_content_l.sort()
return DEFINES_H_FORMAT.format(u'\n'.join(define_content_l))
def write_cpp(code_s):
path = CORE.relative_build_path('src', 'main.cpp')
path = CORE.relative_src_path('main.cpp')
if os.path.isfile(path):
try:
with codecs.open(path, 'r', encoding='utf-8') as f_handle:
text = f_handle.read()
except OSError:
raise EsphomeError(u"Could not read C++ file at {}".format(path))
prev_file = text
text = read_file(path)
code_format = find_begin_end(text, CPP_AUTO_GENERATE_BEGIN, CPP_AUTO_GENERATE_END)
code_format_ = find_begin_end(code_format[0], CPP_INCLUDE_BEGIN, CPP_INCLUDE_END)
code_format = (code_format_[0], code_format_[1], code_format[1])
else:
prev_file = None
mkdir_p(os.path.dirname(path))
code_format = CPP_BASE_FORMAT
include_s = get_include_text()
include_s = copy_src_tree()
global_s = include_s + u'\n'
global_s += CORE.cpp_global_section
full_file = code_format[0] + CPP_INCLUDE_BEGIN + u'\n' + include_s + CPP_INCLUDE_END
full_file = code_format[0] + CPP_INCLUDE_BEGIN + u'\n' + global_s + CPP_INCLUDE_END
full_file += code_format[1] + CPP_AUTO_GENERATE_BEGIN + u'\n' + code_s + CPP_AUTO_GENERATE_END
full_file += code_format[2]
if prev_file == full_file:
return
with codecs.open(path, 'w+', encoding='utf-8') as f_handle:
f_handle.write(full_file)
write_file_if_changed(full_file, path)
def clean_build():