sascha/esphome
1
0
Fork 0
mirror of https://github.com/esphome/esphome.git synced 2025-09-02 03:12:20 +01:00
Code Issues Releases Wiki Activity

🏗 Merge C++ into python codebase (#504)

Browse Source 
## Description:

Move esphome-core codebase into esphome (and a bunch of other refactors). See https://github.com/esphome/feature-requests/issues/97

Yes this is a shit ton of work and no there's no way to automate it :( But it will be worth it 👍

Progress:
- Core support (file copy etc): 80%
- Base Abstractions (light, switch): ~50%
- Integrations: ~10%
- Working? Yes, (but only with ported components).

Other refactors:
- Moves all codegen related stuff into a single class: `esphome.codegen` (imported as `cg`)
- Rework coroutine syntax
- Move from `component/platform.py` to `domain/component.py` structure as with HA
- Move all defaults out of C++ and into config validation.
- Remove `make_...` helpers from Application class. Reason: Merge conflicts with every single new integration.
- Pointer Variables are stored globally instead of locally in setup(). Reason: stack size limit.

Future work:
- Rework const.py - Move all `CONF_...` into a conf class (usage `conf.UPDATE_INTERVAL` vs `CONF_UPDATE_INTERVAL`). Reason: Less convoluted import block
- Enable loading from `custom_components` folder.

**Related issue (if applicable):** https://github.com/esphome/feature-requests/issues/97

**Pull request in [esphome-docs](https://github.com/esphome/esphome-docs) with documentation (if applicable):** esphome/esphome-docs#<esphome-docs PR number goes here>

## Checklist:
  - [ ] The code change is tested and works locally.
  - [ ] Tests have been added to verify that the new code works (under `tests/` folder).

If user exposed functionality or configuration variables are added/changed:
  - [ ] Documentation added/updated in [esphomedocs](https://github.com/OttoWinter/esphomedocs).
This commit is contained in:
Otto Winter
2019-04-17 12:06:00 +02:00
committed by GitHub
parent 049807e3ab
commit 6682c43dfa
817 changed files with 54156 additions and 10830 deletions
Download Patch File Download Diff File Expand all files Collapse all files

26
script/.neopixelbus.patch Normal file
View File

@@ -0,0 +1,26 @@
--- .piolibdeps/NeoPixelBus_ID547/src/internal/NeoEsp8266DmaMethod.h 2018-12-25 06:37:53.000000000 +0100
+++ .piolibdeps/NeoPixelBus_ID547/src/internal/NeoEsp8266DmaMethod.h.2 2019-03-01 22:18:10.000000000 +0100
@@ -169,7 +169,7 @@
_i2sBufDesc[indexDesc].sub_sof = 0;
_i2sBufDesc[indexDesc].datalen = blockSize;
_i2sBufDesc[indexDesc].blocksize = blockSize;
- _i2sBufDesc[indexDesc].buf_ptr = (uint32_t)is2Buffer;
+ _i2sBufDesc[indexDesc].buf_ptr = is2Buffer;
_i2sBufDesc[indexDesc].unused = 0;
_i2sBufDesc[indexDesc].next_link_ptr = (uint32_t)&(_i2sBufDesc[indexDesc + 1]);
@@ -329,11 +329,13 @@
case NeoDmaState_Sending:
{
slc_queue_item* finished_item = (slc_queue_item*)SLCRXEDA;
+ uint32_t **ptr = reinterpret_cast<uint32_t **>(&finished_item);
+ uint32_t dat = *reinterpret_cast<uint32_t *>(ptr);
// the data block had actual data sent
// point last state block to first state block thus
// just looping and not sending the data blocks
- (finished_item + 1)->next_link_ptr = (uint32_t)(finished_item);
+ (finished_item + 1)->next_link_ptr = dat;
s_this->_dmaState = NeoDmaState_Idle;
}

53
script/ci-custom.py Executable file
View File

@@ -0,0 +1,53 @@
#!/usr/bin/env python
from __future__ import print_function
import codecs
import collections
import os.path
import sys
def find_all(a_str, sub):
for i, line in enumerate(a_str.splitlines()):
column = 0
while True:
column = line.find(sub, column)
if column == -1:
break
yield i, column
column += len(sub)
files = []
for root, _, fs in os.walk('esphome'):
for f in fs:
_, ext = os.path.splitext(f)
if ext in ('.h', '.c', '.cpp', '.tcc', '.py'):
files.append(os.path.join(root, f))
files.sort()
errors = collections.defaultdict(list)
for f in files:
try:
with codecs.open(f, 'r', encoding='utf-8') as f_handle:
content = f_handle.read()
except UnicodeDecodeError:
errors[f].append("File is not readable as UTF-8. Please set your editor to UTF-8 mode.")
continue
for line, col in find_all(content, '\t'):
errors[f].append("File contains tab character on line {}:{}. "
"Please convert tabs to spaces.".format(line, col))
for line, col in find_all(content, '\r'):
errors[f].append("File contains windows newline on line {}:{}. "
"Please set your editor to unix newline mode.".format(line, col))
if content and not content.endswith('\n'):
errors[f].append("File does not end with a newline, please add an empty line at the end of "
"the file.")
for f, errs in errors.items():
print("\033[0;32m************* File \033[1;32m{}\033[0m".format(f))
for err in errs:
print(err)
print()
sys.exit(len(errors))

21
script/ci-suggest-changes Executable file
View File

@@ -0,0 +1,21 @@
#!/usr/bin/env bash
set -e
if git diff-index --quiet HEAD --; then
echo "No changes detected, formatting is correct!"
exit 0
else
echo "========================================================="
echo "Your formatting is not correct, ESPHome uses clang-format to format"
echo "all source files in a unified way. Please apply the changes listed below"
echo
echo "The following files need to be changed:"
git diff HEAD --name-only | sed 's/^/ /'
echo
echo
echo "========================================================="
echo
git diff HEAD
exit 1
fi

164
script/clang-format.py Executable file
View File

@@ -0,0 +1,164 @@
#!/usr/bin/env python
from __future__ import print_function
import multiprocessing
import os
import re
import subprocess
import sys
import argparse
import click
import threading
is_py2 = sys.version[0] == '2'
if is_py2:
import Queue as queue
else:
import queue as queue
root_path = os.path.abspath(os.path.normpath(os.path.join(__file__, '..', '..')))
basepath = os.path.join(root_path, 'esphome')
rel_basepath = os.path.relpath(basepath, os.getcwd())
def run_format(args, queue, lock):
"""Takes filenames out of queue and runs clang-tidy on them."""
while True:
path = queue.get()
invocation = ['clang-format-7']
if args.inplace:
invocation.append('-i')
invocation.append(path)
proc = subprocess.Popen(invocation, stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
output, err = proc.communicate()
with lock:
if proc.returncode != 0:
print(' '.join(invocation))
print(output.decode('utf-8'))
print(err.decode('utf-8'))
queue.task_done()
def progress_bar_show(value):
if value is None:
return ''
return value
def walk_files(path):
for root, _, files in os.walk(path):
for name in files:
yield os.path.join(root, name)
def get_output(*args):
proc = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
output, err = proc.communicate()
return output.decode('utf-8')
def splitlines_no_ends(string):
return [s.strip() for s in string.splitlines()]
def filter_changed(files):
for remote in ('upstream', 'origin'):
command = ['git', 'merge-base', '{}/dev'.format(remote), 'HEAD']
try:
merge_base = splitlines_no_ends(get_output(*command))[0]
break
except:
pass
else:
return files
command = ['git', 'diff', merge_base, '--name-only']
changed = splitlines_no_ends(get_output(*command))
changed = {os.path.relpath(f, os.getcwd()) for f in changed}
print("Changed Files:")
files = [p for p in files if p in changed]
for p in files:
print(" {}".format(p))
if not files:
print(" No changed files")
return files
def main():
parser = argparse.ArgumentParser()
parser.add_argument('-j', '--jobs', type=int,
default=multiprocessing.cpu_count(),
help='number of tidy instances to be run in parallel.')
parser.add_argument('files', nargs='*', default=[],
help='files to be processed (regex on path)')
parser.add_argument('-i', '--inplace', action='store_true',
help='apply fix-its')
parser.add_argument('-q', '--quiet', action='store_false',
help='Run clang-tidy in quiet mode')
parser.add_argument('-c', '--changed', action='store_true',
help='Only run on changed files')
args = parser.parse_args()
try:
get_output('clang-format-7', '-version')
except:
print("""
Oops. It looks like clang-format is not installed.
Please check you can run "clang-format-7 -version" in your terminal and install
clang-format (v7) if necessary.
Note you can also upload your code as a pull request on GitHub and see the CI check
output to apply clang-format.
""")
return 1
files = []
for path in walk_files(basepath):
filetypes = ('.cpp', '.h', '.tcc')
ext = os.path.splitext(path)[1]
if ext in filetypes:
path = os.path.relpath(path, os.getcwd())
files.append(path)
# Match against re
file_name_re = re.compile('|'.join(args.files))
files = [p for p in files if file_name_re.search(p)]
if args.changed:
files = filter_changed(files)
files.sort()
return_code = 0
try:
task_queue = queue.Queue(args.jobs)
lock = threading.Lock()
for _ in range(args.jobs):
t = threading.Thread(target=run_format,
args=(args, task_queue, lock))
t.daemon = True
t.start()
# Fill the queue with files.
with click.progressbar(files, width=30, file=sys.stderr,
item_show_func=progress_bar_show) as bar:
for name in bar:
task_queue.put(name)
# Wait for all threads to be done.
task_queue.join()
except KeyboardInterrupt:
print()
print('Ctrl-C detected, goodbye.')
os.kill(0, 9)
sys.exit(return_code)
if __name__ == '__main__':
main()

281
script/clang-tidy.py Executable file
View File

@@ -0,0 +1,281 @@
#!/usr/bin/env python
from __future__ import print_function
import codecs
import json
import multiprocessing
import os
import re
import pexpect
import shutil
import subprocess
import sys
import tempfile
import argparse
import click
import threading
is_py2 = sys.version[0] == '2'
if is_py2:
import Queue as queue
else:
import queue as queue
root_path = os.path.abspath(os.path.normpath(os.path.join(__file__, '..', '..')))
basepath = os.path.join(root_path, 'esphome')
rel_basepath = os.path.relpath(basepath, os.getcwd())
temp_header_file = os.path.join(root_path, '.temp-clang-tidy.cpp')
def run_tidy(args, tmpdir, queue, lock, failed_files):
while True:
path = queue.get()
invocation = ['clang-tidy-7', '-header-filter=^{}/.*'.format(re.escape(basepath))]
if tmpdir is not None:
invocation.append('-export-fixes')
# Get a temporary file. We immediately close the handle so clang-tidy can
# overwrite it.
(handle, name) = tempfile.mkstemp(suffix='.yaml', dir=tmpdir)
os.close(handle)
invocation.append(name)
invocation.append('-p=.')
if args.quiet:
invocation.append('-quiet')
invocation.append(os.path.abspath(path))
invocation_s = ' '.join(shlex_quote(x) for x in invocation)
# Use pexpect for a pseudy-TTY with colored output
output, rc = pexpect.run(invocation_s, withexitstatus=True, encoding='utf-8',
timeout=15*60)
with lock:
if rc != 0:
print()
print("\033[0;32m************* File \033[1;32m{}\033[0m".format(path))
print(invocation_s)
print(output)
print()
failed_files.append(path)
queue.task_done()
def progress_bar_show(value):
if value is None:
return ''
return value
def walk_files(path):
for root, _, files in os.walk(path):
for name in files:
yield os.path.join(root, name)
def get_output(*args):
proc = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
output, err = proc.communicate()
return output.decode('utf-8')
def splitlines_no_ends(string):
return [s.strip() for s in string.splitlines()]
def filter_changed(files):
for remote in ('upstream', 'origin'):
command = ['git', 'merge-base', '{}/dev'.format(remote), 'HEAD']
try:
merge_base = splitlines_no_ends(get_output(*command))[0]
break
except:
pass
else:
return files
command = ['git', 'diff', merge_base, '--name-only']
changed = splitlines_no_ends(get_output(*command))
changed = {os.path.relpath(f, os.getcwd()) for f in changed}
print("Changed Files:")
files = [p for p in files if p in changed]
for p in files:
print(" {}".format(p))
if not files:
print(" No changed files")
return files
def shlex_quote(s):
if not s:
return u"''"
if re.search(r'[^\w@%+=:,./-]', s) is None:
return s
return u"'" + s.replace(u"'", u"'\"'\"'") + u"'"
def build_compile_commands():
gcc_flags_json = os.path.join(root_path, '.gcc-flags.json')
if not os.path.isfile(gcc_flags_json):
print("Could not find {} file which is required for clang-tidy.")
print('Please run "pio init --ide atom" in the root esphome folder to generate that file.')
sys.exit(1)
with codecs.open(gcc_flags_json, 'r', encoding='utf-8') as f:
gcc_flags = json.load(f)
exec_path = gcc_flags['execPath']
include_paths = gcc_flags['gccIncludePaths'].split(',')
includes = ['-I{}'.format(p) for p in include_paths]
cpp_flags = gcc_flags['gccDefaultCppFlags'].split(' ')
defines = [flag for flag in cpp_flags if flag.startswith('-D')]
command = [exec_path]
command.extend(includes)
command.extend(defines)
command.append('-std=gnu++11')
source_files = []
for path in walk_files(basepath):
filetypes = ('.cpp',)
ext = os.path.splitext(path)[1]
if ext in filetypes:
source_files.append(os.path.abspath(path))
source_files.append(temp_header_file)
source_files.sort()
compile_commands = [{
'directory': root_path,
'command': ' '.join(shlex_quote(x) for x in (command + ['-o', p + '.o', '-c', p])),
'file': p
} for p in source_files]
compile_commands_json = os.path.join(root_path, 'compile_commands.json')
if os.path.isfile(compile_commands_json):
with codecs.open(compile_commands_json, 'r', encoding='utf-8') as f:
try:
if json.load(f) == compile_commands:
return
except:
pass
with codecs.open(compile_commands_json, 'w', encoding='utf-8') as f:
json.dump(compile_commands, f, indent=2)
def build_all_include():
# Build a cpp file that includes all header files in this repo.
# Otherwise header-only integrations would not be tested by clang-tidy
headers = []
for path in walk_files(basepath):
filetypes = ('.h',)
ext = os.path.splitext(path)[1]
if ext in filetypes:
path = os.path.relpath(path, root_path)
include_p = path.replace(os.path.sep, '/')
headers.append('#include "{}"'.format(include_p))
headers.sort()
headers.append('')
content = '\n'.join(headers)
with codecs.open(temp_header_file, 'w', encoding='utf-8') as f:
f.write(content)
def main():
parser = argparse.ArgumentParser()
parser.add_argument('-j', '--jobs', type=int,
default=multiprocessing.cpu_count(),
help='number of tidy instances to be run in parallel.')
parser.add_argument('files', nargs='*', default=[],
help='files to be processed (regex on path)')
parser.add_argument('--fix', action='store_true', help='apply fix-its')
parser.add_argument('-q', '--quiet', action='store_false',
help='Run clang-tidy in quiet mode')
parser.add_argument('-c', '--changed', action='store_true',
help='Only run on changed files')
parser.add_argument('--all-headers', action='store_true',
help='Create a dummy file that checks all headers')
args = parser.parse_args()
try:
get_output('clang-tidy-7', '-version')
except:
print("""
Oops. It looks like clang-tidy is not installed.
Please check you can run "clang-tidy-7 -version" in your terminal and install
clang-tidy (v7) if necessary.
Note you can also upload your code as a pull request on GitHub and see the CI check
output to apply clang-tidy.
""")
return 1
build_compile_commands()
files = []
for path in walk_files(basepath):
filetypes = ('.cpp',)
ext = os.path.splitext(path)[1]
if ext in filetypes:
path = os.path.relpath(path, os.getcwd())
files.append(path)
# Match against re
file_name_re = re.compile('|'.join(args.files))
files = [p for p in files if file_name_re.search(p)]
if args.changed:
files = filter_changed(files)
files.sort()
if args.all_headers:
build_all_include()
files.insert(0, temp_header_file)
tmpdir = None
if args.fix:
tmpdir = tempfile.mkdtemp()
failed_files = []
return_code = 0
try:
task_queue = queue.Queue(args.jobs)
lock = threading.Lock()
for _ in range(args.jobs):
t = threading.Thread(target=run_tidy,
args=(args, tmpdir, task_queue, lock, failed_files))
t.daemon = True
t.start()
# Fill the queue with files.
with click.progressbar(files, width=30, file=sys.stderr,
item_show_func=progress_bar_show) as bar:
for name in bar:
task_queue.put(name)
# Wait for all threads to be done.
task_queue.join()
return_code = len(failed_files)
except KeyboardInterrupt:
print()
print('Ctrl-C detected, goodbye.')
if tmpdir:
shutil.rmtree(tmpdir)
if os.path.exists(temp_header_file):
os.remove(temp_header_file)
os.kill(0, 9)
if args.fix and failed_files:
print('Applying fixes ...')
try:
subprocess.call(['clang-apply-replacements-7', tmpdir])
except:
print('Error applying fixes.\n', file=sys.stderr)
if os.path.exists(temp_header_file):
os.remove(temp_header_file)
raise
if os.path.exists(temp_header_file):
os.remove(temp_header_file)
sys.exit(return_code)
if __name__ == '__main__':
main()

16
script/lint-cpp Executable file
View File

@@ -0,0 +1,16 @@
#!/usr/bin/env bash
set -e
cd "$(dirname "$0")/.."
if [[ ! -e ".gcc-flags.json" ]]; then
pio init --ide atom
fi
if ! patch -R -p0 -s -f --dry-run <script/.neopixelbus.patch; then
patch -p0 < script/.neopixelbus.patch
fi
set -x
script/clang-tidy.py -c --fix
script/clang-format.py -c -i

10
script/lint-python Executable file
View File

@@ -0,0 +1,10 @@
#!/usr/bin/env bash
set -e
cd "$(dirname "$0")/.."
set -x
script/ci-custom.py
flake8 esphome
pylint esphome

8
script/setup Executable file
View File

@@ -0,0 +1,8 @@
#!/usr/bin/env bash
# Set up ESPHome dev environment
set -e
cd "$(dirname "$0")/.."
pip install -r requirements_test.txt
pip install -e .

11
script/test Executable file
View File

@@ -0,0 +1,11 @@
#!/usr/bin/env bash
set -e
cd "$(dirname "$0")/.."
set -x
esphome tests/test1.yaml compile
esphome tests/test2.yaml compile
esphome tests/test3.yaml compile