Merge pull request #2302 from esphome/bump-2021.9.0b4

2021.9.0b4

docker/build.py

@@ -24,7 +24,7 @@ TYPE_LINT = 'lint'
 TYPES = [TYPE_DOCKER, TYPE_HA_ADDON, TYPE_LINT]
 
 
-BASE_VERSION = "3.6.0"
+BASE_VERSION = "4.2.0"
 
 
 parser = argparse.ArgumentParser()

esphome/__main__.py

@@ -256,7 +256,7 @@ def show_logs(config, args, port):
         run_miniterm(config, port)
         return 0
     if get_port_type(port) == "NETWORK" and "api" in config:
-        from esphome.api.client import run_logs
+        from esphome.components.api.client import run_logs
 
         return run_logs(config, port)
     if get_port_type(port) == "MQTT" and "mqtt" in config:
@@ -483,75 +483,9 @@ def parse_args(argv):
         metavar=("key", "value"),
     )
 
-    # Keep backward compatibility with the old command line format of
-    # esphome <config> <command>.
-    #
-    # Unfortunately this can't be done by adding another configuration argument to the
-    # main config parser, as argparse is greedy when parsing arguments, so in regular
-    # usage it'll eat the command as the configuration argument and error out out
-    # because it can't parse the configuration as a command.
-    #
-    # Instead, construct an ad-hoc parser for the old format that doesn't actually
-    # process the arguments, but parses them enough to let us figure out if the old
-    # format is used. In that case, swap the command and configuration in the arguments
-    # and continue on with the normal parser (after raising a deprecation warning).
-    #
-    # Disable argparse's built-in help option and add it manually to prevent this
-    # parser from printing the help messagefor the old format when invoked with -h.
-    compat_parser = argparse.ArgumentParser(parents=[options_parser], add_help=False)
-    compat_parser.add_argument("-h", "--help")
-    compat_parser.add_argument("configuration", nargs="*")
-    compat_parser.add_argument(
-        "command",
-        choices=[
-            "config",
-            "compile",
-            "upload",
-            "logs",
-            "run",
-            "clean-mqtt",
-            "wizard",
-            "mqtt-fingerprint",
-            "version",
-            "clean",
-            "dashboard",
-            "vscode",
-            "update-all",
-        ],
-    )
-
-    # on Python 3.9+ we can simply set exit_on_error=False in the constructor
-    def _raise(x):
-        raise argparse.ArgumentError(None, x)
-
-    compat_parser.error = _raise
-
-    deprecated_argv_suggestion = None
-
-    if ["dashboard", "config"] == argv[1:3] or ["version"] == argv[1:2]:
-        # this is most likely meant in new-style arg format. do not try compat parsing
-        pass
-    else:
-        try:
-            result, unparsed = compat_parser.parse_known_args(argv[1:])
-            last_option = len(argv) - len(unparsed) - 1 - len(result.configuration)
-            unparsed = [
-                "--device" if arg in ("--upload-port", "--serial-port") else arg
-                for arg in unparsed
-            ]
-            argv = (
-                argv[0:last_option] + [result.command] + result.configuration + unparsed
-            )
-            deprecated_argv_suggestion = argv
-        except argparse.ArgumentError:
-            # This is not an old-style command line, so we don't have to do anything.
-            pass
-
-    # And continue on with regular parsing
     parser = argparse.ArgumentParser(
         description=f"ESPHome v{const.__version__}", parents=[options_parser]
     )
-    parser.set_defaults(deprecated_argv_suggestion=deprecated_argv_suggestion)
 
     mqtt_options = argparse.ArgumentParser(add_help=False)
     mqtt_options.add_argument("--topic", help="Manually set the MQTT topic.")
@@ -701,7 +635,83 @@ def parse_args(argv):
         "configuration", help="Your YAML configuration file directories.", nargs="+"
     )
 
-    return parser.parse_args(argv[1:])
+    # Keep backward compatibility with the old command line format of
+    # esphome <config> <command>.
+    #
+    # Unfortunately this can't be done by adding another configuration argument to the
+    # main config parser, as argparse is greedy when parsing arguments, so in regular
+    # usage it'll eat the command as the configuration argument and error out out
+    # because it can't parse the configuration as a command.
+    #
+    # Instead, if parsing using the current format fails, construct an ad-hoc parser
+    # that doesn't actually process the arguments, but parses them enough to let us
+    # figure out if the old format is used. In that case, swap the command and
+    # configuration in the arguments and retry with the normal parser (and raise
+    # a deprecation warning).
+    arguments = argv[1:]
+
+    # On Python 3.9+ we can simply set exit_on_error=False in the constructor
+    def _raise(x):
+        raise argparse.ArgumentError(None, x)
+
+    # First, try new-style parsing, but don't exit in case of failure
+    try:
+        # duplicate parser so that we can use the original one to raise errors later on
+        current_parser = argparse.ArgumentParser(add_help=False, parents=[parser])
+        current_parser.set_defaults(deprecated_argv_suggestion=None)
+        current_parser.error = _raise
+        return current_parser.parse_args(arguments)
+    except argparse.ArgumentError:
+        pass
+
+    # Second, try compat parsing and rearrange the command-line if it succeeds
+    # Disable argparse's built-in help option and add it manually to prevent this
+    # parser from printing the help messagefor the old format when invoked with -h.
+    compat_parser = argparse.ArgumentParser(parents=[options_parser], add_help=False)
+    compat_parser.add_argument("-h", "--help", action="store_true")
+    compat_parser.add_argument("configuration", nargs="*")
+    compat_parser.add_argument(
+        "command",
+        choices=[
+            "config",
+            "compile",
+            "upload",
+            "logs",
+            "run",
+            "clean-mqtt",
+            "wizard",
+            "mqtt-fingerprint",
+            "version",
+            "clean",
+            "dashboard",
+            "vscode",
+            "update-all",
+        ],
+    )
+
+    try:
+        compat_parser.error = _raise
+        result, unparsed = compat_parser.parse_known_args(argv[1:])
+        last_option = len(arguments) - len(unparsed) - 1 - len(result.configuration)
+        unparsed = [
+            "--device" if arg in ("--upload-port", "--serial-port") else arg
+            for arg in unparsed
+        ]
+        arguments = (
+            arguments[0:last_option]
+            + [result.command]
+            + result.configuration
+            + unparsed
+        )
+        deprecated_argv_suggestion = arguments
+    except argparse.ArgumentError:
+        # old-style parsing failed, don't suggest any argument
+        deprecated_argv_suggestion = None
+
+    # Finally, run the new-style parser again with the possibly swapped arguments,
+    # and let it error out if the command is unparsable.
+    parser.set_defaults(deprecated_argv_suggestion=deprecated_argv_suggestion)
+    return parser.parse_args(arguments)
 
 
 def run_esphome(argv):
@@ -715,7 +725,7 @@ def run_esphome(argv):
             "and will be removed in the future. "
         )
         _LOGGER.warning("Please instead use:")
-        _LOGGER.warning("   esphome %s", " ".join(args.deprecated_argv_suggestion[1:]))
+        _LOGGER.warning("   esphome %s", " ".join(args.deprecated_argv_suggestion))
 
     if sys.version_info < (3, 7, 0):
         _LOGGER.error(

esphome/api/client.py

@@ -1,518 +0,0 @@
-from datetime import datetime
-import functools
-import logging
-import socket
-import threading
-import time
-
-# pylint: disable=unused-import
-from typing import Optional  # noqa
-from google.protobuf import message  # noqa
-
-from esphome import const
-import esphome.api.api_pb2 as pb
-from esphome.const import CONF_PASSWORD, CONF_PORT
-from esphome.core import EsphomeError
-from esphome.helpers import resolve_ip_address, indent
-from esphome.log import color, Fore
-from esphome.util import safe_print
-
-_LOGGER = logging.getLogger(__name__)
-
-
-class APIConnectionError(EsphomeError):
-    pass
-
-
-MESSAGE_TYPE_TO_PROTO = {
-    1: pb.HelloRequest,
-    2: pb.HelloResponse,
-    3: pb.ConnectRequest,
-    4: pb.ConnectResponse,
-    5: pb.DisconnectRequest,
-    6: pb.DisconnectResponse,
-    7: pb.PingRequest,
-    8: pb.PingResponse,
-    9: pb.DeviceInfoRequest,
-    10: pb.DeviceInfoResponse,
-    11: pb.ListEntitiesRequest,
-    12: pb.ListEntitiesBinarySensorResponse,
-    13: pb.ListEntitiesCoverResponse,
-    14: pb.ListEntitiesFanResponse,
-    15: pb.ListEntitiesLightResponse,
-    16: pb.ListEntitiesSensorResponse,
-    17: pb.ListEntitiesSwitchResponse,
-    18: pb.ListEntitiesTextSensorResponse,
-    19: pb.ListEntitiesDoneResponse,
-    20: pb.SubscribeStatesRequest,
-    21: pb.BinarySensorStateResponse,
-    22: pb.CoverStateResponse,
-    23: pb.FanStateResponse,
-    24: pb.LightStateResponse,
-    25: pb.SensorStateResponse,
-    26: pb.SwitchStateResponse,
-    27: pb.TextSensorStateResponse,
-    28: pb.SubscribeLogsRequest,
-    29: pb.SubscribeLogsResponse,
-    30: pb.CoverCommandRequest,
-    31: pb.FanCommandRequest,
-    32: pb.LightCommandRequest,
-    33: pb.SwitchCommandRequest,
-    34: pb.SubscribeServiceCallsRequest,
-    35: pb.ServiceCallResponse,
-    36: pb.GetTimeRequest,
-    37: pb.GetTimeResponse,
-}
-
-
-def _varuint_to_bytes(value):
-    if value <= 0x7F:
-        return bytes([value])
-
-    ret = bytes()
-    while value:
-        temp = value & 0x7F
-        value >>= 7
-        if value:
-            ret += bytes([temp | 0x80])
-        else:
-            ret += bytes([temp])
-
-    return ret
-
-
-def _bytes_to_varuint(value):
-    result = 0
-    bitpos = 0
-    for val in value:
-        result |= (val & 0x7F) << bitpos
-        bitpos += 7
-        if (val & 0x80) == 0:
-            return result
-    return None
-
-
-# pylint: disable=too-many-instance-attributes,not-callable
-class APIClient(threading.Thread):
-    def __init__(self, address, port, password):
-        threading.Thread.__init__(self)
-        self._address = address  # type: str
-        self._port = port  # type: int
-        self._password = password  # type: Optional[str]
-        self._socket = None  # type: Optional[socket.socket]
-        self._socket_open_event = threading.Event()
-        self._socket_write_lock = threading.Lock()
-        self._connected = False
-        self._authenticated = False
-        self._message_handlers = []
-        self._keepalive = 5
-        self._ping_timer = None
-
-        self.on_disconnect = None
-        self.on_connect = None
-        self.on_login = None
-        self.auto_reconnect = False
-        self._running_event = threading.Event()
-        self._stop_event = threading.Event()
-
-    @property
-    def stopped(self):
-        return self._stop_event.is_set()
-
-    def _refresh_ping(self):
-        if self._ping_timer is not None:
-            self._ping_timer.cancel()
-            self._ping_timer = None
-
-        def func():
-            self._ping_timer = None
-
-            if self._connected:
-                try:
-                    self.ping()
-                except APIConnectionError as err:
-                    self._fatal_error(err)
-                else:
-                    self._refresh_ping()
-
-        self._ping_timer = threading.Timer(self._keepalive, func)
-        self._ping_timer.start()
-
-    def _cancel_ping(self):
-        if self._ping_timer is not None:
-            self._ping_timer.cancel()
-            self._ping_timer = None
-
-    def _close_socket(self):
-        self._cancel_ping()
-        if self._socket is not None:
-            self._socket.close()
-            self._socket = None
-        self._socket_open_event.clear()
-        self._connected = False
-        self._authenticated = False
-        self._message_handlers = []
-
-    def stop(self, force=False):
-        if self.stopped:
-            raise ValueError
-
-        if self._connected and not force:
-            try:
-                self.disconnect()
-            except APIConnectionError:
-                pass
-        self._close_socket()
-
-        self._stop_event.set()
-        if not force:
-            self.join()
-
-    def connect(self):
-        if not self._running_event.wait(0.1):
-            raise APIConnectionError("You need to call start() first!")
-
-        if self._connected:
-            self.disconnect(on_disconnect=False)
-
-        try:
-            ip = resolve_ip_address(self._address)
-        except EsphomeError as err:
-            _LOGGER.warning(
-                "Error resolving IP address of %s. Is it connected to WiFi?",
-                self._address,
-            )
-            _LOGGER.warning(
-                "(If this error persists, please set a static IP address: "
-                "https://esphome.io/components/wifi.html#manual-ips)"
-            )
-            raise APIConnectionError(err) from err
-
-        _LOGGER.info("Connecting to %s:%s (%s)", self._address, self._port, ip)
-        self._socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
-        self._socket.settimeout(10.0)
-        self._socket.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
-        try:
-            self._socket.connect((ip, self._port))
-        except OSError as err:
-            err = APIConnectionError(f"Error connecting to {ip}: {err}")
-            self._fatal_error(err)
-            raise err
-        self._socket.settimeout(0.1)
-
-        self._socket_open_event.set()
-
-        hello = pb.HelloRequest()
-        hello.client_info = f"ESPHome v{const.__version__}"
-        try:
-            resp = self._send_message_await_response(hello, pb.HelloResponse)
-        except APIConnectionError as err:
-            self._fatal_error(err)
-            raise err
-        _LOGGER.debug(
-            "Successfully connected to %s ('%s' API=%s.%s)",
-            self._address,
-            resp.server_info,
-            resp.api_version_major,
-            resp.api_version_minor,
-        )
-        self._connected = True
-        self._refresh_ping()
-        if self.on_connect is not None:
-            self.on_connect()
-
-    def _check_connected(self):
-        if not self._connected:
-            err = APIConnectionError("Must be connected!")
-            self._fatal_error(err)
-            raise err
-
-    def login(self):
-        self._check_connected()
-        if self._authenticated:
-            raise APIConnectionError("Already logged in!")
-
-        connect = pb.ConnectRequest()
-        if self._password is not None:
-            connect.password = self._password
-        resp = self._send_message_await_response(connect, pb.ConnectResponse)
-        if resp.invalid_password:
-            raise APIConnectionError("Invalid password!")
-
-        self._authenticated = True
-        if self.on_login is not None:
-            self.on_login()
-
-    def _fatal_error(self, err):
-        was_connected = self._connected
-
-        self._close_socket()
-
-        if was_connected and self.on_disconnect is not None:
-            self.on_disconnect(err)
-
-    def _write(self, data):  # type: (bytes) -> None
-        if self._socket is None:
-            raise APIConnectionError("Socket closed")
-
-        # _LOGGER.debug("Write: %s", format_bytes(data))
-        with self._socket_write_lock:
-            try:
-                self._socket.sendall(data)
-            except OSError as err:
-                err = APIConnectionError(f"Error while writing data: {err}")
-                self._fatal_error(err)
-                raise err
-
-    def _send_message(self, msg):
-        # type: (message.Message) -> None
-        for message_type, klass in MESSAGE_TYPE_TO_PROTO.items():
-            if isinstance(msg, klass):
-                break
-        else:
-            raise ValueError
-
-        encoded = msg.SerializeToString()
-        _LOGGER.debug("Sending %s:\n%s", type(msg), indent(str(msg)))
-        req = bytes([0])
-        req += _varuint_to_bytes(len(encoded))
-        req += _varuint_to_bytes(message_type)
-        req += encoded
-        self._write(req)
-
-    def _send_message_await_response_complex(
-        self, send_msg, do_append, do_stop, timeout=5
-    ):
-        event = threading.Event()
-        responses = []
-
-        def on_message(resp):
-            if do_append(resp):
-                responses.append(resp)
-            if do_stop(resp):
-                event.set()
-
-        self._message_handlers.append(on_message)
-        self._send_message(send_msg)
-        ret = event.wait(timeout)
-        try:
-            self._message_handlers.remove(on_message)
-        except ValueError:
-            pass
-        if not ret:
-            raise APIConnectionError("Timeout while waiting for message response!")
-        return responses
-
-    def _send_message_await_response(self, send_msg, response_type, timeout=5):
-        def is_response(msg):
-            return isinstance(msg, response_type)
-
-        return self._send_message_await_response_complex(
-            send_msg, is_response, is_response, timeout
-        )[0]
-
-    def device_info(self):
-        self._check_connected()
-        return self._send_message_await_response(
-            pb.DeviceInfoRequest(), pb.DeviceInfoResponse
-        )
-
-    def ping(self):
-        self._check_connected()
-        return self._send_message_await_response(pb.PingRequest(), pb.PingResponse)
-
-    def disconnect(self, on_disconnect=True):
-        self._check_connected()
-
-        try:
-            self._send_message_await_response(
-                pb.DisconnectRequest(), pb.DisconnectResponse
-            )
-        except APIConnectionError:
-            pass
-        self._close_socket()
-
-        if self.on_disconnect is not None and on_disconnect:
-            self.on_disconnect(None)
-
-    def _check_authenticated(self):
-        if not self._authenticated:
-            raise APIConnectionError("Must login first!")
-
-    def subscribe_logs(self, on_log, log_level=7, dump_config=False):
-        self._check_authenticated()
-
-        def on_msg(msg):
-            if isinstance(msg, pb.SubscribeLogsResponse):
-                on_log(msg)
-
-        self._message_handlers.append(on_msg)
-        req = pb.SubscribeLogsRequest(dump_config=dump_config)
-        req.level = log_level
-        self._send_message(req)
-
-    def _recv(self, amount):
-        ret = bytes()
-        if amount == 0:
-            return ret
-
-        while len(ret) < amount:
-            if self.stopped:
-                raise APIConnectionError("Stopped!")
-            if not self._socket_open_event.is_set():
-                raise APIConnectionError("No socket!")
-            try:
-                val = self._socket.recv(amount - len(ret))
-            except AttributeError as err:
-                raise APIConnectionError("Socket was closed") from err
-            except socket.timeout:
-                continue
-            except OSError as err:
-                raise APIConnectionError(f"Error while receiving data: {err}") from err
-            ret += val
-        return ret
-
-    def _recv_varint(self):
-        raw = bytes()
-        while not raw or raw[-1] & 0x80:
-            raw += self._recv(1)
-        return _bytes_to_varuint(raw)
-
-    def _run_once(self):
-        if not self._socket_open_event.wait(0.1):
-            return
-
-        # Preamble
-        if self._recv(1)[0] != 0x00:
-            raise APIConnectionError("Invalid preamble")
-
-        length = self._recv_varint()
-        msg_type = self._recv_varint()
-
-        raw_msg = self._recv(length)
-        if msg_type not in MESSAGE_TYPE_TO_PROTO:
-            _LOGGER.debug("Skipping message type %s", msg_type)
-            return
-
-        msg = MESSAGE_TYPE_TO_PROTO[msg_type]()
-        msg.ParseFromString(raw_msg)
-        _LOGGER.debug("Got message: %s:\n%s", type(msg), indent(str(msg)))
-        for msg_handler in self._message_handlers[:]:
-            msg_handler(msg)
-        self._handle_internal_messages(msg)
-
-    def run(self):
-        self._running_event.set()
-        while not self.stopped:
-            try:
-                self._run_once()
-            except APIConnectionError as err:
-                if self.stopped:
-                    break
-                if self._connected:
-                    _LOGGER.error("Error while reading incoming messages: %s", err)
-                    self._fatal_error(err)
-        self._running_event.clear()
-
-    def _handle_internal_messages(self, msg):
-        if isinstance(msg, pb.DisconnectRequest):
-            self._send_message(pb.DisconnectResponse())
-            if self._socket is not None:
-                self._socket.close()
-                self._socket = None
-            self._connected = False
-            if self.on_disconnect is not None:
-                self.on_disconnect(None)
-        elif isinstance(msg, pb.PingRequest):
-            self._send_message(pb.PingResponse())
-        elif isinstance(msg, pb.GetTimeRequest):
-            resp = pb.GetTimeResponse()
-            resp.epoch_seconds = int(time.time())
-            self._send_message(resp)
-
-
-def run_logs(config, address):
-    conf = config["api"]
-    port = conf[CONF_PORT]
-    password = conf[CONF_PASSWORD]
-    _LOGGER.info("Starting log output from %s using esphome API", address)
-
-    cli = APIClient(address, port, password)
-    stopping = False
-    retry_timer = []
-
-    has_connects = []
-
-    def try_connect(err, tries=0):
-        if stopping:
-            return
-
-        if err:
-            _LOGGER.warning("Disconnected from API: %s", err)
-
-        while retry_timer:
-            retry_timer.pop(0).cancel()
-
-        error = None
-        try:
-            cli.connect()
-            cli.login()
-        except APIConnectionError as err2:  # noqa
-            error = err2
-
-        if error is None:
-            _LOGGER.info("Successfully connected to %s", address)
-            return
-
-        wait_time = int(min(1.5 ** min(tries, 100), 30))
-        if not has_connects:
-            _LOGGER.warning(
-                "Initial connection failed. The ESP might not be connected "
-                "to WiFi yet (%s). Re-Trying in %s seconds",
-                error,
-                wait_time,
-            )
-        else:
-            _LOGGER.warning(
-                "Couldn't connect to API (%s). Trying to reconnect in %s seconds",
-                error,
-                wait_time,
-            )
-        timer = threading.Timer(
-            wait_time, functools.partial(try_connect, None, tries + 1)
-        )
-        timer.start()
-        retry_timer.append(timer)
-
-    def on_log(msg):
-        time_ = datetime.now().time().strftime("[%H:%M:%S]")
-        text = msg.message
-        if msg.send_failed:
-            text = color(
-                Fore.WHITE,
-                "(Message skipped because it was too big to fit in "
-                "TCP buffer - This is only cosmetic)",
-            )
-        safe_print(time_ + text)
-
-    def on_login():
-        try:
-            cli.subscribe_logs(on_log, dump_config=not has_connects)
-            has_connects.append(True)
-        except APIConnectionError:
-            cli.disconnect()
-
-    cli.on_disconnect = try_connect
-    cli.on_login = on_login
-    cli.start()
-
-    try:
-        try_connect(None)
-        while True:
-            time.sleep(1)
-    except KeyboardInterrupt:
-        stopping = True
-        cli.stop(True)
-        while retry_timer:
-            retry_timer.pop(0).cancel()
-    return 0

esphome/components/api/api_connection.cpp

@@ -36,19 +36,14 @@ void APIConnection::start() {
 
   APIError err = helper_->init();
   if (err != APIError::OK) {
-    ESP_LOGW(TAG, "Helper init failed: %d errno=%d", (int) err, errno);
-    remove_ = true;
+    on_fatal_error();
+    ESP_LOGW(TAG, "%s: Helper init failed: %s errno=%d", client_info_.c_str(), api_error_to_str(err), errno);
     return;
   }
   client_info_ = helper_->getpeername();
   helper_->set_log_info(client_info_);
 }
 
-void APIConnection::force_disconnect_client() {
-  this->helper_->close();
-  this->remove_ = true;
-}
-
 void APIConnection::loop() {
   if (this->remove_)
     return;
@@ -57,9 +52,11 @@ void APIConnection::loop() {
     // when network is disconnected force disconnect immediately
     // don't wait for timeout
     this->on_fatal_error();
+    ESP_LOGW(TAG, "%s: Network unavailable, disconnecting", client_info_.c_str());
     return;
   }
   if (this->next_close_) {
+    // requested a disconnect
     this->helper_->close();
     this->remove_ = true;
     return;
@@ -68,7 +65,7 @@ void APIConnection::loop() {
   APIError err = helper_->loop();
   if (err != APIError::OK) {
     on_fatal_error();
-    ESP_LOGW(TAG, "%s: Socket operation failed: %d", client_info_.c_str(), (int) err);
+    ESP_LOGW(TAG, "%s: Socket operation failed: %s errno=%d", client_info_.c_str(), api_error_to_str(err), errno);
     return;
   }
   ReadPacketBuffer buffer;
@@ -77,7 +74,11 @@ void APIConnection::loop() {
     // pass
   } else if (err != APIError::OK) {
     on_fatal_error();
-    ESP_LOGW(TAG, "%s: Reading failed: %d", client_info_.c_str(), (int) err);
+    if (err == APIError::SOCKET_READ_FAILED && errno == ECONNRESET) {
+      ESP_LOGW(TAG, "%s: Connection reset", client_info_.c_str());
+    } else {
+      ESP_LOGW(TAG, "%s: Reading failed: %s errno=%d", client_info_.c_str(), api_error_to_str(err), errno);
+    }
     return;
   } else {
     this->last_traffic_ = millis();
@@ -95,8 +96,8 @@ void APIConnection::loop() {
   if (this->sent_ping_) {
     // Disconnect if not responded within 2.5*keepalive
     if (now - this->last_traffic_ > (keepalive * 5) / 2) {
-      this->force_disconnect_client();
-      ESP_LOGW(TAG, "'%s' didn't respond to ping request in time. Disconnecting...", this->client_info_.c_str());
+      on_fatal_error();
+      ESP_LOGW(TAG, "%s didn't respond to ping request in time. Disconnecting...", this->client_info_.c_str());
     }
   } else if (now - this->last_traffic_ > keepalive) {
     this->sent_ping_ = true;
@@ -124,12 +125,40 @@ void APIConnection::loop() {
     }
   }
 #endif
+
+  if (state_subs_at_ != -1) {
+    const auto &subs = this->parent_->get_state_subs();
+    if (state_subs_at_ >= subs.size()) {
+      state_subs_at_ = -1;
+    } else {
+      auto &it = subs[state_subs_at_];
+      SubscribeHomeAssistantStateResponse resp;
+      resp.entity_id = it.entity_id;
+      resp.attribute = it.attribute.value();
+      if (this->send_subscribe_home_assistant_state_response(resp)) {
+        state_subs_at_++;
+      }
+    }
+  }
 }
 
 std::string get_default_unique_id(const std::string &component_type, Nameable *nameable) {
   return App.get_name() + component_type + nameable->get_object_id();
 }
 
+DisconnectResponse APIConnection::disconnect(const DisconnectRequest &msg) {
+  // remote initiated disconnect_client
+  // don't close yet, we still need to send the disconnect response
+  // close will happen on next loop
+  ESP_LOGD(TAG, "%s requested disconnected", client_info_.c_str());
+  this->next_close_ = true;
+  DisconnectResponse resp;
+  return resp;
+}
+void APIConnection::on_disconnect_response(const DisconnectResponse &value) {
+  // pass
+}
+
 #ifdef USE_BINARY_SENSOR
 bool APIConnection::send_binary_sensor_state(binary_sensor::BinarySensor *binary_sensor, bool state) {
   if (!this->state_subscription_)
@@ -214,6 +243,9 @@ void APIConnection::cover_command(const CoverCommandRequest &msg) {
 #endif
 
 #ifdef USE_FAN
+// Shut-up about usage of deprecated speed_level_to_enum/speed_enum_to_level functions for a bit.
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wdeprecated-declarations"
 bool APIConnection::send_fan_state(fan::FanState *fan) {
   if (!this->state_subscription_)
     return false;
@@ -262,13 +294,13 @@ void APIConnection::fan_command(const FanCommandRequest &msg) {
     // Prefer level
     call.set_speed(msg.speed_level);
   } else if (msg.has_speed) {
-    // NOLINTNEXTLINE(clang-diagnostic-deprecated-declarations)
     call.set_speed(fan::speed_enum_to_level(static_cast<fan::FanSpeed>(msg.speed), traits.supported_speed_count()));
   }
   if (msg.has_direction)
     call.set_direction(static_cast<fan::FanDirection>(msg.direction));
   call.perform();
 }
+#pragma GCC diagnostic pop
 #endif
 
 #ifdef USE_LIGHT
@@ -700,7 +732,7 @@ ConnectResponse APIConnection::connect(const ConnectRequest &msg) {
   // bool invalid_password = 1;
   resp.invalid_password = !correct;
   if (correct) {
-    ESP_LOGD(TAG, "Client '%s' connected successfully!", this->client_info_.c_str());
+    ESP_LOGD(TAG, "%s: Connected successfully", this->client_info_.c_str());
     this->connection_state_ = ConnectionState::AUTHENTICATED;
 
 #ifdef USE_HOMEASSISTANT_TIME
@@ -746,15 +778,7 @@ void APIConnection::execute_service(const ExecuteServiceRequest &msg) {
   }
 }
 void APIConnection::subscribe_home_assistant_states(const SubscribeHomeAssistantStatesRequest &msg) {
-  for (auto &it : this->parent_->get_state_subs()) {
-    SubscribeHomeAssistantStateResponse resp;
-    resp.entity_id = it.entity_id;
-    resp.attribute = it.attribute.value();
-    if (!this->send_subscribe_home_assistant_state_response(resp)) {
-      this->on_fatal_error();
-      return;
-    }
-  }
+  state_subs_at_ = 0;
 }
 bool APIConnection::send_buffer(ProtoWriteBuffer buffer, uint32_t message_type) {
   if (this->remove_)
@@ -767,7 +791,11 @@ bool APIConnection::send_buffer(ProtoWriteBuffer buffer, uint32_t message_type)
     return false;
   if (err != APIError::OK) {
     on_fatal_error();
-    ESP_LOGW(TAG, "%s: Packet write failed %d errno=%d", client_info_.c_str(), (int) err, errno);
+    if (err == APIError::SOCKET_WRITE_FAILED && errno == ECONNRESET) {
+      ESP_LOGW(TAG, "%s: Connection reset", client_info_.c_str());
+    } else {
+      ESP_LOGW(TAG, "%s: Packet write failed %s errno=%d", client_info_.c_str(), api_error_to_str(err), errno);
+    }
     return false;
   }
   this->last_traffic_ = millis();
@@ -775,14 +803,13 @@ bool APIConnection::send_buffer(ProtoWriteBuffer buffer, uint32_t message_type)
 }
 void APIConnection::on_unauthenticated_access() {
   this->on_fatal_error();
-  ESP_LOGD(TAG, "'%s' tried to access without authentication.", this->client_info_.c_str());
+  ESP_LOGD(TAG, "%s: tried to access without authentication.", this->client_info_.c_str());
 }
 void APIConnection::on_no_setup_connection() {
   this->on_fatal_error();
-  ESP_LOGD(TAG, "'%s' tried to access without full connection.", this->client_info_.c_str());
+  ESP_LOGD(TAG, "%s: tried to access without full connection.", this->client_info_.c_str());
 }
 void APIConnection::on_fatal_error() {
-  ESP_LOGV(TAG, "Error: Disconnecting %s", this->client_info_.c_str());
   this->helper_->close();
   this->remove_ = true;
 }

esphome/components/api/api_connection.h

@@ -16,7 +16,6 @@ class APIConnection : public APIServerConnection {
   virtual ~APIConnection() = default;
 
   void start();
-  void force_disconnect_client();
   void loop();
 
   bool send_list_info_done() {
@@ -88,10 +87,7 @@ class APIConnection : public APIServerConnection {
   }
 #endif
 
-  void on_disconnect_response(const DisconnectResponse &value) override {
-    this->helper_->close();
-    this->remove_ = true;
-  }
+  void on_disconnect_response(const DisconnectResponse &value) override;
   void on_ping_response(const PingResponse &value) override {
     // we initiated ping
     this->sent_ping_ = false;
@@ -102,14 +98,7 @@ class APIConnection : public APIServerConnection {
 #endif
   HelloResponse hello(const HelloRequest &msg) override;
   ConnectResponse connect(const ConnectRequest &msg) override;
-  DisconnectResponse disconnect(const DisconnectRequest &msg) override {
-    // remote initiated disconnect_client
-    // don't close yet, we still need to send the disconnect response
-    // close will happen on next loop
-    this->next_close_ = true;
-    DisconnectResponse resp;
-    return resp;
-  }
+  DisconnectResponse disconnect(const DisconnectRequest &msg) override;
   PingResponse ping(const PingRequest &msg) override { return {}; }
   DeviceInfoResponse device_info(const DeviceInfoRequest &msg) override;
   void list_entities(const ListEntitiesRequest &msg) override { this->list_entities_iterator_.begin(); }
@@ -177,6 +166,7 @@ class APIConnection : public APIServerConnection {
   APIServer *parent_;
   InitialStateIterator initial_state_iterator_;
   ListEntitiesIterator list_entities_iterator_;
+  int state_subs_at_ = -1;
 };
 
 }  // namespace api

esphome/components/api/api_frame_helper.cpp

@@ -17,7 +17,59 @@ bool is_would_block(ssize_t ret) {
   return ret == 0;
 }
 
+const char *api_error_to_str(APIError err) {
+  // not using switch to ensure compiler doesn't try to build a big table out of it
+  if (err == APIError::OK) {
+    return "OK";
+  } else if (err == APIError::WOULD_BLOCK) {
+    return "WOULD_BLOCK";
+  } else if (err == APIError::BAD_HANDSHAKE_PACKET_LEN) {
+    return "BAD_HANDSHAKE_PACKET_LEN";
+  } else if (err == APIError::BAD_INDICATOR) {
+    return "BAD_INDICATOR";
+  } else if (err == APIError::BAD_DATA_PACKET) {
+    return "BAD_DATA_PACKET";
+  } else if (err == APIError::TCP_NODELAY_FAILED) {
+    return "TCP_NODELAY_FAILED";
+  } else if (err == APIError::TCP_NONBLOCKING_FAILED) {
+    return "TCP_NONBLOCKING_FAILED";
+  } else if (err == APIError::CLOSE_FAILED) {
+    return "CLOSE_FAILED";
+  } else if (err == APIError::SHUTDOWN_FAILED) {
+    return "SHUTDOWN_FAILED";
+  } else if (err == APIError::BAD_STATE) {
+    return "BAD_STATE";
+  } else if (err == APIError::BAD_ARG) {
+    return "BAD_ARG";
+  } else if (err == APIError::SOCKET_READ_FAILED) {
+    return "SOCKET_READ_FAILED";
+  } else if (err == APIError::SOCKET_WRITE_FAILED) {
+    return "SOCKET_WRITE_FAILED";
+  } else if (err == APIError::HANDSHAKESTATE_READ_FAILED) {
+    return "HANDSHAKESTATE_READ_FAILED";
+  } else if (err == APIError::HANDSHAKESTATE_WRITE_FAILED) {
+    return "HANDSHAKESTATE_WRITE_FAILED";
+  } else if (err == APIError::HANDSHAKESTATE_BAD_STATE) {
+    return "HANDSHAKESTATE_BAD_STATE";
+  } else if (err == APIError::CIPHERSTATE_DECRYPT_FAILED) {
+    return "CIPHERSTATE_DECRYPT_FAILED";
+  } else if (err == APIError::CIPHERSTATE_ENCRYPT_FAILED) {
+    return "CIPHERSTATE_ENCRYPT_FAILED";
+  } else if (err == APIError::OUT_OF_MEMORY) {
+    return "OUT_OF_MEMORY";
+  } else if (err == APIError::HANDSHAKESTATE_SETUP_FAILED) {
+    return "HANDSHAKESTATE_SETUP_FAILED";
+  } else if (err == APIError::HANDSHAKESTATE_SPLIT_FAILED) {
+    return "HANDSHAKESTATE_SPLIT_FAILED";
+  } else if (err == APIError::BAD_HANDSHAKE_ERROR_BYTE) {
+    return "BAD_HANDSHAKE_ERROR_BYTE";
+  }
+  return "UNKNOWN";
+}
+
 #define HELPER_LOG(msg, ...) ESP_LOGVV(TAG, "%s: " msg, info_.c_str(), ##__VA_ARGS__)
+// uncomment to log raw packets
+//#define HELPER_LOG_PACKETS
 
 #ifdef USE_API_NOISE
 static const char *const PROLOGUE_INIT = "NoiseAPIInit";
@@ -188,7 +240,9 @@ APIError APINoiseFrameHelper::try_read_frame_(ParsedFrame *frame) {
   }
 
   // uncomment for even more debugging
-  // ESP_LOGVV(TAG, "Received frame: %s", hexencode(rx_buf_).c_str());
+#ifdef HELPER_LOG_PACKETS
+  ESP_LOGVV(TAG, "Received frame: %s", hexencode(rx_buf_).c_str());
+#endif
   frame->msg = std::move(rx_buf_);
   // consume msg
   rx_buf_ = {};
@@ -217,6 +271,14 @@ APIError APINoiseFrameHelper::state_action_() {
     // waiting for client hello
     ParsedFrame frame;
     aerr = try_read_frame_(&frame);
+    if (aerr == APIError::BAD_INDICATOR) {
+      send_explicit_handshake_reject_("Bad indicator byte");
+      return aerr;
+    }
+    if (aerr == APIError::BAD_HANDSHAKE_PACKET_LEN) {
+      send_explicit_handshake_reject_("Bad handshake packet len");
+      return aerr;
+    }
     if (aerr != APIError::OK)
       return aerr;
     // ignore contents, may be used in future for flags
@@ -260,11 +322,11 @@ APIError APINoiseFrameHelper::state_action_() {
 
       if (frame.msg.empty()) {
         send_explicit_handshake_reject_("Empty handshake message");
-        return APIError::BAD_HANDSHAKE_PACKET_LEN;
+        return APIError::BAD_HANDSHAKE_ERROR_BYTE;
       } else if (frame.msg[0] != 0x00) {
         HELPER_LOG("Bad handshake error byte: %u", frame.msg[0]);
         send_explicit_handshake_reject_("Bad handshake error byte");
-        return APIError::BAD_HANDSHAKE_PACKET_LEN;
+        return APIError::BAD_HANDSHAKE_ERROR_BYTE;
       }
 
       NoiseBuffer mbuf;
@@ -272,7 +334,6 @@ APIError APINoiseFrameHelper::state_action_() {
       noise_buffer_set_input(mbuf, frame.msg.data() + 1, frame.msg.size() - 1);
       err = noise_handshakestate_read_message(handshake_, &mbuf, nullptr);
       if (err != 0) {
-        // TODO: explicit rejection
         state_ = State::FAILED;
         HELPER_LOG("noise_handshakestate_read_message failed: %s", noise_err_to_str(err).c_str());
         if (err == NOISE_ERROR_MAC_FAILURE) {
@@ -320,12 +381,16 @@ APIError APINoiseFrameHelper::state_action_() {
 }
 void APINoiseFrameHelper::send_explicit_handshake_reject_(const std::string &reason) {
   std::vector<uint8_t> data;
-  data.reserve(reason.size() + 1);
+  data.resize(reason.length() + 1);
   data[0] = 0x01;  // failure
-  for (size_t i = 0; i < reason.size(); i++) {
+  for (size_t i = 0; i < reason.length(); i++) {
     data[i + 1] = (uint8_t) reason[i];
   }
+  // temporarily remove failed state
+  auto orig_state = state_;
+  state_ = State::EXPLICIT_REJECT;
   write_frame_(data.data(), data.size());
+  state_ = orig_state;
 }
 
 APIError APINoiseFrameHelper::read_packet(ReadPacketBuffer *buffer) {
@@ -468,7 +533,9 @@ APIError APINoiseFrameHelper::write_raw_(const uint8_t *data, size_t len) {
   APIError aerr;
 
   // uncomment for even more debugging
-  // ESP_LOGVV(TAG, "Sending raw: %s", hexencode(data, len).c_str());
+#ifdef HELPER_LOG_PACKETS
+  ESP_LOGVV(TAG, "Sending raw: %s", hexencode(data, len).c_str());
+#endif
 
   if (!tx_buf_.empty()) {
     // try to empty tx_buf_ first
@@ -751,7 +818,9 @@ APIError APIPlaintextFrameHelper::try_read_frame_(ParsedFrame *frame) {
   }
 
   // uncomment for even more debugging
-  // ESP_LOGVV(TAG, "Received frame: %s", hexencode(rx_buf_).c_str());
+#ifdef HELPER_LOG_PACKETS
+  ESP_LOGVV(TAG, "Received frame: %s", hexencode(rx_buf_).c_str());
+#endif
   frame->msg = std::move(rx_buf_);
   // consume msg
   rx_buf_ = {};
@@ -808,14 +877,12 @@ APIError APIPlaintextFrameHelper::try_send_tx_buf_() {
   // try send from tx_buf
   while (state_ != State::CLOSED && !tx_buf_.empty()) {
     ssize_t sent = socket_->write(tx_buf_.data(), tx_buf_.size());
-    if (sent == -1) {
-      if (errno == EWOULDBLOCK || errno == EAGAIN)
-        break;
+    if (is_would_block(sent)) {
+      break;
+    } else if (sent == -1) {
       state_ = State::FAILED;
       HELPER_LOG("Socket write failed with errno %d", errno);
       return APIError::SOCKET_WRITE_FAILED;
-    } else if (sent == 0) {
-      break;
     }
     // TODO: inefficient if multiple packets in txbuf
     // replace with deque of buffers
@@ -836,7 +903,9 @@ APIError APIPlaintextFrameHelper::write_raw_(const uint8_t *data, size_t len) {
   APIError aerr;
 
   // uncomment for even more debugging
-  // ESP_LOGVV(TAG, "Sending raw: %s", hexencode(data, len).c_str());
+#ifdef HELPER_LOG_PACKETS
+  ESP_LOGVV(TAG, "Sending raw: %s", hexencode(data, len).c_str());
+#endif
 
   if (!tx_buf_.empty()) {
     // try to empty tx_buf_ first
@@ -869,20 +938,6 @@ APIError APIPlaintextFrameHelper::write_raw_(const uint8_t *data, size_t len) {
   // fully sent
   return APIError::OK;
 }
-APIError APIPlaintextFrameHelper::write_frame_(const uint8_t *data, size_t len) {
-  APIError aerr;
-
-  uint8_t header[3];
-  header[0] = 0x01;  // indicator
-  header[1] = (uint8_t)(len >> 8);
-  header[2] = (uint8_t) len;
-
-  aerr = write_raw_(header, 3);
-  if (aerr != APIError::OK)
-    return aerr;
-  aerr = write_raw_(data, len);
-  return aerr;
-}
 
 APIError APIPlaintextFrameHelper::close() {
   state_ = State::CLOSED;

esphome/components/api/api_frame_helper.h

@@ -51,8 +51,11 @@ enum class APIError : int {
   OUT_OF_MEMORY = 1018,
   HANDSHAKESTATE_SETUP_FAILED = 1019,
   HANDSHAKESTATE_SPLIT_FAILED = 1020,
+  BAD_HANDSHAKE_ERROR_BYTE = 1021,
 };
 
+const char *api_error_to_str(APIError err);
+
 class APIFrameHelper {
  public:
   virtual APIError init() = 0;
@@ -123,6 +126,7 @@ class APINoiseFrameHelper : public APIFrameHelper {
     DATA = 5,
     CLOSED = 6,
     FAILED = 7,
+    EXPLICIT_REJECT = 8,
   } state_ = State::INITIALIZE;
 };
 #endif  // USE_API_NOISE
@@ -150,7 +154,6 @@ class APIPlaintextFrameHelper : public APIFrameHelper {
 
   APIError try_read_frame_(ParsedFrame *frame);
   APIError try_send_tx_buf_();
-  APIError write_frame_(const uint8_t *data, size_t len);
   APIError write_raw_(const uint8_t *data, size_t len);
 
   std::unique_ptr<socket::Socket> socket_;

esphome/components/api/api_server.cpp

@@ -104,7 +104,7 @@ void APIServer::loop() {
       std::partition(this->clients_.begin(), this->clients_.end(), [](APIConnection *conn) { return !conn->remove_; });
   // print disconnection messages
   for (auto it = new_end; it != this->clients_.end(); ++it) {
-    ESP_LOGD(TAG, "Disconnecting %s", (*it)->client_info_.c_str());
+    ESP_LOGV(TAG, "Removing connection to %s", (*it)->client_info_.c_str());
   }
   // only then delete the pointers, otherwise log routine
   // would access freed memory

esphome/components/api/client.py

@@ -0,0 +1,73 @@
+import asyncio
+import logging
+from datetime import datetime
+from typing import Optional
+
+from aioesphomeapi import APIClient, ReconnectLogic, APIConnectionError, LogLevel
+import zeroconf
+
+from esphome.const import CONF_KEY, CONF_PORT, CONF_PASSWORD, __version__
+from esphome.util import safe_print
+from . import CONF_ENCRYPTION
+
+_LOGGER = logging.getLogger(__name__)
+
+
+async def async_run_logs(config, address):
+    conf = config["api"]
+    port: int = conf[CONF_PORT]
+    password: str = conf[CONF_PASSWORD]
+    noise_psk: Optional[str] = None
+    if CONF_ENCRYPTION in conf:
+        noise_psk = conf[CONF_ENCRYPTION][CONF_KEY]
+    _LOGGER.info("Starting log output from %s using esphome API", address)
+    zc = zeroconf.Zeroconf()
+    cli = APIClient(
+        asyncio.get_event_loop(),
+        address,
+        port,
+        password,
+        client_info=f"ESPHome Logs {__version__}",
+        noise_psk=noise_psk,
+    )
+    first_connect = True
+
+    def on_log(msg):
+        time_ = datetime.now().time().strftime("[%H:%M:%S]")
+        text = msg.message.decode("utf8", "backslashreplace")
+        safe_print(time_ + text)
+
+    async def on_connect():
+        nonlocal first_connect
+        try:
+            await cli.subscribe_logs(
+                on_log,
+                log_level=LogLevel.LOG_LEVEL_VERY_VERBOSE,
+                dump_config=first_connect,
+            )
+            first_connect = False
+        except APIConnectionError:
+            cli.disconnect()
+
+    async def on_disconnect():
+        _LOGGER.warning("Disconnected from API")
+
+    zc = zeroconf.Zeroconf()
+    reconnect = ReconnectLogic(
+        client=cli,
+        on_connect=on_connect,
+        on_disconnect=on_disconnect,
+        zeroconf_instance=zc,
+    )
+    await reconnect.start()
+
+    try:
+        while True:
+            await asyncio.sleep(60)
+    except KeyboardInterrupt:
+        await reconnect.stop()
+        zc.close()
+
+
+def run_logs(config, address):
+    asyncio.run(async_run_logs(config, address))

esphome/components/fan/fan_helpers.cpp

@@ -4,14 +4,15 @@
 namespace esphome {
 namespace fan {
 
-// NOLINTNEXTLINE(clang-diagnostic-deprecated-declarations)
+// This whole file is deprecated, don't warn about usage of deprecated types in here.
+#pragma GCC diagnostic ignored "-Wdeprecated-declarations"
+
 FanSpeed speed_level_to_enum(int speed_level, int supported_speed_levels) {
   const auto speed_ratio = static_cast<float>(speed_level) / (supported_speed_levels + 1);
   const auto legacy_level = clamp<int>(static_cast<int>(ceilf(speed_ratio * 3)), 1, 3);
-  return static_cast<FanSpeed>(legacy_level - 1);  // NOLINT(clang-diagnostic-deprecated-declarations)
+  return static_cast<FanSpeed>(legacy_level - 1);
 }
 
-// NOLINTNEXTLINE(clang-diagnostic-deprecated-declarations)
 int speed_enum_to_level(FanSpeed speed, int supported_speed_levels) {
   const auto enum_level = static_cast<int>(speed) + 1;
   const auto speed_level = roundf(enum_level / 3.0f * supported_speed_levels);

esphome/components/fan/fan_helpers.h

@@ -4,8 +4,16 @@
 namespace esphome {
 namespace fan {
 
+// Shut-up about usage of deprecated FanSpeed for a bit.
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wdeprecated-declarations"
+
+ESPDEPRECATED("FanSpeed and speed_level_to_enum() are deprecated.", "2021.9")
 FanSpeed speed_level_to_enum(int speed_level, int supported_speed_levels);
+ESPDEPRECATED("FanSpeed and speed_enum_to_level() are deprecated.", "2021.9")
 int speed_enum_to_level(FanSpeed speed, int supported_speed_levels);
 
+#pragma GCC diagnostic pop
+
 }  // namespace fan
 }  // namespace esphome

esphome/components/fan/fan_state.cpp

@@ -67,6 +67,8 @@ void FanStateCall::perform() const {
   this->state_->state_callback_.call();
 }
 
+// This whole method is deprecated, don't warn about usage of deprecated methods inside of it.
+#pragma GCC diagnostic ignored "-Wdeprecated-declarations"
 FanStateCall &FanStateCall::set_speed(const char *legacy_speed) {
   const auto supported_speed_count = this->state_->get_traits().supported_speed_count();
   if (strcasecmp(legacy_speed, "low") == 0) {

esphome/components/light/base_light_effects.h

@@ -156,7 +156,7 @@ class StrobeLightEffect : public LightEffect {
 
     if (!color.is_on()) {
       // Don't turn the light off, otherwise the light effect will be stopped
-      call.set_brightness_if_supported(0.0f);
+      call.set_brightness(0.0f);
       call.set_state(true);
     }
     call.set_publish(false);

esphome/components/mqtt/mqtt_fan.cpp

@@ -100,6 +100,7 @@ bool MQTTFanComponent::publish_state() {
   auto traits = this->state_->get_traits();
   if (traits.supports_speed()) {
     const char *payload;
+    // NOLINTNEXTLINE(clang-diagnostic-deprecated-declarations)
     switch (fan::speed_level_to_enum(this->state_->speed, traits.supported_speed_count())) {
       case FAN_SPEED_LOW: {  // NOLINT(clang-diagnostic-deprecated-declarations)
         payload = "low";

esphome/components/sntp/time.py

@@ -16,7 +16,7 @@ CONFIG_SCHEMA = time_.TIME_SCHEMA.extend(
     {
         cv.GenerateID(): cv.declare_id(SNTPComponent),
         cv.Optional(CONF_SERVERS, default=DEFAULT_SERVERS): cv.All(
-            cv.ensure_list(cv.domain), cv.Length(min=1, max=3)
+            cv.ensure_list(cv.Any(cv.domain, cv.hostname)), cv.Length(min=1, max=3)
         ),
     }
 ).extend(cv.COMPONENT_SCHEMA)

esphome/components/socket/lwip_raw_tcp_impl.cpp

@@ -109,14 +109,17 @@ class LWIPRawImpl : public Socket {
     LWIP_LOG("tcp_bind(%p ip=%u port=%u)", pcb_, ip.addr, port);
     err_t err = tcp_bind(pcb_, &ip, port);
     if (err == ERR_USE) {
+      LWIP_LOG("  -> err ERR_USE");
       errno = EADDRINUSE;
       return -1;
     }
     if (err == ERR_VAL) {
+      LWIP_LOG("  -> err ERR_VAL");
       errno = EINVAL;
       return -1;
     }
     if (err != ERR_OK) {
+      LWIP_LOG("  -> err %d", err);
       errno = EIO;
       return -1;
     }
@@ -124,12 +127,13 @@ class LWIPRawImpl : public Socket {
   }
   int close() override {
     if (pcb_ == nullptr) {
-      errno = EBADF;
+      errno = ECONNRESET;
       return -1;
     }
     LWIP_LOG("tcp_close(%p)", pcb_);
     err_t err = tcp_close(pcb_);
     if (err != ERR_OK) {
+      LWIP_LOG("  -> err %d", err);
       tcp_abort(pcb_);
       pcb_ = nullptr;
       errno = err == ERR_MEM ? ENOMEM : EIO;
@@ -140,7 +144,7 @@ class LWIPRawImpl : public Socket {
   }
   int shutdown(int how) override {
     if (pcb_ == nullptr) {
-      errno = EBADF;
+      errno = ECONNRESET;
       return -1;
     }
     bool shut_rx = false, shut_tx = false;
@@ -157,6 +161,7 @@ class LWIPRawImpl : public Socket {
     LWIP_LOG("tcp_shutdown(%p shut_rx=%d shut_tx=%d)", pcb_, shut_rx ? 1 : 0, shut_tx ? 1 : 0);
     err_t err = tcp_shutdown(pcb_, shut_rx, shut_tx);
     if (err != ERR_OK) {
+      LWIP_LOG("  -> err %d", err);
       errno = err == ERR_MEM ? ENOMEM : EIO;
       return -1;
     }
@@ -165,7 +170,7 @@ class LWIPRawImpl : public Socket {
 
   int getpeername(struct sockaddr *name, socklen_t *addrlen) override {
     if (pcb_ == nullptr) {
-      errno = EBADF;
+      errno = ECONNRESET;
       return -1;
     }
     if (name == nullptr || addrlen == nullptr) {
@@ -185,7 +190,7 @@ class LWIPRawImpl : public Socket {
   }
   std::string getpeername() override {
     if (pcb_ == nullptr) {
-      errno = EBADF;
+      errno = ECONNRESET;
       return "";
     }
     char buffer[24];
@@ -196,7 +201,7 @@ class LWIPRawImpl : public Socket {
   }
   int getsockname(struct sockaddr *name, socklen_t *addrlen) override {
     if (pcb_ == nullptr) {
-      errno = EBADF;
+      errno = ECONNRESET;
       return -1;
     }
     if (name == nullptr || addrlen == nullptr) {
@@ -216,7 +221,7 @@ class LWIPRawImpl : public Socket {
   }
   std::string getsockname() override {
     if (pcb_ == nullptr) {
-      errno = EBADF;
+      errno = ECONNRESET;
       return "";
     }
     char buffer[24];
@@ -227,7 +232,7 @@ class LWIPRawImpl : public Socket {
   }
   int getsockopt(int level, int optname, void *optval, socklen_t *optlen) override {
     if (pcb_ == nullptr) {
-      errno = EBADF;
+      errno = ECONNRESET;
       return -1;
     }
     if (optlen == nullptr || optval == nullptr) {
@@ -261,7 +266,7 @@ class LWIPRawImpl : public Socket {
   }
   int setsockopt(int level, int optname, const void *optval, socklen_t optlen) override {
     if (pcb_ == nullptr) {
-      errno = EBADF;
+      errno = ECONNRESET;
       return -1;
     }
     if (level == SOL_SOCKET && optname == SO_REUSEADDR) {
@@ -314,7 +319,7 @@ class LWIPRawImpl : public Socket {
   }
   ssize_t read(void *buf, size_t len) override {
     if (pcb_ == nullptr) {
-      errno = EBADF;
+      errno = ECONNRESET;
       return -1;
     }
     if (rx_closed_ && rx_buf_ == nullptr) {
@@ -368,7 +373,7 @@ class LWIPRawImpl : public Socket {
   }
   ssize_t write(const void *buf, size_t len) override {
     if (pcb_ == nullptr) {
-      errno = EBADF;
+      errno = ECONNRESET;
       return -1;
     }
     if (len == 0)
@@ -386,24 +391,37 @@ class LWIPRawImpl : public Socket {
     LWIP_LOG("tcp_write(%p buf=%p %u)", pcb_, buf, to_send);
     err_t err = tcp_write(pcb_, buf, to_send, TCP_WRITE_FLAG_COPY);
     if (err == ERR_MEM) {
+      LWIP_LOG("  -> err ERR_MEM");
       errno = EWOULDBLOCK;
       return -1;
     }
     if (err != ERR_OK) {
-      errno = EIO;
+      LWIP_LOG("  -> err %d", err);
+      errno = ECONNRESET;
       return -1;
     }
-    LWIP_LOG("tcp_output(%p)", pcb_);
-    err = tcp_output(pcb_);
-    if (err != ERR_OK) {
-      errno = EIO;
-      return -1;
+    if (tcp_nagle_disabled(pcb_)) {
+      LWIP_LOG("tcp_output(%p)", pcb_);
+      err = tcp_output(pcb_);
+      if (err == ERR_ABRT) {
+        LWIP_LOG("  -> err ERR_ABRT");
+        // sometimes lwip returns ERR_ABRT for no apparent reason
+        // the connection works fine afterwards, and back with ESPAsyncTCP we
+        // indirectly also ignored this error
+        // FIXME: figure out where this is returned and what it means in this context
+        return to_send;
+      }
+      if (err != ERR_OK) {
+        LWIP_LOG("  -> err %d", err);
+        errno = ECONNRESET;
+        return -1;
+      }
     }
     return to_send;
   }
   int setblocking(bool blocking) override {
     if (pcb_ == nullptr) {
-      errno = EBADF;
+      errno = ECONNRESET;
       return -1;
     }
     if (blocking) {
@@ -466,7 +484,7 @@ class LWIPRawImpl : public Socket {
 
   static void s_err_fn(void *arg, err_t err) {
     LWIPRawImpl *arg_this = reinterpret_cast<LWIPRawImpl *>(arg);
-    return arg_this->err_fn(err);
+    arg_this->err_fn(err);
   }
 
   static err_t s_recv_fn(void *arg, struct tcp_pcb *pcb, struct pbuf *pb, err_t err) {

esphome/components/t6615/t6615.cpp

@@ -6,7 +6,7 @@ namespace t6615 {
 
 static const char *const TAG = "t6615";
 
-static const uint8_t T6615_RESPONSE_BUFFER_LENGTH = 32;
+static const uint32_t T6615_TIMEOUT = 1000;
 static const uint8_t T6615_MAGIC = 0xFF;
 static const uint8_t T6615_ADDR_HOST = 0xFA;
 static const uint8_t T6615_ADDR_SENSOR = 0xFE;
@@ -19,31 +19,49 @@ static const uint8_t T6615_COMMAND_ENABLE_ABC[] = {0xB7, 0x01};
 static const uint8_t T6615_COMMAND_DISABLE_ABC[] = {0xB7, 0x02};
 static const uint8_t T6615_COMMAND_SET_ELEVATION[] = {0x03, 0x0F};
 
-void T6615Component::loop() {
-  if (!this->available())
-    return;
+void T6615Component::send_ppm_command_() {
+  this->command_time_ = millis();
+  this->command_ = T6615Command::GET_PPM;
+  this->write_byte(T6615_MAGIC);
+  this->write_byte(T6615_ADDR_SENSOR);
+  this->write_byte(sizeof(T6615_COMMAND_GET_PPM));
+  this->write_array(T6615_COMMAND_GET_PPM, sizeof(T6615_COMMAND_GET_PPM));
+}
 
-  // Read header
-  uint8_t header[3];
-  this->read_array(header, 3);
-  if (header[0] != T6615_MAGIC || header[1] != T6615_ADDR_HOST) {
-    ESP_LOGW(TAG, "Reading data from T6615 failed!");
-    while (this->available())
-      this->read();  // Clear the incoming buffer
-    this->status_set_warning();
+void T6615Component::loop() {
+  if (this->available() < 5) {
+    if (this->command_ == T6615Command::GET_PPM && millis() - this->command_time_ > T6615_TIMEOUT) {
+      /* command got eaten, clear the buffer and fire another */
+      while (this->available())
+        this->read();
+      this->send_ppm_command_();
+    }
     return;
   }
 
-  // Read body
-  uint8_t length = header[2];
-  uint8_t response[T6615_RESPONSE_BUFFER_LENGTH];
-  this->read_array(response, length);
+  uint8_t response_buffer[6];
+
+  /* by the time we get here, we know we have at least five bytes in the buffer */
+  this->read_array(response_buffer, 5);
+
+  // Read header
+  if (response_buffer[0] != T6615_MAGIC || response_buffer[1] != T6615_ADDR_HOST) {
+    ESP_LOGW(TAG, "Got bad data from T6615! Magic was %02X and address was %02X", response_buffer[0],
+             response_buffer[1]);
+    /* make sure the buffer is empty */
+    while (this->available())
+      this->read();
+    /* try again to read the sensor */
+    this->send_ppm_command_();
+    this->status_set_warning();
+    return;
+  }
 
   this->status_clear_warning();
 
   switch (this->command_) {
     case T6615Command::GET_PPM: {
-      const uint16_t ppm = encode_uint16(response[0], response[1]);
+      const uint16_t ppm = encode_uint16(response_buffer[3], response_buffer[4]);
       ESP_LOGD(TAG, "T6615 Received CO₂=%uppm", ppm);
       this->co2_sensor_->publish_state(ppm);
       break;
@@ -51,23 +69,19 @@ void T6615Component::loop() {
     default:
       break;
   }
-
+  this->command_time_ = 0;
   this->command_ = T6615Command::NONE;
 }
 
 void T6615Component::update() { this->query_ppm_(); }
 
 void T6615Component::query_ppm_() {
-  if (this->co2_sensor_ == nullptr || this->command_ != T6615Command::NONE) {
+  if (this->co2_sensor_ == nullptr ||
+      (this->command_ != T6615Command::NONE && millis() - this->command_time_ < T6615_TIMEOUT)) {
     return;
   }
 
-  this->command_ = T6615Command::GET_PPM;
-
-  this->write_byte(T6615_MAGIC);
-  this->write_byte(T6615_ADDR_SENSOR);
-  this->write_byte(sizeof(T6615_COMMAND_GET_PPM));
-  this->write_array(T6615_COMMAND_GET_PPM, sizeof(T6615_COMMAND_GET_PPM));
+  this->send_ppm_command_();
 }
 
 float T6615Component::get_setup_priority() const { return setup_priority::DATA; }

esphome/components/t6615/t6615.h

@@ -32,8 +32,10 @@ class T6615Component : public PollingComponent, public uart::UARTDevice {
 
  protected:
   void query_ppm_();
+  void send_ppm_command_();
 
   T6615Command command_ = T6615Command::NONE;
+  unsigned long command_time_ = 0;
 
   sensor::Sensor *co2_sensor_{nullptr};
 };

esphome/components/web_server/web_server.cpp

@@ -397,6 +397,7 @@ std::string WebServer::fan_json(fan::FanState *obj) {
     const auto traits = obj->get_traits();
     if (traits.supports_speed()) {
       root["speed_level"] = obj->speed;
+      // NOLINTNEXTLINE(clang-diagnostic-deprecated-declarations)
       switch (fan::speed_level_to_enum(obj->speed, traits.supported_speed_count())) {
         case fan::FAN_SPEED_LOW:  // NOLINT(clang-diagnostic-deprecated-declarations)
           root["speed"] = "low";

esphome/components/wifi/wifi_component_esp32.cpp

@@ -11,6 +11,7 @@
 #endif
 #include "lwip/err.h"
 #include "lwip/dns.h"
+#include "lwip/apps/sntp.h"
 
 #include "esphome/core/helpers.h"
 #include "esphome/core/log.h"
@@ -92,6 +93,11 @@ bool WiFiComponent::wifi_sta_ip_config_(optional<ManualIP> manual_ip) {
   tcpip_adapter_dhcp_status_t dhcp_status;
   tcpip_adapter_dhcpc_get_status(TCPIP_ADAPTER_IF_STA, &dhcp_status);
   if (!manual_ip.has_value()) {
+    // lwIP starts the SNTP client if it gets an SNTP server from DHCP. We don't need the time, and more importantly,
+    // the built-in SNTP client has a memory leak in certain situations. Disable this feature.
+    // https://github.com/esphome/issues/issues/2299
+    sntp_servermode_dhcp(false);
+
     // Use DHCP client
     if (dhcp_status != TCPIP_ADAPTER_DHCP_STARTED) {
       esp_err_t err = tcpip_adapter_dhcpc_start(TCPIP_ADAPTER_IF_STA);

esphome/components/wifi/wifi_component_esp8266.cpp

@@ -16,6 +16,7 @@ extern "C" {
 #include "lwip/dns.h"
 #include "lwip/dhcp.h"
 #include "lwip/init.h"  // LWIP_VERSION_
+#include "lwip/apps/sntp.h"
 #if LWIP_IPV6
 #include "lwip/netif.h"  // struct netif
 #endif
@@ -112,6 +113,11 @@ bool WiFiComponent::wifi_sta_ip_config_(optional<ManualIP> manual_ip) {
 
   enum dhcp_status dhcp_status = wifi_station_dhcpc_status();
   if (!manual_ip.has_value()) {
+    // lwIP starts the SNTP client if it gets an SNTP server from DHCP. We don't need the time, and more importantly,
+    // the built-in SNTP client has a memory leak in certain situations. Disable this feature.
+    // https://github.com/esphome/issues/issues/2299
+    sntp_servermode_dhcp(false);
+
     // Use DHCP client
     if (dhcp_status != DHCP_STARTED) {
       bool ret = wifi_station_dhcpc_start();

esphome/const.py

@@ -1,6 +1,6 @@
 """Constants used by esphome."""
 
-__version__ = "2021.9.0b2"
+__version__ = "2021.9.0b4"
 
 ESP_PLATFORM_ESP32 = "ESP32"
 ESP_PLATFORM_ESP8266 = "ESP8266"

esphome/zeroconf.py

@@ -4,9 +4,6 @@ import time
 from typing import Dict, Optional
 
 from zeroconf import (
-    _CLASS_IN,
-    _FLAGS_QR_QUERY,
-    _TYPE_A,
     DNSAddress,
     DNSOutgoing,
     DNSRecord,
@@ -15,6 +12,10 @@ from zeroconf import (
     Zeroconf,
 )
 
+_CLASS_IN = 1
+_FLAGS_QR_QUERY = 0x0000  # query
+_TYPE_A = 1
+
 
 class HostResolver(RecordUpdateListener):
     def __init__(self, name: str):

requirements.txt

@@ -3,12 +3,11 @@ PyYAML==5.4.1
 paho-mqtt==1.5.1
 colorama==0.4.4
 tornado==6.1
-protobuf==3.17.3
 tzlocal==2.1
 pytz==2021.1
 pyserial==3.5
-ifaddr==0.1.7
-platformio==5.1.1
+platformio==5.2.0
 esptool==3.1
 click==7.1.2
 esphome-dashboard==20210908.0
+aioesphomeapi==9.0.0