From 1586a185a0b7692aa72b61feab332b6e347ee684 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Sun, 19 Oct 2025 08:34:38 -1000 Subject: [PATCH 1/5] [esp32] Automatic CONFIG_LWIP_MAX_SOCKETS configuration based on component needs --- esphome/components/api/__init__.py | 12 ++++ esphome/components/esp32/__init__.py | 66 +++++++++++++++++++ .../esp32_camera_web_server/__init__.py | 29 ++++++-- esphome/components/esphome/ota/__init__.py | 14 +++- esphome/components/mdns/__init__.py | 14 ++++ esphome/components/mqtt/__init__.py | 10 +++ esphome/components/socket/__init__.py | 28 ++++++++ esphome/components/web_server/__init__.py | 13 ++++ 8 files changed, 177 insertions(+), 9 deletions(-) diff --git a/esphome/components/api/__init__.py b/esphome/components/api/__init__.py index e8dacf51bc..e91e922204 100644 --- a/esphome/components/api/__init__.py +++ b/esphome/components/api/__init__.py @@ -155,6 +155,17 @@ def _validate_api_config(config: ConfigType) -> ConfigType: return config +def _consume_api_sockets(config: ConfigType) -> ConfigType: + """Register socket needs for API component.""" + from esphome.components import socket + + # API needs 1 listening socket + typically 3 concurrent client connections + # (not max_connections, which is the upper limit rarely reached) + sockets_needed = 1 + 3 + socket.consume_sockets(sockets_needed, "api")(config) + return config + + CONFIG_SCHEMA = cv.All( cv.Schema( { @@ -222,6 +233,7 @@ CONFIG_SCHEMA = cv.All( ).extend(cv.COMPONENT_SCHEMA), cv.rename_key(CONF_SERVICES, CONF_ACTIONS), _validate_api_config, + _consume_api_sockets, ) diff --git a/esphome/components/esp32/__init__.py b/esphome/components/esp32/__init__.py index b7dd25e0d8..383bbf19ee 100644 --- a/esphome/components/esp32/__init__.py +++ b/esphome/components/esp32/__init__.py @@ -1,3 +1,4 @@ +import contextlib from dataclasses import dataclass import itertools import logging @@ -102,6 +103,10 @@ COMPILER_OPTIMIZATIONS = { "SIZE": "CONFIG_COMPILER_OPTIMIZATION_SIZE", } +# Socket limit configuration for ESP-IDF +# ESP-IDF CONFIG_LWIP_MAX_SOCKETS has range 1-253, default 10 +DEFAULT_MAX_SOCKETS = 10 # ESP-IDF default + ARDUINO_ALLOWED_VARIANTS = [ VARIANT_ESP32, VARIANT_ESP32C3, @@ -855,6 +860,67 @@ async def to_code(config): add_idf_sdkconfig_option("CONFIG_LWIP_DNS_SUPPORT_MDNS_QUERIES", False) if not advanced.get(CONF_ENABLE_LWIP_BRIDGE_INTERFACE, False): add_idf_sdkconfig_option("CONFIG_LWIP_BRIDGEIF_MAX_PORTS", 0) + + # Calculate and set CONFIG_LWIP_MAX_SOCKETS based on component needs + # Socket component tracks consumer needs via consume_sockets() called during config validation + # This code runs in to_code() after all components have registered their socket needs + # User-provided sdkconfig_options take precedence + from esphome.components.socket import KEY_SOCKET_CONSUMERS + + # Check if user manually specified CONFIG_LWIP_MAX_SOCKETS + user_max_sockets = conf.get(CONF_SDKCONFIG_OPTIONS, {}).get( + "CONFIG_LWIP_MAX_SOCKETS" + ) + + socket_consumers: dict[str, int] = CORE.data.get(KEY_SOCKET_CONSUMERS, {}) + total_sockets = sum(socket_consumers.values()) + components_list = ( + ", ".join(f"{name}={count}" for name, count in sorted(socket_consumers.items())) + if total_sockets > 0 + else "" + ) + + if user_max_sockets is None: + # Auto-calculate based on component needs + # Use at least the ESP-IDF default (10), or the total needed by components + max_sockets = max(DEFAULT_MAX_SOCKETS, total_sockets) + + if total_sockets > 0: + log_level = ( + logging.INFO if max_sockets > DEFAULT_MAX_SOCKETS else logging.DEBUG + ) + _LOGGER.log( + log_level, + "Setting CONFIG_LWIP_MAX_SOCKETS to %d (registered: %s)", + max_sockets, + components_list, + ) + + add_idf_sdkconfig_option("CONFIG_LWIP_MAX_SOCKETS", max_sockets) + else: + # User specified their own value - respect it + _LOGGER.info( + "Using user-provided CONFIG_LWIP_MAX_SOCKETS: %s", + user_max_sockets, + ) + + # Warn if user's value is less than what components need + if total_sockets > 0: + user_sockets_int = 0 + with contextlib.suppress(ValueError, TypeError): + user_sockets_int = int(user_max_sockets) + + if user_sockets_int < total_sockets: + _LOGGER.warning( + "CONFIG_LWIP_MAX_SOCKETS is set to %d but your configuration needs %d sockets (registered: %s). " + "You may experience socket exhaustion errors. Consider increasing to at least %d.", + user_sockets_int, + total_sockets, + components_list, + total_sockets, + ) + # User's value already added via sdkconfig_options processing + if advanced.get(CONF_EXECUTE_FROM_PSRAM, False): add_idf_sdkconfig_option("CONFIG_SPIRAM_FETCH_INSTRUCTIONS", True) add_idf_sdkconfig_option("CONFIG_SPIRAM_RODATA", True) diff --git a/esphome/components/esp32_camera_web_server/__init__.py b/esphome/components/esp32_camera_web_server/__init__.py index a6a7ac3630..315cd649d1 100644 --- a/esphome/components/esp32_camera_web_server/__init__.py +++ b/esphome/components/esp32_camera_web_server/__init__.py @@ -1,6 +1,7 @@ import esphome.codegen as cg import esphome.config_validation as cv from esphome.const import CONF_ID, CONF_MODE, CONF_PORT +from esphome.types import ConfigType CODEOWNERS = ["@ayufan"] AUTO_LOAD = ["camera"] @@ -13,13 +14,27 @@ Mode = esp32_camera_web_server_ns.enum("Mode") MODES = {"STREAM": Mode.STREAM, "SNAPSHOT": Mode.SNAPSHOT} -CONFIG_SCHEMA = cv.Schema( - { - cv.GenerateID(): cv.declare_id(CameraWebServer), - cv.Required(CONF_PORT): cv.port, - cv.Required(CONF_MODE): cv.enum(MODES, upper=True), - }, -).extend(cv.COMPONENT_SCHEMA) + +def _consume_camera_web_server_sockets(config: ConfigType) -> ConfigType: + """Register socket needs for camera web server.""" + from esphome.components import socket + + # Each camera web server instance needs 1 listening socket + 1-2 client connections + sockets_needed = 2 + socket.consume_sockets(sockets_needed, "esp32_camera_web_server")(config) + return config + + +CONFIG_SCHEMA = cv.All( + cv.Schema( + { + cv.GenerateID(): cv.declare_id(CameraWebServer), + cv.Required(CONF_PORT): cv.port, + cv.Required(CONF_MODE): cv.enum(MODES, upper=True), + }, + ).extend(cv.COMPONENT_SCHEMA), + _consume_camera_web_server_sockets, +) async def to_code(config): diff --git a/esphome/components/esphome/ota/__init__.py b/esphome/components/esphome/ota/__init__.py index 69a50a2de9..e56e85b231 100644 --- a/esphome/components/esphome/ota/__init__.py +++ b/esphome/components/esphome/ota/__init__.py @@ -103,7 +103,16 @@ def ota_esphome_final_validate(config): ) -CONFIG_SCHEMA = ( +def _consume_ota_sockets(config: ConfigType) -> ConfigType: + """Register socket needs for OTA component.""" + from esphome.components import socket + + # OTA needs 1 listening socket (client connections are temporary during updates) + socket.consume_sockets(1, "ota")(config) + return config + + +CONFIG_SCHEMA = cv.All( cv.Schema( { cv.GenerateID(): cv.declare_id(ESPHomeOTAComponent), @@ -130,7 +139,8 @@ CONFIG_SCHEMA = ( } ) .extend(BASE_OTA_SCHEMA) - .extend(cv.COMPONENT_SCHEMA) + .extend(cv.COMPONENT_SCHEMA), + _consume_ota_sockets, ) FINAL_VALIDATE_SCHEMA = ota_esphome_final_validate diff --git a/esphome/components/mdns/__init__.py b/esphome/components/mdns/__init__.py index c6a9ee1a0c..6b4578ac23 100644 --- a/esphome/components/mdns/__init__.py +++ b/esphome/components/mdns/__init__.py @@ -46,6 +46,19 @@ SERVICE_SCHEMA = cv.Schema( } ) + +def _consume_mdns_sockets(config): + """Register socket needs for mDNS component.""" + if config.get(CONF_DISABLED): + return config + + from esphome.components import socket + + # mDNS needs 2 sockets (IPv4 + IPv6 multicast) + socket.consume_sockets(2, "mdns")(config) + return config + + CONFIG_SCHEMA = cv.All( cv.Schema( { @@ -55,6 +68,7 @@ CONFIG_SCHEMA = cv.All( } ), _remove_id_if_disabled, + _consume_mdns_sockets, ) diff --git a/esphome/components/mqtt/__init__.py b/esphome/components/mqtt/__init__.py index 814fb566d4..3866e09a24 100644 --- a/esphome/components/mqtt/__init__.py +++ b/esphome/components/mqtt/__init__.py @@ -210,6 +210,15 @@ def validate_fingerprint(value): return value +def _consume_mqtt_sockets(config): + """Register socket needs for MQTT component.""" + from esphome.components import socket + + # MQTT needs 1 socket for the broker connection + socket.consume_sockets(1, "mqtt")(config) + return config + + CONFIG_SCHEMA = cv.All( cv.Schema( { @@ -306,6 +315,7 @@ CONFIG_SCHEMA = cv.All( ), validate_config, cv.only_on([PLATFORM_ESP32, PLATFORM_ESP8266, PLATFORM_BK72XX]), + _consume_mqtt_sockets, ) diff --git a/esphome/components/socket/__init__.py b/esphome/components/socket/__init__.py index e085a09eac..e6a4cfc07f 100644 --- a/esphome/components/socket/__init__.py +++ b/esphome/components/socket/__init__.py @@ -1,3 +1,5 @@ +from collections.abc import Callable, MutableMapping + import esphome.codegen as cg import esphome.config_validation as cv from esphome.core import CORE @@ -9,6 +11,32 @@ IMPLEMENTATION_LWIP_TCP = "lwip_tcp" IMPLEMENTATION_LWIP_SOCKETS = "lwip_sockets" IMPLEMENTATION_BSD_SOCKETS = "bsd_sockets" +# Socket tracking infrastructure +# Components register their socket needs and platforms read this to configure appropriately +KEY_SOCKET_CONSUMERS = "socket_consumers" + + +def consume_sockets( + value: int, consumer: str +) -> Callable[[MutableMapping], MutableMapping]: + """Register socket usage for a component. + + Args: + value: Number of sockets needed by the component + consumer: Name of the component consuming the sockets + + Returns: + A validator function that records the socket usage + """ + + def _consume_sockets(config: MutableMapping) -> MutableMapping: + consumers: dict[str, int] = CORE.data.setdefault(KEY_SOCKET_CONSUMERS, {}) + consumers[consumer] = consumers.get(consumer, 0) + value + return config + + return _consume_sockets + + CONFIG_SCHEMA = cv.Schema( { cv.SplitDefault( diff --git a/esphome/components/web_server/__init__.py b/esphome/components/web_server/__init__.py index 288d928e80..a7fdf30eef 100644 --- a/esphome/components/web_server/__init__.py +++ b/esphome/components/web_server/__init__.py @@ -136,6 +136,18 @@ def _final_validate_sorting(config: ConfigType) -> ConfigType: FINAL_VALIDATE_SCHEMA = _final_validate_sorting + +def _consume_web_server_sockets(config: ConfigType) -> ConfigType: + """Register socket needs for web_server component.""" + from esphome.components import socket + + # Web server needs 1 listening socket + typically 2 concurrent client connections + # (browser makes 2 connections for page + event stream) + sockets_needed = 3 + socket.consume_sockets(sockets_needed, "web_server")(config) + return config + + sorting_group = { cv.Required(CONF_ID): cv.declare_id(cg.int_), cv.Required(CONF_NAME): cv.string, @@ -205,6 +217,7 @@ CONFIG_SCHEMA = cv.All( validate_local, validate_sorting_groups, validate_ota, + _consume_web_server_sockets, ) From 55473991a903a9195e6fa9c96d3b125f4a4da7a0 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Sun, 19 Oct 2025 08:37:43 -1000 Subject: [PATCH 2/5] preen --- esphome/components/mdns/__init__.py | 3 ++- esphome/components/mqtt/__init__.py | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/esphome/components/mdns/__init__.py b/esphome/components/mdns/__init__.py index 6b4578ac23..4776bef22f 100644 --- a/esphome/components/mdns/__init__.py +++ b/esphome/components/mdns/__init__.py @@ -13,6 +13,7 @@ from esphome.const import ( ) from esphome.core import CORE, Lambda, coroutine_with_priority from esphome.coroutine import CoroPriority +from esphome.types import ConfigType CODEOWNERS = ["@esphome/core"] DEPENDENCIES = ["network"] @@ -47,7 +48,7 @@ SERVICE_SCHEMA = cv.Schema( ) -def _consume_mdns_sockets(config): +def _consume_mdns_sockets(config: ConfigType) -> ConfigType: """Register socket needs for mDNS component.""" if config.get(CONF_DISABLED): return config diff --git a/esphome/components/mqtt/__init__.py b/esphome/components/mqtt/__init__.py index 3866e09a24..641c70a367 100644 --- a/esphome/components/mqtt/__init__.py +++ b/esphome/components/mqtt/__init__.py @@ -58,6 +58,7 @@ from esphome.const import ( PlatformFramework, ) from esphome.core import CORE, CoroPriority, coroutine_with_priority +from esphome.types import ConfigType DEPENDENCIES = ["network"] @@ -210,7 +211,7 @@ def validate_fingerprint(value): return value -def _consume_mqtt_sockets(config): +def _consume_mqtt_sockets(config: ConfigType) -> ConfigType: """Register socket needs for MQTT component.""" from esphome.components import socket From 7107f5d984a48ef1f91b6121974389202df51258 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Sun, 19 Oct 2025 08:40:01 -1000 Subject: [PATCH 3/5] preen --- esphome/components/esp32/__init__.py | 126 ++++++++++++++------------- 1 file changed, 67 insertions(+), 59 deletions(-) diff --git a/esphome/components/esp32/__init__.py b/esphome/components/esp32/__init__.py index 383bbf19ee..7fdf6d340a 100644 --- a/esphome/components/esp32/__init__.py +++ b/esphome/components/esp32/__init__.py @@ -751,6 +751,72 @@ CONFIG_SCHEMA = cv.All( FINAL_VALIDATE_SCHEMA = cv.Schema(final_validate) +def _configure_lwip_max_sockets(conf: dict) -> None: + """Calculate and set CONFIG_LWIP_MAX_SOCKETS based on component needs. + + Socket component tracks consumer needs via consume_sockets() called during config validation. + This function runs in to_code() after all components have registered their socket needs. + User-provided sdkconfig_options take precedence. + """ + from esphome.components.socket import KEY_SOCKET_CONSUMERS + + # Check if user manually specified CONFIG_LWIP_MAX_SOCKETS + user_max_sockets = conf.get(CONF_SDKCONFIG_OPTIONS, {}).get( + "CONFIG_LWIP_MAX_SOCKETS" + ) + + socket_consumers: dict[str, int] = CORE.data.get(KEY_SOCKET_CONSUMERS, {}) + total_sockets = sum(socket_consumers.values()) + + # Early return if no sockets registered and no user override + if total_sockets == 0 and user_max_sockets is None: + add_idf_sdkconfig_option("CONFIG_LWIP_MAX_SOCKETS", DEFAULT_MAX_SOCKETS) + return + + components_list = ", ".join( + f"{name}={count}" for name, count in sorted(socket_consumers.items()) + ) + + # User specified their own value - respect it but warn if insufficient + if user_max_sockets is not None: + _LOGGER.info( + "Using user-provided CONFIG_LWIP_MAX_SOCKETS: %s", + user_max_sockets, + ) + + # Warn if user's value is less than what components need + if total_sockets > 0: + user_sockets_int = 0 + with contextlib.suppress(ValueError, TypeError): + user_sockets_int = int(user_max_sockets) + + if user_sockets_int < total_sockets: + _LOGGER.warning( + "CONFIG_LWIP_MAX_SOCKETS is set to %d but your configuration needs %d sockets (registered: %s). " + "You may experience socket exhaustion errors. Consider increasing to at least %d.", + user_sockets_int, + total_sockets, + components_list, + total_sockets, + ) + # User's value already added via sdkconfig_options processing + return + + # Auto-calculate based on component needs + # Use at least the ESP-IDF default (10), or the total needed by components + max_sockets = max(DEFAULT_MAX_SOCKETS, total_sockets) + + log_level = logging.INFO if max_sockets > DEFAULT_MAX_SOCKETS else logging.DEBUG + _LOGGER.log( + log_level, + "Setting CONFIG_LWIP_MAX_SOCKETS to %d (registered: %s)", + max_sockets, + components_list, + ) + + add_idf_sdkconfig_option("CONFIG_LWIP_MAX_SOCKETS", max_sockets) + + async def to_code(config): cg.add_platformio_option("board", config[CONF_BOARD]) cg.add_platformio_option("board_upload.flash_size", config[CONF_FLASH_SIZE]) @@ -861,65 +927,7 @@ async def to_code(config): if not advanced.get(CONF_ENABLE_LWIP_BRIDGE_INTERFACE, False): add_idf_sdkconfig_option("CONFIG_LWIP_BRIDGEIF_MAX_PORTS", 0) - # Calculate and set CONFIG_LWIP_MAX_SOCKETS based on component needs - # Socket component tracks consumer needs via consume_sockets() called during config validation - # This code runs in to_code() after all components have registered their socket needs - # User-provided sdkconfig_options take precedence - from esphome.components.socket import KEY_SOCKET_CONSUMERS - - # Check if user manually specified CONFIG_LWIP_MAX_SOCKETS - user_max_sockets = conf.get(CONF_SDKCONFIG_OPTIONS, {}).get( - "CONFIG_LWIP_MAX_SOCKETS" - ) - - socket_consumers: dict[str, int] = CORE.data.get(KEY_SOCKET_CONSUMERS, {}) - total_sockets = sum(socket_consumers.values()) - components_list = ( - ", ".join(f"{name}={count}" for name, count in sorted(socket_consumers.items())) - if total_sockets > 0 - else "" - ) - - if user_max_sockets is None: - # Auto-calculate based on component needs - # Use at least the ESP-IDF default (10), or the total needed by components - max_sockets = max(DEFAULT_MAX_SOCKETS, total_sockets) - - if total_sockets > 0: - log_level = ( - logging.INFO if max_sockets > DEFAULT_MAX_SOCKETS else logging.DEBUG - ) - _LOGGER.log( - log_level, - "Setting CONFIG_LWIP_MAX_SOCKETS to %d (registered: %s)", - max_sockets, - components_list, - ) - - add_idf_sdkconfig_option("CONFIG_LWIP_MAX_SOCKETS", max_sockets) - else: - # User specified their own value - respect it - _LOGGER.info( - "Using user-provided CONFIG_LWIP_MAX_SOCKETS: %s", - user_max_sockets, - ) - - # Warn if user's value is less than what components need - if total_sockets > 0: - user_sockets_int = 0 - with contextlib.suppress(ValueError, TypeError): - user_sockets_int = int(user_max_sockets) - - if user_sockets_int < total_sockets: - _LOGGER.warning( - "CONFIG_LWIP_MAX_SOCKETS is set to %d but your configuration needs %d sockets (registered: %s). " - "You may experience socket exhaustion errors. Consider increasing to at least %d.", - user_sockets_int, - total_sockets, - components_list, - total_sockets, - ) - # User's value already added via sdkconfig_options processing + _configure_lwip_max_sockets(conf) if advanced.get(CONF_EXECUTE_FROM_PSRAM, False): add_idf_sdkconfig_option("CONFIG_SPIRAM_FETCH_INSTRUCTIONS", True) From 148a78aa015a7723230b6521f865deceea89444a Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Sun, 19 Oct 2025 08:41:21 -1000 Subject: [PATCH 4/5] preen --- esphome/components/esp32/__init__.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/esphome/components/esp32/__init__.py b/esphome/components/esp32/__init__.py index 7fdf6d340a..6764764644 100644 --- a/esphome/components/esp32/__init__.py +++ b/esphome/components/esp32/__init__.py @@ -770,7 +770,6 @@ def _configure_lwip_max_sockets(conf: dict) -> None: # Early return if no sockets registered and no user override if total_sockets == 0 and user_max_sockets is None: - add_idf_sdkconfig_option("CONFIG_LWIP_MAX_SOCKETS", DEFAULT_MAX_SOCKETS) return components_list = ", ".join( @@ -792,8 +791,9 @@ def _configure_lwip_max_sockets(conf: dict) -> None: if user_sockets_int < total_sockets: _LOGGER.warning( - "CONFIG_LWIP_MAX_SOCKETS is set to %d but your configuration needs %d sockets (registered: %s). " - "You may experience socket exhaustion errors. Consider increasing to at least %d.", + "CONFIG_LWIP_MAX_SOCKETS is set to %d but your configuration " + "needs %d sockets (registered: %s). You may experience socket " + "exhaustion errors. Consider increasing to at least %d.", user_sockets_int, total_sockets, components_list, From 4fa908d0b8ed199cdd27020e54751725ddf476cf Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Sun, 19 Oct 2025 08:43:30 -1000 Subject: [PATCH 5/5] preen --- esphome/components/esp32_camera_web_server/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/esphome/components/esp32_camera_web_server/__init__.py b/esphome/components/esp32_camera_web_server/__init__.py index 315cd649d1..ed1aaa2e07 100644 --- a/esphome/components/esp32_camera_web_server/__init__.py +++ b/esphome/components/esp32_camera_web_server/__init__.py @@ -19,8 +19,8 @@ def _consume_camera_web_server_sockets(config: ConfigType) -> ConfigType: """Register socket needs for camera web server.""" from esphome.components import socket - # Each camera web server instance needs 1 listening socket + 1-2 client connections - sockets_needed = 2 + # Each camera web server instance needs 1 listening socket + 2 client connections + sockets_needed = 3 socket.consume_sockets(sockets_needed, "esp32_camera_web_server")(config) return config