From 49e7ccd9375421ba21f2b4c8b6dd8b549c501811 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Sat, 6 Dec 2025 22:25:00 -0600 Subject: [PATCH 1/4] [text_sensor] Use StringRef for filter static data to avoid heap allocation --- esphome/codegen.py | 1 + esphome/components/text_sensor/__init__.py | 12 +- esphome/components/text_sensor/filter.cpp | 12 +- esphome/components/text_sensor/filter.h | 13 +- esphome/cpp_generator.py | 17 ++ .../fixtures/text_sensor_raw_state.yaml | 127 ++++++++++++ .../integration/test_text_sensor_raw_state.py | 190 ++++++++++++++++-- 7 files changed, 341 insertions(+), 31 deletions(-) diff --git a/esphome/codegen.py b/esphome/codegen.py index 6d55c6023d..4ea41ad191 100644 --- a/esphome/codegen.py +++ b/esphome/codegen.py @@ -19,6 +19,7 @@ from esphome.cpp_generator import ( # noqa: F401 RawExpression, RawStatement, Statement, + StringRefLiteral, StructInitializer, TemplateArguments, add, diff --git a/esphome/components/text_sensor/__init__.py b/esphome/components/text_sensor/__init__.py index 0d22400a8e..2be14b36fd 100644 --- a/esphome/components/text_sensor/__init__.py +++ b/esphome/components/text_sensor/__init__.py @@ -86,12 +86,12 @@ async def to_lower_filter_to_code(config, filter_id): @FILTER_REGISTRY.register("append", AppendFilter, cv.string) async def append_filter_to_code(config, filter_id): - return cg.new_Pvariable(filter_id, config) + return cg.new_Pvariable(filter_id, cg.StringRefLiteral(config)) @FILTER_REGISTRY.register("prepend", PrependFilter, cv.string) async def prepend_filter_to_code(config, filter_id): - return cg.new_Pvariable(filter_id, config) + return cg.new_Pvariable(filter_id, cg.StringRefLiteral(config)) def validate_mapping(value): @@ -114,8 +114,8 @@ async def substitute_filter_to_code(config, filter_id): substitutions = [ cg.StructInitializer( cg.MockObj("Substitution", "esphome::text_sensor::"), - ("from", conf[CONF_FROM]), - ("to", conf[CONF_TO]), + ("from", cg.StringRefLiteral(conf[CONF_FROM])), + ("to", cg.StringRefLiteral(conf[CONF_TO])), ) for conf in config ] @@ -127,8 +127,8 @@ async def map_filter_to_code(config, filter_id): mappings = [ cg.StructInitializer( cg.MockObj("Substitution", "esphome::text_sensor::"), - ("from", conf[CONF_FROM]), - ("to", conf[CONF_TO]), + ("from", cg.StringRefLiteral(conf[CONF_FROM])), + ("to", cg.StringRefLiteral(conf[CONF_TO])), ) for conf in config ] diff --git a/esphome/components/text_sensor/filter.cpp b/esphome/components/text_sensor/filter.cpp index 40a37febee..a6d0ae0c93 100644 --- a/esphome/components/text_sensor/filter.cpp +++ b/esphome/components/text_sensor/filter.cpp @@ -56,7 +56,10 @@ optional ToLowerFilter::new_value(std::string value) { } // Append -optional AppendFilter::new_value(std::string value) { return value + this->suffix_; } +optional AppendFilter::new_value(std::string value) { + value += this->suffix_; + return value; +} // Prepend optional PrependFilter::new_value(std::string value) { return this->prefix_ + value; } @@ -68,8 +71,9 @@ SubstituteFilter::SubstituteFilter(const std::initializer_list &su optional SubstituteFilter::new_value(std::string value) { for (const auto &sub : this->substitutions_) { std::size_t pos = 0; - while ((pos = value.find(sub.from, pos)) != std::string::npos) { - value.replace(pos, sub.from.size(), sub.to); + // Use c_str()/size() to avoid temporary std::string allocation from implicit conversion + while ((pos = value.find(sub.from.c_str(), pos, sub.from.size())) != std::string::npos) { + value.replace(pos, sub.from.size(), sub.to.c_str(), sub.to.size()); // Advance past the replacement to avoid infinite loop when // the replacement contains the search pattern (e.g., f -> foo) pos += sub.to.size(); @@ -84,7 +88,7 @@ MapFilter::MapFilter(const std::initializer_list &mappings) : mapp optional MapFilter::new_value(std::string value) { for (const auto &mapping : this->mappings_) { if (mapping.from == value) - return mapping.to; + return mapping.to.str(); } return value; // Pass through if no match } diff --git a/esphome/components/text_sensor/filter.h b/esphome/components/text_sensor/filter.h index 85acac5c8d..472dd87f15 100644 --- a/esphome/components/text_sensor/filter.h +++ b/esphome/components/text_sensor/filter.h @@ -2,6 +2,7 @@ #include "esphome/core/component.h" #include "esphome/core/helpers.h" +#include "esphome/core/string_ref.h" namespace esphome { namespace text_sensor { @@ -92,26 +93,26 @@ class ToLowerFilter : public Filter { /// A simple filter that adds a string to the end of another string class AppendFilter : public Filter { public: - AppendFilter(std::string suffix) : suffix_(std::move(suffix)) {} + explicit AppendFilter(StringRef suffix) : suffix_(suffix) {} optional new_value(std::string value) override; protected: - std::string suffix_; + StringRef suffix_; }; /// A simple filter that adds a string to the start of another string class PrependFilter : public Filter { public: - PrependFilter(std::string prefix) : prefix_(std::move(prefix)) {} + explicit PrependFilter(StringRef prefix) : prefix_(prefix) {} optional new_value(std::string value) override; protected: - std::string prefix_; + StringRef prefix_; }; struct Substitution { - std::string from; - std::string to; + StringRef from; + StringRef to; }; /// A simple filter that replaces a substring with another substring diff --git a/esphome/cpp_generator.py b/esphome/cpp_generator.py index 1a47b346b7..89993d997c 100644 --- a/esphome/cpp_generator.py +++ b/esphome/cpp_generator.py @@ -243,6 +243,23 @@ class LogStringLiteral(Literal): return f"LOG_STR({cpp_string_escape(self.string)})" +class StringRefLiteral(Literal): + """A StringRef literal using from_lit() for compile-time string references. + + Uses StringRef::from_lit() which stores pointer + length (8 bytes) instead of + std::string (24-32 bytes + heap allocation). The string data stays in flash. + """ + + __slots__ = ("string",) + + def __init__(self, string: str) -> None: + super().__init__() + self.string = string + + def __str__(self) -> str: + return f"StringRef::from_lit({cpp_string_escape(self.string)})" + + class IntLiteral(Literal): __slots__ = ("i",) diff --git a/tests/integration/fixtures/text_sensor_raw_state.yaml b/tests/integration/fixtures/text_sensor_raw_state.yaml index 03aece0a04..54ab2e8dcc 100644 --- a/tests/integration/fixtures/text_sensor_raw_state.yaml +++ b/tests/integration/fixtures/text_sensor_raw_state.yaml @@ -20,6 +20,42 @@ text_sensor: filters: - to_upper + # StringRef-based filters (append, prepend, substitute, map) + - platform: template + name: "Append Sensor" + id: append_sensor + filters: + - append: " suffix" + + - platform: template + name: "Prepend Sensor" + id: prepend_sensor + filters: + - prepend: "prefix " + + - platform: template + name: "Substitute Sensor" + id: substitute_sensor + filters: + - substitute: + - foo -> bar + - hello -> world + + - platform: template + name: "Map Sensor" + id: map_sensor + filters: + - map: + - ON -> Active + - OFF -> Inactive + + - platform: template + name: "Chained Sensor" + id: chained_sensor + filters: + - prepend: "[" + - append: "]" + # Button to publish values and log raw_state vs state button: - platform: template @@ -52,3 +88,94 @@ button: args: - id(with_filter_sensor).state.c_str() - id(with_filter_sensor).get_raw_state().c_str() + + - platform: template + name: "Test Append Button" + id: test_append_button + on_press: + - text_sensor.template.publish: + id: append_sensor + state: "test" + - delay: 50ms + - logger.log: + format: "APPEND: state='%s'" + args: + - id(append_sensor).state.c_str() + + - platform: template + name: "Test Prepend Button" + id: test_prepend_button + on_press: + - text_sensor.template.publish: + id: prepend_sensor + state: "test" + - delay: 50ms + - logger.log: + format: "PREPEND: state='%s'" + args: + - id(prepend_sensor).state.c_str() + + - platform: template + name: "Test Substitute Button" + id: test_substitute_button + on_press: + - text_sensor.template.publish: + id: substitute_sensor + state: "foo says hello" + - delay: 50ms + - logger.log: + format: "SUBSTITUTE: state='%s'" + args: + - id(substitute_sensor).state.c_str() + + - platform: template + name: "Test Map ON Button" + id: test_map_on_button + on_press: + - text_sensor.template.publish: + id: map_sensor + state: "ON" + - delay: 50ms + - logger.log: + format: "MAP_ON: state='%s'" + args: + - id(map_sensor).state.c_str() + + - platform: template + name: "Test Map OFF Button" + id: test_map_off_button + on_press: + - text_sensor.template.publish: + id: map_sensor + state: "OFF" + - delay: 50ms + - logger.log: + format: "MAP_OFF: state='%s'" + args: + - id(map_sensor).state.c_str() + + - platform: template + name: "Test Map Unknown Button" + id: test_map_unknown_button + on_press: + - text_sensor.template.publish: + id: map_sensor + state: "UNKNOWN" + - delay: 50ms + - logger.log: + format: "MAP_UNKNOWN: state='%s'" + args: + - id(map_sensor).state.c_str() + + - platform: template + name: "Test Chained Button" + id: test_chained_button + on_press: + - text_sensor.template.publish: + id: chained_sensor + state: "value" + - delay: 50ms + - logger.log: + format: "CHAINED: state='%s'" + args: + - id(chained_sensor).state.c_str() diff --git a/tests/integration/test_text_sensor_raw_state.py b/tests/integration/test_text_sensor_raw_state.py index a53ec8c963..482ebbe9c2 100644 --- a/tests/integration/test_text_sensor_raw_state.py +++ b/tests/integration/test_text_sensor_raw_state.py @@ -1,8 +1,10 @@ -"""Integration test for TextSensor get_raw_state() functionality. +"""Integration test for TextSensor get_raw_state() and StringRef-based filters. -This tests the optimization in PR #12205 where raw_state is only stored -when filters are configured. When no filters exist, get_raw_state() should -return state directly. +This tests: +1. The optimization in PR #12205 where raw_state is only stored when filters + are configured. When no filters exist, get_raw_state() should return state. +2. StringRef-based filters (append, prepend, substitute, map) which store + static string data in flash instead of heap-allocating std::string. """ from __future__ import annotations @@ -21,16 +23,25 @@ async def test_text_sensor_raw_state( run_compiled: RunCompiledFunction, api_client_connected: APIClientConnectedFactory, ) -> None: - """Test that get_raw_state() works correctly with and without filters. + """Test text sensor filters and raw_state behavior. - Without filters: get_raw_state() should return the same value as state - With filters: get_raw_state() should return the original (unfiltered) value + Tests: + 1. get_raw_state() without filters returns same as state + 2. get_raw_state() with filters returns original (unfiltered) value + 3. StringRef-based filters: append, prepend, substitute, map, chained """ loop = asyncio.get_running_loop() # Futures to track log messages no_filter_future: asyncio.Future[tuple[str, str]] = loop.create_future() with_filter_future: asyncio.Future[tuple[str, str]] = loop.create_future() + append_future: asyncio.Future[str] = loop.create_future() + prepend_future: asyncio.Future[str] = loop.create_future() + substitute_future: asyncio.Future[str] = loop.create_future() + map_on_future: asyncio.Future[str] = loop.create_future() + map_off_future: asyncio.Future[str] = loop.create_future() + map_unknown_future: asyncio.Future[str] = loop.create_future() + chained_future: asyncio.Future[str] = loop.create_future() # Patterns to match log output # NO_FILTER: state='hello world' raw_state='hello world' @@ -39,18 +50,47 @@ async def test_text_sensor_raw_state( with_filter_pattern = re.compile( r"WITH_FILTER: state='([^']*)' raw_state='([^']*)'" ) + # StringRef-based filter patterns + append_pattern = re.compile(r"APPEND: state='([^']*)'") + prepend_pattern = re.compile(r"PREPEND: state='([^']*)'") + substitute_pattern = re.compile(r"SUBSTITUTE: state='([^']*)'") + map_on_pattern = re.compile(r"MAP_ON: state='([^']*)'") + map_off_pattern = re.compile(r"MAP_OFF: state='([^']*)'") + map_unknown_pattern = re.compile(r"MAP_UNKNOWN: state='([^']*)'") + chained_pattern = re.compile(r"CHAINED: state='([^']*)'") def check_output(line: str) -> None: """Check log output for expected messages.""" - if not no_filter_future.done(): - match = no_filter_pattern.search(line) - if match: - no_filter_future.set_result((match.group(1), match.group(2))) + if not no_filter_future.done() and (match := no_filter_pattern.search(line)): + no_filter_future.set_result((match.group(1), match.group(2))) - if not with_filter_future.done(): - match = with_filter_pattern.search(line) - if match: - with_filter_future.set_result((match.group(1), match.group(2))) + if not with_filter_future.done() and ( + match := with_filter_pattern.search(line) + ): + with_filter_future.set_result((match.group(1), match.group(2))) + + if not append_future.done() and (match := append_pattern.search(line)): + append_future.set_result(match.group(1)) + + if not prepend_future.done() and (match := prepend_pattern.search(line)): + prepend_future.set_result(match.group(1)) + + if not substitute_future.done() and (match := substitute_pattern.search(line)): + substitute_future.set_result(match.group(1)) + + if not map_on_future.done() and (match := map_on_pattern.search(line)): + map_on_future.set_result(match.group(1)) + + if not map_off_future.done() and (match := map_off_pattern.search(line)): + map_off_future.set_result(match.group(1)) + + if not map_unknown_future.done() and ( + match := map_unknown_pattern.search(line) + ): + map_unknown_future.set_result(match.group(1)) + + if not chained_future.done() and (match := chained_pattern.search(line)): + chained_future.set_result(match.group(1)) async with ( run_compiled(yaml_config, line_callback=check_output), @@ -112,3 +152,123 @@ async def test_text_sensor_raw_state( f"With filters, state and raw_state should differ. " f"state='{state}', raw_state='{raw_state}'" ) + + # Test 3: Append filter (StringRef-based) + # "test" + " suffix" = "test suffix" + append_button = next( + (e for e in entities if "test_append_button" in e.object_id.lower()), + None, + ) + assert append_button is not None, "Test Append Button not found" + client.button_command(append_button.key) + + try: + state = await asyncio.wait_for(append_future, timeout=5.0) + except TimeoutError: + pytest.fail("Timeout waiting for APPEND log message") + + assert state == "test suffix", ( + f"Append failed: expected 'test suffix', got '{state}'" + ) + + # Test 4: Prepend filter (StringRef-based) + # "prefix " + "test" = "prefix test" + prepend_button = next( + (e for e in entities if "test_prepend_button" in e.object_id.lower()), + None, + ) + assert prepend_button is not None, "Test Prepend Button not found" + client.button_command(prepend_button.key) + + try: + state = await asyncio.wait_for(prepend_future, timeout=5.0) + except TimeoutError: + pytest.fail("Timeout waiting for PREPEND log message") + + assert state == "prefix test", ( + f"Prepend failed: expected 'prefix test', got '{state}'" + ) + + # Test 5: Substitute filter (StringRef-based) + # "foo says hello" with foo->bar, hello->world = "bar says world" + substitute_button = next( + (e for e in entities if "test_substitute_button" in e.object_id.lower()), + None, + ) + assert substitute_button is not None, "Test Substitute Button not found" + client.button_command(substitute_button.key) + + try: + state = await asyncio.wait_for(substitute_future, timeout=5.0) + except TimeoutError: + pytest.fail("Timeout waiting for SUBSTITUTE log message") + + assert state == "bar says world", ( + f"Substitute failed: expected 'bar says world', got '{state}'" + ) + + # Test 6: Map filter - "ON" -> "Active" + map_on_button = next( + (e for e in entities if "test_map_on_button" in e.object_id.lower()), + None, + ) + assert map_on_button is not None, "Test Map ON Button not found" + client.button_command(map_on_button.key) + + try: + state = await asyncio.wait_for(map_on_future, timeout=5.0) + except TimeoutError: + pytest.fail("Timeout waiting for MAP_ON log message") + + assert state == "Active", f"Map ON failed: expected 'Active', got '{state}'" + + # Test 7: Map filter - "OFF" -> "Inactive" + map_off_button = next( + (e for e in entities if "test_map_off_button" in e.object_id.lower()), + None, + ) + assert map_off_button is not None, "Test Map OFF Button not found" + client.button_command(map_off_button.key) + + try: + state = await asyncio.wait_for(map_off_future, timeout=5.0) + except TimeoutError: + pytest.fail("Timeout waiting for MAP_OFF log message") + + assert state == "Inactive", ( + f"Map OFF failed: expected 'Inactive', got '{state}'" + ) + + # Test 8: Map filter - passthrough for unknown values + # "UNKNOWN" -> "UNKNOWN" (no match, passes through unchanged) + map_unknown_button = next( + (e for e in entities if "test_map_unknown_button" in e.object_id.lower()), + None, + ) + assert map_unknown_button is not None, "Test Map Unknown Button not found" + client.button_command(map_unknown_button.key) + + try: + state = await asyncio.wait_for(map_unknown_future, timeout=5.0) + except TimeoutError: + pytest.fail("Timeout waiting for MAP_UNKNOWN log message") + + assert state == "UNKNOWN", ( + f"Map passthrough failed: expected 'UNKNOWN', got '{state}'" + ) + + # Test 9: Chained filters (prepend "[" + append "]") + # "[" + "value" + "]" = "[value]" + chained_button = next( + (e for e in entities if "test_chained_button" in e.object_id.lower()), + None, + ) + assert chained_button is not None, "Test Chained Button not found" + client.button_command(chained_button.key) + + try: + state = await asyncio.wait_for(chained_future, timeout=5.0) + except TimeoutError: + pytest.fail("Timeout waiting for CHAINED log message") + + assert state == "[value]", f"Chained failed: expected '[value]', got '{state}'" From 8fd7c006130e832e5ac0cc71c611a5c05840b987 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Sat, 6 Dec 2025 22:29:24 -0600 Subject: [PATCH 2/4] [text_sensor] Use StringRef for filter static data to avoid heap allocation --- esphome/components/text_sensor/filter.cpp | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/esphome/components/text_sensor/filter.cpp b/esphome/components/text_sensor/filter.cpp index a6d0ae0c93..52c2bd394e 100644 --- a/esphome/components/text_sensor/filter.cpp +++ b/esphome/components/text_sensor/filter.cpp @@ -62,7 +62,10 @@ optional AppendFilter::new_value(std::string value) { } // Prepend -optional PrependFilter::new_value(std::string value) { return this->prefix_ + value; } +optional PrependFilter::new_value(std::string value) { + value.insert(0, this->prefix_.c_str(), this->prefix_.size()); + return value; +} // Substitute SubstituteFilter::SubstituteFilter(const std::initializer_list &substitutions) From 81f4add32458473110bd18a9700367a0ee1ffcda Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Sat, 6 Dec 2025 22:31:21 -0600 Subject: [PATCH 3/4] [text_sensor] Use StringRef for filter static data to avoid heap allocation --- esphome/components/text_sensor/filter.cpp | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/esphome/components/text_sensor/filter.cpp b/esphome/components/text_sensor/filter.cpp index 52c2bd394e..d9afaf80f1 100644 --- a/esphome/components/text_sensor/filter.cpp +++ b/esphome/components/text_sensor/filter.cpp @@ -90,8 +90,10 @@ MapFilter::MapFilter(const std::initializer_list &mappings) : mapp optional MapFilter::new_value(std::string value) { for (const auto &mapping : this->mappings_) { - if (mapping.from == value) - return mapping.to.str(); + if (mapping.from == value) { + value.assign(mapping.to.c_str(), mapping.to.size()); + return value; + } } return value; // Pass through if no match } From f3a039e70f92681c2d7e8bc42229ce8ab96ba4ad Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Sat, 6 Dec 2025 22:55:28 -0600 Subject: [PATCH 4/4] cover --- tests/unit_tests/test_cpp_generator.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tests/unit_tests/test_cpp_generator.py b/tests/unit_tests/test_cpp_generator.py index 2c9f760c8e..428e7626a1 100644 --- a/tests/unit_tests/test_cpp_generator.py +++ b/tests/unit_tests/test_cpp_generator.py @@ -248,6 +248,12 @@ class TestLiterals: (cg.FloatLiteral(4.2), "4.2f"), (cg.FloatLiteral(1.23456789), "1.23456789f"), (cg.FloatLiteral(math.nan), "NAN"), + (cg.StringRefLiteral("foo"), 'StringRef::from_lit("foo")'), + (cg.StringRefLiteral(""), 'StringRef::from_lit("")'), + ( + cg.StringRefLiteral('with "quotes"'), + 'StringRef::from_lit("with \\042quotes\\042")', + ), ), ) def test_str__simple(self, target: cg.Literal, expected: str):