From d7832c44bc114461778e1d8aa8ee827e78e71b38 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Wed, 15 Oct 2025 17:45:37 -1000 Subject: [PATCH 01/31] [sensor] Fix sliding window filter memory fragmentation with FixedVector ring buffer --- esphome/components/sensor/filter.cpp | 235 +++++++++------------------ esphome/components/sensor/filter.h | 144 +++++++++------- 2 files changed, 165 insertions(+), 214 deletions(-) diff --git a/esphome/components/sensor/filter.cpp b/esphome/components/sensor/filter.cpp index 3241ae28af..900acd281a 100644 --- a/esphome/components/sensor/filter.cpp +++ b/esphome/components/sensor/filter.cpp @@ -32,50 +32,73 @@ void Filter::initialize(Sensor *parent, Filter *next) { this->next_ = next; } -// MedianFilter -MedianFilter::MedianFilter(size_t window_size, size_t send_every, size_t send_first_at) - : send_every_(send_every), send_at_(send_every - send_first_at), window_size_(window_size) {} -void MedianFilter::set_send_every(size_t send_every) { this->send_every_ = send_every; } -void MedianFilter::set_window_size(size_t window_size) { this->window_size_ = window_size; } -optional MedianFilter::new_value(float value) { - while (this->queue_.size() >= this->window_size_) { - this->queue_.pop_front(); - } - this->queue_.push_back(value); - ESP_LOGVV(TAG, "MedianFilter(%p)::new_value(%f)", this, value); +// SlidingWindowFilter +SlidingWindowFilter::SlidingWindowFilter(size_t window_size, size_t send_every, size_t send_first_at) + : window_size_(window_size), send_every_(send_every), send_at_(send_every - send_first_at) { + // Allocate ring buffer once at initialization + this->window_.init(window_size); +} +void SlidingWindowFilter::set_window_size(size_t window_size) { + this->window_size_ = window_size; + // Reallocate buffer with new size + this->window_.init(window_size); + this->window_head_ = 0; + this->window_count_ = 0; +} + +optional SlidingWindowFilter::new_value(float value) { + // Add value to ring buffer + if (this->window_count_ < this->window_size_) { + // Buffer not yet full - just append + this->window_.push_back(value); + this->window_count_++; + this->window_head_ = this->window_count_; + } else { + // Buffer full - overwrite oldest value (ring buffer) + this->window_[this->window_head_] = value; + this->window_head_ = (this->window_head_ + 1) % this->window_size_; + } + + // Check if we should send a result if (++this->send_at_ >= this->send_every_) { this->send_at_ = 0; - - float median = NAN; - if (!this->queue_.empty()) { - // Copy queue without NaN values - std::vector median_queue; - median_queue.reserve(this->queue_.size()); - for (auto v : this->queue_) { - if (!std::isnan(v)) { - median_queue.push_back(v); - } - } - - sort(median_queue.begin(), median_queue.end()); - - size_t queue_size = median_queue.size(); - if (queue_size) { - if (queue_size % 2) { - median = median_queue[queue_size / 2]; - } else { - median = (median_queue[queue_size / 2] + median_queue[(queue_size / 2) - 1]) / 2.0f; - } - } - } - - ESP_LOGVV(TAG, "MedianFilter(%p)::new_value(%f) SENDING %f", this, value, median); - return median; + float result = this->compute_result_(); + ESP_LOGVV(TAG, "SlidingWindowFilter(%p)::new_value(%f) SENDING %f", this, value, result); + return result; } return {}; } +// SortedWindowFilter +FixedVector SortedWindowFilter::get_sorted_values_() { + // Copy window without NaN values using FixedVector (no heap allocation) + FixedVector sorted_values; + sorted_values.init(this->window_count_); + for (size_t i = 0; i < this->window_count_; i++) { + float v = this->window_[i]; + if (!std::isnan(v)) { + sorted_values.push_back(v); + } + } + sort(sorted_values.begin(), sorted_values.end()); + return sorted_values; +} + +// MedianFilter +float MedianFilter::compute_result_() { + FixedVector sorted_values = this->get_sorted_values_(); + if (sorted_values.empty()) + return NAN; + + size_t size = sorted_values.size(); + if (size % 2) { + return sorted_values[size / 2]; + } else { + return (sorted_values[size / 2] + sorted_values[(size / 2) - 1]) / 2.0f; + } +} + // SkipInitialFilter SkipInitialFilter::SkipInitialFilter(size_t num_to_ignore) : num_to_ignore_(num_to_ignore) {} optional SkipInitialFilter::new_value(float value) { @@ -91,136 +114,36 @@ optional SkipInitialFilter::new_value(float value) { // QuantileFilter QuantileFilter::QuantileFilter(size_t window_size, size_t send_every, size_t send_first_at, float quantile) - : send_every_(send_every), send_at_(send_every - send_first_at), window_size_(window_size), quantile_(quantile) {} -void QuantileFilter::set_send_every(size_t send_every) { this->send_every_ = send_every; } -void QuantileFilter::set_window_size(size_t window_size) { this->window_size_ = window_size; } -void QuantileFilter::set_quantile(float quantile) { this->quantile_ = quantile; } -optional QuantileFilter::new_value(float value) { - while (this->queue_.size() >= this->window_size_) { - this->queue_.pop_front(); - } - this->queue_.push_back(value); - ESP_LOGVV(TAG, "QuantileFilter(%p)::new_value(%f), quantile:%f", this, value, this->quantile_); + : SortedWindowFilter(window_size, send_every, send_first_at), quantile_(quantile) {} - if (++this->send_at_ >= this->send_every_) { - this->send_at_ = 0; +float QuantileFilter::compute_result_() { + FixedVector sorted_values = this->get_sorted_values_(); + if (sorted_values.empty()) + return NAN; - float result = NAN; - if (!this->queue_.empty()) { - // Copy queue without NaN values - std::vector quantile_queue; - for (auto v : this->queue_) { - if (!std::isnan(v)) { - quantile_queue.push_back(v); - } - } - - sort(quantile_queue.begin(), quantile_queue.end()); - - size_t queue_size = quantile_queue.size(); - if (queue_size) { - size_t position = ceilf(queue_size * this->quantile_) - 1; - ESP_LOGVV(TAG, "QuantileFilter(%p)::position: %zu/%zu", this, position + 1, queue_size); - result = quantile_queue[position]; - } - } - - ESP_LOGVV(TAG, "QuantileFilter(%p)::new_value(%f) SENDING %f", this, value, result); - return result; - } - return {}; + size_t position = ceilf(sorted_values.size() * this->quantile_) - 1; + ESP_LOGVV(TAG, "QuantileFilter(%p)::position: %zu/%zu", this, position + 1, sorted_values.size()); + return sorted_values[position]; } // MinFilter -MinFilter::MinFilter(size_t window_size, size_t send_every, size_t send_first_at) - : send_every_(send_every), send_at_(send_every - send_first_at), window_size_(window_size) {} -void MinFilter::set_send_every(size_t send_every) { this->send_every_ = send_every; } -void MinFilter::set_window_size(size_t window_size) { this->window_size_ = window_size; } -optional MinFilter::new_value(float value) { - while (this->queue_.size() >= this->window_size_) { - this->queue_.pop_front(); - } - this->queue_.push_back(value); - ESP_LOGVV(TAG, "MinFilter(%p)::new_value(%f)", this, value); - - if (++this->send_at_ >= this->send_every_) { - this->send_at_ = 0; - - float min = NAN; - for (auto v : this->queue_) { - if (!std::isnan(v)) { - min = std::isnan(min) ? v : std::min(min, v); - } - } - - ESP_LOGVV(TAG, "MinFilter(%p)::new_value(%f) SENDING %f", this, value, min); - return min; - } - return {}; -} +float MinFilter::compute_result_() { return this->find_extremum_>(); } // MaxFilter -MaxFilter::MaxFilter(size_t window_size, size_t send_every, size_t send_first_at) - : send_every_(send_every), send_at_(send_every - send_first_at), window_size_(window_size) {} -void MaxFilter::set_send_every(size_t send_every) { this->send_every_ = send_every; } -void MaxFilter::set_window_size(size_t window_size) { this->window_size_ = window_size; } -optional MaxFilter::new_value(float value) { - while (this->queue_.size() >= this->window_size_) { - this->queue_.pop_front(); - } - this->queue_.push_back(value); - ESP_LOGVV(TAG, "MaxFilter(%p)::new_value(%f)", this, value); - - if (++this->send_at_ >= this->send_every_) { - this->send_at_ = 0; - - float max = NAN; - for (auto v : this->queue_) { - if (!std::isnan(v)) { - max = std::isnan(max) ? v : std::max(max, v); - } - } - - ESP_LOGVV(TAG, "MaxFilter(%p)::new_value(%f) SENDING %f", this, value, max); - return max; - } - return {}; -} +float MaxFilter::compute_result_() { return this->find_extremum_>(); } // SlidingWindowMovingAverageFilter -SlidingWindowMovingAverageFilter::SlidingWindowMovingAverageFilter(size_t window_size, size_t send_every, - size_t send_first_at) - : send_every_(send_every), send_at_(send_every - send_first_at), window_size_(window_size) {} -void SlidingWindowMovingAverageFilter::set_send_every(size_t send_every) { this->send_every_ = send_every; } -void SlidingWindowMovingAverageFilter::set_window_size(size_t window_size) { this->window_size_ = window_size; } -optional SlidingWindowMovingAverageFilter::new_value(float value) { - while (this->queue_.size() >= this->window_size_) { - this->queue_.pop_front(); - } - this->queue_.push_back(value); - ESP_LOGVV(TAG, "SlidingWindowMovingAverageFilter(%p)::new_value(%f)", this, value); - - if (++this->send_at_ >= this->send_every_) { - this->send_at_ = 0; - - float sum = 0; - size_t valid_count = 0; - for (auto v : this->queue_) { - if (!std::isnan(v)) { - sum += v; - valid_count++; - } +float SlidingWindowMovingAverageFilter::compute_result_() { + float sum = 0; + size_t valid_count = 0; + for (size_t i = 0; i < this->window_count_; i++) { + float v = this->window_[i]; + if (!std::isnan(v)) { + sum += v; + valid_count++; } - - float average = NAN; - if (valid_count) { - average = sum / valid_count; - } - - ESP_LOGVV(TAG, "SlidingWindowMovingAverageFilter(%p)::new_value(%f) SENDING %f", this, value, average); - return average; } - return {}; + return valid_count ? sum / valid_count : NAN; } // ExponentialMovingAverageFilter diff --git a/esphome/components/sensor/filter.h b/esphome/components/sensor/filter.h index 49d83e5b4b..0154cb8321 100644 --- a/esphome/components/sensor/filter.h +++ b/esphome/components/sensor/filter.h @@ -44,11 +44,78 @@ class Filter { Sensor *parent_{nullptr}; }; +/** Base class for filters that use a sliding window of values. + * + * Uses a ring buffer to efficiently maintain a fixed-size sliding window without + * reallocations or pop_front() overhead. Eliminates deque fragmentation issues. + */ +class SlidingWindowFilter : public Filter { + public: + SlidingWindowFilter(size_t window_size, size_t send_every, size_t send_first_at); + + void set_send_every(size_t send_every) { this->send_every_ = send_every; } + void set_window_size(size_t window_size); + + optional new_value(float value) final; + + protected: + /// Called by new_value() to compute the filtered result from the current window + virtual float compute_result_() = 0; + + /// Access the sliding window values (ring buffer implementation) + /// Use: for (size_t i = 0; i < window_count_; i++) { float val = window_[i]; } + FixedVector window_; + size_t window_head_{0}; ///< Index where next value will be written + size_t window_count_{0}; ///< Number of valid values in window (0 to window_size_) + size_t window_size_; ///< Maximum window size + size_t send_every_; ///< Send result every N values + size_t send_at_; ///< Counter for send_every +}; + +/** Base class for Min/Max filters. + * + * Provides a templated helper to find extremum values efficiently. + */ +class MinMaxFilter : public SlidingWindowFilter { + public: + using SlidingWindowFilter::SlidingWindowFilter; + + protected: + /// Helper to find min or max value in window, skipping NaN values + /// Usage: find_extremum_>() for min, find_extremum_>() for max + template float find_extremum_() { + float result = NAN; + Compare comp; + for (size_t i = 0; i < this->window_count_; i++) { + float v = this->window_[i]; + if (!std::isnan(v)) { + result = std::isnan(result) ? v : (comp(v, result) ? v : result); + } + } + return result; + } +}; + +/** Base class for filters that need a sorted window (Median, Quantile). + * + * Extends SlidingWindowFilter to provide a helper that creates a sorted copy + * of non-NaN values from the window. + */ +class SortedWindowFilter : public SlidingWindowFilter { + public: + using SlidingWindowFilter::SlidingWindowFilter; + + protected: + /// Helper to get sorted non-NaN values from the window + /// Returns empty FixedVector if all values are NaN + FixedVector get_sorted_values_(); +}; + /** Simple quantile filter. * - * Takes the quantile of the last values and pushes it out every . + * Takes the quantile of the last values and pushes it out every . */ -class QuantileFilter : public Filter { +class QuantileFilter : public SortedWindowFilter { public: /** Construct a QuantileFilter. * @@ -61,25 +128,18 @@ class QuantileFilter : public Filter { */ explicit QuantileFilter(size_t window_size, size_t send_every, size_t send_first_at, float quantile); - optional new_value(float value) override; - - void set_send_every(size_t send_every); - void set_window_size(size_t window_size); - void set_quantile(float quantile); + void set_quantile(float quantile) { this->quantile_ = quantile; } protected: - std::deque queue_; - size_t send_every_; - size_t send_at_; - size_t window_size_; + float compute_result_() override; float quantile_; }; /** Simple median filter. * - * Takes the median of the last values and pushes it out every . + * Takes the median of the last values and pushes it out every . */ -class MedianFilter : public Filter { +class MedianFilter : public SortedWindowFilter { public: /** Construct a MedianFilter. * @@ -89,18 +149,10 @@ class MedianFilter : public Filter { * on startup being published on the first *raw* value, so with no filter applied. Must be less than or equal to * send_every. */ - explicit MedianFilter(size_t window_size, size_t send_every, size_t send_first_at); - - optional new_value(float value) override; - - void set_send_every(size_t send_every); - void set_window_size(size_t window_size); + using SortedWindowFilter::SortedWindowFilter; protected: - std::deque queue_; - size_t send_every_; - size_t send_at_; - size_t window_size_; + float compute_result_() override; }; /** Simple skip filter. @@ -123,9 +175,9 @@ class SkipInitialFilter : public Filter { /** Simple min filter. * - * Takes the min of the last values and pushes it out every . + * Takes the min of the last values and pushes it out every . */ -class MinFilter : public Filter { +class MinFilter : public MinMaxFilter { public: /** Construct a MinFilter. * @@ -135,25 +187,17 @@ class MinFilter : public Filter { * on startup being published on the first *raw* value, so with no filter applied. Must be less than or equal to * send_every. */ - explicit MinFilter(size_t window_size, size_t send_every, size_t send_first_at); - - optional new_value(float value) override; - - void set_send_every(size_t send_every); - void set_window_size(size_t window_size); + using MinMaxFilter::MinMaxFilter; protected: - std::deque queue_; - size_t send_every_; - size_t send_at_; - size_t window_size_; + float compute_result_() override; }; /** Simple max filter. * - * Takes the max of the last values and pushes it out every . + * Takes the max of the last values and pushes it out every . */ -class MaxFilter : public Filter { +class MaxFilter : public MinMaxFilter { public: /** Construct a MaxFilter. * @@ -163,18 +207,10 @@ class MaxFilter : public Filter { * on startup being published on the first *raw* value, so with no filter applied. Must be less than or equal to * send_every. */ - explicit MaxFilter(size_t window_size, size_t send_every, size_t send_first_at); - - optional new_value(float value) override; - - void set_send_every(size_t send_every); - void set_window_size(size_t window_size); + using MinMaxFilter::MinMaxFilter; protected: - std::deque queue_; - size_t send_every_; - size_t send_at_; - size_t window_size_; + float compute_result_() override; }; /** Simple sliding window moving average filter. @@ -182,7 +218,7 @@ class MaxFilter : public Filter { * Essentially just takes takes the average of the last window_size values and pushes them out * every send_every. */ -class SlidingWindowMovingAverageFilter : public Filter { +class SlidingWindowMovingAverageFilter : public SlidingWindowFilter { public: /** Construct a SlidingWindowMovingAverageFilter. * @@ -192,18 +228,10 @@ class SlidingWindowMovingAverageFilter : public Filter { * on startup being published on the first *raw* value, so with no filter applied. Must be less than or equal to * send_every. */ - explicit SlidingWindowMovingAverageFilter(size_t window_size, size_t send_every, size_t send_first_at); - - optional new_value(float value) override; - - void set_send_every(size_t send_every); - void set_window_size(size_t window_size); + using SlidingWindowFilter::SlidingWindowFilter; protected: - std::deque queue_; - size_t send_every_; - size_t send_at_; - size_t window_size_; + float compute_result_() override; }; /** Simple exponential moving average filter. From 12874187dd80c0853aa6ce727b7f317ad7c88e2c Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Wed, 15 Oct 2025 17:50:27 -1000 Subject: [PATCH 02/31] fix --- esphome/components/sensor/filter.h | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/esphome/components/sensor/filter.h b/esphome/components/sensor/filter.h index 0154cb8321..9c2710bc93 100644 --- a/esphome/components/sensor/filter.h +++ b/esphome/components/sensor/filter.h @@ -60,7 +60,7 @@ class SlidingWindowFilter : public Filter { protected: /// Called by new_value() to compute the filtered result from the current window - virtual float compute_result_() = 0; + virtual float compute_result() = 0; /// Access the sliding window values (ring buffer implementation) /// Use: for (size_t i = 0; i < window_count_; i++) { float val = window_[i]; } @@ -131,7 +131,7 @@ class QuantileFilter : public SortedWindowFilter { void set_quantile(float quantile) { this->quantile_ = quantile; } protected: - float compute_result_() override; + float compute_result() override; float quantile_; }; @@ -152,7 +152,7 @@ class MedianFilter : public SortedWindowFilter { using SortedWindowFilter::SortedWindowFilter; protected: - float compute_result_() override; + float compute_result() override; }; /** Simple skip filter. @@ -190,7 +190,7 @@ class MinFilter : public MinMaxFilter { using MinMaxFilter::MinMaxFilter; protected: - float compute_result_() override; + float compute_result() override; }; /** Simple max filter. @@ -210,7 +210,7 @@ class MaxFilter : public MinMaxFilter { using MinMaxFilter::MinMaxFilter; protected: - float compute_result_() override; + float compute_result() override; }; /** Simple sliding window moving average filter. From 36f851130950d071ba914efaba95849e836d5bad Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Wed, 15 Oct 2025 17:50:32 -1000 Subject: [PATCH 03/31] fix --- esphome/components/sensor/filter.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/esphome/components/sensor/filter.h b/esphome/components/sensor/filter.h index 9c2710bc93..b391048521 100644 --- a/esphome/components/sensor/filter.h +++ b/esphome/components/sensor/filter.h @@ -231,7 +231,7 @@ class SlidingWindowMovingAverageFilter : public SlidingWindowFilter { using SlidingWindowFilter::SlidingWindowFilter; protected: - float compute_result_() override; + float compute_result() override; }; /** Simple exponential moving average filter. From cd252a33f9c0a16c67a8edc14f0311182eaf4c75 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Wed, 15 Oct 2025 17:51:03 -1000 Subject: [PATCH 04/31] fix --- esphome/components/sensor/filter.cpp | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/esphome/components/sensor/filter.cpp b/esphome/components/sensor/filter.cpp index 900acd281a..6406d670c1 100644 --- a/esphome/components/sensor/filter.cpp +++ b/esphome/components/sensor/filter.cpp @@ -63,7 +63,7 @@ optional SlidingWindowFilter::new_value(float value) { // Check if we should send a result if (++this->send_at_ >= this->send_every_) { this->send_at_ = 0; - float result = this->compute_result_(); + float result = this->compute_result(); ESP_LOGVV(TAG, "SlidingWindowFilter(%p)::new_value(%f) SENDING %f", this, value, result); return result; } @@ -86,7 +86,7 @@ FixedVector SortedWindowFilter::get_sorted_values_() { } // MedianFilter -float MedianFilter::compute_result_() { +float MedianFilter::compute_result() { FixedVector sorted_values = this->get_sorted_values_(); if (sorted_values.empty()) return NAN; @@ -116,7 +116,7 @@ optional SkipInitialFilter::new_value(float value) { QuantileFilter::QuantileFilter(size_t window_size, size_t send_every, size_t send_first_at, float quantile) : SortedWindowFilter(window_size, send_every, send_first_at), quantile_(quantile) {} -float QuantileFilter::compute_result_() { +float QuantileFilter::compute_result() { FixedVector sorted_values = this->get_sorted_values_(); if (sorted_values.empty()) return NAN; @@ -127,10 +127,10 @@ float QuantileFilter::compute_result_() { } // MinFilter -float MinFilter::compute_result_() { return this->find_extremum_>(); } +float MinFilter::compute_result() { return this->find_extremum_>(); } // MaxFilter -float MaxFilter::compute_result_() { return this->find_extremum_>(); } +float MaxFilter::compute_result() { return this->find_extremum_>(); } // SlidingWindowMovingAverageFilter float SlidingWindowMovingAverageFilter::compute_result_() { From 4c24545b826215fedf2e4c4aaa0cd15d01e2b25d Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Wed, 15 Oct 2025 17:51:08 -1000 Subject: [PATCH 05/31] fix --- esphome/components/sensor/filter.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/esphome/components/sensor/filter.cpp b/esphome/components/sensor/filter.cpp index 6406d670c1..a6819dd73c 100644 --- a/esphome/components/sensor/filter.cpp +++ b/esphome/components/sensor/filter.cpp @@ -133,7 +133,7 @@ float MinFilter::compute_result() { return this->find_extremum_ float MaxFilter::compute_result() { return this->find_extremum_>(); } // SlidingWindowMovingAverageFilter -float SlidingWindowMovingAverageFilter::compute_result_() { +float SlidingWindowMovingAverageFilter::compute_result() { float sum = 0; size_t valid_count = 0; for (size_t i = 0; i < this->window_count_; i++) { From b074ca8a1ed6c7658049bff8bc4e47e862d50b09 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Wed, 15 Oct 2025 18:00:33 -1000 Subject: [PATCH 06/31] fix --- esphome/components/sensor/filter.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/esphome/components/sensor/filter.cpp b/esphome/components/sensor/filter.cpp index a6819dd73c..0e52f9d94f 100644 --- a/esphome/components/sensor/filter.cpp +++ b/esphome/components/sensor/filter.cpp @@ -53,7 +53,7 @@ optional SlidingWindowFilter::new_value(float value) { // Buffer not yet full - just append this->window_.push_back(value); this->window_count_++; - this->window_head_ = this->window_count_; + this->window_head_ = this->window_count_ % this->window_size_; } else { // Buffer full - overwrite oldest value (ring buffer) this->window_[this->window_head_] = value; @@ -81,7 +81,7 @@ FixedVector SortedWindowFilter::get_sorted_values_() { sorted_values.push_back(v); } } - sort(sorted_values.begin(), sorted_values.end()); + std::sort(sorted_values.begin(), sorted_values.end()); return sorted_values; } From 9b6707c1c0b1a6e38bd1ff30d9a0621dd6aff7a9 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Wed, 15 Oct 2025 18:25:42 -1000 Subject: [PATCH 07/31] tests --- .../fixtures/sensor_filters_nan_handling.yaml | 50 +++ ...sensor_filters_ring_buffer_wraparound.yaml | 36 ++ .../sensor_filters_sliding_window.yaml | 78 ++++ .../test_sensor_filters_sliding_window.py | 385 ++++++++++++++++++ 4 files changed, 549 insertions(+) create mode 100644 tests/integration/fixtures/sensor_filters_nan_handling.yaml create mode 100644 tests/integration/fixtures/sensor_filters_ring_buffer_wraparound.yaml create mode 100644 tests/integration/fixtures/sensor_filters_sliding_window.yaml create mode 100644 tests/integration/test_sensor_filters_sliding_window.py diff --git a/tests/integration/fixtures/sensor_filters_nan_handling.yaml b/tests/integration/fixtures/sensor_filters_nan_handling.yaml new file mode 100644 index 0000000000..20fae64e8b --- /dev/null +++ b/tests/integration/fixtures/sensor_filters_nan_handling.yaml @@ -0,0 +1,50 @@ +esphome: + name: test-nan-handling + +host: +api: + batch_delay: 0ms # Disable batching to receive all state updates +logger: + level: DEBUG + +sensor: + - platform: template + name: "Source NaN Sensor" + id: source_nan_sensor + accuracy_decimals: 2 + + - platform: copy + source_id: source_nan_sensor + name: "Min NaN Sensor" + id: min_nan_sensor + filters: + - min: + window_size: 5 + send_every: 5 + + - platform: copy + source_id: source_nan_sensor + name: "Max NaN Sensor" + id: max_nan_sensor + filters: + - max: + window_size: 5 + send_every: 5 + +button: + - platform: template + name: "Publish NaN Values Button" + id: publish_nan_button + on_press: + - lambda: |- + // Publish 10 values with NaN mixed in: 10, NaN, 5, NaN, 15, 8, NaN, 12, 3, NaN + id(source_nan_sensor).publish_state(10.0); + id(source_nan_sensor).publish_state(NAN); + id(source_nan_sensor).publish_state(5.0); + id(source_nan_sensor).publish_state(NAN); + id(source_nan_sensor).publish_state(15.0); + id(source_nan_sensor).publish_state(8.0); + id(source_nan_sensor).publish_state(NAN); + id(source_nan_sensor).publish_state(12.0); + id(source_nan_sensor).publish_state(3.0); + id(source_nan_sensor).publish_state(NAN); diff --git a/tests/integration/fixtures/sensor_filters_ring_buffer_wraparound.yaml b/tests/integration/fixtures/sensor_filters_ring_buffer_wraparound.yaml new file mode 100644 index 0000000000..1ff9ec542a --- /dev/null +++ b/tests/integration/fixtures/sensor_filters_ring_buffer_wraparound.yaml @@ -0,0 +1,36 @@ +esphome: + name: test-ring-buffer-wraparound + +host: +api: + batch_delay: 0ms # Disable batching to receive all state updates +logger: + level: DEBUG + +sensor: + - platform: template + name: "Source Wraparound Sensor" + id: source_wraparound + accuracy_decimals: 2 + + - platform: copy + source_id: source_wraparound + name: "Wraparound Min Sensor" + id: wraparound_min_sensor + filters: + - min: + window_size: 3 + send_every: 3 + +button: + - platform: template + name: "Publish Wraparound Button" + id: publish_wraparound_button + on_press: + - lambda: |- + // Publish 9 values to test ring buffer wraparound + // Values: 10, 20, 30, 5, 25, 15, 40, 35, 20 + float values[] = {10.0, 20.0, 30.0, 5.0, 25.0, 15.0, 40.0, 35.0, 20.0}; + for (int i = 0; i < 9; i++) { + id(source_wraparound).publish_state(values[i]); + } diff --git a/tests/integration/fixtures/sensor_filters_sliding_window.yaml b/tests/integration/fixtures/sensor_filters_sliding_window.yaml new file mode 100644 index 0000000000..a2ae3182b8 --- /dev/null +++ b/tests/integration/fixtures/sensor_filters_sliding_window.yaml @@ -0,0 +1,78 @@ +esphome: + name: test-sliding-window-filters + +host: +api: + batch_delay: 0ms # Disable batching to receive all state updates +logger: + level: DEBUG + +# Template sensor that we'll use to publish values +sensor: + - platform: template + name: "Source Sensor" + id: source_sensor + accuracy_decimals: 2 + + # Min filter sensor + - platform: copy + source_id: source_sensor + name: "Min Sensor" + id: min_sensor + filters: + - min: + window_size: 5 + send_every: 5 + + # Max filter sensor + - platform: copy + source_id: source_sensor + name: "Max Sensor" + id: max_sensor + filters: + - max: + window_size: 5 + send_every: 5 + + # Median filter sensor + - platform: copy + source_id: source_sensor + name: "Median Sensor" + id: median_sensor + filters: + - median: + window_size: 5 + send_every: 5 + + # Quantile filter sensor (90th percentile) + - platform: copy + source_id: source_sensor + name: "Quantile Sensor" + id: quantile_sensor + filters: + - quantile: + window_size: 5 + send_every: 5 + quantile: 0.9 + + # Moving average filter sensor + - platform: copy + source_id: source_sensor + name: "Moving Avg Sensor" + id: moving_avg_sensor + filters: + - sliding_window_moving_average: + window_size: 5 + send_every: 5 + +# Button to trigger publishing test values +button: + - platform: template + name: "Publish Values Button" + id: publish_button + on_press: + - lambda: |- + // Publish 10 values: 1.0, 2.0, ..., 10.0 + for (int i = 1; i <= 10; i++) { + id(source_sensor).publish_state(float(i)); + } diff --git a/tests/integration/test_sensor_filters_sliding_window.py b/tests/integration/test_sensor_filters_sliding_window.py new file mode 100644 index 0000000000..943502c38f --- /dev/null +++ b/tests/integration/test_sensor_filters_sliding_window.py @@ -0,0 +1,385 @@ +"""Test sensor sliding window filter functionality.""" + +from __future__ import annotations + +import asyncio + +from aioesphomeapi import EntityInfo, EntityState, SensorState +import pytest + +from .types import APIClientConnectedFactory, RunCompiledFunction + + +def build_key_to_sensor_mapping( + entities: list[EntityInfo], sensor_names: list[str] +) -> dict[int, str]: + """Build a mapping from entity keys to sensor names. + + Args: + entities: List of entity info objects from the API + sensor_names: List of sensor names to search for in object_ids + + Returns: + Dictionary mapping entity keys to sensor names + """ + key_to_sensor: dict[int, str] = {} + for entity in entities: + obj_id = entity.object_id.lower() + for sensor_name in sensor_names: + if sensor_name in obj_id: + key_to_sensor[entity.key] = sensor_name + break + return key_to_sensor + + +@pytest.mark.asyncio +async def test_sensor_filters_sliding_window( + yaml_config: str, + run_compiled: RunCompiledFunction, + api_client_connected: APIClientConnectedFactory, +) -> None: + """Test that sliding window filters (min, max, median, quantile, moving_average) work correctly.""" + loop = asyncio.get_running_loop() + + # Track state changes for each sensor + sensor_states: dict[str, list[float]] = { + "min_sensor": [], + "max_sensor": [], + "median_sensor": [], + "quantile_sensor": [], + "moving_avg_sensor": [], + } + + # Futures to track when we receive expected values + min_received = loop.create_future() + max_received = loop.create_future() + median_received = loop.create_future() + quantile_received = loop.create_future() + moving_avg_received = loop.create_future() + + def on_state(state: EntityState) -> None: + """Track sensor state updates.""" + if not isinstance(state, SensorState): + return + + # Skip NaN values (initial states) + if state.missing_state: + return + + # Get the sensor name from the key mapping + sensor_name = key_to_sensor.get(state.key) + if sensor_name and sensor_name in sensor_states: + sensor_states[sensor_name].append(state.state) + + # Check if we received the expected final value + # After publishing 10 values [1.0, 2.0, ..., 10.0], the window has the last 5: [2, 3, 4, 5, 6] + # Filters send at position 1 and position 6 (send_every=5 means every 5th value after first) + if ( + sensor_name == "min_sensor" + and abs(state.state - 2.0) < 0.01 + and not min_received.done() + ): + min_received.set_result(True) + elif ( + sensor_name == "max_sensor" + and abs(state.state - 6.0) < 0.01 + and not max_received.done() + ): + max_received.set_result(True) + elif ( + sensor_name == "median_sensor" + and abs(state.state - 4.0) < 0.01 + and not median_received.done() + ): + # Median of [2, 3, 4, 5, 6] = 4 + median_received.set_result(True) + elif ( + sensor_name == "quantile_sensor" + and abs(state.state - 6.0) < 0.01 + and not quantile_received.done() + ): + # 90th percentile of [2, 3, 4, 5, 6] = 6 + quantile_received.set_result(True) + elif ( + sensor_name == "moving_avg_sensor" + and abs(state.state - 4.0) < 0.01 + and not moving_avg_received.done() + ): + # Average of [2, 3, 4, 5, 6] = 4 + moving_avg_received.set_result(True) + + async with ( + run_compiled(yaml_config), + api_client_connected() as client, + ): + # Get entities first to build key mapping + entities, services = await client.list_entities_services() + + # Build key-to-sensor mapping + key_to_sensor = build_key_to_sensor_mapping( + entities, + [ + "min_sensor", + "max_sensor", + "median_sensor", + "quantile_sensor", + "moving_avg_sensor", + ], + ) + + # Subscribe to state changes AFTER building mapping + client.subscribe_states(on_state) + + # Find the publish button + publish_button = next( + (e for e in entities if "publish_values_button" in e.object_id.lower()), + None, + ) + assert publish_button is not None, "Publish Values Button not found" + + # Press the button to publish test values + client.button_command(publish_button.key) + + # Wait for all sensors to receive their final values + try: + await asyncio.wait_for( + asyncio.gather( + min_received, + max_received, + median_received, + quantile_received, + moving_avg_received, + ), + timeout=10.0, + ) + except TimeoutError: + # Provide detailed failure info + pytest.fail( + f"Timeout waiting for expected values. Received states:\n" + f" min: {sensor_states['min_sensor']}\n" + f" max: {sensor_states['max_sensor']}\n" + f" median: {sensor_states['median_sensor']}\n" + f" quantile: {sensor_states['quantile_sensor']}\n" + f" moving_avg: {sensor_states['moving_avg_sensor']}" + ) + + # Verify we got the expected values + # With batch_delay: 0ms, we should receive all outputs + # Filters output at positions 1 and 6 (send_every: 5) + assert len(sensor_states["min_sensor"]) == 2, ( + f"Min sensor should have 2 values, got {len(sensor_states['min_sensor'])}: {sensor_states['min_sensor']}" + ) + assert len(sensor_states["max_sensor"]) == 2, ( + f"Max sensor should have 2 values, got {len(sensor_states['max_sensor'])}: {sensor_states['max_sensor']}" + ) + assert len(sensor_states["median_sensor"]) == 2 + assert len(sensor_states["quantile_sensor"]) == 2 + assert len(sensor_states["moving_avg_sensor"]) == 2 + + # Verify the first output (after 1 value: [1]) + assert abs(sensor_states["min_sensor"][0] - 1.0) < 0.01, ( + f"First min should be 1.0, got {sensor_states['min_sensor'][0]}" + ) + assert abs(sensor_states["max_sensor"][0] - 1.0) < 0.01, ( + f"First max should be 1.0, got {sensor_states['max_sensor'][0]}" + ) + assert abs(sensor_states["median_sensor"][0] - 1.0) < 0.01, ( + f"First median should be 1.0, got {sensor_states['median_sensor'][0]}" + ) + assert abs(sensor_states["moving_avg_sensor"][0] - 1.0) < 0.01, ( + f"First moving avg should be 1.0, got {sensor_states['moving_avg_sensor'][0]}" + ) + + # Verify the second output (after 6 values, window has [2, 3, 4, 5, 6]) + assert abs(sensor_states["min_sensor"][1] - 2.0) < 0.01, ( + f"Second min should be 2.0, got {sensor_states['min_sensor'][1]}" + ) + assert abs(sensor_states["max_sensor"][1] - 6.0) < 0.01, ( + f"Second max should be 6.0, got {sensor_states['max_sensor'][1]}" + ) + assert abs(sensor_states["median_sensor"][1] - 4.0) < 0.01, ( + f"Second median should be 4.0, got {sensor_states['median_sensor'][1]}" + ) + assert abs(sensor_states["moving_avg_sensor"][1] - 4.0) < 0.01, ( + f"Second moving avg should be 4.0, got {sensor_states['moving_avg_sensor'][1]}" + ) + + +@pytest.mark.asyncio +async def test_sensor_filters_nan_handling( + yaml_config: str, + run_compiled: RunCompiledFunction, + api_client_connected: APIClientConnectedFactory, +) -> None: + """Test that sliding window filters handle NaN values correctly.""" + loop = asyncio.get_running_loop() + + # Track states + min_states: list[float] = [] + max_states: list[float] = [] + + # Future to track completion + filters_completed = loop.create_future() + + def on_state(state: EntityState) -> None: + """Track sensor state updates.""" + if not isinstance(state, SensorState): + return + + # Skip NaN values (initial states) + if state.missing_state: + return + + sensor_name = key_to_sensor.get(state.key) + if sensor_name == "min_nan": + min_states.append(state.state) + elif sensor_name == "max_nan": + max_states.append(state.state) + + # Check if both have received their final values + # With batch_delay: 0ms, we should receive 2 outputs each + if ( + len(min_states) >= 2 + and len(max_states) >= 2 + and not filters_completed.done() + ): + filters_completed.set_result(True) + + async with ( + run_compiled(yaml_config), + api_client_connected() as client, + ): + # Get entities first to build key mapping + entities, services = await client.list_entities_services() + + # Build key-to-sensor mapping + key_to_sensor = build_key_to_sensor_mapping(entities, ["min_nan", "max_nan"]) + + # Subscribe to state changes AFTER building mapping + client.subscribe_states(on_state) + + # Find the publish button + publish_button = next( + (e for e in entities if "publish_nan_values_button" in e.object_id.lower()), + None, + ) + assert publish_button is not None, "Publish NaN Values Button not found" + + # Press the button + client.button_command(publish_button.key) + + # Wait for filters to process + try: + await asyncio.wait_for(filters_completed, timeout=10.0) + except TimeoutError: + pytest.fail( + f"Timeout waiting for NaN handling. Received:\n" + f" min_states: {min_states}\n" + f" max_states: {max_states}" + ) + + # Verify NaN values were ignored + # With batch_delay: 0ms, we should receive both outputs (at positions 1 and 6) + # Position 1: window=[10], min=10, max=10 + # Position 6: window=[NaN, 5, NaN, 15, 8], ignoring NaN -> [5, 15, 8], min=5, max=15 + assert len(min_states) == 2, ( + f"Should have 2 min states, got {len(min_states)}: {min_states}" + ) + assert len(max_states) == 2, ( + f"Should have 2 max states, got {len(max_states)}: {max_states}" + ) + + # First output + assert abs(min_states[0] - 10.0) < 0.01, ( + f"First min should be 10.0, got {min_states[0]}" + ) + assert abs(max_states[0] - 10.0) < 0.01, ( + f"First max should be 10.0, got {max_states[0]}" + ) + + # Second output - verify NaN values were ignored + assert abs(min_states[1] - 5.0) < 0.01, ( + f"Second min should ignore NaN and return 5.0, got {min_states[1]}" + ) + assert abs(max_states[1] - 15.0) < 0.01, ( + f"Second max should ignore NaN and return 15.0, got {max_states[1]}" + ) + + +@pytest.mark.asyncio +async def test_sensor_filters_ring_buffer_wraparound( + yaml_config: str, + run_compiled: RunCompiledFunction, + api_client_connected: APIClientConnectedFactory, +) -> None: + """Test that ring buffer correctly wraps around when window fills up.""" + loop = asyncio.get_running_loop() + + min_states: list[float] = [] + + test_completed = loop.create_future() + + def on_state(state: EntityState) -> None: + """Track min sensor states.""" + if not isinstance(state, SensorState): + return + + # Skip NaN values (initial states) + if state.missing_state: + return + + sensor_name = key_to_sensor.get(state.key) + if sensor_name == "wraparound_min": + min_states.append(state.state) + # With batch_delay: 0ms, we should receive all 3 outputs + if len(min_states) >= 3 and not test_completed.done(): + test_completed.set_result(True) + + async with ( + run_compiled(yaml_config), + api_client_connected() as client, + ): + # Get entities first to build key mapping + entities, services = await client.list_entities_services() + + # Build key-to-sensor mapping + key_to_sensor = build_key_to_sensor_mapping(entities, ["wraparound_min"]) + + # Subscribe to state changes AFTER building mapping + client.subscribe_states(on_state) + + # Find the publish button + publish_button = next( + (e for e in entities if "publish_wraparound_button" in e.object_id.lower()), + None, + ) + assert publish_button is not None, "Publish Wraparound Button not found" + + # Press the button + # Will publish: 10, 20, 30, 5, 25, 15, 40, 35, 20 + client.button_command(publish_button.key) + + # Wait for completion + try: + await asyncio.wait_for(test_completed, timeout=10.0) + except TimeoutError: + pytest.fail(f"Timeout waiting for wraparound test. Received: {min_states}") + + # Verify outputs + # With window_size=3, send_every=3, we get outputs at positions 1, 4, 7 + # Position 1: window=[10], min=10 + # Position 4: window=[20, 30, 5], min=5 + # Position 7: window=[15, 40, 35], min=15 + # With batch_delay: 0ms, we should receive all 3 outputs + assert len(min_states) == 3, ( + f"Should have 3 states, got {len(min_states)}: {min_states}" + ) + assert abs(min_states[0] - 10.0) < 0.01, ( + f"First min should be 10.0, got {min_states[0]}" + ) + assert abs(min_states[1] - 5.0) < 0.01, ( + f"Second min should be 5.0, got {min_states[1]}" + ) + assert abs(min_states[2] - 15.0) < 0.01, ( + f"Third min should be 15.0, got {min_states[2]}" + ) From 447ee3da39ef1327693ac999c2c80c75fae90b05 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Wed, 15 Oct 2025 18:26:23 -1000 Subject: [PATCH 08/31] tests --- .../test_sensor_filters_sliding_window.py | 78 ++++++++++--------- 1 file changed, 40 insertions(+), 38 deletions(-) diff --git a/tests/integration/test_sensor_filters_sliding_window.py b/tests/integration/test_sensor_filters_sliding_window.py index 943502c38f..0c7aec70aa 100644 --- a/tests/integration/test_sensor_filters_sliding_window.py +++ b/tests/integration/test_sensor_filters_sliding_window.py @@ -68,45 +68,47 @@ async def test_sensor_filters_sliding_window( # Get the sensor name from the key mapping sensor_name = key_to_sensor.get(state.key) - if sensor_name and sensor_name in sensor_states: - sensor_states[sensor_name].append(state.state) + if not sensor_name or sensor_name not in sensor_states: + return - # Check if we received the expected final value - # After publishing 10 values [1.0, 2.0, ..., 10.0], the window has the last 5: [2, 3, 4, 5, 6] - # Filters send at position 1 and position 6 (send_every=5 means every 5th value after first) - if ( - sensor_name == "min_sensor" - and abs(state.state - 2.0) < 0.01 - and not min_received.done() - ): - min_received.set_result(True) - elif ( - sensor_name == "max_sensor" - and abs(state.state - 6.0) < 0.01 - and not max_received.done() - ): - max_received.set_result(True) - elif ( - sensor_name == "median_sensor" - and abs(state.state - 4.0) < 0.01 - and not median_received.done() - ): - # Median of [2, 3, 4, 5, 6] = 4 - median_received.set_result(True) - elif ( - sensor_name == "quantile_sensor" - and abs(state.state - 6.0) < 0.01 - and not quantile_received.done() - ): - # 90th percentile of [2, 3, 4, 5, 6] = 6 - quantile_received.set_result(True) - elif ( - sensor_name == "moving_avg_sensor" - and abs(state.state - 4.0) < 0.01 - and not moving_avg_received.done() - ): - # Average of [2, 3, 4, 5, 6] = 4 - moving_avg_received.set_result(True) + sensor_states[sensor_name].append(state.state) + + # Check if we received the expected final value + # After publishing 10 values [1.0, 2.0, ..., 10.0], the window has the last 5: [2, 3, 4, 5, 6] + # Filters send at position 1 and position 6 (send_every=5 means every 5th value after first) + if ( + sensor_name == "min_sensor" + and abs(state.state - 2.0) < 0.01 + and not min_received.done() + ): + min_received.set_result(True) + elif ( + sensor_name == "max_sensor" + and abs(state.state - 6.0) < 0.01 + and not max_received.done() + ): + max_received.set_result(True) + elif ( + sensor_name == "median_sensor" + and abs(state.state - 4.0) < 0.01 + and not median_received.done() + ): + # Median of [2, 3, 4, 5, 6] = 4 + median_received.set_result(True) + elif ( + sensor_name == "quantile_sensor" + and abs(state.state - 6.0) < 0.01 + and not quantile_received.done() + ): + # 90th percentile of [2, 3, 4, 5, 6] = 6 + quantile_received.set_result(True) + elif ( + sensor_name == "moving_avg_sensor" + and abs(state.state - 4.0) < 0.01 + and not moving_avg_received.done() + ): + # Average of [2, 3, 4, 5, 6] = 4 + moving_avg_received.set_result(True) async with ( run_compiled(yaml_config), From a4b14902db2f0d7fdba63d866b456a77c68b8655 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Wed, 15 Oct 2025 18:44:37 -1000 Subject: [PATCH 09/31] perf --- esphome/components/sensor/filter.cpp | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/esphome/components/sensor/filter.cpp b/esphome/components/sensor/filter.cpp index 0e52f9d94f..4863c00a29 100644 --- a/esphome/components/sensor/filter.cpp +++ b/esphome/components/sensor/filter.cpp @@ -53,11 +53,13 @@ optional SlidingWindowFilter::new_value(float value) { // Buffer not yet full - just append this->window_.push_back(value); this->window_count_++; - this->window_head_ = this->window_count_ % this->window_size_; } else { // Buffer full - overwrite oldest value (ring buffer) this->window_[this->window_head_] = value; - this->window_head_ = (this->window_head_ + 1) % this->window_size_; + this->window_head_++; + if (this->window_head_ >= this->window_size_) { + this->window_head_ = 0; + } } // Check if we should send a result From e3089ff0f6b7c015e10adc5ca7df05060177a647 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Wed, 15 Oct 2025 19:21:33 -1000 Subject: [PATCH 10/31] tweak --- esphome/components/sensor/__init__.py | 42 +++++++++++++++ esphome/components/sensor/filter.cpp | 73 +++++++++++++++++++++++++ esphome/components/sensor/filter.h | 76 +++++++++++++++++++++++++++ 3 files changed, 191 insertions(+) diff --git a/esphome/components/sensor/__init__.py b/esphome/components/sensor/__init__.py index 2b99f68ac0..1585a6342f 100644 --- a/esphome/components/sensor/__init__.py +++ b/esphome/components/sensor/__init__.py @@ -249,6 +249,9 @@ MaxFilter = sensor_ns.class_("MaxFilter", Filter) SlidingWindowMovingAverageFilter = sensor_ns.class_( "SlidingWindowMovingAverageFilter", Filter ) +StreamingMinFilter = sensor_ns.class_("StreamingMinFilter", Filter) +StreamingMaxFilter = sensor_ns.class_("StreamingMaxFilter", Filter) +StreamingMovingAverageFilter = sensor_ns.class_("StreamingMovingAverageFilter", Filter) ExponentialMovingAverageFilter = sensor_ns.class_( "ExponentialMovingAverageFilter", Filter ) @@ -452,6 +455,19 @@ async def skip_initial_filter_to_code(config, filter_id): @FILTER_REGISTRY.register("min", MinFilter, MIN_SCHEMA) async def min_filter_to_code(config, filter_id): + window_size = config[CONF_WINDOW_SIZE] + send_every = config[CONF_SEND_EVERY] + send_first_at = config[CONF_SEND_FIRST_AT] + + # Optimization: Use streaming filter for batch windows (window_size == send_every) + # Saves 99.98% memory for large windows (e.g., 20KB → 4 bytes for window_size=5000) + if window_size == send_every: + return cg.new_Pvariable( + filter_id, + StreamingMinFilter, + window_size, + send_first_at, + ) return cg.new_Pvariable( filter_id, config[CONF_WINDOW_SIZE], @@ -474,6 +490,19 @@ MAX_SCHEMA = cv.All( @FILTER_REGISTRY.register("max", MaxFilter, MAX_SCHEMA) async def max_filter_to_code(config, filter_id): + window_size = config[CONF_WINDOW_SIZE] + send_every = config[CONF_SEND_EVERY] + send_first_at = config[CONF_SEND_FIRST_AT] + + # Optimization: Use streaming filter for batch windows (window_size == send_every) + # Saves 99.98% memory for large windows (e.g., 20KB → 4 bytes for window_size=5000) + if window_size == send_every: + return cg.new_Pvariable( + filter_id, + StreamingMaxFilter, + window_size, + send_first_at, + ) return cg.new_Pvariable( filter_id, config[CONF_WINDOW_SIZE], @@ -500,6 +529,19 @@ SLIDING_AVERAGE_SCHEMA = cv.All( SLIDING_AVERAGE_SCHEMA, ) async def sliding_window_moving_average_filter_to_code(config, filter_id): + window_size = config[CONF_WINDOW_SIZE] + send_every = config[CONF_SEND_EVERY] + send_first_at = config[CONF_SEND_FIRST_AT] + + # Optimization: Use streaming filter for batch windows (window_size == send_every) + # Saves 99.94% memory for large windows (e.g., 20KB → 12 bytes for window_size=5000) + if window_size == send_every: + return cg.new_Pvariable( + filter_id, + StreamingMovingAverageFilter, + window_size, + send_first_at, + ) return cg.new_Pvariable( filter_id, config[CONF_WINDOW_SIZE], diff --git a/esphome/components/sensor/filter.cpp b/esphome/components/sensor/filter.cpp index 4863c00a29..c804125dcc 100644 --- a/esphome/components/sensor/filter.cpp +++ b/esphome/components/sensor/filter.cpp @@ -468,5 +468,78 @@ optional ToNTCTemperatureFilter::new_value(float value) { return temp; } +// StreamingFilter (base class) +StreamingFilter::StreamingFilter(size_t window_size, size_t send_first_at) + : window_size_(window_size), send_first_at_(send_first_at) {} + +optional StreamingFilter::new_value(float value) { + // Process the value (child class tracks min/max/sum/etc) + this->process_value(value); + + this->count_++; + + // Check if we should send (handle send_first_at for first value) + bool should_send = false; + if (this->first_send_ && this->count_ >= this->send_first_at_) { + should_send = true; + this->first_send_ = false; + } else if (!this->first_send_ && this->count_ >= this->window_size_) { + should_send = true; + } + + if (should_send) { + float result = this->compute_batch_result(); + // Reset for next batch + this->count_ = 0; + this->reset_batch(); + ESP_LOGVV(TAG, "StreamingFilter(%p)::new_value(%f) SENDING %f", this, value, result); + return result; + } + + return {}; +} + +// StreamingMinFilter +void StreamingMinFilter::process_value(float value) { + // Update running minimum (ignore NaN values) + if (!std::isnan(value)) { + this->current_min_ = std::isnan(this->current_min_) ? value : std::min(this->current_min_, value); + } +} + +float StreamingMinFilter::compute_batch_result() { return this->current_min_; } + +void StreamingMinFilter::reset_batch() { this->current_min_ = NAN; } + +// StreamingMaxFilter +void StreamingMaxFilter::process_value(float value) { + // Update running maximum (ignore NaN values) + if (!std::isnan(value)) { + this->current_max_ = std::isnan(this->current_max_) ? value : std::max(this->current_max_, value); + } +} + +float StreamingMaxFilter::compute_batch_result() { return this->current_max_; } + +void StreamingMaxFilter::reset_batch() { this->current_max_ = NAN; } + +// StreamingMovingAverageFilter +void StreamingMovingAverageFilter::process_value(float value) { + // Accumulate sum (ignore NaN values) + if (!std::isnan(value)) { + this->sum_ += value; + this->valid_count_++; + } +} + +float StreamingMovingAverageFilter::compute_batch_result() { + return this->valid_count_ > 0 ? this->sum_ / this->valid_count_ : NAN; +} + +void StreamingMovingAverageFilter::reset_batch() { + this->sum_ = 0.0f; + this->valid_count_ = 0; +} + } // namespace sensor } // namespace esphome diff --git a/esphome/components/sensor/filter.h b/esphome/components/sensor/filter.h index b391048521..c9b39b73c3 100644 --- a/esphome/components/sensor/filter.h +++ b/esphome/components/sensor/filter.h @@ -504,5 +504,81 @@ class ToNTCTemperatureFilter : public Filter { double c_; }; +/** Base class for streaming filters (batch windows where window_size == send_every). + * + * When window_size equals send_every, we don't need a sliding window. + * This base class handles the common batching logic. + */ +class StreamingFilter : public Filter { + public: + StreamingFilter(size_t window_size, size_t send_first_at); + + optional new_value(float value) final; + + protected: + /// Called by new_value() to process each value in the batch + virtual void process_value(float value) = 0; + + /// Called by new_value() to compute the result after collecting window_size values + virtual float compute_batch_result() = 0; + + /// Called by new_value() to reset internal state after sending a result + virtual void reset_batch() = 0; + + size_t window_size_; + size_t count_{0}; + size_t send_first_at_; + bool first_send_{true}; +}; + +/** Streaming min filter for batch windows (window_size == send_every). + * + * Uses O(1) memory instead of O(n) by tracking only the minimum value. + */ +class StreamingMinFilter : public StreamingFilter { + public: + using StreamingFilter::StreamingFilter; + + protected: + void process_value(float value) override; + float compute_batch_result() override; + void reset_batch() override; + + float current_min_{NAN}; +}; + +/** Streaming max filter for batch windows (window_size == send_every). + * + * Uses O(1) memory instead of O(n) by tracking only the maximum value. + */ +class StreamingMaxFilter : public StreamingFilter { + public: + using StreamingFilter::StreamingFilter; + + protected: + void process_value(float value) override; + float compute_batch_result() override; + void reset_batch() override; + + float current_max_{NAN}; +}; + +/** Streaming moving average filter for batch windows (window_size == send_every). + * + * Uses O(1) memory instead of O(n) by tracking only sum and count. + */ +class StreamingMovingAverageFilter : public StreamingFilter { + public: + using StreamingFilter::StreamingFilter; + + protected: + void process_value(float value) override; + float compute_batch_result() override; + void reset_batch() override; + + float sum_{0.0f}; + size_t valid_count_{0}; +}; + } // namespace sensor } // namespace esphome From a72c494b758389b145191d5b1827e705d0c8e1ea Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Wed, 15 Oct 2025 19:23:01 -1000 Subject: [PATCH 11/31] tweak --- esphome/components/sensor/__init__.py | 68 ++++++++------------------- 1 file changed, 20 insertions(+), 48 deletions(-) diff --git a/esphome/components/sensor/__init__.py b/esphome/components/sensor/__init__.py index 1585a6342f..ec5bf1364d 100644 --- a/esphome/components/sensor/__init__.py +++ b/esphome/components/sensor/__init__.py @@ -453,8 +453,12 @@ async def skip_initial_filter_to_code(config, filter_id): return cg.new_Pvariable(filter_id, config) -@FILTER_REGISTRY.register("min", MinFilter, MIN_SCHEMA) -async def min_filter_to_code(config, filter_id): +def _create_sliding_window_filter(config, filter_id, sliding_class, streaming_class): + """Helper to create sliding window or streaming filter based on config. + + When window_size == send_every, use streaming filter (O(1) memory). + Otherwise, use sliding window filter (O(n) memory). + """ window_size = config[CONF_WINDOW_SIZE] send_every = config[CONF_SEND_EVERY] send_first_at = config[CONF_SEND_FIRST_AT] @@ -462,17 +466,14 @@ async def min_filter_to_code(config, filter_id): # Optimization: Use streaming filter for batch windows (window_size == send_every) # Saves 99.98% memory for large windows (e.g., 20KB → 4 bytes for window_size=5000) if window_size == send_every: - return cg.new_Pvariable( - filter_id, - StreamingMinFilter, - window_size, - send_first_at, - ) - return cg.new_Pvariable( - filter_id, - config[CONF_WINDOW_SIZE], - config[CONF_SEND_EVERY], - config[CONF_SEND_FIRST_AT], + return cg.new_Pvariable(filter_id, streaming_class, window_size, send_first_at) + return cg.new_Pvariable(filter_id, window_size, send_every, send_first_at) + + +@FILTER_REGISTRY.register("min", MinFilter, MIN_SCHEMA) +async def min_filter_to_code(config, filter_id): + return _create_sliding_window_filter( + config, filter_id, MinFilter, StreamingMinFilter ) @@ -490,24 +491,8 @@ MAX_SCHEMA = cv.All( @FILTER_REGISTRY.register("max", MaxFilter, MAX_SCHEMA) async def max_filter_to_code(config, filter_id): - window_size = config[CONF_WINDOW_SIZE] - send_every = config[CONF_SEND_EVERY] - send_first_at = config[CONF_SEND_FIRST_AT] - - # Optimization: Use streaming filter for batch windows (window_size == send_every) - # Saves 99.98% memory for large windows (e.g., 20KB → 4 bytes for window_size=5000) - if window_size == send_every: - return cg.new_Pvariable( - filter_id, - StreamingMaxFilter, - window_size, - send_first_at, - ) - return cg.new_Pvariable( - filter_id, - config[CONF_WINDOW_SIZE], - config[CONF_SEND_EVERY], - config[CONF_SEND_FIRST_AT], + return _create_sliding_window_filter( + config, filter_id, MaxFilter, StreamingMaxFilter ) @@ -529,24 +514,11 @@ SLIDING_AVERAGE_SCHEMA = cv.All( SLIDING_AVERAGE_SCHEMA, ) async def sliding_window_moving_average_filter_to_code(config, filter_id): - window_size = config[CONF_WINDOW_SIZE] - send_every = config[CONF_SEND_EVERY] - send_first_at = config[CONF_SEND_FIRST_AT] - - # Optimization: Use streaming filter for batch windows (window_size == send_every) - # Saves 99.94% memory for large windows (e.g., 20KB → 12 bytes for window_size=5000) - if window_size == send_every: - return cg.new_Pvariable( - filter_id, - StreamingMovingAverageFilter, - window_size, - send_first_at, - ) - return cg.new_Pvariable( + return _create_sliding_window_filter( + config, filter_id, - config[CONF_WINDOW_SIZE], - config[CONF_SEND_EVERY], - config[CONF_SEND_FIRST_AT], + SlidingWindowMovingAverageFilter, + StreamingMovingAverageFilter, ) From 5a8558e1c557cd4308264d9d18d027d464886791 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Wed, 15 Oct 2025 19:23:35 -1000 Subject: [PATCH 12/31] tweak --- esphome/components/sensor/__init__.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/esphome/components/sensor/__init__.py b/esphome/components/sensor/__init__.py index ec5bf1364d..738ccf2ac6 100644 --- a/esphome/components/sensor/__init__.py +++ b/esphome/components/sensor/__init__.py @@ -467,7 +467,9 @@ def _create_sliding_window_filter(config, filter_id, sliding_class, streaming_cl # Saves 99.98% memory for large windows (e.g., 20KB → 4 bytes for window_size=5000) if window_size == send_every: return cg.new_Pvariable(filter_id, streaming_class, window_size, send_first_at) - return cg.new_Pvariable(filter_id, window_size, send_every, send_first_at) + return cg.new_Pvariable( + filter_id, sliding_class, window_size, send_every, send_first_at + ) @FILTER_REGISTRY.register("min", MinFilter, MIN_SCHEMA) From 589c25e65a3be2bf8d192fd8c580357d4e2241be Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Wed, 15 Oct 2025 19:24:44 -1000 Subject: [PATCH 13/31] tweak --- esphome/components/sensor/__init__.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/esphome/components/sensor/__init__.py b/esphome/components/sensor/__init__.py index 738ccf2ac6..b2c81f6239 100644 --- a/esphome/components/sensor/__init__.py +++ b/esphome/components/sensor/__init__.py @@ -472,7 +472,7 @@ def _create_sliding_window_filter(config, filter_id, sliding_class, streaming_cl ) -@FILTER_REGISTRY.register("min", MinFilter, MIN_SCHEMA) +@FILTER_REGISTRY.register("min", Filter, MIN_SCHEMA) async def min_filter_to_code(config, filter_id): return _create_sliding_window_filter( config, filter_id, MinFilter, StreamingMinFilter @@ -491,7 +491,7 @@ MAX_SCHEMA = cv.All( ) -@FILTER_REGISTRY.register("max", MaxFilter, MAX_SCHEMA) +@FILTER_REGISTRY.register("max", Filter, MAX_SCHEMA) async def max_filter_to_code(config, filter_id): return _create_sliding_window_filter( config, filter_id, MaxFilter, StreamingMaxFilter @@ -512,7 +512,7 @@ SLIDING_AVERAGE_SCHEMA = cv.All( @FILTER_REGISTRY.register( "sliding_window_moving_average", - SlidingWindowMovingAverageFilter, + Filter, SLIDING_AVERAGE_SCHEMA, ) async def sliding_window_moving_average_filter_to_code(config, filter_id): From 92d54ffb09131521432ffa6e22be9b81c10cb6e7 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Wed, 15 Oct 2025 19:28:51 -1000 Subject: [PATCH 14/31] tweak --- esphome/components/sensor/__init__.py | 60 +++++++++++++-------------- 1 file changed, 29 insertions(+), 31 deletions(-) diff --git a/esphome/components/sensor/__init__.py b/esphome/components/sensor/__init__.py index b2c81f6239..feb7d0374d 100644 --- a/esphome/components/sensor/__init__.py +++ b/esphome/components/sensor/__init__.py @@ -453,30 +453,19 @@ async def skip_initial_filter_to_code(config, filter_id): return cg.new_Pvariable(filter_id, config) -def _create_sliding_window_filter(config, filter_id, sliding_class, streaming_class): - """Helper to create sliding window or streaming filter based on config. - - When window_size == send_every, use streaming filter (O(1) memory). - Otherwise, use sliding window filter (O(n) memory). - """ - window_size = config[CONF_WINDOW_SIZE] - send_every = config[CONF_SEND_EVERY] - send_first_at = config[CONF_SEND_FIRST_AT] +@FILTER_REGISTRY.register("min", MinFilter, MIN_SCHEMA) +async def min_filter_to_code(config, filter_id): + window_size: int = config[CONF_WINDOW_SIZE] + send_every: int = config[CONF_SEND_EVERY] + send_first_at: int = config[CONF_SEND_FIRST_AT] # Optimization: Use streaming filter for batch windows (window_size == send_every) # Saves 99.98% memory for large windows (e.g., 20KB → 4 bytes for window_size=5000) if window_size == send_every: - return cg.new_Pvariable(filter_id, streaming_class, window_size, send_first_at) - return cg.new_Pvariable( - filter_id, sliding_class, window_size, send_every, send_first_at - ) - - -@FILTER_REGISTRY.register("min", Filter, MIN_SCHEMA) -async def min_filter_to_code(config, filter_id): - return _create_sliding_window_filter( - config, filter_id, MinFilter, StreamingMinFilter - ) + # Use streaming filter - O(1) memory instead of O(n) + return cg.Pvariable(filter_id, StreamingMinFilter, window_size, send_first_at) + # Use sliding window filter - maintains ring buffer + return cg.new_Pvariable(filter_id, window_size, send_every, send_first_at) MAX_SCHEMA = cv.All( @@ -491,11 +480,16 @@ MAX_SCHEMA = cv.All( ) -@FILTER_REGISTRY.register("max", Filter, MAX_SCHEMA) +@FILTER_REGISTRY.register("max", MaxFilter, MAX_SCHEMA) async def max_filter_to_code(config, filter_id): - return _create_sliding_window_filter( - config, filter_id, MaxFilter, StreamingMaxFilter - ) + window_size: int = config[CONF_WINDOW_SIZE] + send_every: int = config[CONF_SEND_EVERY] + send_first_at: int = config[CONF_SEND_FIRST_AT] + + # Optimization: Use streaming filter for batch windows (window_size == send_every) + if window_size == send_every: + return cg.Pvariable(filter_id, StreamingMaxFilter, window_size, send_first_at) + return cg.new_Pvariable(filter_id, window_size, send_every, send_first_at) SLIDING_AVERAGE_SCHEMA = cv.All( @@ -512,16 +506,20 @@ SLIDING_AVERAGE_SCHEMA = cv.All( @FILTER_REGISTRY.register( "sliding_window_moving_average", - Filter, + SlidingWindowMovingAverageFilter, SLIDING_AVERAGE_SCHEMA, ) async def sliding_window_moving_average_filter_to_code(config, filter_id): - return _create_sliding_window_filter( - config, - filter_id, - SlidingWindowMovingAverageFilter, - StreamingMovingAverageFilter, - ) + window_size: int = config[CONF_WINDOW_SIZE] + send_every: int = config[CONF_SEND_EVERY] + send_first_at: int = config[CONF_SEND_FIRST_AT] + + # Optimization: Use streaming filter for batch windows (window_size == send_every) + if window_size == send_every: + return cg.Pvariable( + filter_id, StreamingMovingAverageFilter, window_size, send_first_at + ) + return cg.new_Pvariable(filter_id, window_size, send_every, send_first_at) EXPONENTIAL_AVERAGE_SCHEMA = cv.All( From a999349fa5726720ffd0a8e3f098ad937ba4d17f Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Wed, 15 Oct 2025 19:29:55 -1000 Subject: [PATCH 15/31] tweak --- esphome/components/sensor/__init__.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/esphome/components/sensor/__init__.py b/esphome/components/sensor/__init__.py index feb7d0374d..a7a92d3968 100644 --- a/esphome/components/sensor/__init__.py +++ b/esphome/components/sensor/__init__.py @@ -463,7 +463,8 @@ async def min_filter_to_code(config, filter_id): # Saves 99.98% memory for large windows (e.g., 20KB → 4 bytes for window_size=5000) if window_size == send_every: # Use streaming filter - O(1) memory instead of O(n) - return cg.Pvariable(filter_id, StreamingMinFilter, window_size, send_first_at) + rhs = cg.new_Pvariable(StreamingMinFilter, window_size, send_first_at) + return cg.Pvariable(filter_id, rhs) # Use sliding window filter - maintains ring buffer return cg.new_Pvariable(filter_id, window_size, send_every, send_first_at) @@ -488,7 +489,8 @@ async def max_filter_to_code(config, filter_id): # Optimization: Use streaming filter for batch windows (window_size == send_every) if window_size == send_every: - return cg.Pvariable(filter_id, StreamingMaxFilter, window_size, send_first_at) + rhs = cg.new_Pvariable(StreamingMaxFilter, window_size, send_first_at) + return cg.Pvariable(filter_id, rhs) return cg.new_Pvariable(filter_id, window_size, send_every, send_first_at) @@ -516,9 +518,8 @@ async def sliding_window_moving_average_filter_to_code(config, filter_id): # Optimization: Use streaming filter for batch windows (window_size == send_every) if window_size == send_every: - return cg.Pvariable( - filter_id, StreamingMovingAverageFilter, window_size, send_first_at - ) + rhs = cg.new_Pvariable(StreamingMovingAverageFilter, window_size, send_first_at) + return cg.Pvariable(filter_id, rhs) return cg.new_Pvariable(filter_id, window_size, send_every, send_first_at) From f75f11b550b50089c57b25c1f939da3457ec9744 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Wed, 15 Oct 2025 19:57:29 -1000 Subject: [PATCH 16/31] add --- esphome/components/sensor/__init__.py | 27 ++++--- .../fixtures/sensor_filters_batch_window.yaml | 58 ++++++++++++++ .../fixtures/sensor_filters_ring_buffer.yaml | 75 +++++++++++++++++++ 3 files changed, 148 insertions(+), 12 deletions(-) create mode 100644 tests/integration/fixtures/sensor_filters_batch_window.yaml create mode 100644 tests/integration/fixtures/sensor_filters_ring_buffer.yaml diff --git a/esphome/components/sensor/__init__.py b/esphome/components/sensor/__init__.py index a7a92d3968..0538531354 100644 --- a/esphome/components/sensor/__init__.py +++ b/esphome/components/sensor/__init__.py @@ -453,7 +453,7 @@ async def skip_initial_filter_to_code(config, filter_id): return cg.new_Pvariable(filter_id, config) -@FILTER_REGISTRY.register("min", MinFilter, MIN_SCHEMA) +@FILTER_REGISTRY.register("min", Filter, MIN_SCHEMA) async def min_filter_to_code(config, filter_id): window_size: int = config[CONF_WINDOW_SIZE] send_every: int = config[CONF_SEND_EVERY] @@ -463,10 +463,11 @@ async def min_filter_to_code(config, filter_id): # Saves 99.98% memory for large windows (e.g., 20KB → 4 bytes for window_size=5000) if window_size == send_every: # Use streaming filter - O(1) memory instead of O(n) - rhs = cg.new_Pvariable(StreamingMinFilter, window_size, send_first_at) - return cg.Pvariable(filter_id, rhs) + rhs = StreamingMinFilter.new(window_size, send_first_at) + return cg.Pvariable(filter_id, rhs, StreamingMinFilter) # Use sliding window filter - maintains ring buffer - return cg.new_Pvariable(filter_id, window_size, send_every, send_first_at) + rhs = MinFilter.new(window_size, send_every, send_first_at) + return cg.Pvariable(filter_id, rhs, MinFilter) MAX_SCHEMA = cv.All( @@ -481,7 +482,7 @@ MAX_SCHEMA = cv.All( ) -@FILTER_REGISTRY.register("max", MaxFilter, MAX_SCHEMA) +@FILTER_REGISTRY.register("max", Filter, MAX_SCHEMA) async def max_filter_to_code(config, filter_id): window_size: int = config[CONF_WINDOW_SIZE] send_every: int = config[CONF_SEND_EVERY] @@ -489,9 +490,10 @@ async def max_filter_to_code(config, filter_id): # Optimization: Use streaming filter for batch windows (window_size == send_every) if window_size == send_every: - rhs = cg.new_Pvariable(StreamingMaxFilter, window_size, send_first_at) - return cg.Pvariable(filter_id, rhs) - return cg.new_Pvariable(filter_id, window_size, send_every, send_first_at) + rhs = StreamingMaxFilter.new(window_size, send_first_at) + return cg.Pvariable(filter_id, rhs, StreamingMaxFilter) + rhs = MaxFilter.new(window_size, send_every, send_first_at) + return cg.Pvariable(filter_id, rhs, MaxFilter) SLIDING_AVERAGE_SCHEMA = cv.All( @@ -508,7 +510,7 @@ SLIDING_AVERAGE_SCHEMA = cv.All( @FILTER_REGISTRY.register( "sliding_window_moving_average", - SlidingWindowMovingAverageFilter, + Filter, SLIDING_AVERAGE_SCHEMA, ) async def sliding_window_moving_average_filter_to_code(config, filter_id): @@ -518,9 +520,10 @@ async def sliding_window_moving_average_filter_to_code(config, filter_id): # Optimization: Use streaming filter for batch windows (window_size == send_every) if window_size == send_every: - rhs = cg.new_Pvariable(StreamingMovingAverageFilter, window_size, send_first_at) - return cg.Pvariable(filter_id, rhs) - return cg.new_Pvariable(filter_id, window_size, send_every, send_first_at) + rhs = StreamingMovingAverageFilter.new(window_size, send_first_at) + return cg.Pvariable(filter_id, rhs, StreamingMovingAverageFilter) + rhs = SlidingWindowMovingAverageFilter.new(window_size, send_every, send_first_at) + return cg.Pvariable(filter_id, rhs, SlidingWindowMovingAverageFilter) EXPONENTIAL_AVERAGE_SCHEMA = cv.All( diff --git a/tests/integration/fixtures/sensor_filters_batch_window.yaml b/tests/integration/fixtures/sensor_filters_batch_window.yaml new file mode 100644 index 0000000000..58a254c215 --- /dev/null +++ b/tests/integration/fixtures/sensor_filters_batch_window.yaml @@ -0,0 +1,58 @@ +esphome: + name: test-batch-window-filters + +host: +api: + batch_delay: 0ms # Disable batching to receive all state updates +logger: + level: DEBUG + +# Template sensor that we'll use to publish values +sensor: + - platform: template + name: "Source Sensor" + id: source_sensor + accuracy_decimals: 2 + + # Batch window filters (window_size == send_every) - use streaming filters + - platform: copy + source_id: source_sensor + name: "Min Sensor" + id: min_sensor + filters: + - min: + window_size: 5 + send_every: 5 + send_first_at: 1 + + - platform: copy + source_id: source_sensor + name: "Max Sensor" + id: max_sensor + filters: + - max: + window_size: 5 + send_every: 5 + send_first_at: 1 + + - platform: copy + source_id: source_sensor + name: "Moving Avg Sensor" + id: moving_avg_sensor + filters: + - sliding_window_moving_average: + window_size: 5 + send_every: 5 + send_first_at: 1 + +# Button to trigger publishing test values +button: + - platform: template + name: "Publish Values Button" + id: publish_button + on_press: + - lambda: |- + // Publish 10 values: 1.0, 2.0, ..., 10.0 + for (int i = 1; i <= 10; i++) { + id(source_sensor).publish_state(float(i)); + } diff --git a/tests/integration/fixtures/sensor_filters_ring_buffer.yaml b/tests/integration/fixtures/sensor_filters_ring_buffer.yaml new file mode 100644 index 0000000000..0d603ee9ce --- /dev/null +++ b/tests/integration/fixtures/sensor_filters_ring_buffer.yaml @@ -0,0 +1,75 @@ +esphome: + name: test-sliding-window-filters + +host: +api: + batch_delay: 0ms # Disable batching to receive all state updates +logger: + level: DEBUG + +# Template sensor that we'll use to publish values +sensor: + - platform: template + name: "Source Sensor" + id: source_sensor + accuracy_decimals: 2 + + # ACTUAL sliding window filters (window_size != send_every) - use ring buffers + # Window of 5, send every 2 values + - platform: copy + source_id: source_sensor + name: "Sliding Min Sensor" + id: sliding_min_sensor + filters: + - min: + window_size: 5 + send_every: 2 + send_first_at: 1 + + - platform: copy + source_id: source_sensor + name: "Sliding Max Sensor" + id: sliding_max_sensor + filters: + - max: + window_size: 5 + send_every: 2 + send_first_at: 1 + + - platform: copy + source_id: source_sensor + name: "Sliding Median Sensor" + id: sliding_median_sensor + filters: + - median: + window_size: 5 + send_every: 2 + send_first_at: 1 + + - platform: copy + source_id: source_sensor + name: "Sliding Moving Avg Sensor" + id: sliding_moving_avg_sensor + filters: + - sliding_window_moving_average: + window_size: 5 + send_every: 2 + send_first_at: 1 + +# Button to trigger publishing test values +button: + - platform: template + name: "Publish Values Button" + id: publish_button + on_press: + - lambda: |- + // Publish 10 values: 1.0, 2.0, ..., 10.0 + // With window_size=5, send_every=2, send_first_at=1: + // - Output at position 1: window=[1], min=1, max=1, median=1, avg=1 + // - Output at position 3: window=[1,2,3], min=1, max=3, median=2, avg=2 + // - Output at position 5: window=[1,2,3,4,5], min=1, max=5, median=3, avg=3 + // - Output at position 7: window=[3,4,5,6,7], min=3, max=7, median=5, avg=5 + // - Output at position 9: window=[5,6,7,8,9], min=5, max=9, median=7, avg=7 + for (int i = 1; i <= 10; i++) { + id(source_sensor).publish_state(float(i)); + } From 855df423ee8c03d30d9a0d5430edbb09c59e5bd7 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Wed, 15 Oct 2025 19:58:18 -1000 Subject: [PATCH 17/31] add --- .../test_sensor_filters_ring_buffer.py | 163 ++++++++++++++++++ 1 file changed, 163 insertions(+) create mode 100644 tests/integration/test_sensor_filters_ring_buffer.py diff --git a/tests/integration/test_sensor_filters_ring_buffer.py b/tests/integration/test_sensor_filters_ring_buffer.py new file mode 100644 index 0000000000..e138f93e7e --- /dev/null +++ b/tests/integration/test_sensor_filters_ring_buffer.py @@ -0,0 +1,163 @@ +"""Test sensor ring buffer filter functionality (window_size != send_every).""" + +from __future__ import annotations + +import asyncio + +from aioesphomeapi import EntityInfo, EntityState, SensorState +import pytest + +from .types import APIClientConnectedFactory, RunCompiledFunction + + +def build_key_to_sensor_mapping( + entities: list[EntityInfo], sensor_names: list[str] +) -> dict[int, str]: + """Build a mapping from entity keys to sensor names. + + Args: + entities: List of entity info objects from the API + sensor_names: List of sensor names to search for in object_ids + + Returns: + Dictionary mapping entity keys to sensor names + """ + key_to_sensor: dict[int, str] = {} + for entity in entities: + obj_id = entity.object_id.lower() + for sensor_name in sensor_names: + if sensor_name in obj_id: + key_to_sensor[entity.key] = sensor_name + break + return key_to_sensor + + +@pytest.mark.asyncio +async def test_sensor_filters_ring_buffer( + yaml_config: str, + run_compiled: RunCompiledFunction, + api_client_connected: APIClientConnectedFactory, +) -> None: + """Test that ring buffer filters (window_size != send_every) work correctly.""" + loop = asyncio.get_running_loop() + + # Track state changes for each sensor + sensor_states: dict[str, list[float]] = { + "sliding_min": [], + "sliding_max": [], + "sliding_median": [], + "sliding_moving_avg": [], + } + + # Futures to track when we receive expected values + all_updates_received = loop.create_future() + + def on_state(state: EntityState) -> None: + """Track sensor state updates.""" + if not isinstance(state, SensorState): + return + + # Skip NaN values (initial states) + if state.missing_state: + return + + # Get the sensor name from the key mapping + sensor_name = key_to_sensor.get(state.key) + if not sensor_name or sensor_name not in sensor_states: + return + + sensor_states[sensor_name].append(state.state) + + # Check if we've received enough updates from all sensors + # With send_every=2, send_first_at=1, we expect 5 outputs per sensor + if ( + len(sensor_states["sliding_min"]) >= 5 + and len(sensor_states["sliding_max"]) >= 5 + and len(sensor_states["sliding_median"]) >= 5 + and len(sensor_states["sliding_moving_avg"]) >= 5 + and not all_updates_received.done() + ): + all_updates_received.set_result(True) + + async with ( + run_compiled(yaml_config), + api_client_connected() as client, + ): + # Get entities first to build key mapping + entities, services = await client.list_entities_services() + + # Build key-to-sensor mapping + key_to_sensor = build_key_to_sensor_mapping( + entities, + [ + "sliding_min", + "sliding_max", + "sliding_median", + "sliding_moving_avg", + ], + ) + + # Subscribe to state changes AFTER building mapping + client.subscribe_states(on_state) + + # Find the publish button + publish_button = next( + (e for e in entities if "publish_values_button" in e.object_id.lower()), + None, + ) + assert publish_button is not None, "Publish Values Button not found" + + # Press the button to publish test values + client.button_command(publish_button.key) + + # Wait for all sensors to receive their values + try: + await asyncio.wait_for(all_updates_received, timeout=10.0) + except TimeoutError: + # Provide detailed failure info + pytest.fail( + f"Timeout waiting for updates. Received states:\n" + f" min: {sensor_states['sliding_min']}\n" + f" max: {sensor_states['sliding_max']}\n" + f" median: {sensor_states['sliding_median']}\n" + f" moving_avg: {sensor_states['sliding_moving_avg']}" + ) + + # Verify we got 5 outputs per sensor (positions 1, 3, 5, 7, 9) + assert len(sensor_states["sliding_min"]) == 5, ( + f"Min sensor should have 5 values, got {len(sensor_states['sliding_min'])}: {sensor_states['sliding_min']}" + ) + assert len(sensor_states["sliding_max"]) == 5 + assert len(sensor_states["sliding_median"]) == 5 + assert len(sensor_states["sliding_moving_avg"]) == 5 + + # Verify the values at each output position + # Position 1: window=[1] + assert abs(sensor_states["sliding_min"][0] - 1.0) < 0.01 + assert abs(sensor_states["sliding_max"][0] - 1.0) < 0.01 + assert abs(sensor_states["sliding_median"][0] - 1.0) < 0.01 + assert abs(sensor_states["sliding_moving_avg"][0] - 1.0) < 0.01 + + # Position 3: window=[1,2,3] + assert abs(sensor_states["sliding_min"][1] - 1.0) < 0.01 + assert abs(sensor_states["sliding_max"][1] - 3.0) < 0.01 + assert abs(sensor_states["sliding_median"][1] - 2.0) < 0.01 + assert abs(sensor_states["sliding_moving_avg"][1] - 2.0) < 0.01 + + # Position 5: window=[1,2,3,4,5] + assert abs(sensor_states["sliding_min"][2] - 1.0) < 0.01 + assert abs(sensor_states["sliding_max"][2] - 5.0) < 0.01 + assert abs(sensor_states["sliding_median"][2] - 3.0) < 0.01 + assert abs(sensor_states["sliding_moving_avg"][2] - 3.0) < 0.01 + + # Position 7: window=[3,4,5,6,7] (ring buffer wrapped) + assert abs(sensor_states["sliding_min"][3] - 3.0) < 0.01 + assert abs(sensor_states["sliding_max"][3] - 7.0) < 0.01 + assert abs(sensor_states["sliding_median"][3] - 5.0) < 0.01 + assert abs(sensor_states["sliding_moving_avg"][3] - 5.0) < 0.01 + + # Position 9: window=[5,6,7,8,9] (ring buffer wrapped) + assert abs(sensor_states["sliding_min"][4] - 5.0) < 0.01 + assert abs(sensor_states["sliding_max"][4] - 9.0) < 0.01 + assert abs(sensor_states["sliding_median"][4] - 7.0) < 0.01 + assert abs(sensor_states["sliding_moving_avg"][4] - 7.0) < 0.01 From 784183ca8d25cf1be8129824755f9fff55d7764e Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Wed, 15 Oct 2025 21:38:02 -1000 Subject: [PATCH 18/31] [datetime] Fix DateTimeStateTrigger compilation when time component is not used --- esphome/components/datetime/datetime_base.h | 2 -- 1 file changed, 2 deletions(-) diff --git a/esphome/components/datetime/datetime_base.h b/esphome/components/datetime/datetime_base.h index b7645f5539..b5f54ac96f 100644 --- a/esphome/components/datetime/datetime_base.h +++ b/esphome/components/datetime/datetime_base.h @@ -30,14 +30,12 @@ class DateTimeBase : public EntityBase { #endif }; -#ifdef USE_TIME class DateTimeStateTrigger : public Trigger { public: explicit DateTimeStateTrigger(DateTimeBase *parent) { parent->add_on_state_callback([this, parent]() { this->trigger(parent->state_as_esptime()); }); } }; -#endif } // namespace datetime } // namespace esphome From 7027ae983319c24071b8cfc3c3b0b62c6beddc7d Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Wed, 15 Oct 2025 21:44:38 -1000 Subject: [PATCH 19/31] race --- .../fixtures/sensor_filters_nan_handling.yaml | 11 +++++++++++ .../fixtures/sensor_filters_ring_buffer.yaml | 2 ++ .../sensor_filters_ring_buffer_wraparound.yaml | 2 ++ .../fixtures/sensor_filters_sliding_window.yaml | 2 ++ 4 files changed, 17 insertions(+) diff --git a/tests/integration/fixtures/sensor_filters_nan_handling.yaml b/tests/integration/fixtures/sensor_filters_nan_handling.yaml index 20fae64e8b..5fc3d1db29 100644 --- a/tests/integration/fixtures/sensor_filters_nan_handling.yaml +++ b/tests/integration/fixtures/sensor_filters_nan_handling.yaml @@ -38,13 +38,24 @@ button: on_press: - lambda: |- // Publish 10 values with NaN mixed in: 10, NaN, 5, NaN, 15, 8, NaN, 12, 3, NaN + // Small delay to ensure API can process each state update id(source_nan_sensor).publish_state(10.0); + delay(10); id(source_nan_sensor).publish_state(NAN); + delay(10); id(source_nan_sensor).publish_state(5.0); + delay(10); id(source_nan_sensor).publish_state(NAN); + delay(10); id(source_nan_sensor).publish_state(15.0); + delay(10); id(source_nan_sensor).publish_state(8.0); + delay(10); id(source_nan_sensor).publish_state(NAN); + delay(10); id(source_nan_sensor).publish_state(12.0); + delay(10); id(source_nan_sensor).publish_state(3.0); + delay(10); id(source_nan_sensor).publish_state(NAN); + delay(10); diff --git a/tests/integration/fixtures/sensor_filters_ring_buffer.yaml b/tests/integration/fixtures/sensor_filters_ring_buffer.yaml index 0d603ee9ce..fb502dc999 100644 --- a/tests/integration/fixtures/sensor_filters_ring_buffer.yaml +++ b/tests/integration/fixtures/sensor_filters_ring_buffer.yaml @@ -70,6 +70,8 @@ button: // - Output at position 5: window=[1,2,3,4,5], min=1, max=5, median=3, avg=3 // - Output at position 7: window=[3,4,5,6,7], min=3, max=7, median=5, avg=5 // - Output at position 9: window=[5,6,7,8,9], min=5, max=9, median=7, avg=7 + // Small delay to ensure API can process each state update for (int i = 1; i <= 10; i++) { id(source_sensor).publish_state(float(i)); + delay(10); } diff --git a/tests/integration/fixtures/sensor_filters_ring_buffer_wraparound.yaml b/tests/integration/fixtures/sensor_filters_ring_buffer_wraparound.yaml index 1ff9ec542a..ec8917c2e2 100644 --- a/tests/integration/fixtures/sensor_filters_ring_buffer_wraparound.yaml +++ b/tests/integration/fixtures/sensor_filters_ring_buffer_wraparound.yaml @@ -30,7 +30,9 @@ button: - lambda: |- // Publish 9 values to test ring buffer wraparound // Values: 10, 20, 30, 5, 25, 15, 40, 35, 20 + // Small delay to ensure API can process each state update float values[] = {10.0, 20.0, 30.0, 5.0, 25.0, 15.0, 40.0, 35.0, 20.0}; for (int i = 0; i < 9; i++) { id(source_wraparound).publish_state(values[i]); + delay(10); } diff --git a/tests/integration/fixtures/sensor_filters_sliding_window.yaml b/tests/integration/fixtures/sensor_filters_sliding_window.yaml index a2ae3182b8..2b58477aa9 100644 --- a/tests/integration/fixtures/sensor_filters_sliding_window.yaml +++ b/tests/integration/fixtures/sensor_filters_sliding_window.yaml @@ -73,6 +73,8 @@ button: on_press: - lambda: |- // Publish 10 values: 1.0, 2.0, ..., 10.0 + // Small delay to ensure API can process each state update for (int i = 1; i <= 10; i++) { id(source_sensor).publish_state(float(i)); + delay(10); } From 55e03036e23ee87ccca4d60d9002093d998e6cf9 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Wed, 15 Oct 2025 21:46:00 -1000 Subject: [PATCH 20/31] preen --- tests/integration/fixtures/sensor_filters_nan_handling.yaml | 2 ++ .../fixtures/sensor_filters_ring_buffer_wraparound.yaml | 1 + .../integration/fixtures/sensor_filters_sliding_window.yaml | 5 +++++ 3 files changed, 8 insertions(+) diff --git a/tests/integration/fixtures/sensor_filters_nan_handling.yaml b/tests/integration/fixtures/sensor_filters_nan_handling.yaml index 5fc3d1db29..445d20497b 100644 --- a/tests/integration/fixtures/sensor_filters_nan_handling.yaml +++ b/tests/integration/fixtures/sensor_filters_nan_handling.yaml @@ -21,6 +21,7 @@ sensor: - min: window_size: 5 send_every: 5 + send_first_at: 1 - platform: copy source_id: source_nan_sensor @@ -30,6 +31,7 @@ sensor: - max: window_size: 5 send_every: 5 + send_first_at: 1 button: - platform: template diff --git a/tests/integration/fixtures/sensor_filters_ring_buffer_wraparound.yaml b/tests/integration/fixtures/sensor_filters_ring_buffer_wraparound.yaml index ec8917c2e2..4757d78aeb 100644 --- a/tests/integration/fixtures/sensor_filters_ring_buffer_wraparound.yaml +++ b/tests/integration/fixtures/sensor_filters_ring_buffer_wraparound.yaml @@ -21,6 +21,7 @@ sensor: - min: window_size: 3 send_every: 3 + send_first_at: 1 button: - platform: template diff --git a/tests/integration/fixtures/sensor_filters_sliding_window.yaml b/tests/integration/fixtures/sensor_filters_sliding_window.yaml index 2b58477aa9..edcc596f64 100644 --- a/tests/integration/fixtures/sensor_filters_sliding_window.yaml +++ b/tests/integration/fixtures/sensor_filters_sliding_window.yaml @@ -23,6 +23,7 @@ sensor: - min: window_size: 5 send_every: 5 + send_first_at: 1 # Max filter sensor - platform: copy @@ -33,6 +34,7 @@ sensor: - max: window_size: 5 send_every: 5 + send_first_at: 1 # Median filter sensor - platform: copy @@ -43,6 +45,7 @@ sensor: - median: window_size: 5 send_every: 5 + send_first_at: 1 # Quantile filter sensor (90th percentile) - platform: copy @@ -53,6 +56,7 @@ sensor: - quantile: window_size: 5 send_every: 5 + send_first_at: 1 quantile: 0.9 # Moving average filter sensor @@ -64,6 +68,7 @@ sensor: - sliding_window_moving_average: window_size: 5 send_every: 5 + send_first_at: 1 # Button to trigger publishing test values button: From baf117b411fbd43edc66d0ead545ad22b1e62961 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Wed, 15 Oct 2025 22:03:22 -1000 Subject: [PATCH 21/31] fix flakey test --- .../fixtures/sensor_filters_nan_handling.yaml | 67 ++++++++++++------- .../fixtures/sensor_filters_ring_buffer.yaml | 64 ++++++++++++++---- ...sensor_filters_ring_buffer_wraparound.yaml | 51 +++++++++++--- .../sensor_filters_sliding_window.yaml | 52 ++++++++++++-- 4 files changed, 182 insertions(+), 52 deletions(-) diff --git a/tests/integration/fixtures/sensor_filters_nan_handling.yaml b/tests/integration/fixtures/sensor_filters_nan_handling.yaml index 445d20497b..fcb12cfde5 100644 --- a/tests/integration/fixtures/sensor_filters_nan_handling.yaml +++ b/tests/integration/fixtures/sensor_filters_nan_handling.yaml @@ -33,31 +33,52 @@ sensor: send_every: 5 send_first_at: 1 +script: + - id: publish_nan_values_script + then: + - sensor.template.publish: + id: source_nan_sensor + state: 10.0 + - delay: 20ms + - sensor.template.publish: + id: source_nan_sensor + state: !lambda 'return NAN;' + - delay: 20ms + - sensor.template.publish: + id: source_nan_sensor + state: 5.0 + - delay: 20ms + - sensor.template.publish: + id: source_nan_sensor + state: !lambda 'return NAN;' + - delay: 20ms + - sensor.template.publish: + id: source_nan_sensor + state: 15.0 + - delay: 20ms + - sensor.template.publish: + id: source_nan_sensor + state: 8.0 + - delay: 20ms + - sensor.template.publish: + id: source_nan_sensor + state: !lambda 'return NAN;' + - delay: 20ms + - sensor.template.publish: + id: source_nan_sensor + state: 12.0 + - delay: 20ms + - sensor.template.publish: + id: source_nan_sensor + state: 3.0 + - delay: 20ms + - sensor.template.publish: + id: source_nan_sensor + state: !lambda 'return NAN;' + button: - platform: template name: "Publish NaN Values Button" id: publish_nan_button on_press: - - lambda: |- - // Publish 10 values with NaN mixed in: 10, NaN, 5, NaN, 15, 8, NaN, 12, 3, NaN - // Small delay to ensure API can process each state update - id(source_nan_sensor).publish_state(10.0); - delay(10); - id(source_nan_sensor).publish_state(NAN); - delay(10); - id(source_nan_sensor).publish_state(5.0); - delay(10); - id(source_nan_sensor).publish_state(NAN); - delay(10); - id(source_nan_sensor).publish_state(15.0); - delay(10); - id(source_nan_sensor).publish_state(8.0); - delay(10); - id(source_nan_sensor).publish_state(NAN); - delay(10); - id(source_nan_sensor).publish_state(12.0); - delay(10); - id(source_nan_sensor).publish_state(3.0); - delay(10); - id(source_nan_sensor).publish_state(NAN); - delay(10); + - script.execute: publish_nan_values_script diff --git a/tests/integration/fixtures/sensor_filters_ring_buffer.yaml b/tests/integration/fixtures/sensor_filters_ring_buffer.yaml index fb502dc999..ea7a326b8d 100644 --- a/tests/integration/fixtures/sensor_filters_ring_buffer.yaml +++ b/tests/integration/fixtures/sensor_filters_ring_buffer.yaml @@ -57,21 +57,59 @@ sensor: send_first_at: 1 # Button to trigger publishing test values +script: + - id: publish_values_script + then: + # Publish 10 values: 1.0, 2.0, ..., 10.0 + # With window_size=5, send_every=2, send_first_at=1: + # - Output at position 1: window=[1], min=1, max=1, median=1, avg=1 + # - Output at position 3: window=[1,2,3], min=1, max=3, median=2, avg=2 + # - Output at position 5: window=[1,2,3,4,5], min=1, max=5, median=3, avg=3 + # - Output at position 7: window=[3,4,5,6,7], min=3, max=7, median=5, avg=5 + # - Output at position 9: window=[5,6,7,8,9], min=5, max=9, median=7, avg=7 + - sensor.template.publish: + id: source_sensor + state: 1.0 + - delay: 20ms + - sensor.template.publish: + id: source_sensor + state: 2.0 + - delay: 20ms + - sensor.template.publish: + id: source_sensor + state: 3.0 + - delay: 20ms + - sensor.template.publish: + id: source_sensor + state: 4.0 + - delay: 20ms + - sensor.template.publish: + id: source_sensor + state: 5.0 + - delay: 20ms + - sensor.template.publish: + id: source_sensor + state: 6.0 + - delay: 20ms + - sensor.template.publish: + id: source_sensor + state: 7.0 + - delay: 20ms + - sensor.template.publish: + id: source_sensor + state: 8.0 + - delay: 20ms + - sensor.template.publish: + id: source_sensor + state: 9.0 + - delay: 20ms + - sensor.template.publish: + id: source_sensor + state: 10.0 + button: - platform: template name: "Publish Values Button" id: publish_button on_press: - - lambda: |- - // Publish 10 values: 1.0, 2.0, ..., 10.0 - // With window_size=5, send_every=2, send_first_at=1: - // - Output at position 1: window=[1], min=1, max=1, median=1, avg=1 - // - Output at position 3: window=[1,2,3], min=1, max=3, median=2, avg=2 - // - Output at position 5: window=[1,2,3,4,5], min=1, max=5, median=3, avg=3 - // - Output at position 7: window=[3,4,5,6,7], min=3, max=7, median=5, avg=5 - // - Output at position 9: window=[5,6,7,8,9], min=5, max=9, median=7, avg=7 - // Small delay to ensure API can process each state update - for (int i = 1; i <= 10; i++) { - id(source_sensor).publish_state(float(i)); - delay(10); - } + - script.execute: publish_values_script diff --git a/tests/integration/fixtures/sensor_filters_ring_buffer_wraparound.yaml b/tests/integration/fixtures/sensor_filters_ring_buffer_wraparound.yaml index 4757d78aeb..bd5980160b 100644 --- a/tests/integration/fixtures/sensor_filters_ring_buffer_wraparound.yaml +++ b/tests/integration/fixtures/sensor_filters_ring_buffer_wraparound.yaml @@ -23,17 +23,50 @@ sensor: send_every: 3 send_first_at: 1 +script: + - id: publish_wraparound_script + then: + # Publish 9 values to test ring buffer wraparound + # Values: 10, 20, 30, 5, 25, 15, 40, 35, 20 + - sensor.template.publish: + id: source_wraparound + state: 10.0 + - delay: 20ms + - sensor.template.publish: + id: source_wraparound + state: 20.0 + - delay: 20ms + - sensor.template.publish: + id: source_wraparound + state: 30.0 + - delay: 20ms + - sensor.template.publish: + id: source_wraparound + state: 5.0 + - delay: 20ms + - sensor.template.publish: + id: source_wraparound + state: 25.0 + - delay: 20ms + - sensor.template.publish: + id: source_wraparound + state: 15.0 + - delay: 20ms + - sensor.template.publish: + id: source_wraparound + state: 40.0 + - delay: 20ms + - sensor.template.publish: + id: source_wraparound + state: 35.0 + - delay: 20ms + - sensor.template.publish: + id: source_wraparound + state: 20.0 + button: - platform: template name: "Publish Wraparound Button" id: publish_wraparound_button on_press: - - lambda: |- - // Publish 9 values to test ring buffer wraparound - // Values: 10, 20, 30, 5, 25, 15, 40, 35, 20 - // Small delay to ensure API can process each state update - float values[] = {10.0, 20.0, 30.0, 5.0, 25.0, 15.0, 40.0, 35.0, 20.0}; - for (int i = 0; i < 9; i++) { - id(source_wraparound).publish_state(values[i]); - delay(10); - } + - script.execute: publish_wraparound_script diff --git a/tests/integration/fixtures/sensor_filters_sliding_window.yaml b/tests/integration/fixtures/sensor_filters_sliding_window.yaml index edcc596f64..2055118811 100644 --- a/tests/integration/fixtures/sensor_filters_sliding_window.yaml +++ b/tests/integration/fixtures/sensor_filters_sliding_window.yaml @@ -70,16 +70,54 @@ sensor: send_every: 5 send_first_at: 1 +# Script to publish values with delays +script: + - id: publish_values_script + then: + - sensor.template.publish: + id: source_sensor + state: 1.0 + - delay: 20ms + - sensor.template.publish: + id: source_sensor + state: 2.0 + - delay: 20ms + - sensor.template.publish: + id: source_sensor + state: 3.0 + - delay: 20ms + - sensor.template.publish: + id: source_sensor + state: 4.0 + - delay: 20ms + - sensor.template.publish: + id: source_sensor + state: 5.0 + - delay: 20ms + - sensor.template.publish: + id: source_sensor + state: 6.0 + - delay: 20ms + - sensor.template.publish: + id: source_sensor + state: 7.0 + - delay: 20ms + - sensor.template.publish: + id: source_sensor + state: 8.0 + - delay: 20ms + - sensor.template.publish: + id: source_sensor + state: 9.0 + - delay: 20ms + - sensor.template.publish: + id: source_sensor + state: 10.0 + # Button to trigger publishing test values button: - platform: template name: "Publish Values Button" id: publish_button on_press: - - lambda: |- - // Publish 10 values: 1.0, 2.0, ..., 10.0 - // Small delay to ensure API can process each state update - for (int i = 1; i <= 10; i++) { - id(source_sensor).publish_state(float(i)); - delay(10); - } + - script.execute: publish_values_script From febe075bb2b6ece497b8b3ae3d345061434bb596 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Wed, 15 Oct 2025 23:17:08 -1000 Subject: [PATCH 22/31] helper --- esphome/components/datetime/datetime_base.h | 2 ++ tests/integration/sensor_test_utils.py | 27 +++++++++++++++++++ .../test_sensor_filters_ring_buffer.py | 25 ++--------------- .../test_sensor_filters_sliding_window.py | 25 ++--------------- 4 files changed, 33 insertions(+), 46 deletions(-) create mode 100644 tests/integration/sensor_test_utils.py diff --git a/esphome/components/datetime/datetime_base.h b/esphome/components/datetime/datetime_base.h index b5f54ac96f..b7645f5539 100644 --- a/esphome/components/datetime/datetime_base.h +++ b/esphome/components/datetime/datetime_base.h @@ -30,12 +30,14 @@ class DateTimeBase : public EntityBase { #endif }; +#ifdef USE_TIME class DateTimeStateTrigger : public Trigger { public: explicit DateTimeStateTrigger(DateTimeBase *parent) { parent->add_on_state_callback([this, parent]() { this->trigger(parent->state_as_esptime()); }); } }; +#endif } // namespace datetime } // namespace esphome diff --git a/tests/integration/sensor_test_utils.py b/tests/integration/sensor_test_utils.py new file mode 100644 index 0000000000..c3843a26ab --- /dev/null +++ b/tests/integration/sensor_test_utils.py @@ -0,0 +1,27 @@ +"""Shared utilities for sensor integration tests.""" + +from __future__ import annotations + +from aioesphomeapi import EntityInfo + + +def build_key_to_sensor_mapping( + entities: list[EntityInfo], sensor_names: list[str] +) -> dict[int, str]: + """Build a mapping from entity keys to sensor names. + + Args: + entities: List of entity info objects from the API + sensor_names: List of sensor names to search for in object_ids + + Returns: + Dictionary mapping entity keys to sensor names + """ + key_to_sensor: dict[int, str] = {} + for entity in entities: + obj_id = entity.object_id.lower() + for sensor_name in sensor_names: + if sensor_name in obj_id: + key_to_sensor[entity.key] = sensor_name + break + return key_to_sensor diff --git a/tests/integration/test_sensor_filters_ring_buffer.py b/tests/integration/test_sensor_filters_ring_buffer.py index e138f93e7e..8edb1600d9 100644 --- a/tests/integration/test_sensor_filters_ring_buffer.py +++ b/tests/integration/test_sensor_filters_ring_buffer.py @@ -4,34 +4,13 @@ from __future__ import annotations import asyncio -from aioesphomeapi import EntityInfo, EntityState, SensorState +from aioesphomeapi import EntityState, SensorState import pytest +from .sensor_test_utils import build_key_to_sensor_mapping from .types import APIClientConnectedFactory, RunCompiledFunction -def build_key_to_sensor_mapping( - entities: list[EntityInfo], sensor_names: list[str] -) -> dict[int, str]: - """Build a mapping from entity keys to sensor names. - - Args: - entities: List of entity info objects from the API - sensor_names: List of sensor names to search for in object_ids - - Returns: - Dictionary mapping entity keys to sensor names - """ - key_to_sensor: dict[int, str] = {} - for entity in entities: - obj_id = entity.object_id.lower() - for sensor_name in sensor_names: - if sensor_name in obj_id: - key_to_sensor[entity.key] = sensor_name - break - return key_to_sensor - - @pytest.mark.asyncio async def test_sensor_filters_ring_buffer( yaml_config: str, diff --git a/tests/integration/test_sensor_filters_sliding_window.py b/tests/integration/test_sensor_filters_sliding_window.py index 0c7aec70aa..2183946134 100644 --- a/tests/integration/test_sensor_filters_sliding_window.py +++ b/tests/integration/test_sensor_filters_sliding_window.py @@ -4,34 +4,13 @@ from __future__ import annotations import asyncio -from aioesphomeapi import EntityInfo, EntityState, SensorState +from aioesphomeapi import EntityState, SensorState import pytest +from .sensor_test_utils import build_key_to_sensor_mapping from .types import APIClientConnectedFactory, RunCompiledFunction -def build_key_to_sensor_mapping( - entities: list[EntityInfo], sensor_names: list[str] -) -> dict[int, str]: - """Build a mapping from entity keys to sensor names. - - Args: - entities: List of entity info objects from the API - sensor_names: List of sensor names to search for in object_ids - - Returns: - Dictionary mapping entity keys to sensor names - """ - key_to_sensor: dict[int, str] = {} - for entity in entities: - obj_id = entity.object_id.lower() - for sensor_name in sensor_names: - if sensor_name in obj_id: - key_to_sensor[entity.key] = sensor_name - break - return key_to_sensor - - @pytest.mark.asyncio async def test_sensor_filters_sliding_window( yaml_config: str, From b4ba2aff30f1491efccde9de808d4d4401731d25 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Wed, 15 Oct 2025 23:30:45 -1000 Subject: [PATCH 23/31] remove dead unreachable code --- esphome/components/sensor/filter.cpp | 8 -------- esphome/components/sensor/filter.h | 3 --- 2 files changed, 11 deletions(-) diff --git a/esphome/components/sensor/filter.cpp b/esphome/components/sensor/filter.cpp index c804125dcc..1cc744e3b5 100644 --- a/esphome/components/sensor/filter.cpp +++ b/esphome/components/sensor/filter.cpp @@ -39,14 +39,6 @@ SlidingWindowFilter::SlidingWindowFilter(size_t window_size, size_t send_every, this->window_.init(window_size); } -void SlidingWindowFilter::set_window_size(size_t window_size) { - this->window_size_ = window_size; - // Reallocate buffer with new size - this->window_.init(window_size); - this->window_head_ = 0; - this->window_count_ = 0; -} - optional SlidingWindowFilter::new_value(float value) { // Add value to ring buffer if (this->window_count_ < this->window_size_) { diff --git a/esphome/components/sensor/filter.h b/esphome/components/sensor/filter.h index c9b39b73c3..d99cd79f05 100644 --- a/esphome/components/sensor/filter.h +++ b/esphome/components/sensor/filter.h @@ -53,9 +53,6 @@ class SlidingWindowFilter : public Filter { public: SlidingWindowFilter(size_t window_size, size_t send_every, size_t send_first_at); - void set_send_every(size_t send_every) { this->send_every_ = send_every; } - void set_window_size(size_t window_size); - optional new_value(float value) final; protected: From 3ba2212cfc22d7369a11f91035fc7476889b323f Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Thu, 16 Oct 2025 06:01:32 -1000 Subject: [PATCH 24/31] fix flakey --- .../test_sensor_filters_ring_buffer.py | 16 +++++-- .../test_sensor_filters_sliding_window.py | 48 +++++++++++++++---- 2 files changed, 52 insertions(+), 12 deletions(-) diff --git a/tests/integration/test_sensor_filters_ring_buffer.py b/tests/integration/test_sensor_filters_ring_buffer.py index 8edb1600d9..da4862c14b 100644 --- a/tests/integration/test_sensor_filters_ring_buffer.py +++ b/tests/integration/test_sensor_filters_ring_buffer.py @@ -8,6 +8,7 @@ from aioesphomeapi import EntityState, SensorState import pytest from .sensor_test_utils import build_key_to_sensor_mapping +from .state_utils import InitialStateHelper from .types import APIClientConnectedFactory, RunCompiledFunction @@ -36,7 +37,7 @@ async def test_sensor_filters_ring_buffer( if not isinstance(state, SensorState): return - # Skip NaN values (initial states) + # Skip NaN values if state.missing_state: return @@ -76,8 +77,17 @@ async def test_sensor_filters_ring_buffer( ], ) - # Subscribe to state changes AFTER building mapping - client.subscribe_states(on_state) + # Set up initial state helper with all entities + initial_state_helper = InitialStateHelper(entities) + + # Subscribe to state changes with wrapper + client.subscribe_states(initial_state_helper.on_state_wrapper(on_state)) + + # Wait for initial states to be sent before pressing button + try: + await initial_state_helper.wait_for_initial_states() + except TimeoutError: + pytest.fail("Timeout waiting for initial states") # Find the publish button publish_button = next( diff --git a/tests/integration/test_sensor_filters_sliding_window.py b/tests/integration/test_sensor_filters_sliding_window.py index 2183946134..389cbf2659 100644 --- a/tests/integration/test_sensor_filters_sliding_window.py +++ b/tests/integration/test_sensor_filters_sliding_window.py @@ -8,6 +8,7 @@ from aioesphomeapi import EntityState, SensorState import pytest from .sensor_test_utils import build_key_to_sensor_mapping +from .state_utils import InitialStateHelper from .types import APIClientConnectedFactory, RunCompiledFunction @@ -41,7 +42,7 @@ async def test_sensor_filters_sliding_window( if not isinstance(state, SensorState): return - # Skip NaN values (initial states) + # Skip NaN values if state.missing_state: return @@ -108,8 +109,17 @@ async def test_sensor_filters_sliding_window( ], ) - # Subscribe to state changes AFTER building mapping - client.subscribe_states(on_state) + # Set up initial state helper with all entities + initial_state_helper = InitialStateHelper(entities) + + # Subscribe to state changes with wrapper + client.subscribe_states(initial_state_helper.on_state_wrapper(on_state)) + + # Wait for initial states to be sent before pressing button + try: + await initial_state_helper.wait_for_initial_states() + except TimeoutError: + pytest.fail("Timeout waiting for initial states") # Find the publish button publish_button = next( @@ -207,11 +217,12 @@ async def test_sensor_filters_nan_handling( if not isinstance(state, SensorState): return - # Skip NaN values (initial states) + # Skip NaN values if state.missing_state: return sensor_name = key_to_sensor.get(state.key) + if sensor_name == "min_nan": min_states.append(state.state) elif sensor_name == "max_nan": @@ -236,8 +247,17 @@ async def test_sensor_filters_nan_handling( # Build key-to-sensor mapping key_to_sensor = build_key_to_sensor_mapping(entities, ["min_nan", "max_nan"]) - # Subscribe to state changes AFTER building mapping - client.subscribe_states(on_state) + # Set up initial state helper with all entities + initial_state_helper = InitialStateHelper(entities) + + # Subscribe to state changes with wrapper + client.subscribe_states(initial_state_helper.on_state_wrapper(on_state)) + + # Wait for initial states + try: + await initial_state_helper.wait_for_initial_states() + except TimeoutError: + pytest.fail("Timeout waiting for initial states") # Find the publish button publish_button = next( @@ -305,11 +325,12 @@ async def test_sensor_filters_ring_buffer_wraparound( if not isinstance(state, SensorState): return - # Skip NaN values (initial states) + # Skip NaN values if state.missing_state: return sensor_name = key_to_sensor.get(state.key) + if sensor_name == "wraparound_min": min_states.append(state.state) # With batch_delay: 0ms, we should receive all 3 outputs @@ -326,8 +347,17 @@ async def test_sensor_filters_ring_buffer_wraparound( # Build key-to-sensor mapping key_to_sensor = build_key_to_sensor_mapping(entities, ["wraparound_min"]) - # Subscribe to state changes AFTER building mapping - client.subscribe_states(on_state) + # Set up initial state helper with all entities + initial_state_helper = InitialStateHelper(entities) + + # Subscribe to state changes with wrapper + client.subscribe_states(initial_state_helper.on_state_wrapper(on_state)) + + # Wait for initial state + try: + await initial_state_helper.wait_for_initial_states() + except TimeoutError: + pytest.fail("Timeout waiting for initial state") # Find the publish button publish_button = next( From 44ad787cb3da66ab61190abb27450abf8d990672 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Thu, 16 Oct 2025 06:04:42 -1000 Subject: [PATCH 25/31] fix flakey --- tests/integration/state_utils.py | 146 ++++++++++++++++++ .../test_sensor_filters_ring_buffer.py | 40 ++--- .../test_sensor_filters_sliding_window.py | 40 ++--- 3 files changed, 186 insertions(+), 40 deletions(-) create mode 100644 tests/integration/state_utils.py diff --git a/tests/integration/state_utils.py b/tests/integration/state_utils.py new file mode 100644 index 0000000000..7392393501 --- /dev/null +++ b/tests/integration/state_utils.py @@ -0,0 +1,146 @@ +"""Shared utilities for ESPHome integration tests - state handling.""" + +from __future__ import annotations + +import asyncio +import logging + +from aioesphomeapi import ButtonInfo, EntityInfo, EntityState + +_LOGGER = logging.getLogger(__name__) + + +class InitialStateHelper: + """Helper to wait for initial states before processing test states. + + When an API client connects, ESPHome sends the current state of all entities. + This helper wraps the user's state callback and swallows the first state for + each entity, then forwards all subsequent states to the user callback. + + Usage: + entities, services = await client.list_entities_services() + helper = InitialStateHelper(entities) + client.subscribe_states(helper.on_state_wrapper(user_callback)) + await helper.wait_for_initial_states() + """ + + def __init__(self, entities: list[EntityInfo]) -> None: + """Initialize the helper. + + Args: + entities: All entities from list_entities_services() + """ + # Set of (device_id, key) tuples waiting for initial state + # Buttons are stateless, so exclude them + self._wait_initial_states = { + (entity.device_id, entity.key) + for entity in entities + if not isinstance(entity, ButtonInfo) + } + # Keep entity info for debugging - use (device_id, key) tuple + self._entities_by_id = { + (entity.device_id, entity.key): entity for entity in entities + } + + # Log all entities + _LOGGER.debug( + "InitialStateHelper: Found %d total entities: %s", + len(entities), + [(type(e).__name__, e.object_id) for e in entities], + ) + + # Log which ones we're waiting for + _LOGGER.debug( + "InitialStateHelper: Waiting for %d entities (excluding ButtonInfo): %s", + len(self._wait_initial_states), + [self._entities_by_id[k].object_id for k in self._wait_initial_states], + ) + + # Log which ones we're NOT waiting for + not_waiting = { + (e.device_id, e.key) for e in entities + } - self._wait_initial_states + _LOGGER.debug( + "InitialStateHelper: NOT waiting for %d entities: %s", + len(not_waiting), + [ + ( + type(self._entities_by_id[k]).__name__, + self._entities_by_id[k].object_id, + ) + for k in not_waiting + ], + ) + + # Create future in the running event loop + self._initial_states_received = asyncio.get_running_loop().create_future() + # If no entities to wait for, mark complete immediately + if not self._wait_initial_states: + self._initial_states_received.set_result(True) + + def on_state_wrapper(self, user_callback): + """Wrap a user callback to track initial states. + + Args: + user_callback: The user's state callback function + + Returns: + Wrapped callback that swallows first state per entity, forwards rest + """ + + def wrapper(state: EntityState) -> None: + """Swallow initial state per entity, forward subsequent states.""" + # Create entity identifier tuple + entity_id = (state.device_id, state.key) + + # Log which entity is sending state + if entity_id in self._entities_by_id: + entity = self._entities_by_id[entity_id] + _LOGGER.debug( + "Received state for %s (type: %s, device_id: %s, key: %d)", + entity.object_id, + type(entity).__name__, + state.device_id, + state.key, + ) + + # If this entity is waiting for initial state + if entity_id in self._wait_initial_states: + # Remove from waiting set + self._wait_initial_states.discard(entity_id) + + _LOGGER.debug( + "Swallowed initial state for %s, %d entities remaining", + self._entities_by_id[entity_id].object_id + if entity_id in self._entities_by_id + else entity_id, + len(self._wait_initial_states), + ) + + # Check if we've now seen all entities + if ( + not self._wait_initial_states + and not self._initial_states_received.done() + ): + _LOGGER.debug("All initial states received") + self._initial_states_received.set_result(True) + + # Don't forward initial state to user + return + + # Forward subsequent states to user callback + _LOGGER.debug("Forwarding state to user callback") + user_callback(state) + + return wrapper + + async def wait_for_initial_states(self, timeout: float = 5.0) -> None: + """Wait for all initial states to be received. + + Args: + timeout: Maximum time to wait in seconds + + Raises: + asyncio.TimeoutError: If initial states aren't received within timeout + """ + await asyncio.wait_for(self._initial_states_received, timeout=timeout) diff --git a/tests/integration/test_sensor_filters_ring_buffer.py b/tests/integration/test_sensor_filters_ring_buffer.py index da4862c14b..5d00986cc2 100644 --- a/tests/integration/test_sensor_filters_ring_buffer.py +++ b/tests/integration/test_sensor_filters_ring_buffer.py @@ -122,31 +122,31 @@ async def test_sensor_filters_ring_buffer( # Verify the values at each output position # Position 1: window=[1] - assert abs(sensor_states["sliding_min"][0] - 1.0) < 0.01 - assert abs(sensor_states["sliding_max"][0] - 1.0) < 0.01 - assert abs(sensor_states["sliding_median"][0] - 1.0) < 0.01 - assert abs(sensor_states["sliding_moving_avg"][0] - 1.0) < 0.01 + assert sensor_states["sliding_min"][0] == pytest.approx(1.0) + assert sensor_states["sliding_max"][0] == pytest.approx(1.0) + assert sensor_states["sliding_median"][0] == pytest.approx(1.0) + assert sensor_states["sliding_moving_avg"][0] == pytest.approx(1.0) # Position 3: window=[1,2,3] - assert abs(sensor_states["sliding_min"][1] - 1.0) < 0.01 - assert abs(sensor_states["sliding_max"][1] - 3.0) < 0.01 - assert abs(sensor_states["sliding_median"][1] - 2.0) < 0.01 - assert abs(sensor_states["sliding_moving_avg"][1] - 2.0) < 0.01 + assert sensor_states["sliding_min"][1] == pytest.approx(1.0) + assert sensor_states["sliding_max"][1] == pytest.approx(3.0) + assert sensor_states["sliding_median"][1] == pytest.approx(2.0) + assert sensor_states["sliding_moving_avg"][1] == pytest.approx(2.0) # Position 5: window=[1,2,3,4,5] - assert abs(sensor_states["sliding_min"][2] - 1.0) < 0.01 - assert abs(sensor_states["sliding_max"][2] - 5.0) < 0.01 - assert abs(sensor_states["sliding_median"][2] - 3.0) < 0.01 - assert abs(sensor_states["sliding_moving_avg"][2] - 3.0) < 0.01 + assert sensor_states["sliding_min"][2] == pytest.approx(1.0) + assert sensor_states["sliding_max"][2] == pytest.approx(5.0) + assert sensor_states["sliding_median"][2] == pytest.approx(3.0) + assert sensor_states["sliding_moving_avg"][2] == pytest.approx(3.0) # Position 7: window=[3,4,5,6,7] (ring buffer wrapped) - assert abs(sensor_states["sliding_min"][3] - 3.0) < 0.01 - assert abs(sensor_states["sliding_max"][3] - 7.0) < 0.01 - assert abs(sensor_states["sliding_median"][3] - 5.0) < 0.01 - assert abs(sensor_states["sliding_moving_avg"][3] - 5.0) < 0.01 + assert sensor_states["sliding_min"][3] == pytest.approx(3.0) + assert sensor_states["sliding_max"][3] == pytest.approx(7.0) + assert sensor_states["sliding_median"][3] == pytest.approx(5.0) + assert sensor_states["sliding_moving_avg"][3] == pytest.approx(5.0) # Position 9: window=[5,6,7,8,9] (ring buffer wrapped) - assert abs(sensor_states["sliding_min"][4] - 5.0) < 0.01 - assert abs(sensor_states["sliding_max"][4] - 9.0) < 0.01 - assert abs(sensor_states["sliding_median"][4] - 7.0) < 0.01 - assert abs(sensor_states["sliding_moving_avg"][4] - 7.0) < 0.01 + assert sensor_states["sliding_min"][4] == pytest.approx(5.0) + assert sensor_states["sliding_max"][4] == pytest.approx(9.0) + assert sensor_states["sliding_median"][4] == pytest.approx(7.0) + assert sensor_states["sliding_moving_avg"][4] == pytest.approx(7.0) diff --git a/tests/integration/test_sensor_filters_sliding_window.py b/tests/integration/test_sensor_filters_sliding_window.py index 389cbf2659..57ab65acd4 100644 --- a/tests/integration/test_sensor_filters_sliding_window.py +++ b/tests/integration/test_sensor_filters_sliding_window.py @@ -58,33 +58,33 @@ async def test_sensor_filters_sliding_window( # Filters send at position 1 and position 6 (send_every=5 means every 5th value after first) if ( sensor_name == "min_sensor" - and abs(state.state - 2.0) < 0.01 + and state.state == pytest.approx(2.0) and not min_received.done() ): min_received.set_result(True) elif ( sensor_name == "max_sensor" - and abs(state.state - 6.0) < 0.01 + and state.state == pytest.approx(6.0) and not max_received.done() ): max_received.set_result(True) elif ( sensor_name == "median_sensor" - and abs(state.state - 4.0) < 0.01 + and state.state == pytest.approx(4.0) and not median_received.done() ): # Median of [2, 3, 4, 5, 6] = 4 median_received.set_result(True) elif ( sensor_name == "quantile_sensor" - and abs(state.state - 6.0) < 0.01 + and state.state == pytest.approx(6.0) and not quantile_received.done() ): # 90th percentile of [2, 3, 4, 5, 6] = 6 quantile_received.set_result(True) elif ( sensor_name == "moving_avg_sensor" - and abs(state.state - 4.0) < 0.01 + and state.state == pytest.approx(4.0) and not moving_avg_received.done() ): # Average of [2, 3, 4, 5, 6] = 4 @@ -168,30 +168,30 @@ async def test_sensor_filters_sliding_window( assert len(sensor_states["moving_avg_sensor"]) == 2 # Verify the first output (after 1 value: [1]) - assert abs(sensor_states["min_sensor"][0] - 1.0) < 0.01, ( + assert sensor_states["min_sensor"][0] == pytest.approx(1.0), ( f"First min should be 1.0, got {sensor_states['min_sensor'][0]}" ) - assert abs(sensor_states["max_sensor"][0] - 1.0) < 0.01, ( + assert sensor_states["max_sensor"][0] == pytest.approx(1.0), ( f"First max should be 1.0, got {sensor_states['max_sensor'][0]}" ) - assert abs(sensor_states["median_sensor"][0] - 1.0) < 0.01, ( + assert sensor_states["median_sensor"][0] == pytest.approx(1.0), ( f"First median should be 1.0, got {sensor_states['median_sensor'][0]}" ) - assert abs(sensor_states["moving_avg_sensor"][0] - 1.0) < 0.01, ( + assert sensor_states["moving_avg_sensor"][0] == pytest.approx(1.0), ( f"First moving avg should be 1.0, got {sensor_states['moving_avg_sensor'][0]}" ) # Verify the second output (after 6 values, window has [2, 3, 4, 5, 6]) - assert abs(sensor_states["min_sensor"][1] - 2.0) < 0.01, ( + assert sensor_states["min_sensor"][1] == pytest.approx(2.0), ( f"Second min should be 2.0, got {sensor_states['min_sensor'][1]}" ) - assert abs(sensor_states["max_sensor"][1] - 6.0) < 0.01, ( + assert sensor_states["max_sensor"][1] == pytest.approx(6.0), ( f"Second max should be 6.0, got {sensor_states['max_sensor'][1]}" ) - assert abs(sensor_states["median_sensor"][1] - 4.0) < 0.01, ( + assert sensor_states["median_sensor"][1] == pytest.approx(4.0), ( f"Second median should be 4.0, got {sensor_states['median_sensor'][1]}" ) - assert abs(sensor_states["moving_avg_sensor"][1] - 4.0) < 0.01, ( + assert sensor_states["moving_avg_sensor"][1] == pytest.approx(4.0), ( f"Second moving avg should be 4.0, got {sensor_states['moving_avg_sensor'][1]}" ) @@ -291,18 +291,18 @@ async def test_sensor_filters_nan_handling( ) # First output - assert abs(min_states[0] - 10.0) < 0.01, ( + assert min_states[0] == pytest.approx(10.0), ( f"First min should be 10.0, got {min_states[0]}" ) - assert abs(max_states[0] - 10.0) < 0.01, ( + assert max_states[0] == pytest.approx(10.0), ( f"First max should be 10.0, got {max_states[0]}" ) # Second output - verify NaN values were ignored - assert abs(min_states[1] - 5.0) < 0.01, ( + assert min_states[1] == pytest.approx(5.0), ( f"Second min should ignore NaN and return 5.0, got {min_states[1]}" ) - assert abs(max_states[1] - 15.0) < 0.01, ( + assert max_states[1] == pytest.approx(15.0), ( f"Second max should ignore NaN and return 15.0, got {max_states[1]}" ) @@ -385,12 +385,12 @@ async def test_sensor_filters_ring_buffer_wraparound( assert len(min_states) == 3, ( f"Should have 3 states, got {len(min_states)}: {min_states}" ) - assert abs(min_states[0] - 10.0) < 0.01, ( + assert min_states[0] == pytest.approx(10.0), ( f"First min should be 10.0, got {min_states[0]}" ) - assert abs(min_states[1] - 5.0) < 0.01, ( + assert min_states[1] == pytest.approx(5.0), ( f"Second min should be 5.0, got {min_states[1]}" ) - assert abs(min_states[2] - 15.0) < 0.01, ( + assert min_states[2] == pytest.approx(15.0), ( f"Third min should be 15.0, got {min_states[2]}" ) From 0200d7c358a4b8a23db1361d7aabe7c510ad536b Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Thu, 16 Oct 2025 06:05:39 -1000 Subject: [PATCH 26/31] fix flakey --- tests/integration/README.md | 76 +++++++++++++++++++++++++++++++++++++ 1 file changed, 76 insertions(+) diff --git a/tests/integration/README.md b/tests/integration/README.md index 8fce81bb80..a2ffb1358b 100644 --- a/tests/integration/README.md +++ b/tests/integration/README.md @@ -7,6 +7,8 @@ This directory contains end-to-end integration tests for ESPHome, focusing on te - `conftest.py` - Common fixtures and utilities - `const.py` - Constants used throughout the integration tests - `types.py` - Type definitions for fixtures and functions +- `state_utils.py` - State handling utilities (e.g., `InitialStateHelper`) +- `sensor_test_utils.py` - Sensor-specific test utilities - `fixtures/` - YAML configuration files for tests - `test_*.py` - Individual test files @@ -26,6 +28,32 @@ The `yaml_config` fixture automatically loads YAML configurations based on the t - `reserved_tcp_port` - Reserves a TCP port by holding the socket open until ESPHome needs it - `unused_tcp_port` - Provides the reserved port number for each test +### Helper Utilities + +#### InitialStateHelper (`state_utils.py`) + +The `InitialStateHelper` class solves a common problem in integration tests: when an API client connects, ESPHome automatically broadcasts the current state of all entities. This can interfere with tests that want to track only new state changes triggered by test actions. + +**What it does:** +- Tracks all entities (except stateless ones like buttons) +- Swallows the first state broadcast for each entity +- Forwards all subsequent state changes to your test callback +- Provides `wait_for_initial_states()` to synchronize before test actions + +**When to use it:** +- Any test that triggers entity state changes and needs to verify them +- Tests that would otherwise see duplicate or unexpected states +- Tests that need clean separation between initial state and test-triggered changes + +**Implementation details:** +- Uses `(device_id, key)` tuples to uniquely identify entities across devices +- Automatically excludes `ButtonInfo` entities (stateless) +- Provides debug logging to track state reception (use `--log-cli-level=DEBUG`) +- Safe for concurrent use with multiple entity types + +**Future work:** +Consider converting existing integration tests to use `InitialStateHelper` for more reliable state tracking and to eliminate race conditions related to initial state broadcasts. + ### Writing Tests The simplest way to write a test is to use the `run_compiled` and `api_client_connected` fixtures: @@ -125,6 +153,54 @@ async def test_my_sensor( ``` ##### State Subscription Pattern + +**Recommended: Using InitialStateHelper** + +When an API client connects, ESPHome automatically sends the current state of all entities. The `InitialStateHelper` (from `state_utils.py`) handles this by swallowing these initial states and only forwarding subsequent state changes to your test callback: + +```python +from .state_utils import InitialStateHelper + +# Track state changes with futures +loop = asyncio.get_running_loop() +states: dict[int, EntityState] = {} +state_future: asyncio.Future[EntityState] = loop.create_future() + +def on_state(state: EntityState) -> None: + """This callback only receives NEW state changes, not initial states.""" + states[state.key] = state + # Check for specific condition using isinstance + if isinstance(state, SensorState) and state.state == expected_value: + if not state_future.done(): + state_future.set_result(state) + +# Get entities and set up state synchronization +entities, services = await client.list_entities_services() +initial_state_helper = InitialStateHelper(entities) + +# Subscribe with the wrapper that filters initial states +client.subscribe_states(initial_state_helper.on_state_wrapper(on_state)) + +# Wait for all initial states to be broadcast +try: + await initial_state_helper.wait_for_initial_states() +except TimeoutError: + pytest.fail("Timeout waiting for initial states") + +# Now perform your test actions - on_state will only receive new changes +# ... trigger state changes ... + +# Wait for expected state +try: + result = await asyncio.wait_for(state_future, timeout=5.0) +except asyncio.TimeoutError: + pytest.fail(f"Expected state not received. Got: {list(states.values())}") +``` + +**Legacy: Manual State Tracking** + +If you need to handle initial states manually (not recommended for new tests): + ```python # Track state changes with futures loop = asyncio.get_running_loop() From b5c4dc13e010f2f7d9b37dec91aed88f254b8217 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Thu, 16 Oct 2025 06:07:41 -1000 Subject: [PATCH 27/31] fix flakey --- tests/integration/state_utils.py | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/tests/integration/state_utils.py b/tests/integration/state_utils.py index 7392393501..5f34bb61d4 100644 --- a/tests/integration/state_utils.py +++ b/tests/integration/state_utils.py @@ -60,17 +60,16 @@ class InitialStateHelper: not_waiting = { (e.device_id, e.key) for e in entities } - self._wait_initial_states - _LOGGER.debug( - "InitialStateHelper: NOT waiting for %d entities: %s", - len(not_waiting), - [ - ( - type(self._entities_by_id[k]).__name__, - self._entities_by_id[k].object_id, - ) + if not_waiting: + not_waiting_info = [ + f"{type(self._entities_by_id[k]).__name__}:{self._entities_by_id[k].object_id}" for k in not_waiting - ], - ) + ] + _LOGGER.debug( + "InitialStateHelper: NOT waiting for %d entities: %s", + len(not_waiting), + not_waiting_info, + ) # Create future in the running event loop self._initial_states_received = asyncio.get_running_loop().create_future() From 7be04916acb1750e1b23817ae0488a1c23851d4e Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Thu, 16 Oct 2025 06:09:38 -1000 Subject: [PATCH 28/31] fix flakey --- tests/integration/README.md | 2 +- .../{sensor_test_utils.py => sensor_utils.py} | 0 tests/integration/state_utils.py | 22 +++++++++++++++++++ .../test_sensor_filters_ring_buffer.py | 5 ++--- .../test_sensor_filters_sliding_window.py | 9 ++++---- 5 files changed, 29 insertions(+), 9 deletions(-) rename tests/integration/{sensor_test_utils.py => sensor_utils.py} (100%) diff --git a/tests/integration/README.md b/tests/integration/README.md index a2ffb1358b..11c33fc5db 100644 --- a/tests/integration/README.md +++ b/tests/integration/README.md @@ -8,7 +8,7 @@ This directory contains end-to-end integration tests for ESPHome, focusing on te - `const.py` - Constants used throughout the integration tests - `types.py` - Type definitions for fixtures and functions - `state_utils.py` - State handling utilities (e.g., `InitialStateHelper`) -- `sensor_test_utils.py` - Sensor-specific test utilities +- `sensor_utils.py` - Sensor-specific test utilities - `fixtures/` - YAML configuration files for tests - `test_*.py` - Individual test files diff --git a/tests/integration/sensor_test_utils.py b/tests/integration/sensor_utils.py similarity index 100% rename from tests/integration/sensor_test_utils.py rename to tests/integration/sensor_utils.py diff --git a/tests/integration/state_utils.py b/tests/integration/state_utils.py index 5f34bb61d4..58d6d2790f 100644 --- a/tests/integration/state_utils.py +++ b/tests/integration/state_utils.py @@ -10,6 +10,28 @@ from aioesphomeapi import ButtonInfo, EntityInfo, EntityState _LOGGER = logging.getLogger(__name__) +def build_key_to_entity_mapping( + entities: list[EntityInfo], entity_names: list[str] +) -> dict[int, str]: + """Build a mapping from entity keys to entity names. + + Args: + entities: List of entity info objects from the API + entity_names: List of entity names to search for in object_ids + + Returns: + Dictionary mapping entity keys to entity names + """ + key_to_entity: dict[int, str] = {} + for entity in entities: + obj_id = entity.object_id.lower() + for entity_name in entity_names: + if entity_name in obj_id: + key_to_entity[entity.key] = entity_name + break + return key_to_entity + + class InitialStateHelper: """Helper to wait for initial states before processing test states. diff --git a/tests/integration/test_sensor_filters_ring_buffer.py b/tests/integration/test_sensor_filters_ring_buffer.py index 5d00986cc2..c8be8edce0 100644 --- a/tests/integration/test_sensor_filters_ring_buffer.py +++ b/tests/integration/test_sensor_filters_ring_buffer.py @@ -7,8 +7,7 @@ import asyncio from aioesphomeapi import EntityState, SensorState import pytest -from .sensor_test_utils import build_key_to_sensor_mapping -from .state_utils import InitialStateHelper +from .state_utils import InitialStateHelper, build_key_to_entity_mapping from .types import APIClientConnectedFactory, RunCompiledFunction @@ -67,7 +66,7 @@ async def test_sensor_filters_ring_buffer( entities, services = await client.list_entities_services() # Build key-to-sensor mapping - key_to_sensor = build_key_to_sensor_mapping( + key_to_sensor = build_key_to_entity_mapping( entities, [ "sliding_min", diff --git a/tests/integration/test_sensor_filters_sliding_window.py b/tests/integration/test_sensor_filters_sliding_window.py index 57ab65acd4..b0688a6536 100644 --- a/tests/integration/test_sensor_filters_sliding_window.py +++ b/tests/integration/test_sensor_filters_sliding_window.py @@ -7,8 +7,7 @@ import asyncio from aioesphomeapi import EntityState, SensorState import pytest -from .sensor_test_utils import build_key_to_sensor_mapping -from .state_utils import InitialStateHelper +from .state_utils import InitialStateHelper, build_key_to_entity_mapping from .types import APIClientConnectedFactory, RunCompiledFunction @@ -98,7 +97,7 @@ async def test_sensor_filters_sliding_window( entities, services = await client.list_entities_services() # Build key-to-sensor mapping - key_to_sensor = build_key_to_sensor_mapping( + key_to_sensor = build_key_to_entity_mapping( entities, [ "min_sensor", @@ -245,7 +244,7 @@ async def test_sensor_filters_nan_handling( entities, services = await client.list_entities_services() # Build key-to-sensor mapping - key_to_sensor = build_key_to_sensor_mapping(entities, ["min_nan", "max_nan"]) + key_to_sensor = build_key_to_entity_mapping(entities, ["min_nan", "max_nan"]) # Set up initial state helper with all entities initial_state_helper = InitialStateHelper(entities) @@ -345,7 +344,7 @@ async def test_sensor_filters_ring_buffer_wraparound( entities, services = await client.list_entities_services() # Build key-to-sensor mapping - key_to_sensor = build_key_to_sensor_mapping(entities, ["wraparound_min"]) + key_to_sensor = build_key_to_entity_mapping(entities, ["wraparound_min"]) # Set up initial state helper with all entities initial_state_helper = InitialStateHelper(entities) From 0cff6acdf4a7b8f8ab763535f57c3c9d1147e194 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Thu, 16 Oct 2025 06:09:44 -1000 Subject: [PATCH 29/31] fix flakey --- tests/integration/sensor_utils.py | 27 --------------------------- 1 file changed, 27 deletions(-) delete mode 100644 tests/integration/sensor_utils.py diff --git a/tests/integration/sensor_utils.py b/tests/integration/sensor_utils.py deleted file mode 100644 index c3843a26ab..0000000000 --- a/tests/integration/sensor_utils.py +++ /dev/null @@ -1,27 +0,0 @@ -"""Shared utilities for sensor integration tests.""" - -from __future__ import annotations - -from aioesphomeapi import EntityInfo - - -def build_key_to_sensor_mapping( - entities: list[EntityInfo], sensor_names: list[str] -) -> dict[int, str]: - """Build a mapping from entity keys to sensor names. - - Args: - entities: List of entity info objects from the API - sensor_names: List of sensor names to search for in object_ids - - Returns: - Dictionary mapping entity keys to sensor names - """ - key_to_sensor: dict[int, str] = {} - for entity in entities: - obj_id = entity.object_id.lower() - for sensor_name in sensor_names: - if sensor_name in obj_id: - key_to_sensor[entity.key] = sensor_name - break - return key_to_sensor From 1118ef32c3437f29e8c8ac02dcc6885437b279d4 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Thu, 16 Oct 2025 06:16:37 -1000 Subject: [PATCH 30/31] preen --- tests/integration/README.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/integration/README.md b/tests/integration/README.md index 11c33fc5db..2a6b6fe564 100644 --- a/tests/integration/README.md +++ b/tests/integration/README.md @@ -7,8 +7,7 @@ This directory contains end-to-end integration tests for ESPHome, focusing on te - `conftest.py` - Common fixtures and utilities - `const.py` - Constants used throughout the integration tests - `types.py` - Type definitions for fixtures and functions -- `state_utils.py` - State handling utilities (e.g., `InitialStateHelper`) -- `sensor_utils.py` - Sensor-specific test utilities +- `state_utils.py` - State handling utilities (e.g., `InitialStateHelper`, `build_key_to_entity_mapping`) - `fixtures/` - YAML configuration files for tests - `test_*.py` - Individual test files From 5e1ee92754c3262d6dc8af99830d2fc6099ec2a8 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Fri, 17 Oct 2025 16:34:25 -1000 Subject: [PATCH 31/31] add tests --- tests/components/sensor/common.yaml | 101 ++++++++++++++++++ tests/components/sensor/test.esp8266-ard.yaml | 1 + 2 files changed, 102 insertions(+) create mode 100644 tests/components/sensor/common.yaml create mode 100644 tests/components/sensor/test.esp8266-ard.yaml diff --git a/tests/components/sensor/common.yaml b/tests/components/sensor/common.yaml new file mode 100644 index 0000000000..ace7d0a38a --- /dev/null +++ b/tests/components/sensor/common.yaml @@ -0,0 +1,101 @@ +sensor: + # Source sensor for testing filters + - platform: template + name: "Source Sensor" + id: source_sensor + lambda: return 42.0; + update_interval: 1s + + # Streaming filters (window_size == send_every) - uses StreamingFilter base class + - platform: copy + source_id: source_sensor + name: "Streaming Min Filter" + filters: + - min: + window_size: 10 + send_every: 10 # Batch window → StreamingMinFilter + + - platform: copy + source_id: source_sensor + name: "Streaming Max Filter" + filters: + - max: + window_size: 10 + send_every: 10 # Batch window → StreamingMaxFilter + + - platform: copy + source_id: source_sensor + name: "Streaming Moving Average Filter" + filters: + - sliding_window_moving_average: + window_size: 10 + send_every: 10 # Batch window → StreamingMovingAverageFilter + + # Sliding window filters (window_size != send_every) - uses SlidingWindowFilter base class with ring buffer + - platform: copy + source_id: source_sensor + name: "Sliding Min Filter" + filters: + - min: + window_size: 10 + send_every: 5 # Sliding window → MinFilter with ring buffer + + - platform: copy + source_id: source_sensor + name: "Sliding Max Filter" + filters: + - max: + window_size: 10 + send_every: 5 # Sliding window → MaxFilter with ring buffer + + - platform: copy + source_id: source_sensor + name: "Sliding Median Filter" + filters: + - median: + window_size: 10 + send_every: 5 # Sliding window → MedianFilter with ring buffer + + - platform: copy + source_id: source_sensor + name: "Sliding Quantile Filter" + filters: + - quantile: + window_size: 10 + send_every: 5 + quantile: 0.9 # Sliding window → QuantileFilter with ring buffer + + - platform: copy + source_id: source_sensor + name: "Sliding Moving Average Filter" + filters: + - sliding_window_moving_average: + window_size: 10 + send_every: 5 # Sliding window → SlidingWindowMovingAverageFilter with ring buffer + + # Edge cases + - platform: copy + source_id: source_sensor + name: "Large Batch Window Min" + filters: + - min: + window_size: 1000 + send_every: 1000 # Large batch → StreamingMinFilter (4 bytes, not 4KB) + + - platform: copy + source_id: source_sensor + name: "Small Sliding Window" + filters: + - median: + window_size: 3 + send_every: 1 # Frequent output → MedianFilter with 3-element ring buffer + + # send_first_at parameter test + - platform: copy + source_id: source_sensor + name: "Early Send Filter" + filters: + - max: + window_size: 10 + send_every: 10 + send_first_at: 1 # Send after first value diff --git a/tests/components/sensor/test.esp8266-ard.yaml b/tests/components/sensor/test.esp8266-ard.yaml new file mode 100644 index 0000000000..dade44d145 --- /dev/null +++ b/tests/components/sensor/test.esp8266-ard.yaml @@ -0,0 +1 @@ +<<: !include common.yaml