Merge pull request #6132 from esphome/bump-2023.12.9

2023.12.9

docker/Dockerfile

@@ -34,7 +34,7 @@ RUN \
         python3-wheel=0.38.4-2 \
         iputils-ping=3:20221126-1 \
         git=1:2.39.2-1.1 \
-        curl=7.88.1-10+deb12u4 \
+        curl=7.88.1-10+deb12u5 \
         openssh-client=1:9.2p1-2+deb12u1 \
         python3-cffi=1.15.1-5 \
         libcairo2=1.16.0-7 \

esphome/components/api/api_server.cpp

@@ -319,7 +319,7 @@ void APIServer::set_reboot_timeout(uint32_t reboot_timeout) { this->reboot_timeo
 #ifdef USE_HOMEASSISTANT_TIME
 void APIServer::request_time() {
   for (auto &client : this->clients_) {
-    if (!client->remove_ && client->connection_state_ == APIConnection::ConnectionState::CONNECTED)
+    if (!client->remove_ && client->is_authenticated())
       client->send_time_request();
   }
 }

esphome/components/bl0940/sensor.py

@@ -18,6 +18,7 @@ from esphome.const import (
     UNIT_KILOWATT_HOURS,
     UNIT_VOLT,
     UNIT_WATT,
+    STATE_CLASS_TOTAL_INCREASING,
 )
 
 DEPENDENCIES = ["uart"]
@@ -54,6 +55,7 @@ CONFIG_SCHEMA = (
                 unit_of_measurement=UNIT_KILOWATT_HOURS,
                 accuracy_decimals=0,
                 device_class=DEVICE_CLASS_ENERGY,
+                state_class=STATE_CLASS_TOTAL_INCREASING,
             ),
             cv.Optional(CONF_INTERNAL_TEMPERATURE): sensor.sensor_schema(
                 unit_of_measurement=UNIT_CELSIUS,

esphome/components/bl0942/sensor.py

@@ -19,6 +19,7 @@ from esphome.const import (
     UNIT_VOLT,
     UNIT_WATT,
     UNIT_HERTZ,
+    STATE_CLASS_TOTAL_INCREASING,
 )
 
 DEPENDENCIES = ["uart"]
@@ -52,6 +53,7 @@ CONFIG_SCHEMA = (
                 unit_of_measurement=UNIT_KILOWATT_HOURS,
                 accuracy_decimals=0,
                 device_class=DEVICE_CLASS_ENERGY,
+                state_class=STATE_CLASS_TOTAL_INCREASING,
             ),
             cv.Optional(CONF_FREQUENCY): sensor.sensor_schema(
                 unit_of_measurement=UNIT_HERTZ,

esphome/components/dht/dht.cpp

@@ -217,8 +217,12 @@ bool HOT IRAM_ATTR DHT::read_sensor_(float *temperature, float *humidity, bool r
     uint16_t raw_humidity = (uint16_t(data[0] & 0xFF) << 8) | (data[1] & 0xFF);
     uint16_t raw_temperature = (uint16_t(data[2] & 0xFF) << 8) | (data[3] & 0xFF);
 
-    if (this->model_ != DHT_MODEL_DHT22_TYPE2 && (raw_temperature & 0x8000) != 0)
-      raw_temperature = ~(raw_temperature & 0x7FFF);
+    if (raw_temperature & 0x8000) {
+      if (!(raw_temperature & 0x4000))
+        raw_temperature = ~(raw_temperature & 0x7FFF);
+    } else if (raw_temperature & 0x800) {
+      raw_temperature |= 0xf000;
+    }
 
     if (raw_temperature == 1 && raw_humidity == 10) {
       if (report_errors) {

esphome/components/display/__init__.py

@@ -145,7 +145,7 @@ async def display_page_show_to_code(config, action_id, template_arg, args):
     DisplayPageShowNextAction,
     maybe_simple_id(
         {
-            cv.Required(CONF_ID): cv.templatable(cv.use_id(DisplayBuffer)),
+            cv.Required(CONF_ID): cv.templatable(cv.use_id(Display)),
         }
     ),
 )
@@ -159,7 +159,7 @@ async def display_page_show_next_to_code(config, action_id, template_arg, args):
     DisplayPageShowPrevAction,
     maybe_simple_id(
         {
-            cv.Required(CONF_ID): cv.templatable(cv.use_id(DisplayBuffer)),
+            cv.Required(CONF_ID): cv.templatable(cv.use_id(Display)),
         }
     ),
 )
@@ -173,7 +173,7 @@ async def display_page_show_previous_to_code(config, action_id, template_arg, ar
     DisplayIsDisplayingPageCondition,
     cv.maybe_simple_value(
         {
-            cv.GenerateID(CONF_ID): cv.use_id(DisplayBuffer),
+            cv.GenerateID(CONF_ID): cv.use_id(Display),
             cv.Required(CONF_PAGE_ID): cv.use_id(DisplayPage),
         },
         key=CONF_PAGE_ID,

esphome/components/esp32_rmt_led_strip/led_strip.cpp

@@ -13,6 +13,8 @@ namespace esp32_rmt_led_strip {
 
 static const char *const TAG = "esp32_rmt_led_strip";
 
+static const uint32_t RMT_CLK_FREQ = 80000000;
+
 static const uint8_t RMT_CLK_DIV = 2;
 
 void ESP32RMTLEDStripLightOutput::setup() {
@@ -65,7 +67,7 @@ void ESP32RMTLEDStripLightOutput::setup() {
 
 void ESP32RMTLEDStripLightOutput::set_led_params(uint32_t bit0_high, uint32_t bit0_low, uint32_t bit1_high,
                                                  uint32_t bit1_low) {
-  float ratio = (float) APB_CLK_FREQ / RMT_CLK_DIV / 1e09f;
+  float ratio = (float) RMT_CLK_FREQ / RMT_CLK_DIV / 1e09f;
 
   // 0-bit
   this->bit0_.duration0 = (uint32_t) (ratio * bit0_high);

esphome/components/improv_serial/improv_serial_component.cpp

@@ -52,7 +52,7 @@ optional<uint8_t> ImprovSerialComponent::read_byte_() {
         size_t available;
         uart_get_buffered_data_len(this->uart_num_, &available);
         if (available) {
-          uart_read_bytes(this->uart_num_, &data, 1, 20 / portTICK_PERIOD_MS);
+          uart_read_bytes(this->uart_num_, &data, 1, 0);
           byte = data;
         }
       }
@@ -71,7 +71,7 @@ optional<uint8_t> ImprovSerialComponent::read_byte_() {
 #endif  // USE_ESP32_VARIANT_ESP32S2 || USE_ESP32_VARIANT_ESP32S3
 #if defined(USE_ESP32_VARIANT_ESP32C3) || defined(USE_ESP32_VARIANT_ESP32C6) || defined(USE_ESP32_VARIANT_ESP32S3)
     case logger::UART_SELECTION_USB_SERIAL_JTAG: {
-      if (usb_serial_jtag_read_bytes((char *) &data, 1, 20 / portTICK_PERIOD_MS)) {
+      if (usb_serial_jtag_read_bytes((char *) &data, 1, 0)) {
         byte = data;
       }
       break;

esphome/components/inkplate6/inkplate.cpp

@@ -55,6 +55,9 @@ void Inkplate6::setup() {
   this->wakeup_pin_->digital_write(false);
 }
 
+/**
+ * Allocate buffers. May be called after setup to re-initialise if e.g. greyscale is changed.
+ */
 void Inkplate6::initialize_() {
   ExternalRAMAllocator<uint8_t> allocator(ExternalRAMAllocator<uint8_t>::ALLOW_FAILURE);
   ExternalRAMAllocator<uint32_t> allocator32(ExternalRAMAllocator<uint32_t>::ALLOW_FAILURE);

esphome/components/inkplate6/inkplate.h

@@ -68,8 +68,9 @@ class Inkplate6 : public display::DisplayBuffer, public i2c::I2CDevice {
 
   void set_greyscale(bool greyscale) {
     this->greyscale_ = greyscale;
-    this->initialize_();
     this->block_partial_ = true;
+    if (this->is_ready())
+      this->initialize_();
   }
   void set_partial_updating(bool partial_updating) { this->partial_updating_ = partial_updating; }
   void set_full_update_every(uint32_t full_update_every) { this->full_update_every_ = full_update_every; }

esphome/components/libretiny/__init__.py

@@ -309,7 +309,7 @@ async def component_to_code(config):
         lt_options["LT_UART_SILENT_ENABLED"] = 0
         lt_options["LT_UART_SILENT_ALL"] = 0
     # set default UART port
-    if uart_port := framework.get(CONF_UART_PORT, None) is not None:
+    if (uart_port := framework.get(CONF_UART_PORT, None)) is not None:
         lt_options["LT_UART_DEFAULT_PORT"] = uart_port
     # add custom options
     lt_options.update(framework[CONF_OPTIONS])

esphome/components/logger/__init__.py

@@ -84,7 +84,7 @@ UART_SELECTION_ESP32 = {
     VARIANT_ESP32: [UART0, UART1, UART2],
     VARIANT_ESP32S2: [UART0, UART1, USB_CDC],
     VARIANT_ESP32S3: [UART0, UART1, USB_CDC, USB_SERIAL_JTAG],
-    VARIANT_ESP32C3: [UART0, UART1, USB_SERIAL_JTAG],
+    VARIANT_ESP32C3: [UART0, UART1, USB_CDC, USB_SERIAL_JTAG],
     VARIANT_ESP32C2: [UART0, UART1],
     VARIANT_ESP32C6: [UART0, UART1, USB_CDC, USB_SERIAL_JTAG],
     VARIANT_ESP32H2: [UART0, UART1, USB_CDC, USB_SERIAL_JTAG],
@@ -172,9 +172,10 @@ CONFIG_SCHEMA = cv.All(
                 esp8266=UART0,
                 esp32=UART0,
                 esp32_s2=USB_CDC,
-                esp32_s3_idf=USB_SERIAL_JTAG,
-                esp32_c3_idf=USB_SERIAL_JTAG,
                 esp32_s3_arduino=USB_CDC,
+                esp32_s3_idf=USB_SERIAL_JTAG,
+                esp32_c3_arduino=USB_CDC,
+                esp32_c3_idf=USB_SERIAL_JTAG,
                 rp2040=USB_CDC,
                 bk72xx=DEFAULT,
                 rtl87xx=DEFAULT,
@@ -265,6 +266,8 @@ async def to_code(config):
     if CORE.using_arduino:
         if config[CONF_HARDWARE_UART] == USB_CDC:
             cg.add_build_flag("-DARDUINO_USB_CDC_ON_BOOT=1")
+            if CORE.is_esp32 and get_esp32_variant() == VARIANT_ESP32C3:
+                cg.add_build_flag("-DARDUINO_USB_MODE=1")
 
     if CORE.using_esp_idf:
         if config[CONF_HARDWARE_UART] == USB_CDC:

esphome/components/logger/logger.cpp

@@ -272,17 +272,13 @@ void Logger::pre_setup() {
 #endif
 #if defined(USE_ESP32) && \
     (defined(USE_ESP32_VARIANT_ESP32S2) || defined(USE_ESP32_VARIANT_ESP32S3) || defined(USE_ESP32_VARIANT_ESP32C3))
-#if defined(USE_ESP32_VARIANT_ESP32S2) || defined(USE_ESP32_VARIANT_ESP32S3)
+#if defined(USE_ESP32_VARIANT_ESP32S2) || defined(USE_ESP32_VARIANT_ESP32S3) || defined(USE_ESP32_VARIANT_ESP32C3)
       case UART_SELECTION_USB_CDC:
-#endif  // USE_ESP32_VARIANT_ESP32S2 || USE_ESP32_VARIANT_ESP32S3
+#endif  // USE_ESP32_VARIANT_ESP32S2 || USE_ESP32_VARIANT_ESP32S3 || USE_ESP32_VARIANT_ESP32C3
 #if defined(USE_ESP32_VARIANT_ESP32C3) || defined(USE_ESP32_VARIANT_ESP32S3)
       case UART_SELECTION_USB_SERIAL_JTAG:
 #endif  // USE_ESP32_VARIANT_ESP32C3 || USE_ESP32_VARIANT_ESP32S3
-#ifdef USE_ESP32_VARIANT_ESP32C3
-        this->hw_serial_ = &Serial;
-        Serial.begin(this->baud_rate_);
-#endif  // USE_ESP32_VARIANT_ESP32C3
-#if defined(USE_ESP32_VARIANT_ESP32S2) || defined(USE_ESP32_VARIANT_ESP32S3)
+#if defined(USE_ESP32_VARIANT_ESP32S2) || defined(USE_ESP32_VARIANT_ESP32S3) || defined(USE_ESP32_VARIANT_ESP32C3)
 #if ARDUINO_USB_CDC_ON_BOOT
         this->hw_serial_ = &Serial;
         Serial.setTxTimeoutMs(0);  // workaround for 2.0.9 crash when there's no data connection
@@ -291,7 +287,7 @@ void Logger::pre_setup() {
         this->hw_serial_ = &Serial;
         Serial.begin(this->baud_rate_);
 #endif  // ARDUINO_USB_CDC_ON_BOOT
-#endif  // USE_ESP32_VARIANT_ESP32S2 || USE_ESP32_VARIANT_ESP32S3
+#endif  // USE_ESP32_VARIANT_ESP32S2 || USE_ESP32_VARIANT_ESP32S3 || USE_ESP32_VARIANT_ESP32C3
         break;
 #endif  // USE_ESP32 && (USE_ESP32_VARIANT_ESP32S2 || USE_ESP32_VARIANT_ESP32S3 || USE_ESP32_VARIANT_ESP32C3)
 #ifdef USE_RP2040

esphome/components/logger/logger.h

@@ -45,9 +45,10 @@ enum UARTSelection {
   UART_SELECTION_UART2,
 #endif  // !USE_ESP32_VARIANT_ESP32C3 && !USE_ESP32_VARIANT_ESP32C6 && !USE_ESP32_VARIANT_ESP32S2 &&
         // !USE_ESP32_VARIANT_ESP32S3 && !USE_ESP32_VARIANT_ESP32H2
-#if defined(USE_ESP32_VARIANT_ESP32S2) || defined(USE_ESP32_VARIANT_ESP32S3)
+#if defined(USE_ESP32_VARIANT_ESP32S2) || defined(USE_ESP32_VARIANT_ESP32S3) || \
+    (defined(USE_ESP32_VARIANT_ESP32C3) && defined(USE_ARDUINO))
   UART_SELECTION_USB_CDC,
-#endif  // USE_ESP32_VARIANT_ESP32S2 || USE_ESP32_VARIANT_ESP32S3
+#endif  // USE_ESP32_VARIANT_ESP32S2 || USE_ESP32_VARIANT_ESP32S3 || USE_ESP32_VARIANT_ESP32C3
 #if defined(USE_ESP32_VARIANT_ESP32C3) || defined(USE_ESP32_VARIANT_ESP32C6) || defined(USE_ESP32_VARIANT_ESP32S3) || \
     defined(USE_ESP32_VARIANT_ESP32H2)
   UART_SELECTION_USB_SERIAL_JTAG,

esphome/components/pmsx003/pmsx003.cpp

@@ -195,7 +195,7 @@ void PMSX003Component::send_command_(uint8_t cmd, uint16_t data) {
 void PMSX003Component::parse_data_() {
   switch (this->type_) {
     case PMSX003_TYPE_5003ST: {
-      float temperature = this->get_16_bit_uint_(30) / 10.0f;
+      float temperature = (int16_t) this->get_16_bit_uint_(30) / 10.0f;
       float humidity = this->get_16_bit_uint_(32) / 10.0f;
 
       ESP_LOGD(TAG, "Got Temperature: %.1f°C, Humidity: %.1f%%", temperature, humidity);
@@ -279,7 +279,7 @@ void PMSX003Component::parse_data_() {
       // Note the pm particles 50um & 100um are not returned,
       // as PMS5003T uses those data values for temperature and humidity.
 
-      float temperature = this->get_16_bit_uint_(24) / 10.0f;
+      float temperature = (int16_t) this->get_16_bit_uint_(24) / 10.0f;
       float humidity = this->get_16_bit_uint_(26) / 10.0f;
 
       ESP_LOGD(TAG,

esphome/components/pylontech/__init__.py

@@ -19,7 +19,7 @@ PylontechComponent = pylontech_ns.class_(
 )
 PylontechBattery = pylontech_ns.class_("PylontechBattery")
 
-CV_NUM_BATTERIES = cv.int_range(1, 6)
+CV_NUM_BATTERIES = cv.int_range(1, 16)
 
 PYLONTECH_COMPONENT_SCHEMA = cv.Schema(
     {

esphome/components/pylontech/pylontech.cpp

@@ -1,5 +1,6 @@
 #include "pylontech.h"
 #include "esphome/core/log.h"
+#include "esphome/core/helpers.h"
 
 namespace esphome {
 namespace pylontech {
@@ -34,26 +35,30 @@ void PylontechComponent::setup() {
 void PylontechComponent::update() { this->write_str("pwr\n"); }
 
 void PylontechComponent::loop() {
-  uint8_t data;
-
-  // pylontech sends a lot of data very suddenly
-  // we need to quickly put it all into our own buffer, otherwise the uart's buffer will overflow
-  while (this->available() > 0) {
-    if (this->read_byte(&data)) {
-      buffer_[buffer_index_write_] += (char) data;
-      if (buffer_[buffer_index_write_].back() == static_cast<char>(ASCII_LF) ||
-          buffer_[buffer_index_write_].length() >= MAX_DATA_LENGTH_BYTES) {
-        // complete line received
-        buffer_index_write_ = (buffer_index_write_ + 1) % NUM_BUFFERS;
+  if (this->available() > 0) {
+    // pylontech sends a lot of data very suddenly
+    // we need to quickly put it all into our own buffer, otherwise the uart's buffer will overflow
+    uint8_t data;
+    int recv = 0;
+    while (this->available() > 0) {
+      if (this->read_byte(&data)) {
+        buffer_[buffer_index_write_] += (char) data;
+        recv++;
+        if (buffer_[buffer_index_write_].back() == static_cast<char>(ASCII_LF) ||
+            buffer_[buffer_index_write_].length() >= MAX_DATA_LENGTH_BYTES) {
+          // complete line received
+          buffer_index_write_ = (buffer_index_write_ + 1) % NUM_BUFFERS;
+        }
       }
     }
-  }
-
-  // only process one line per call of loop() to not block esphome for too long
-  if (buffer_index_read_ != buffer_index_write_) {
-    this->process_line_(buffer_[buffer_index_read_]);
-    buffer_[buffer_index_read_].clear();
-    buffer_index_read_ = (buffer_index_read_ + 1) % NUM_BUFFERS;
+    ESP_LOGV(TAG, "received %d bytes", recv);
+  } else {
+    // only process one line per call of loop() to not block esphome for too long
+    if (buffer_index_read_ != buffer_index_write_) {
+      this->process_line_(buffer_[buffer_index_read_]);
+      buffer_[buffer_index_read_].clear();
+      buffer_index_read_ = (buffer_index_read_ + 1) % NUM_BUFFERS;
+    }
   }
 }
 
@@ -66,10 +71,11 @@ void PylontechComponent::process_line_(std::string &buffer) {
   // clang-format on
 
   PylontechListener::LineContents l{};
-  const int parsed = sscanf(                                                                                  // NOLINT
-      buffer.c_str(), "%d %d %d %d %d %d %d %d %7s %7s %7s %7s %d%% %*d-%*d-%*d %*d:%*d:%*d %*s %*s %d %*s",  // NOLINT
-      &l.bat_num, &l.volt, &l.curr, &l.tempr, &l.tlow, &l.thigh, &l.vlow, &l.vhigh, l.base_st, l.volt_st,     // NOLINT
-      l.curr_st, l.temp_st, &l.coulomb, &l.mostempr);                                                         // NOLINT
+  char mostempr_s[6];
+  const int parsed = sscanf(                                                                                   // NOLINT
+      buffer.c_str(), "%d %d %d %d %d %d %d %d %7s %7s %7s %7s %d%% %*d-%*d-%*d %*d:%*d:%*d %*s %*s %5s %*s",  // NOLINT
+      &l.bat_num, &l.volt, &l.curr, &l.tempr, &l.tlow, &l.thigh, &l.vlow, &l.vhigh, l.base_st, l.volt_st,      // NOLINT
+      l.curr_st, l.temp_st, &l.coulomb, mostempr_s);                                                           // NOLINT
 
   if (l.bat_num <= 0) {
     ESP_LOGD(TAG, "invalid bat_num in line %s", buffer.substr(0, buffer.size() - 2).c_str());
@@ -79,6 +85,13 @@ void PylontechComponent::process_line_(std::string &buffer) {
     ESP_LOGW(TAG, "invalid line: found only %d items in %s", parsed, buffer.substr(0, buffer.size() - 2).c_str());
     return;
   }
+  auto mostempr_parsed = parse_number<int>(mostempr_s);
+  if (mostempr_parsed.has_value()) {
+    l.mostempr = mostempr_parsed.value();
+  } else {
+    l.mostempr = -300;
+    ESP_LOGW(TAG, "bat_num %d: received no mostempr", l.bat_num);
+  }
 
   for (PylontechListener *listener : this->listeners_) {
     listener->on_line_read(&l);

esphome/components/remote_base/pronto_protocol.cpp

@@ -227,16 +227,17 @@ optional<ProntoData> ProntoProtocol::decode(RemoteReceiveData src) {
 }
 
 void ProntoProtocol::dump(const ProntoData &data) {
-  std::string first, rest;
-  if (data.data.size() < 230) {
-    first = data.data;
-  } else {
-    first = data.data.substr(0, 229);
-    rest = data.data.substr(230);
-  }
-  ESP_LOGI(TAG, "Received Pronto: data=%s", first.c_str());
-  if (!rest.empty()) {
-    ESP_LOGI(TAG, "%s", rest.c_str());
+  std::string rest;
+
+  rest = data.data;
+  ESP_LOGI(TAG, "Received Pronto: data=");
+  while (true) {
+    ESP_LOGI(TAG, "%s", rest.substr(0, 230).c_str());
+    if (rest.size() > 230) {
+      rest = rest.substr(230);
+    } else {
+      break;
+    }
   }
 }
 

esphome/components/sen5x/sen5x.cpp

@@ -352,7 +352,7 @@ void SEN5XComponent::update() {
     float humidity = measurements[4] / 100.0;
     if (measurements[4] == 0xFFFF)
       humidity = NAN;
-    float temperature = measurements[5] / 200.0;
+    float temperature = (int16_t) measurements[5] / 200.0;
     if (measurements[5] == 0xFFFF)
       temperature = NAN;
     float voc = measurements[6] / 10.0;

esphome/components/voice_assistant/voice_assistant.cpp

@@ -17,7 +17,7 @@ static const char *const TAG = "voice_assistant";
 
 static const size_t SAMPLE_RATE_HZ = 16000;
 static const size_t INPUT_BUFFER_SIZE = 32 * SAMPLE_RATE_HZ / 1000;  // 32ms * 16kHz / 1000ms
-static const size_t BUFFER_SIZE = 1000 * SAMPLE_RATE_HZ / 1000;      // 1s
+static const size_t BUFFER_SIZE = 1024 * SAMPLE_RATE_HZ / 1000;
 static const size_t SEND_BUFFER_SIZE = INPUT_BUFFER_SIZE * sizeof(int16_t);
 static const size_t RECEIVE_SIZE = 1024;
 static const size_t SPEAKER_BUFFER_SIZE = 16 * RECEIVE_SIZE;
@@ -86,14 +86,14 @@ void VoiceAssistant::setup() {
 
 #ifdef USE_ESP_ADF
   this->vad_instance_ = vad_create(VAD_MODE_4);
+#endif
 
-  this->ring_buffer_ = rb_create(BUFFER_SIZE, sizeof(int16_t));
+  this->ring_buffer_ = RingBuffer::create(BUFFER_SIZE * sizeof(int16_t));
   if (this->ring_buffer_ == nullptr) {
     ESP_LOGW(TAG, "Could not allocate ring buffer");
     this->mark_failed();
     return;
   }
-#endif
 
   ExternalRAMAllocator<uint8_t> send_allocator(ExternalRAMAllocator<uint8_t>::ALLOW_FAILURE);
   this->send_buffer_ = send_allocator.allocate(SEND_BUFFER_SIZE);
@@ -112,14 +112,8 @@ int VoiceAssistant::read_microphone_() {
       memset(this->input_buffer_, 0, INPUT_BUFFER_SIZE * sizeof(int16_t));
       return 0;
     }
-#ifdef USE_ESP_ADF
     // Write audio into ring buffer
-    int available = rb_bytes_available(this->ring_buffer_);
-    if (available < bytes_read) {
-      rb_read(this->ring_buffer_, nullptr, bytes_read - available, 0);
-    }
-    rb_write(this->ring_buffer_, (char *) this->input_buffer_, bytes_read, 0);
-#endif
+    this->ring_buffer_->write((void *) this->input_buffer_, bytes_read);
   } else {
     ESP_LOGD(TAG, "microphone not running");
   }
@@ -141,9 +135,9 @@ void VoiceAssistant::loop() {
   switch (this->state_) {
     case State::IDLE: {
       if (this->continuous_ && this->desired_state_ == State::IDLE) {
+        this->ring_buffer_->reset();
 #ifdef USE_ESP_ADF
         if (this->use_wake_word_) {
-          rb_reset(this->ring_buffer_);
           this->set_state_(State::START_MICROPHONE, State::WAIT_FOR_VAD);
         } else
 #endif
@@ -236,19 +230,15 @@ void VoiceAssistant::loop() {
       break;  // State changed when udp server port received
     }
     case State::STREAMING_MICROPHONE: {
-      size_t bytes_read = this->read_microphone_();
-#ifdef USE_ESP_ADF
-      if (rb_bytes_filled(this->ring_buffer_) >= SEND_BUFFER_SIZE) {
-        rb_read(this->ring_buffer_, (char *) this->send_buffer_, SEND_BUFFER_SIZE, 0);
-        this->socket_->sendto(this->send_buffer_, SEND_BUFFER_SIZE, 0, (struct sockaddr *) &this->dest_addr_,
+      this->read_microphone_();
+      size_t available = this->ring_buffer_->available();
+      while (available >= SEND_BUFFER_SIZE) {
+        size_t read_bytes = this->ring_buffer_->read((void *) this->send_buffer_, SEND_BUFFER_SIZE, 0);
+        this->socket_->sendto(this->send_buffer_, read_bytes, 0, (struct sockaddr *) &this->dest_addr_,
                               sizeof(this->dest_addr_));
+        available = this->ring_buffer_->available();
       }
-#else
-      if (bytes_read > 0) {
-        this->socket_->sendto(this->input_buffer_, bytes_read, 0, (struct sockaddr *) &this->dest_addr_,
-                              sizeof(this->dest_addr_));
-      }
-#endif
+
       break;
     }
     case State::STOP_MICROPHONE: {
@@ -473,9 +463,9 @@ void VoiceAssistant::request_start(bool continuous, bool silence_detection) {
   if (this->state_ == State::IDLE) {
     this->continuous_ = continuous;
     this->silence_detection_ = silence_detection;
+    this->ring_buffer_->reset();
 #ifdef USE_ESP_ADF
     if (this->use_wake_word_) {
-      rb_reset(this->ring_buffer_);
       this->set_state_(State::START_MICROPHONE, State::WAIT_FOR_VAD);
     } else
 #endif
@@ -618,9 +608,9 @@ void VoiceAssistant::on_event(const api::VoiceAssistantEventResponse &msg) {
     case api::enums::VOICE_ASSISTANT_RUN_END: {
       ESP_LOGD(TAG, "Assist Pipeline ended");
       if (this->state_ == State::STREAMING_MICROPHONE) {
+        this->ring_buffer_->reset();
 #ifdef USE_ESP_ADF
         if (this->use_wake_word_) {
-          rb_reset(this->ring_buffer_);
           // No need to stop the microphone since we didn't use the speaker
           this->set_state_(State::WAIT_FOR_VAD, State::WAITING_FOR_VAD);
         } else

esphome/components/voice_assistant/voice_assistant.h

@@ -7,6 +7,7 @@
 #include "esphome/core/automation.h"
 #include "esphome/core/component.h"
 #include "esphome/core/helpers.h"
+#include "esphome/core/ring_buffer.h"
 
 #include "esphome/components/api/api_connection.h"
 #include "esphome/components/api/api_pb2.h"
@@ -21,7 +22,6 @@
 
 #ifdef USE_ESP_ADF
 #include <esp_vad.h>
-#include <ringbuf.h>
 #endif
 
 namespace esphome {
@@ -177,10 +177,10 @@ class VoiceAssistant : public Component {
 
 #ifdef USE_ESP_ADF
   vad_handle_t vad_instance_;
-  ringbuf_handle_t ring_buffer_;
   uint8_t vad_threshold_{5};
   uint8_t vad_counter_{0};
 #endif
+  std::unique_ptr<RingBuffer> ring_buffer_;
 
   bool use_wake_word_;
   uint8_t noise_suppression_level_;

esphome/const.py

@@ -1,6 +1,6 @@
 """Constants used by esphome."""
 
-__version__ = "2023.12.4"
+__version__ = "2023.12.9"
 
 ALLOWED_NAME_CHARS = "abcdefghijklmnopqrstuvwxyz0123456789-_"
 VALID_SUBSTITUTIONS_CHARACTERS = (

esphome/core/ring_buffer.cpp

@@ -0,0 +1,50 @@
+#include "ring_buffer.h"
+
+#include "esphome/core/helpers.h"
+#include "esphome/core/log.h"
+
+#ifdef USE_ESP32
+
+#include "helpers.h"
+
+namespace esphome {
+
+static const char *const TAG = "ring_buffer";
+
+std::unique_ptr<RingBuffer> RingBuffer::create(size_t len) {
+  std::unique_ptr<RingBuffer> rb = make_unique<RingBuffer>();
+
+  ExternalRAMAllocator<uint8_t> allocator(ExternalRAMAllocator<uint8_t>::ALLOW_FAILURE);
+  rb->storage_ = allocator.allocate(len + 1);
+  if (rb->storage_ == nullptr) {
+    return nullptr;
+  }
+
+  rb->handle_ = xStreamBufferCreateStatic(len + 1, 0, rb->storage_, &rb->structure_);
+  ESP_LOGD(TAG, "Created ring buffer with size %u", len);
+  return rb;
+}
+
+size_t RingBuffer::read(void *data, size_t len, TickType_t ticks_to_wait) {
+  return xStreamBufferReceive(this->handle_, data, len, ticks_to_wait);
+}
+
+size_t RingBuffer::write(void *data, size_t len) {
+  size_t free = this->free();
+  if (free < len) {
+    size_t needed = len - free;
+    uint8_t discard[needed];
+    xStreamBufferReceive(this->handle_, discard, needed, 0);
+  }
+  return xStreamBufferSend(this->handle_, data, len, 0);
+}
+
+size_t RingBuffer::available() const { return xStreamBufferBytesAvailable(this->handle_); }
+
+size_t RingBuffer::free() const { return xStreamBufferSpacesAvailable(this->handle_); }
+
+BaseType_t RingBuffer::reset() { return xStreamBufferReset(this->handle_); }
+
+}  // namespace esphome
+
+#endif

esphome/core/ring_buffer.h

@@ -0,0 +1,34 @@
+#pragma once
+
+#ifdef USE_ESP32
+
+#include <freertos/FreeRTOS.h>
+#include <freertos/stream_buffer.h>
+
+#include <cinttypes>
+#include <memory>
+
+namespace esphome {
+
+class RingBuffer {
+ public:
+  size_t read(void *data, size_t len, TickType_t ticks_to_wait = 0);
+
+  size_t write(void *data, size_t len);
+
+  size_t available() const;
+  size_t free() const;
+
+  BaseType_t reset();
+
+  static std::unique_ptr<RingBuffer> create(size_t len);
+
+ protected:
+  StreamBufferHandle_t handle_;
+  StaticStreamBuffer_t structure_;
+  uint8_t *storage_;
+};
+
+}  // namespace esphome
+
+#endif

esphome/yaml_util.py

@@ -1,36 +1,37 @@
+from __future__ import annotations
+
 import fnmatch
 import functools
 import inspect
 import logging
 import math
 import os
-
 import uuid
+from typing import Any
+
 import yaml
 import yaml.constructor
+from yaml import SafeLoader as PurePythonLoader
+
+try:
+    from yaml import CSafeLoader as FastestAvailableSafeLoader
+except ImportError:
+    FastestAvailableSafeLoader = PurePythonLoader
 
 from esphome import core
-from esphome.config_helpers import read_config_file, Extend, Remove
+from esphome.config_helpers import Extend, Remove, read_config_file
 from esphome.core import (
+    CORE,
+    DocumentRange,
     EsphomeError,
     IPAddress,
     Lambda,
     MACAddress,
     TimePeriod,
-    DocumentRange,
-    CORE,
 )
 from esphome.helpers import add_class_to_obj
 from esphome.util import OrderedDict, filter_yaml_files
 
-try:
-    from yaml import CSafeLoader as FastestAvailableSafeLoader
-except ImportError:
-    from yaml import (  # type: ignore[assignment]
-        SafeLoader as FastestAvailableSafeLoader,
-    )
-
-
 _LOGGER = logging.getLogger(__name__)
 
 # Mostly copied from Home Assistant because that code works fine and
@@ -97,7 +98,7 @@ def _add_data_ref(fn):
     return wrapped
 
 
-class ESPHomeLoader(FastestAvailableSafeLoader):
+class ESPHomeLoaderMixin:
     """Loader class that keeps track of line numbers."""
 
     @_add_data_ref
@@ -282,8 +283,8 @@ class ESPHomeLoader(FastestAvailableSafeLoader):
             return file, vars
 
         def substitute_vars(config, vars):
-            from esphome.const import CONF_SUBSTITUTIONS
             from esphome.components import substitutions
+            from esphome.const import CONF_SUBSTITUTIONS
 
             org_subs = None
             result = config
@@ -367,50 +368,64 @@ class ESPHomeLoader(FastestAvailableSafeLoader):
         return Remove(str(node.value))
 
 
-ESPHomeLoader.add_constructor("tag:yaml.org,2002:int", ESPHomeLoader.construct_yaml_int)
-ESPHomeLoader.add_constructor(
-    "tag:yaml.org,2002:float", ESPHomeLoader.construct_yaml_float
-)
-ESPHomeLoader.add_constructor(
-    "tag:yaml.org,2002:binary", ESPHomeLoader.construct_yaml_binary
-)
-ESPHomeLoader.add_constructor(
-    "tag:yaml.org,2002:omap", ESPHomeLoader.construct_yaml_omap
-)
-ESPHomeLoader.add_constructor("tag:yaml.org,2002:str", ESPHomeLoader.construct_yaml_str)
-ESPHomeLoader.add_constructor("tag:yaml.org,2002:seq", ESPHomeLoader.construct_yaml_seq)
-ESPHomeLoader.add_constructor("tag:yaml.org,2002:map", ESPHomeLoader.construct_yaml_map)
-ESPHomeLoader.add_constructor("!env_var", ESPHomeLoader.construct_env_var)
-ESPHomeLoader.add_constructor("!secret", ESPHomeLoader.construct_secret)
-ESPHomeLoader.add_constructor("!include", ESPHomeLoader.construct_include)
-ESPHomeLoader.add_constructor(
-    "!include_dir_list", ESPHomeLoader.construct_include_dir_list
-)
-ESPHomeLoader.add_constructor(
-    "!include_dir_merge_list", ESPHomeLoader.construct_include_dir_merge_list
-)
-ESPHomeLoader.add_constructor(
-    "!include_dir_named", ESPHomeLoader.construct_include_dir_named
-)
-ESPHomeLoader.add_constructor(
-    "!include_dir_merge_named", ESPHomeLoader.construct_include_dir_merge_named
-)
-ESPHomeLoader.add_constructor("!lambda", ESPHomeLoader.construct_lambda)
-ESPHomeLoader.add_constructor("!force", ESPHomeLoader.construct_force)
-ESPHomeLoader.add_constructor("!extend", ESPHomeLoader.construct_extend)
-ESPHomeLoader.add_constructor("!remove", ESPHomeLoader.construct_remove)
+class ESPHomeLoader(ESPHomeLoaderMixin, FastestAvailableSafeLoader):
+    """Loader class that keeps track of line numbers."""
 
 
-def load_yaml(fname, clear_secrets=True):
+class ESPHomePurePythonLoader(ESPHomeLoaderMixin, PurePythonLoader):
+    """Loader class that keeps track of line numbers."""
+
+
+for _loader in (ESPHomeLoader, ESPHomePurePythonLoader):
+    _loader.add_constructor("tag:yaml.org,2002:int", _loader.construct_yaml_int)
+    _loader.add_constructor("tag:yaml.org,2002:float", _loader.construct_yaml_float)
+    _loader.add_constructor("tag:yaml.org,2002:binary", _loader.construct_yaml_binary)
+    _loader.add_constructor("tag:yaml.org,2002:omap", _loader.construct_yaml_omap)
+    _loader.add_constructor("tag:yaml.org,2002:str", _loader.construct_yaml_str)
+    _loader.add_constructor("tag:yaml.org,2002:seq", _loader.construct_yaml_seq)
+    _loader.add_constructor("tag:yaml.org,2002:map", _loader.construct_yaml_map)
+    _loader.add_constructor("!env_var", _loader.construct_env_var)
+    _loader.add_constructor("!secret", _loader.construct_secret)
+    _loader.add_constructor("!include", _loader.construct_include)
+    _loader.add_constructor("!include_dir_list", _loader.construct_include_dir_list)
+    _loader.add_constructor(
+        "!include_dir_merge_list", _loader.construct_include_dir_merge_list
+    )
+    _loader.add_constructor("!include_dir_named", _loader.construct_include_dir_named)
+    _loader.add_constructor(
+        "!include_dir_merge_named", _loader.construct_include_dir_merge_named
+    )
+    _loader.add_constructor("!lambda", _loader.construct_lambda)
+    _loader.add_constructor("!force", _loader.construct_force)
+    _loader.add_constructor("!extend", _loader.construct_extend)
+    _loader.add_constructor("!remove", _loader.construct_remove)
+
+
+def load_yaml(fname: str, clear_secrets: bool = True) -> Any:
     if clear_secrets:
         _SECRET_VALUES.clear()
         _SECRET_CACHE.clear()
     return _load_yaml_internal(fname)
 
 
-def _load_yaml_internal(fname):
+def _load_yaml_internal(fname: str) -> Any:
+    """Load a YAML file."""
     content = read_config_file(fname)
-    loader = ESPHomeLoader(content)
+    try:
+        return _load_yaml_internal_with_type(ESPHomeLoader, fname, content)
+    except EsphomeError:
+        # Loading failed, so we now load with the Python loader which has more
+        # readable exceptions
+        return _load_yaml_internal_with_type(ESPHomePurePythonLoader, fname, content)
+
+
+def _load_yaml_internal_with_type(
+    loader_type: type[ESPHomeLoader] | type[ESPHomePurePythonLoader],
+    fname: str,
+    content: str,
+) -> Any:
+    """Load a YAML file."""
+    loader = loader_type(content)
     loader.name = fname
     try:
         return loader.get_single_data() or OrderedDict()

tests/unit_tests/fixtures/yaml_util/broken_includetest.yaml

@@ -0,0 +1,18 @@
+---
+substitutions:
+  name: original
+
+wifi: !include
+  file: includes/broken_included.yaml.txt
+  vars:
+    name: my_custom_ssid
+
+esphome:
+  # should be substituted as 'original',
+  # not overwritten by vars in the !include above
+  name: ${name}
+  name_add_mac_suffix: true
+  platform: esp8266
+  board: !include {file: includes/scalar.yaml, vars: {var1: nodemcu}}
+
+  libraries: !include {file: includes/list.yaml, vars: {var1: Wire}}

tests/unit_tests/fixtures/yaml_util/includes/broken_included.yaml.txt

@@ -0,0 +1,5 @@
+---
+# yamllint disable-line
+   ssid: ${name}
+# yamllint disable-line
+       fdf: error

tests/unit_tests/test_yaml_util.py

@@ -1,5 +1,6 @@
 from esphome import yaml_util
 from esphome.components import substitutions
+from esphome.core import EsphomeError
 
 
 def test_include_with_vars(fixture_path):
@@ -11,3 +12,13 @@ def test_include_with_vars(fixture_path):
     assert actual["esphome"]["libraries"][0] == "Wire"
     assert actual["esphome"]["board"] == "nodemcu"
     assert actual["wifi"]["ssid"] == "my_custom_ssid"
+
+
+def test_loading_a_broken_yaml_file(fixture_path):
+    """Ensure we fallback to pure python to give good errors."""
+    yaml_file = fixture_path / "yaml_util" / "broken_includetest.yaml"
+
+    try:
+        yaml_util.load_yaml(yaml_file)
+    except EsphomeError as err:
+        assert "broken_included.yaml" in str(err)